alarsyo/spot
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

python: upgrade notebook format to v4

Browse source 
Fixes #311.

* tests/python/ipnbdoctest.py: Adjust to process the new format,
with a lot of inspiration from Vcsn's copy of this file.
* tests/python/_altscc.ipynb, tests/python/_aux.ipynb,
tests/python/acc_cond.ipynb, tests/python/accparse.ipynb,
tests/python/alternation.ipynb, tests/python/atva16-fig2a.ipynb,
tests/python/atva16-fig2b.ipynb, tests/python/automata-io.ipynb,
tests/python/automata.ipynb, tests/python/decompose.ipynb,
tests/python/formulas.ipynb, tests/python/gen.ipynb,
tests/python/highlighting.ipynb, tests/python/ltsmin-dve.ipynb,
tests/python/ltsmin-pml.ipynb, tests/python/parity.ipynb,
tests/python/piperead.ipynb, tests/python/product.ipynb,
tests/python/randaut.ipynb, tests/python/randltl.ipynb,
tests/python/stutter-inv.ipynb, tests/python/testingaut.ipynb,
tests/python/word.ipynb: Upgrade to the new format.
* NEWS: Mention the change.
This commit is contained in:
Alexandre Duret-Lutz 2018-01-07 10:33:38 +01:00
parent 020c981188
commit 58e64e752c
25 changed files with 33431 additions and 33104 deletions
Download patch file Download diff file Expand all files Collapse all files

214
tests/python/ipnbdoctest.py
View file

@ -8,11 +8,14 @@ Each cell is submitted to the kernel, and the outputs are compared
with those stored in the notebook.
"""
# Derived from https://gist.github.com/minrk/2620735
from __future__ import print_function
import os,sys,time
import base64
import re
import pprint
from difflib import unified_diff as diff
from collections import defaultdict
@ -44,9 +47,9 @@ except ImportError:
# Until Debian Stable ships IPython >3.0, we stick to the v3 format.
try:
from nbformat import v3 as nbformat
from nbformat import v4 as nbformat
except ImportError:
from IPython.nbformat import v3 as nbformat
from IPython.nbformat import v4 as nbformat
def compare_png(a64, b64):
"""compare two b64 PNGs (incomplete)"""
@ -58,7 +61,7 @@ def compare_png(a64, b64):
bdata = base64.decodestring(b64)
return True
def sanitize(s):
def canonicalize(s, type, ignores):
"""sanitize a string for comparison.
fix universal newlines, strip trailing newlines, and normalize likely
@ -85,6 +88,9 @@ def sanitize(s):
s = re.sub(r'SpinS Promela Compiler.*Compiled C .* to .*pml.spins',
'SpinS output', s, flags=re.DOTALL)
# %%file writes `Writing`, or `Overwriting` if the file exists.
s = re.sub(r'^Overwriting ', 'Writing ', s)
# SVG generated by graphviz may put note at different positions
# depending on the graphviz build. Let's just strip anything that
# look like a position.
@ -115,47 +121,58 @@ def sanitize(s):
# CalledProcessError message has a final dot in Python 3.6
s = re.sub(r"(' returned non-zero exit status \d+)\.", r'\1', s)
for n, p in enumerate(ignores):
s = re.sub(p, 'IGN{}'.format(n), s)
return s
def consolidate_outputs(outputs):
"""consolidate outputs into a summary dict (incomplete)"""
data = defaultdict(list)
data['stdout'] = ''
data['stderr'] = ''
def canonical_dict(dict, ignores):
'''Neutralize gratuitous differences in a Jupyter dictionary.
for out in outputs:
if out.type == 'stream':
data[out.stream] += out.text
elif out.type == 'pyerr':
data['pyerr'] = dict(ename=out.ename, evalue=out.evalue)
else:
for key in ('png', 'svg', 'latex', 'html',
'javascript', 'text', 'jpeg',):
if key in out:
data[key].append(out[key])
return data
For instance, neutralize different Graphviz layouts in SVG.
'''
if 'text' in dict:
dict['text'] = canonicalize(dict['text'], 'text', ignores)
if 'data' in dict:
for k in dict['data']:
dict['data'][k] = canonicalize(dict['data'][k], k, ignores)
if ('ename' in dict and
dict['ename'] == 'SystemExit' and dict['evalue'] == '77'):
# sys.exit(77) is used to Skip the test.
sys.exit(77)
if 'transient' in dict:
del dict['transient']
if 'execution_count' in dict:
del dict['execution_count']
if 'traceback' in dict:
del dict['traceback']
return dict
def compare_outputs(test, ref, skip_cmp=('png', 'traceback',
'latex', 'prompt_number')):
for key in ref:
if key not in test:
print("missing key: %s != %s" % (test.keys(), ref.keys()))
return False
elif key not in skip_cmp:
exp = sanitize(ref[key])
eff = sanitize(test[key])
if exp != eff:
print("mismatch %s:" % key)
if exp[:-1] != '\n':
exp += '\n'
if eff[:-1] != '\n':
eff += '\n'
print(''.join(diff(exp.splitlines(1), eff.splitlines(1),
fromfile='expected', tofile='effective')))
return False
return True
def compare_outputs(ref, test, ignores=[]):
'''Check that two lists of outputs are equivalent and report the
result.'''
# There can be several outputs. For instance wnen the cell both
# prints a result (goes to "stdout") and displays an automaton
# (goes to "data").
exp = pprint.pformat([canonical_dict(d, ignores) for d in ref], width=132)
eff = pprint.pformat([canonical_dict(d, ignores) for d in test], width=132)
if exp[:-1] != '\n':
exp += '\n'
if eff[:-1] != '\n':
eff += '\n'
if exp == eff:
return True
else:
print(''.join(diff(exp.splitlines(1), eff.splitlines(1),
fromfile='expected', tofile='effective')))
return False
def _wait_for_ready_backport(kc):
"""Backport BlockingKernelClient.wait_for_ready from IPython 3"""
@ -173,8 +190,7 @@ def _wait_for_ready_backport(kc):
break
def run_cell(kc, cell):
# print cell.input
kc.execute(cell.input)
kc.execute(cell.source)
# wait for finish, maximum 20s
reply = kc.get_shell_msg(timeout=20)
outs = []
@ -184,52 +200,38 @@ def run_cell(kc, cell):
msg = kc.get_iopub_msg(timeout=0.2)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'pyin', 'execute_input'):
content = msg['content']
if msg_type == 'status' and content['execution_state'] == 'idle':
break
if msg_type in ('status', 'pyin', 'execute_input',
'comm_open', 'comm_msg'):
continue
if msg_type == 'stream':
if 'Widget' in content['text']:
continue
# If the last stream had the same name, then outputs are
# appended.
if outs:
last = outs[-1]
if last['output_type'] == 'stream' and \
last['name'] == content['name']:
last['text'] += content['text']
continue
elif msg_type == 'clear_output':
outs = []
continue
content = msg['content']
# print (msg_type, content)
if msg_type == 'execute_result':
msg_type = 'pyout'
elif msg_type == 'error':
msg_type = 'pyerr'
out = nbformat.NotebookNode(output_type=msg_type)
if msg_type == 'stream':
out.stream = content['name']
if 'text' in content:
out.text = content['text']
else:
out.text = content['data']
elif msg_type in ('display_data', 'pyout'):
out['metadata'] = content['metadata']
for mime, data in content['data'].items():
attr = mime.split('/')[-1].lower()
# this gets most right, but fix svg+html, plain
attr = attr.replace('+xml', '').replace('plain', 'text')
setattr(out, attr, data)
if 'execution_count' in content:
out.prompt_number = content['execution_count']
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
# sys.exit(77) is used to Skip the test.
if out.ename == 'SystemExit' and out.evalue == '77':
sys.exit(77)
else:
print("unhandled iopub msg:", msg_type)
outs.append(out)
content['output_type'] = msg_type
outs.append(content)
return outs
def test_notebook(nb):
def test_notebook(ipynb):
with open(ipynb, encoding='utf-8') as f:
nb = nbformat.reads_json(f.read())
km = KernelManager()
# Do not save the history to disk, as it can yield spurious lock errors.
# See https://github.com/ipython/ipython/issues/2845
@ -246,36 +248,34 @@ def test_notebook(nb):
successes = 0
failures = 0
errors = 0
for ws in nb.worksheets:
for i, cell in enumerate(ws.cells):
if cell.cell_type != 'code' or cell.input.startswith('%timeit'):
continue
try:
outs = run_cell(kc, cell)
except Exception as e:
print("failed to run cell:", repr(e))
print(cell.input)
errors += 1
continue
for i, cell in enumerate(nb.cells):
if cell.cell_type != 'code' or cell.source.startswith('%timeit'):
continue
try:
outs = run_cell(kc, cell)
except Exception as e:
print("failed to run cell:", repr(e))
print(cell.input)
errors += 1
continue
failed = False
if len(outs) != len(cell.outputs):
print("output length mismatch (expected {}, got {})".format(
len(cell.outputs), len(outs)))
failed = True
for out, ref in zip(outs, cell.outputs):
if not compare_outputs(out, ref):
failed = True
print("cell %d: " % i, end="")
if failed:
print("FAIL")
failures += 1
else:
print("OK")
successes += 1
failed = False
if len(outs) != len(cell.outputs):
print("output length mismatch (expected {}, got {})".format(
len(cell.outputs), len(outs)))
failed = True
if not compare_outputs(outs, cell.outputs):
failed = True
print("cell %d: " % i, end="")
if failed:
print("FAIL")
failures += 1
else:
print("OK")
successes += 1
print()
print("tested notebook %s" % nb.metadata.name)
print("tested notebook %s" % ipynb)
print(" %3i cells successfully replicated" % successes)
if failures:
print(" %3i cells mismatched output" % failures)
@ -290,6 +290,4 @@ def test_notebook(nb):
if __name__ == '__main__':
for ipynb in sys.argv[1:]:
print("testing %s" % ipynb)
with open(ipynb) as f:
nb = nbformat.reads_json(f.read())
test_notebook(nb)
test_notebook(ipynb)