From 5dc6da0b20169d71093f81696543ca061847d246 Mon Sep 17 00:00:00 2001 From: Alexandre Duret-Lutz Date: Sat, 4 Jan 2020 17:59:45 +0100 Subject: [PATCH] ipnbdoctest: attempt to restart when the kernel dies * tests/python/ipnbdoctest.py: Catch kernel deaths, wait a random number of seconds, and try again up to three times. --- tests/python/ipnbdoctest.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/python/ipnbdoctest.py b/tests/python/ipnbdoctest.py index 26918a883..5d7d00d10 100755 --- a/tests/python/ipnbdoctest.py +++ b/tests/python/ipnbdoctest.py @@ -18,6 +18,7 @@ import time import base64 import re import pprint +import random from difflib import unified_diff as diff from collections import defaultdict @@ -324,5 +325,20 @@ def test_notebook(ipynb): if __name__ == '__main__': for ipynb in sys.argv[1:]: - print("testing %s" % ipynb) - test_notebook(ipynb) + tries=3 + while tries: + print("testing %s" % ipynb) + try: + test_notebook(ipynb) + break + except RuntimeError as e: + # If the Kernel dies, try again. It seems we have spurious + # failures when multiple instances of jupyter start in parallel. + if 'Kernel died' in str(e): + tries -= 1 + if tries: + s = random.randint(1, 5) + print("trying again in", s, "seconds...") + time.sleep(s) + else: + raise e