ltlcross, autcross, ltldo: support --fail-on-timeout
Suggested by Tobias Meggendorfer. Fixes #294. * bin/autcross.cc, bin/ltlcross.cc, bin/ltldo.cc: Add the option. * tests/core/autcross3.test, tests/core/ltlcross3.test, tests/core/ltldo.test: Test it. * tests/Makefile.am: Add autcross3.test. * NEWS, doc/org/autcross.org, doc/org/ltlcross.org, doc/org/ltldo.org: Mention the option. * THANKS: Add Tobias.
This commit is contained in:
Alexandre Duret-Lutz
parent
0a2bca1377
commit
183ec1fb4e
12 changed files with 147 additions and 21 deletions
|
|
@ -59,7 +59,7 @@ Call several tools that process automata and cross-compare their output \
|
|||
to detect bugs, or to gather statistics. The list of automata to use \
|
||||
should be supplied on standard input, or using the -F option.\v\
|
||||
Exit status:\n\
|
||||
0 everything went fine (timeouts are OK too)\n\
|
||||
0 everything went fine (without --fail-on-timeout, timeouts are OK)\n\
|
||||
1 some tools failed to output something we understand, or failed\n\
|
||||
sanity checks (statistics were output nonetheless)\n\
|
||||
2 autcross aborted on error\n\
|
||||
|
|
@ -69,6 +69,7 @@ enum {
|
|||
OPT_BOGUS = 256,
|
||||
OPT_CSV,
|
||||
OPT_HIGH,
|
||||
OPT_FAIL_ON_TIMEOUT,
|
||||
OPT_IGNORE_EXEC_FAIL,
|
||||
OPT_LANG,
|
||||
OPT_LOW,
|
||||
|
|
@ -92,6 +93,8 @@ static const argp_option options[] =
|
|||
{ "ignore-execution-failures", OPT_IGNORE_EXEC_FAIL, nullptr, 0,
|
||||
"ignore automata from translators that return with a non-zero exit code,"
|
||||
" but do not flag this as an error", 0 },
|
||||
{ "fail-on-timeout", OPT_FAIL_ON_TIMEOUT, nullptr, 0,
|
||||
"consider timeouts as errors", 0 },
|
||||
{ "language-preserved", OPT_LANG, nullptr, 0,
|
||||
"expect that each tool preserves the input language", 0 },
|
||||
{ "no-checks", OPT_NOCHECKS, nullptr, 0,
|
||||
|
|
@ -137,6 +140,7 @@ const struct argp_child children[] =
|
|||
static bool verbose = false;
|
||||
static bool ignore_exec_fail = false;
|
||||
static unsigned ignored_exec_fail = 0;
|
||||
static bool fail_on_timeout = false;
|
||||
static bool stop_on_error = false;
|
||||
static bool no_checks = false;
|
||||
static bool opt_language_preserved = false;
|
||||
|
|
@ -163,6 +167,9 @@ parse_opt(int key, char* arg, struct argp_state*)
|
|||
case OPT_CSV:
|
||||
csv_output = arg ? arg : "-";
|
||||
break;
|
||||
case OPT_FAIL_ON_TIMEOUT:
|
||||
fail_on_timeout = true;
|
||||
break;
|
||||
case OPT_HIGH:
|
||||
level = spot::postprocessor::High;
|
||||
level_set = true;
|
||||
|
|
@ -392,11 +399,18 @@ namespace
|
|||
spot::twa_graph_ptr res = nullptr;
|
||||
if (timed_out)
|
||||
{
|
||||
// This is not considered to be a global error.
|
||||
std::cerr << "warning: timeout during execution of command\n";
|
||||
++timeout_count;
|
||||
if (fail_on_timeout)
|
||||
{
|
||||
global_error() << "error: timeout during execution of command\n";
|
||||
end_error();
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cerr << "warning: timeout during execution of command\n";
|
||||
++timeout_count;
|
||||
}
|
||||
status_str = "timeout";
|
||||
problem = false; // A timeout is not a sign of a bug
|
||||
problem = fail_on_timeout;
|
||||
es = -1;
|
||||
}
|
||||
else if (WIFSIGNALED(es))
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ Call several LTL/PSL translators and cross-compare their output to detect \
|
|||
bugs, or to gather statistics. The list of formulas to use should be \
|
||||
supplied on standard input, or using the -f or -F options.\v\
|
||||
Exit status:\n\
|
||||
0 everything went fine (timeouts are OK too)\n\
|
||||
0 everything went fine (without --fail-on-timeout, timeouts are OK)\n\
|
||||
1 some translator failed to output something we understand, or failed\n\
|
||||
sanity checks (statistics were output nonetheless)\n\
|
||||
2 ltlcross aborted on error\n\
|
||||
|
|
@ -87,6 +87,7 @@ enum {
|
|||
OPT_CSV,
|
||||
OPT_DENSITY,
|
||||
OPT_DUPS,
|
||||
OPT_FAIL_ON_TIMEOUT,
|
||||
OPT_GRIND,
|
||||
OPT_IGNORE_EXEC_FAIL,
|
||||
OPT_JSON,
|
||||
|
|
@ -121,6 +122,8 @@ static const argp_option options[] =
|
|||
{ "ignore-execution-failures", OPT_IGNORE_EXEC_FAIL, nullptr, 0,
|
||||
"ignore automata from translators that return with a non-zero exit code,"
|
||||
" but do not flag this as an error", 0 },
|
||||
{ "fail-on-timeout", OPT_FAIL_ON_TIMEOUT, nullptr, 0,
|
||||
"consider timeouts as errors", 0 },
|
||||
/**************************************************/
|
||||
{ nullptr, 0, nullptr, 0, "State-space generation:", 6 },
|
||||
{ "states", OPT_STATES, "INT", 0,
|
||||
|
|
@ -199,6 +202,7 @@ static output_file* grind_output = nullptr;
|
|||
static bool verbose = false;
|
||||
static bool ignore_exec_fail = false;
|
||||
static unsigned ignored_exec_fail = 0;
|
||||
static bool fail_on_timeout = false;
|
||||
static bool opt_automata = false;
|
||||
static bool opt_strength = false;
|
||||
static bool opt_ambiguous = false;
|
||||
|
|
@ -445,6 +449,9 @@ parse_opt(int key, char* arg, struct argp_state*)
|
|||
case OPT_DUPS:
|
||||
allow_dups = true;
|
||||
break;
|
||||
case OPT_FAIL_ON_TIMEOUT:
|
||||
fail_on_timeout = true;
|
||||
break;
|
||||
case OPT_GRIND:
|
||||
grind_output = new output_file(arg);
|
||||
break;
|
||||
|
|
@ -530,11 +537,18 @@ namespace
|
|||
spot::twa_graph_ptr res = nullptr;
|
||||
if (timed_out)
|
||||
{
|
||||
// This is not considered to be a global error.
|
||||
std::cerr << "warning: timeout during execution of command\n";
|
||||
++timeout_count;
|
||||
if (fail_on_timeout)
|
||||
{
|
||||
global_error() << "error: timeout during execution of command\n";
|
||||
end_error();
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cerr << "warning: timeout during execution of command\n";
|
||||
++timeout_count;
|
||||
}
|
||||
status_str = "timeout";
|
||||
problem = false; // A timeout is not a sign of a bug
|
||||
problem = fail_on_timeout;
|
||||
es = -1;
|
||||
}
|
||||
else if (WIFSIGNALED(es))
|
||||
|
|
|
|||
21
bin/ltldo.cc
21
bin/ltldo.cc
|
|
@ -53,6 +53,7 @@ enum {
|
|||
OPT_ERRORS = 256,
|
||||
OPT_SMALLEST,
|
||||
OPT_GREATEST,
|
||||
OPT_FAIL_ON_TIMEOUT,
|
||||
};
|
||||
|
||||
static const argp_option options[] =
|
||||
|
|
@ -62,6 +63,8 @@ static const argp_option options[] =
|
|||
{ "errors", OPT_ERRORS, "abort|warn|ignore", 0,
|
||||
"how to deal with tools returning with non-zero exit codes or "
|
||||
"automata that ltldo cannot parse (default: abort)", 0 },
|
||||
{ "fail-on-timeout", OPT_FAIL_ON_TIMEOUT, nullptr, 0,
|
||||
"consider timeouts as errors", 0 },
|
||||
/**************************************************/
|
||||
{ nullptr, 0, nullptr, 0, "Output selection:", 5 },
|
||||
{ "smallest", OPT_SMALLEST, "FORMAT", OPTION_ARG_OPTIONAL,
|
||||
|
|
@ -122,7 +125,7 @@ build_percent_list()
|
|||
|
||||
enum errors_type { errors_abort, errors_warn, errors_ignore };
|
||||
static errors_type errors_opt;
|
||||
|
||||
static bool fail_on_timeout = false;
|
||||
static int best_type = 0; // -1 smallest, 1 greatest, 0 no selection
|
||||
static const char* best_format = "%s,%e";
|
||||
static int opt_max_count = -1;
|
||||
|
|
@ -162,6 +165,9 @@ parse_opt(int key, char* arg, struct argp_state*)
|
|||
case OPT_ERRORS:
|
||||
errors_opt = XARGMATCH("--errors", arg, errors_args, errors_types);
|
||||
break;
|
||||
case OPT_FAIL_ON_TIMEOUT:
|
||||
fail_on_timeout = true;
|
||||
break;
|
||||
case OPT_GREATEST:
|
||||
best_type = 1;
|
||||
if (arg)
|
||||
|
|
@ -218,9 +224,18 @@ namespace
|
|||
if (timed_out)
|
||||
{
|
||||
// A timeout is considered benign
|
||||
std::cerr << "warning: timeout during execution of command \""
|
||||
if (fail_on_timeout)
|
||||
{
|
||||
std::cerr << "error:";
|
||||
problem = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cerr << "warning:";
|
||||
++timeout_count;
|
||||
}
|
||||
std::cerr << " timeout during execution of command \""
|
||||
<< cmd << "\"\n";
|
||||
++timeout_count;
|
||||
}
|
||||
else if (WIFSIGNALED(es))
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue