ltlcross: report exit_status and exit_code columns in CSV and JSON
* src/bin/ltlcross.cc: Report exit_status and exit_code columns in CSV
and JSON files. Also output lines for failed translations, and add
a --omit-missing option to disable that. Move the time column right
after exit_status and exit_code.
* src/bin/man/ltlcross.x: Document each column of the output.
* bench/ltl2tgba/tools: Use the "{name}cmd" notation.
* bench/ltl2tgba/sum.py: Adjust to the new columns.
* bench/ltl2tgba/README: Update to point to the man page for a
description of the columns.
* bench/ltl2tgba/Makefile.am: Build results.pdf as said announced in
README.
* bench/spin13/html.bottom: Update code to ignore these two new
columns and lines with null values.
* src/tgbatest/ltlcross3.test: Add tests.
* doc/org/ltlcross.org: Adjust examples.
* NEWS: Mention this.
This commit is contained in:
Alexandre Duret-Lutz
parent
686a45484d
commit
f65c621a55
10 changed files with 430 additions and 202 deletions
|
|
@ -93,6 +93,7 @@ Exit status:\n\
|
|||
#define OPT_PRODUCTS 9
|
||||
#define OPT_COLOR 10
|
||||
#define OPT_NOCOMP 11
|
||||
#define OPT_OMIT 12
|
||||
|
||||
static const argp_option options[] =
|
||||
{
|
||||
|
|
@ -150,6 +151,8 @@ static const argp_option options[] =
|
|||
"output statistics as JSON in FILENAME or on standard output", 0 },
|
||||
{ "csv", OPT_CSV, "FILENAME", OPTION_ARG_OPTIONAL,
|
||||
"output statistics as CSV in FILENAME or on standard output", 0 },
|
||||
{ "omit-missing", OPT_OMIT, 0, 0,
|
||||
"do not output statistics for timeouts or failed translations", 0 },
|
||||
/**************************************************/
|
||||
{ 0, 0, 0, 0, "Miscellaneous options:", -1 },
|
||||
{ "color", OPT_COLOR, "WHEN", OPTION_ARG_OPTIONAL,
|
||||
|
|
@ -166,7 +169,6 @@ const struct argp_child children[] =
|
|||
{ 0, 0, 0, 0 }
|
||||
};
|
||||
|
||||
|
||||
enum color_type { color_never, color_always, color_if_tty };
|
||||
|
||||
static char const *const color_args[] =
|
||||
|
|
@ -201,7 +203,7 @@ bool no_complement = false;
|
|||
bool stop_on_error = false;
|
||||
int seed = 0;
|
||||
unsigned products = 1;
|
||||
|
||||
bool opt_omit = false;
|
||||
|
||||
struct translator_spec
|
||||
{
|
||||
|
|
@ -289,6 +291,9 @@ struct statistics
|
|||
{
|
||||
statistics()
|
||||
: ok(false),
|
||||
status_str(0),
|
||||
status_code(0),
|
||||
time(0),
|
||||
states(0),
|
||||
transitions(0),
|
||||
acc(0),
|
||||
|
|
@ -302,14 +307,18 @@ struct statistics
|
|||
terminal_aut(false),
|
||||
weak_aut(false),
|
||||
strong_aut(false),
|
||||
time(0),
|
||||
product_states(0),
|
||||
product_transitions(0),
|
||||
product_scc(0)
|
||||
{
|
||||
}
|
||||
|
||||
// If OK is false, only the status_str, status_code, and time fields
|
||||
// should be valid.
|
||||
bool ok;
|
||||
const char* status_str;
|
||||
int status_code;
|
||||
double time;
|
||||
unsigned states;
|
||||
unsigned edges;
|
||||
unsigned transitions;
|
||||
|
|
@ -324,15 +333,17 @@ struct statistics
|
|||
bool terminal_aut;
|
||||
bool weak_aut;
|
||||
bool strong_aut;
|
||||
double time;
|
||||
double product_states;
|
||||
double product_transitions;
|
||||
double product_scc;
|
||||
|
||||
static void
|
||||
fields(std::ostream& os)
|
||||
fields(std::ostream& os, bool all)
|
||||
{
|
||||
os << ("\"states\","
|
||||
if (all)
|
||||
os << "\"exit_status\",\"exit_code\",";
|
||||
os << ("\"time\","
|
||||
"\"states\","
|
||||
"\"edges\","
|
||||
"\"transitions\","
|
||||
"\"acc\","
|
||||
|
|
@ -346,33 +357,53 @@ struct statistics
|
|||
"\"terminal_aut\","
|
||||
"\"weak_aut\","
|
||||
"\"strong_aut\","
|
||||
"\"time\","
|
||||
"\"product_states\","
|
||||
"\"product_transitions\","
|
||||
"\"product_scc\"");
|
||||
}
|
||||
|
||||
void
|
||||
to_csv(std::ostream& os)
|
||||
to_csv(std::ostream& os, bool all, const char* na = "")
|
||||
{
|
||||
os << states << ','
|
||||
<< edges << ','
|
||||
<< transitions << ','
|
||||
<< acc << ','
|
||||
<< scc << ','
|
||||
<< nonacc_scc << ','
|
||||
<< terminal_scc << ','
|
||||
<< weak_scc << ','
|
||||
<< strong_scc << ','
|
||||
<< nondetstates << ','
|
||||
<< nondeterministic << ','
|
||||
<< terminal_aut << ','
|
||||
<< weak_aut << ','
|
||||
<< strong_aut << ','
|
||||
<< time << ','
|
||||
<< product_states << ','
|
||||
<< product_transitions << ','
|
||||
<< product_scc;
|
||||
if (all)
|
||||
os << '"' << status_str << "\"," << status_code << ',';
|
||||
os << time << ',';
|
||||
if (ok)
|
||||
os << states << ','
|
||||
<< edges << ','
|
||||
<< transitions << ','
|
||||
<< acc << ','
|
||||
<< scc << ','
|
||||
<< nonacc_scc << ','
|
||||
<< terminal_scc << ','
|
||||
<< weak_scc << ','
|
||||
<< strong_scc << ','
|
||||
<< nondetstates << ','
|
||||
<< nondeterministic << ','
|
||||
<< terminal_aut << ','
|
||||
<< weak_aut << ','
|
||||
<< strong_aut << ','
|
||||
<< product_states << ','
|
||||
<< product_transitions << ','
|
||||
<< product_scc;
|
||||
else
|
||||
os << na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na << ','
|
||||
<< na;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -470,6 +501,9 @@ parse_opt(int key, char* arg, struct argp_state*)
|
|||
case OPT_NOCOMP:
|
||||
no_complement = true;
|
||||
break;
|
||||
case OPT_OMIT:
|
||||
opt_omit = true;
|
||||
break;
|
||||
case OPT_SEED:
|
||||
seed = to_pos_int(arg);
|
||||
break;
|
||||
|
|
@ -784,27 +818,37 @@ namespace
|
|||
int es = exec_with_timeout(cmd.c_str());
|
||||
xtime_t after = gethrxtime();
|
||||
|
||||
const char* status_str = 0;
|
||||
|
||||
const spot::tgba* res = 0;
|
||||
if (timed_out)
|
||||
{
|
||||
// This is not considered to be a global error.
|
||||
std::cerr << "warning: timeout during execution of command\n";
|
||||
++timeout_count;
|
||||
status_str = "timeout";
|
||||
es = -1;
|
||||
}
|
||||
else if (WIFSIGNALED(es))
|
||||
{
|
||||
status_str = "signal";
|
||||
es = WTERMSIG(es);
|
||||
global_error() << "error: execution terminated by signal "
|
||||
<< WTERMSIG(es) << ".\n";
|
||||
<< es << ".\n";
|
||||
end_error();
|
||||
}
|
||||
else if (WIFEXITED(es) && WEXITSTATUS(es) != 0)
|
||||
{
|
||||
es = WEXITSTATUS(es);
|
||||
status_str = "exit code";
|
||||
global_error() << "error: execution returned exit code "
|
||||
<< WEXITSTATUS(es) << ".\n";
|
||||
<< es << ".\n";
|
||||
end_error();
|
||||
}
|
||||
else
|
||||
{
|
||||
status_str = "ok";
|
||||
es = 0;
|
||||
switch (output.format)
|
||||
{
|
||||
case printable_result_filename::Spin:
|
||||
|
|
@ -814,6 +858,8 @@ namespace
|
|||
res = spot::neverclaim_parse(filename, pel, &dict);
|
||||
if (!pel.empty())
|
||||
{
|
||||
status_str = "parse error";
|
||||
es = -1;
|
||||
std::ostream& err = global_error();
|
||||
err << "error: failed to parse the produced neverclaim.\n";
|
||||
spot::format_neverclaim_parse_errors(err, filename, pel);
|
||||
|
|
@ -829,6 +875,8 @@ namespace
|
|||
std::ifstream f(output.val()->name());
|
||||
if (!f)
|
||||
{
|
||||
status_str = "no output";
|
||||
es = -1;
|
||||
global_error() << "Cannot open " << output.val()
|
||||
<< std::endl;
|
||||
end_error();
|
||||
|
|
@ -838,6 +886,8 @@ namespace
|
|||
res = spot::lbtt_parse(f, error, &dict);
|
||||
if (!res)
|
||||
{
|
||||
status_str = "parse error";
|
||||
es = -1;
|
||||
global_error() << ("error: failed to parse output in "
|
||||
"LBTT format: ")
|
||||
<< error << std::endl;
|
||||
|
|
@ -854,6 +904,8 @@ namespace
|
|||
aut = spot::dstar_parse(filename, pel, &dict);
|
||||
if (!pel.empty())
|
||||
{
|
||||
status_str = "parse error";
|
||||
es = -1;
|
||||
std::ostream& err = global_error();
|
||||
err << "error: failed to parse the produced DSTAR"
|
||||
" output.\n";
|
||||
|
|
@ -873,41 +925,48 @@ namespace
|
|||
assert(!"unreachable code");
|
||||
}
|
||||
}
|
||||
// Compute statistics.
|
||||
if (res && want_stats)
|
||||
|
||||
if (want_stats)
|
||||
{
|
||||
statistics* st = &(*fstats)[translator_num];
|
||||
st->ok = true;
|
||||
spot::tgba_sub_statistics s = sub_stats_reachable(res);
|
||||
st->states = s.states;
|
||||
st->edges = s.transitions;
|
||||
st->transitions = s.sub_transitions;
|
||||
st->acc = res->number_of_acceptance_conditions();
|
||||
spot::scc_map m(res);
|
||||
m.build_map();
|
||||
unsigned c = m.scc_count();
|
||||
st->scc = m.scc_count();
|
||||
st->nondetstates = spot::count_nondet_states(res);
|
||||
st->nondeterministic = st->nondetstates != 0;
|
||||
for (unsigned n = 0; n < c; ++n)
|
||||
{
|
||||
if (!m.accepting(n))
|
||||
++st->nonacc_scc;
|
||||
else if (is_terminal_scc(m, n))
|
||||
++st->terminal_scc;
|
||||
else if (is_weak_scc(m, n))
|
||||
++st->weak_scc;
|
||||
else
|
||||
++st->strong_scc;
|
||||
}
|
||||
if (st->strong_scc)
|
||||
st->strong_aut = true;
|
||||
else if (st->weak_scc)
|
||||
st->weak_aut = true;
|
||||
else
|
||||
st->terminal_aut = true;
|
||||
double prec = XTIME_PRECISION;
|
||||
st->status_str = status_str;
|
||||
st->status_code = es;
|
||||
double prec = XTIME_PRECISION;
|
||||
st->time = (after - before) / prec;
|
||||
|
||||
// Compute statistics.
|
||||
if (res)
|
||||
{
|
||||
st->ok = true;
|
||||
spot::tgba_sub_statistics s = sub_stats_reachable(res);
|
||||
st->states = s.states;
|
||||
st->edges = s.transitions;
|
||||
st->transitions = s.sub_transitions;
|
||||
st->acc = res->number_of_acceptance_conditions();
|
||||
spot::scc_map m(res);
|
||||
m.build_map();
|
||||
unsigned c = m.scc_count();
|
||||
st->scc = m.scc_count();
|
||||
st->nondetstates = spot::count_nondet_states(res);
|
||||
st->nondeterministic = st->nondetstates != 0;
|
||||
for (unsigned n = 0; n < c; ++n)
|
||||
{
|
||||
if (!m.accepting(n))
|
||||
++st->nonacc_scc;
|
||||
else if (is_terminal_scc(m, n))
|
||||
++st->terminal_scc;
|
||||
else if (is_weak_scc(m, n))
|
||||
++st->weak_scc;
|
||||
else
|
||||
++st->strong_scc;
|
||||
}
|
||||
if (st->strong_scc)
|
||||
st->strong_aut = true;
|
||||
else if (st->weak_scc)
|
||||
st->weak_aut = true;
|
||||
else
|
||||
st->terminal_aut = true;
|
||||
}
|
||||
}
|
||||
output.cleanup();
|
||||
return res;
|
||||
|
|
@ -1401,18 +1460,18 @@ print_stats_csv(const char* filename)
|
|||
assert(rounds == formulas.size());
|
||||
|
||||
*out << "\"formula\",\"tool\",";
|
||||
statistics::fields(*out);
|
||||
statistics::fields(*out, !opt_omit);
|
||||
*out << "\r\n";
|
||||
for (unsigned r = 0; r < rounds; ++r)
|
||||
for (unsigned t = 0; t < ntrans; ++t)
|
||||
if (vstats[r][t].ok)
|
||||
if (!opt_omit || vstats[r][t].ok)
|
||||
{
|
||||
*out << "\"";
|
||||
spot::escape_rfc4180(*out, formulas[r]);
|
||||
*out << "\",\"";
|
||||
spot::escape_rfc4180(*out, translators[t].name);
|
||||
*out << "\",";
|
||||
vstats[r][t].to_csv(*out);
|
||||
vstats[r][t].to_csv(*out, !opt_omit);
|
||||
*out << "\r\n";
|
||||
}
|
||||
delete outfile;
|
||||
|
|
@ -1453,19 +1512,19 @@ print_stats_json(const char* filename)
|
|||
spot::escape_str(*out, formulas[r]);
|
||||
}
|
||||
*out << ("\"\n ],\n \"fields\": [\n \"formula\",\"tool\",");
|
||||
statistics::fields(*out);
|
||||
statistics::fields(*out, !opt_omit);
|
||||
*out << "\n ],\n \"inputs\": [ 0, 1 ],";
|
||||
*out << "\n \"results\": [";
|
||||
bool notfirst = false;
|
||||
for (unsigned r = 0; r < rounds; ++r)
|
||||
for (unsigned t = 0; t < ntrans; ++t)
|
||||
if (vstats[r][t].ok)
|
||||
if (!opt_omit || vstats[r][t].ok)
|
||||
{
|
||||
if (notfirst)
|
||||
*out << ',';
|
||||
notfirst = true;
|
||||
*out << "\n [ " << r << ',' << t << ',';
|
||||
vstats[r][t].to_csv(*out);
|
||||
vstats[r][t].to_csv(*out, !opt_omit, "null");
|
||||
*out << " ]";
|
||||
}
|
||||
*out << "\n ]\n}\n";
|
||||
|
|
|
|||
|
|
@ -68,6 +68,110 @@ When this variable is defined, temporary files are not removed.
|
|||
This is mostly useful for debugging.
|
||||
|
||||
|
||||
[OUTPUT DATA]
|
||||
|
||||
The following columns are output in the CSV or JSON files.
|
||||
|
||||
.TP 7
|
||||
\fBformula\fR
|
||||
The formula translated.
|
||||
|
||||
.TP
|
||||
\fBtool\fR
|
||||
The tool used to translate this formula. This is either the value of the
|
||||
full \fICOMMANDFMT\fR string specified on the command-line, or,
|
||||
if \fICOMMANDFMT\fR has the form \f(CW{\fISHORTNAME\fR\f(CW}\fR\FiCMD\fR,
|
||||
the value of \fISHORTNAME\fR.
|
||||
|
||||
.TP
|
||||
\fBexit_status\fR, \fBexit_code\fR
|
||||
Information about how the execution of the translator went. If the
|
||||
option \fB\-\-omit\-missing\fR is given, these two columns are omitted
|
||||
and only the lines corresponding to successful translation are output.
|
||||
Otherwise, \fBexit_status\fR is a string that can take the following
|
||||
values:
|
||||
|
||||
.RS
|
||||
.TP
|
||||
\f(CW"ok"\fR
|
||||
The translator ran succesfully (this does not imply that the produced
|
||||
automaton is correct) and ltlcross could parse the resulting
|
||||
automaton. In this case \fBexit_code\fR is always 0.
|
||||
|
||||
.TP
|
||||
\f(CW"timeout"\fR
|
||||
The translator ran for more than the number of seconds
|
||||
specified with the \fB\-\-timeout\fR option. In this
|
||||
case \fBexit_code\fR is always -1.
|
||||
|
||||
.TP
|
||||
\f(CW"exit code"\fR
|
||||
The translator terminated with a non-zero exit code.
|
||||
\fBexit_code\fR contains that value.
|
||||
|
||||
.TP
|
||||
\f(CW"signal"\fR
|
||||
The translator terminated with a signal.
|
||||
\fBexit_code\fR contains that signal's number.
|
||||
|
||||
.TP
|
||||
\f(CW"parse error"\fR
|
||||
The translator terminated normally, but ltlcross could not
|
||||
parse its output. In this case \fBexit_code\fR is always -1.
|
||||
|
||||
.TP
|
||||
\f(CW"no output"\fR
|
||||
The translator terminated normally, but without creating the specified
|
||||
output file. In this case \fBexit_code\fR is always -1.
|
||||
.RE
|
||||
|
||||
.TP
|
||||
\fBtime\fR
|
||||
A floating point number giving the run time of the translator in seconds.
|
||||
This is reported for all executions, even failling ones.
|
||||
|
||||
.PP
|
||||
Unless the \fB\-\-omit\-missing\fR option is used, data for all the
|
||||
following columns might be missing.
|
||||
|
||||
.TP
|
||||
\fBstate\fR, \fBedges\fR, \fBtransitions\fR, \fBacc\fR
|
||||
The number of states, edges, transitions, and acceptance sets in the
|
||||
translated automaton. Column \fBedges\fR counts the number of edges
|
||||
(labeled by Boolean formulas) in the automaton seen as a graph, while
|
||||
\fBtransitions\fR counts the number of assignment-labeled transitions
|
||||
that might have been merged into a formula-labeled edge. For instance
|
||||
an edge labeled by \f(CWtrue\fR will be counted as 2^3=8 transitions if
|
||||
the automaton mention 3 atomic propositions.
|
||||
|
||||
.TP
|
||||
\fBscc\fR, \fBnonacc_scc\fR, \fBterminal_scc\fR, \fBweak_scc\fR, \fBstrong_scc\fR
|
||||
The number of strongly connected components in the automaton. The
|
||||
\fBscc\fR column gives the total number, while the other columns only
|
||||
count the SCCs that are non-accepting (a.k.a. transiant), terminal
|
||||
(recognizes and accepts all words), weak (do not recognize all words,
|
||||
but accepts all recognized words), or strong (accept some words, but
|
||||
reject some recognized words).
|
||||
|
||||
.TP
|
||||
\fBnondet_states\fR, \fBnondet_aut\fR
|
||||
The number of nondeterministic states, and a Boolean indicating whether the
|
||||
automaton is nondeterministic.
|
||||
|
||||
.TP
|
||||
\fBterminal_aut\fR, \fBweak_aut\fR, \fBstrong_aut\fR
|
||||
Three Boolean used to indicate whether the automaton is terminal (no
|
||||
weak nor strong SCCs), weak (some weak SCCs but no strong SCCs), or strong
|
||||
(some strong SCCs).
|
||||
|
||||
.TP
|
||||
\fBproduct_states\fR, \fBproduct_transitions\fR, \fBproduct_scc\fR
|
||||
Size of the product between the translated automaton and a randomly
|
||||
generated state-space. For a given formula, the same state-space is
|
||||
of course used the result of each translator. When the
|
||||
\fB\-\-products\fR=\fIN\fR option is used, these values are averaged
|
||||
over the \fIN\fR products performed.
|
||||
|
||||
[SEE ALSO]
|
||||
.BR randltl (1),
|
||||
.BR genltl (1),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue