diff --git a/bench/ltl2tgba/Makefile.am b/bench/ltl2tgba/Makefile.am index 1f1d67bdd..d4e7a567a 100644 --- a/bench/ltl2tgba/Makefile.am +++ b/bench/ltl2tgba/Makefile.am @@ -18,11 +18,12 @@ ## along with this program. If not, see . EXTRA_DIST = \ - algorithms \ big \ formulae.ltl \ known \ small \ + tools \ + tools.sim \ sum.py OUTPUTS = known small big @@ -37,7 +38,7 @@ CLEANFILES = $(OUTCSV) $(OUTJSON) $(OUTLOG) \ run: $(OUTJSON) -deps = $(srcdir)/algorithms \ +deps = $(srcdir)/tools \ $(top_srcdir)/configure.ac \ $(top_builddir)/src/bin/ltl2tgba diff --git a/bench/ltl2tgba/README b/bench/ltl2tgba/README index 519f475fc..6abdf5bfd 100644 --- a/bench/ltl2tgba/README +++ b/bench/ltl2tgba/README @@ -26,25 +26,19 @@ may edit the file 'algorithms' to remove tools or lower the timeout. Here are the different scripts used, in case you want to customize this benchmark. -* algorithms +* tools The configuration of all the translators. This is merely a script that builds the command-line of ltlcross, to be run by the next - three scripts. Most of the $TOOL variables are defined by the - 'defs' file, which is output by 'configure' after checking for - the presence of the said tools. + three scripts. Most of the variables (like $SPIN, $LTL2BA, etc) are + defined by the 'defs' file, which is output by 'configure' after + checking for the presence of the said tools. - If you want to add your own tool to the mix, simply modify this file. + If you want to add your own tool to the mix, simply modify this + 'tools' file. The timeout value, common to the three benchmarks, is also set here. - You can also benchmark some simulations algorithms by setting the - variable "BENCH_SIMULATION". For example by running: - - BENCH_SIMULATION=t make -j3 run - - You run the simulation benchmark on the three kind of formulae. - * small * big * known @@ -161,3 +155,21 @@ For all these values (except count), the sammler number the better. More details about ltlcross (used to produce these outputs) can be found in its man page, and at http://spot.lip6.fr/userdoc/tools.html + + +============================================== + Running differents configurations / toolsets +============================================== + +Instead of modifying the 'tools' file, you can also set the TOOLS +environment variable to point to another file. + +For instance try + + TOOLS=./tools.sim make -j3 run + +to benchmark several simulation-related options. + + + + diff --git a/bench/ltl2tgba/algorithms b/bench/ltl2tgba/algorithms deleted file mode 100644 index 12c6bc0e5..000000000 --- a/bench/ltl2tgba/algorithms +++ /dev/null @@ -1,47 +0,0 @@ -# Fill "$@" with the list of translators we want to benchmark. - -# Add a dummy initial argument to clear "$@" and also in case one of -# the tools starts with "-". -set dummy - - -if test -n "$BENCH_SIMULATION"; then - translator="../../src/tgbatest/ltl2tgba" - set "$@" "$translator -R3 -Rm -r7 -t %s >%T" \ - "$translator -t -RDS -Rm -r7 -R3 %s >%T" \ - "$translator -t -RDCS -r7 -Rm -R3 %s >%T" \ - "$translator -t -RRS -r7 -R3 -Rm %s >%T" \ - "$translator -t -RIS -r7 -R3 -Rm %s >%T" \ - "$translator -t -RDCIS -r7 -Rm -R3 %s >%T" \ - "$translator -t -R3 -Rm -r7 -DS %s >%T" \ - "$translator -t -r7 -R3 -Rm -RDS -DS %s >%T" \ - "$translator -t -RDCS -r7 -Rm -R3 -DS %s >%T" \ - "$translator -t -RRS -r7 -R3 -Rm -DS %s >%T" \ - "$translator -t -RIS -r7 -R3 -Rm -DS %s >%T" \ - "$translator -t -RDCIS -r7 -R3 -Rm -DS %s >%T" - -else -# Add third-party tools if they are available - test -n "$SPIN" && set "$@" "$SPIN -f %s >%N" - test -n "$LTL2BA" && set "$@" "$LTL2BA -f %s >%N" - test -n "$LTL3BA" && set "$@" "$LTL3BA -f %s >%N" \ - "$LTL3BA -M -f %s >%N" \ - "$LTL3BA -S -f %s >%N" \ - "$LTL3BA -S -M -f %s >%N" - -# Use -s to output a neverclaim, like the other tools. - set "$@" "$LTL2TGBA --det -s %s >%N" \ - "$LTL2TGBA --small -s %s >%N" -fi; -# If you want to add your own tool, you can add it here. -# See 'man ltlcross' for the list of %-escape you may use -# to specify input formula and output automaton. -# -# set "$@" "tool options %... > %..." - - -# Set the timeout to 5 minutes -set "$@" --timeout=300 - -# Finaly remove the dummy initial argument -shift diff --git a/bench/ltl2tgba/big b/bench/ltl2tgba/big index e1e7f5fd4..e5134cc9c 100755 --- a/bench/ltl2tgba/big +++ b/bench/ltl2tgba/big @@ -19,7 +19,6 @@ # along with this program. If not, see . . ./defs -. "$srcdir/algorithms" $RANDLTL -n 100 --tree-size=15..20 p1 p2 p3 p4 p5 p6 p7 p8 | $LTLFILT --nnf | $LTLCROSS "$@" --csv=big.csv --json=big.json 2>&1 | tee big.log diff --git a/bench/ltl2tgba/defs.in b/bench/ltl2tgba/defs.in index cc4543111..ad2de6b6f 100644 --- a/bench/ltl2tgba/defs.in +++ b/bench/ltl2tgba/defs.in @@ -46,3 +46,5 @@ LTL2TGBA="@top_builddir@/src/bin/ltl2tgba@EXEEXT@" MODELLA="@MODELLA@" SPIN="@SPIN@" WRING2LBTT="@WRING2LBTT@" + +. ${TOOLS-$srcdir/tools} diff --git a/bench/ltl2tgba/known b/bench/ltl2tgba/known index f1bd3fb55..8688afc1e 100755 --- a/bench/ltl2tgba/known +++ b/bench/ltl2tgba/known @@ -19,7 +19,6 @@ # along with this program. If not, see . . ./defs -. "$srcdir/algorithms" $LTLCROSS "$@" --csv=known.csv --json=known.json \ < "$srcdir/formulae.ltl" 2>&1 | diff --git a/bench/ltl2tgba/small b/bench/ltl2tgba/small index d5995f3a8..a07790596 100755 --- a/bench/ltl2tgba/small +++ b/bench/ltl2tgba/small @@ -19,7 +19,6 @@ # along with this program. If not, see . . ./defs -. "$srcdir/algorithms" $RANDLTL -n 100 --tree-size=10 p1 p2 p3 p4 | $LTLFILT --nnf | $LTLCROSS "$@" --csv=small.csv --json=small.json 2>&1 | tee small.log diff --git a/bench/ltl2tgba/tools b/bench/ltl2tgba/tools new file mode 100644 index 000000000..26d6129b7 --- /dev/null +++ b/bench/ltl2tgba/tools @@ -0,0 +1,50 @@ +## -*- coding: utf-8; mode: sh -*- +## Copyright (C) 2013 Laboratoire de Recherche et Développement de +## l'Epita (LRDE). +## +## This file is part of Spot, a model checking library. +## +## Spot is free software; you can redistribute it and/or modify it +## under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## Spot is distributed in the hope that it will be useful, but WITHOUT +## ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +## or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +## License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program. If not, see . + +# The purpose of this script is to fill "$@" with the list of +# translator tools we want to benchmark. + +# Add a dummy initial argument to clear "$@" and also in case one of +# the tools starts with "-". +set dummy + +# Add third-party tools if they are available. +test -n "$SPIN" && set "$@" "$SPIN -f %s >%N" +test -n "$LTL2BA" && set "$@" "$LTL2BA -f %s >%N" +test -n "$LTL3BA" && set "$@" "$LTL3BA -f %s >%N" \ + "$LTL3BA -M -f %s >%N" \ + "$LTL3BA -S -f %s >%N" \ + "$LTL3BA -S -M -f %s >%N" + +# Use -s to output a neverclaim, like the other tools. +set "$@" \ + "$LTL2TGBA --det -s %s >%N" \ + "$LTL2TGBA --small -s %s >%N" + +# If you want to add your own tool, you can add it here. +# See 'man ltlcross' for the list of %-escapes you may use +# to specify input formula and output automaton. +# +# set "$@" "tool options %... > %..." + +# Set the timeout to 5 minutes +set "$@" --timeout=300 + +# Finaly remove the dummy initial argument +shift diff --git a/bench/ltl2tgba/tools.sim b/bench/ltl2tgba/tools.sim new file mode 100644 index 000000000..0438788e4 --- /dev/null +++ b/bench/ltl2tgba/tools.sim @@ -0,0 +1,35 @@ +## -*- coding: utf-8; mode: sh -*- +## Copyright (C) 2013 Laboratoire de Recherche et Développement de +## l'Epita (LRDE). +## +## This file is part of Spot, a model checking library. +## +## Spot is free software; you can redistribute it and/or modify it +## under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## Spot is distributed in the hope that it will be useful, but WITHOUT +## ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +## or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +## License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program. If not, see . + + +## Test simulations algorithms. + +set dummy \ + "$LTL2TGBA --lbtt -x simul=0 %f >%T" \ + "$LTL2TGBA --lbtt -x simul=1 %f >%T" \ + "$LTL2TGBA --lbtt -x simul=2 %f >%T" \ + "$LTL2TGBA --lbtt -x simul=3 %f >%T" \ + "$LTL2TGBA --lbtt -x simul=4 %f >%T" \ + "$LTL2TGBA --lbtt --ba -x simul=0 %f >%T" \ + "$LTL2TGBA --lbtt --ba -x simul=1 %f >%T" \ + "$LTL2TGBA --lbtt --ba -x simul=2 %f >%T" \ + "$LTL2TGBA --lbtt --ba -x simul=3 %f >%T" \ + "$LTL2TGBA --lbtt --ba -x simul=4 %f >%T" \ + --timeout=300 +shift