bench/ltl2tgba: Adjust to support different config files.

* bench/ltl2tgba/algorithms: Rename as...
* bench/ltl2tgba/tools: ... this.
* bench/ltl2tgba/README, bench/ltl2tgba/Makefile.am,
bench/ltl2tgba/defs.in, bench/ltl2tgba/big, bench/ltl2tgba/small,
bench/ltl2tgba/known: Adjust.
* bench/ltl2tgba/tools.sim: New file, extracted from ./algorithms.

bench/ltl2tgba/Makefile.am

@@ -18,11 +18,12 @@
 ## along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 EXTRA_DIST = \
-  algorithms \
   big \
   formulae.ltl \
   known \
   small \
+  tools \
+  tools.sim \
   sum.py
 
 OUTPUTS = known small big
@@ -37,7 +38,7 @@ CLEANFILES = $(OUTCSV) $(OUTJSON) $(OUTLOG) \
 
 run: $(OUTJSON)
 
-deps = $(srcdir)/algorithms \
+deps = $(srcdir)/tools \
        $(top_srcdir)/configure.ac \
        $(top_builddir)/src/bin/ltl2tgba
 

bench/ltl2tgba/README

@@ -26,25 +26,19 @@ may edit the file 'algorithms' to remove tools or lower the timeout.
 Here are the different scripts used, in case you want to customize
 this benchmark.
 
-* algorithms
+* tools
 
   The configuration of all the translators.  This is merely a script
   that builds the command-line of ltlcross, to be run by the next
-  three scripts.  Most of the $TOOL variables are defined by the
+  three scripts.  Most of the variables (like $SPIN, $LTL2BA, etc) are
-  'defs' file, which is output by 'configure' after checking for
+  defined by the 'defs' file, which is output by 'configure' after
-  the presence of the said tools.
+  checking for the presence of the said tools.
 
-  If you want to add your own tool to the mix, simply modify this file.
+  If you want to add your own tool to the mix, simply modify this
+  'tools' file.
 
   The timeout value, common to the three benchmarks, is also set here.
 
-  You can also benchmark some simulations algorithms by setting the
-  variable "BENCH_SIMULATION". For example by running:
-
-  BENCH_SIMULATION=t make -j3 run
-
-  You run the simulation benchmark on the three kind of formulae.
-
 * small
 * big
 * known
@@ -161,3 +155,21 @@ For all these values (except count), the sammler number the better.
 
 More details about ltlcross (used to produce these outputs) can be
 found in its man page, and at http://spot.lip6.fr/userdoc/tools.html
+
+
+==============================================
+ Running differents configurations / toolsets
+==============================================
+
+Instead of modifying the 'tools' file, you can also set the TOOLS
+environment variable to point to another file.
+
+For instance try
+
+  TOOLS=./tools.sim make -j3 run
+
+to benchmark several simulation-related options.
+
+
+
+

bench/ltl2tgba/algorithms

@@ -1,47 +0,0 @@
-# Fill "$@" with the list of translators we want to benchmark.
-
-# Add a dummy initial argument to clear "$@" and also in case one of
-# the tools starts with "-".
-set dummy
-
-
-if test -n "$BENCH_SIMULATION"; then
-    translator="../../src/tgbatest/ltl2tgba"
-    set "$@" "$translator -R3 -Rm -r7 -t %s >%T" \
-        "$translator -t -RDS -Rm -r7 -R3  %s >%T" \
-        "$translator -t -RDCS -r7 -Rm -R3 %s >%T" \
-        "$translator -t -RRS  -r7 -R3 -Rm  %s >%T" \
-        "$translator -t -RIS  -r7 -R3 -Rm %s >%T" \
-        "$translator -t -RDCIS -r7 -Rm -R3  %s >%T" \
-        "$translator -t -R3 -Rm -r7 -DS %s >%T" \
-        "$translator -t -r7 -R3 -Rm -RDS -DS %s >%T" \
-        "$translator -t -RDCS -r7 -Rm -R3 -DS %s >%T" \
-        "$translator -t -RRS  -r7 -R3 -Rm -DS %s >%T" \
-        "$translator -t -RIS  -r7 -R3 -Rm -DS %s >%T" \
-        "$translator -t -RDCIS -r7 -R3 -Rm -DS %s >%T"
-
-else
-# Add third-party tools if they are available
-    test -n "$SPIN"   && set "$@" "$SPIN -f %s >%N"
-    test -n "$LTL2BA" && set "$@" "$LTL2BA -f %s >%N"
-    test -n "$LTL3BA" && set "$@" "$LTL3BA -f %s >%N" \
-        "$LTL3BA -M -f %s >%N" \
-        "$LTL3BA -S -f %s >%N" \
-        "$LTL3BA -S -M -f %s >%N"
-
-# Use -s to output a neverclaim, like the other tools.
-    set "$@" "$LTL2TGBA --det -s %s >%N" \
-        "$LTL2TGBA --small -s %s >%N"
-fi;
-# If you want to add your own tool, you can add it here.
-# See 'man ltlcross' for the list of %-escape you may use
-# to specify input formula and output automaton.
-#
-# set "$@" "tool options %... > %..."
-
-
-# Set the timeout to 5 minutes
-set "$@" --timeout=300
-
-# Finaly remove the dummy initial argument
-shift

bench/ltl2tgba/big

@@ -19,7 +19,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 . ./defs
-. "$srcdir/algorithms"
 
 $RANDLTL -n 100 --tree-size=15..20 p1 p2 p3 p4 p5 p6 p7 p8 | $LTLFILT --nnf |
 $LTLCROSS "$@" --csv=big.csv --json=big.json 2>&1 | tee big.log

bench/ltl2tgba/defs.in

@@ -46,3 +46,5 @@ LTL2TGBA="@top_builddir@/src/bin/ltl2tgba@EXEEXT@"
 MODELLA="@MODELLA@"
 SPIN="@SPIN@"
 WRING2LBTT="@WRING2LBTT@"
+
+. ${TOOLS-$srcdir/tools}

bench/ltl2tgba/known

@@ -19,7 +19,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 . ./defs
-. "$srcdir/algorithms"
 
 $LTLCROSS "$@" --csv=known.csv --json=known.json \
  < "$srcdir/formulae.ltl" 2>&1 |

bench/ltl2tgba/small

@@ -19,7 +19,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 . ./defs
-. "$srcdir/algorithms"
 
 $RANDLTL -n 100 --tree-size=10 p1 p2 p3 p4 | $LTLFILT --nnf |
 $LTLCROSS "$@" --csv=small.csv --json=small.json 2>&1 | tee small.log

bench/ltl2tgba/tools

@@ -0,0 +1,50 @@
+## -*- coding: utf-8; mode: sh -*-
+## Copyright (C) 2013 Laboratoire de Recherche et Développement de
+## l'Epita (LRDE).
+##
+## This file is part of Spot, a model checking library.
+##
+## Spot is free software; you can redistribute it and/or modify it
+## under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3 of the License, or
+## (at your option) any later version.
+##
+## Spot is distributed in the hope that it will be useful, but WITHOUT
+## ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+## or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
+## License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# The purpose of this script is to fill "$@" with the list of
+# translator tools we want to benchmark.
+
+# Add a dummy initial argument to clear "$@" and also in case one of
+# the tools starts with "-".
+set dummy
+
+# Add third-party tools if they are available.
+test -n "$SPIN"   && set "$@" "$SPIN -f %s >%N"
+test -n "$LTL2BA" && set "$@" "$LTL2BA -f %s >%N"
+test -n "$LTL3BA" && set "$@" "$LTL3BA -f %s >%N" \
+        "$LTL3BA -M -f %s >%N" \
+        "$LTL3BA -S -f %s >%N" \
+        "$LTL3BA -S -M -f %s >%N"
+
+# Use -s to output a neverclaim, like the other tools.
+set "$@" \
+    "$LTL2TGBA --det -s %s >%N" \
+    "$LTL2TGBA --small -s %s >%N"
+
+# If you want to add your own tool, you can add it here.
+# See 'man ltlcross' for the list of %-escapes you may use
+# to specify input formula and output automaton.
+#
+# set "$@" "tool options %... > %..."
+
+# Set the timeout to 5 minutes
+set "$@" --timeout=300
+
+# Finaly remove the dummy initial argument
+shift

bench/ltl2tgba/tools.sim

@@ -0,0 +1,35 @@
+## -*- coding: utf-8; mode: sh -*-
+## Copyright (C) 2013 Laboratoire de Recherche et Développement de
+## l'Epita (LRDE).
+##
+## This file is part of Spot, a model checking library.
+##
+## Spot is free software; you can redistribute it and/or modify it
+## under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3 of the License, or
+## (at your option) any later version.
+##
+## Spot is distributed in the hope that it will be useful, but WITHOUT
+## ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+## or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
+## License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+## Test simulations algorithms.
+
+set dummy \
+    "$LTL2TGBA --lbtt -x simul=0 %f >%T" \
+    "$LTL2TGBA --lbtt -x simul=1 %f >%T" \
+    "$LTL2TGBA --lbtt -x simul=2 %f >%T" \
+    "$LTL2TGBA --lbtt -x simul=3 %f >%T" \
+    "$LTL2TGBA --lbtt -x simul=4 %f >%T" \
+    "$LTL2TGBA --lbtt --ba -x simul=0 %f >%T" \
+    "$LTL2TGBA --lbtt --ba -x simul=1 %f >%T" \
+    "$LTL2TGBA --lbtt --ba -x simul=2 %f >%T" \
+    "$LTL2TGBA --lbtt --ba -x simul=3 %f >%T" \
+    "$LTL2TGBA --lbtt --ba -x simul=4 %f >%T" \
+    --timeout=300
+shift