bench/ltl2tgba: Adjust to support different config files.
* bench/ltl2tgba/algorithms: Rename as... * bench/ltl2tgba/tools: ... this. * bench/ltl2tgba/README, bench/ltl2tgba/Makefile.am, bench/ltl2tgba/defs.in, bench/ltl2tgba/big, bench/ltl2tgba/small, bench/ltl2tgba/known: Adjust. * bench/ltl2tgba/tools.sim: New file, extracted from ./algorithms.
This commit is contained in:
Alexandre Duret-Lutz
parent
c2e96387e4
commit
212cb835fc
9 changed files with 114 additions and 64 deletions
|
|
@ -18,11 +18,12 @@
|
|||
## along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
EXTRA_DIST = \
|
||||
algorithms \
|
||||
big \
|
||||
formulae.ltl \
|
||||
known \
|
||||
small \
|
||||
tools \
|
||||
tools.sim \
|
||||
sum.py
|
||||
|
||||
OUTPUTS = known small big
|
||||
|
|
@ -37,7 +38,7 @@ CLEANFILES = $(OUTCSV) $(OUTJSON) $(OUTLOG) \
|
|||
|
||||
run: $(OUTJSON)
|
||||
|
||||
deps = $(srcdir)/algorithms \
|
||||
deps = $(srcdir)/tools \
|
||||
$(top_srcdir)/configure.ac \
|
||||
$(top_builddir)/src/bin/ltl2tgba
|
||||
|
||||
|
|
|
|||
|
|
@ -26,25 +26,19 @@ may edit the file 'algorithms' to remove tools or lower the timeout.
|
|||
Here are the different scripts used, in case you want to customize
|
||||
this benchmark.
|
||||
|
||||
* algorithms
|
||||
* tools
|
||||
|
||||
The configuration of all the translators. This is merely a script
|
||||
that builds the command-line of ltlcross, to be run by the next
|
||||
three scripts. Most of the $TOOL variables are defined by the
|
||||
'defs' file, which is output by 'configure' after checking for
|
||||
the presence of the said tools.
|
||||
three scripts. Most of the variables (like $SPIN, $LTL2BA, etc) are
|
||||
defined by the 'defs' file, which is output by 'configure' after
|
||||
checking for the presence of the said tools.
|
||||
|
||||
If you want to add your own tool to the mix, simply modify this file.
|
||||
If you want to add your own tool to the mix, simply modify this
|
||||
'tools' file.
|
||||
|
||||
The timeout value, common to the three benchmarks, is also set here.
|
||||
|
||||
You can also benchmark some simulations algorithms by setting the
|
||||
variable "BENCH_SIMULATION". For example by running:
|
||||
|
||||
BENCH_SIMULATION=t make -j3 run
|
||||
|
||||
You run the simulation benchmark on the three kind of formulae.
|
||||
|
||||
* small
|
||||
* big
|
||||
* known
|
||||
|
|
@ -161,3 +155,21 @@ For all these values (except count), the sammler number the better.
|
|||
|
||||
More details about ltlcross (used to produce these outputs) can be
|
||||
found in its man page, and at http://spot.lip6.fr/userdoc/tools.html
|
||||
|
||||
|
||||
==============================================
|
||||
Running differents configurations / toolsets
|
||||
==============================================
|
||||
|
||||
Instead of modifying the 'tools' file, you can also set the TOOLS
|
||||
environment variable to point to another file.
|
||||
|
||||
For instance try
|
||||
|
||||
TOOLS=./tools.sim make -j3 run
|
||||
|
||||
to benchmark several simulation-related options.
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,47 +0,0 @@
|
|||
# Fill "$@" with the list of translators we want to benchmark.
|
||||
|
||||
# Add a dummy initial argument to clear "$@" and also in case one of
|
||||
# the tools starts with "-".
|
||||
set dummy
|
||||
|
||||
|
||||
if test -n "$BENCH_SIMULATION"; then
|
||||
translator="../../src/tgbatest/ltl2tgba"
|
||||
set "$@" "$translator -R3 -Rm -r7 -t %s >%T" \
|
||||
"$translator -t -RDS -Rm -r7 -R3 %s >%T" \
|
||||
"$translator -t -RDCS -r7 -Rm -R3 %s >%T" \
|
||||
"$translator -t -RRS -r7 -R3 -Rm %s >%T" \
|
||||
"$translator -t -RIS -r7 -R3 -Rm %s >%T" \
|
||||
"$translator -t -RDCIS -r7 -Rm -R3 %s >%T" \
|
||||
"$translator -t -R3 -Rm -r7 -DS %s >%T" \
|
||||
"$translator -t -r7 -R3 -Rm -RDS -DS %s >%T" \
|
||||
"$translator -t -RDCS -r7 -Rm -R3 -DS %s >%T" \
|
||||
"$translator -t -RRS -r7 -R3 -Rm -DS %s >%T" \
|
||||
"$translator -t -RIS -r7 -R3 -Rm -DS %s >%T" \
|
||||
"$translator -t -RDCIS -r7 -R3 -Rm -DS %s >%T"
|
||||
|
||||
else
|
||||
# Add third-party tools if they are available
|
||||
test -n "$SPIN" && set "$@" "$SPIN -f %s >%N"
|
||||
test -n "$LTL2BA" && set "$@" "$LTL2BA -f %s >%N"
|
||||
test -n "$LTL3BA" && set "$@" "$LTL3BA -f %s >%N" \
|
||||
"$LTL3BA -M -f %s >%N" \
|
||||
"$LTL3BA -S -f %s >%N" \
|
||||
"$LTL3BA -S -M -f %s >%N"
|
||||
|
||||
# Use -s to output a neverclaim, like the other tools.
|
||||
set "$@" "$LTL2TGBA --det -s %s >%N" \
|
||||
"$LTL2TGBA --small -s %s >%N"
|
||||
fi;
|
||||
# If you want to add your own tool, you can add it here.
|
||||
# See 'man ltlcross' for the list of %-escape you may use
|
||||
# to specify input formula and output automaton.
|
||||
#
|
||||
# set "$@" "tool options %... > %..."
|
||||
|
||||
|
||||
# Set the timeout to 5 minutes
|
||||
set "$@" --timeout=300
|
||||
|
||||
# Finaly remove the dummy initial argument
|
||||
shift
|
||||
|
|
@ -19,7 +19,6 @@
|
|||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
. ./defs
|
||||
. "$srcdir/algorithms"
|
||||
|
||||
$RANDLTL -n 100 --tree-size=15..20 p1 p2 p3 p4 p5 p6 p7 p8 | $LTLFILT --nnf |
|
||||
$LTLCROSS "$@" --csv=big.csv --json=big.json 2>&1 | tee big.log
|
||||
|
|
|
|||
|
|
@ -46,3 +46,5 @@ LTL2TGBA="@top_builddir@/src/bin/ltl2tgba@EXEEXT@"
|
|||
MODELLA="@MODELLA@"
|
||||
SPIN="@SPIN@"
|
||||
WRING2LBTT="@WRING2LBTT@"
|
||||
|
||||
. ${TOOLS-$srcdir/tools}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@
|
|||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
. ./defs
|
||||
. "$srcdir/algorithms"
|
||||
|
||||
$LTLCROSS "$@" --csv=known.csv --json=known.json \
|
||||
< "$srcdir/formulae.ltl" 2>&1 |
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@
|
|||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
. ./defs
|
||||
. "$srcdir/algorithms"
|
||||
|
||||
$RANDLTL -n 100 --tree-size=10 p1 p2 p3 p4 | $LTLFILT --nnf |
|
||||
$LTLCROSS "$@" --csv=small.csv --json=small.json 2>&1 | tee small.log
|
||||
|
|
|
|||
50
bench/ltl2tgba/tools
Normal file
50
bench/ltl2tgba/tools
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
## -*- coding: utf-8; mode: sh -*-
|
||||
## Copyright (C) 2013 Laboratoire de Recherche et Développement de
|
||||
## l'Epita (LRDE).
|
||||
##
|
||||
## This file is part of Spot, a model checking library.
|
||||
##
|
||||
## Spot is free software; you can redistribute it and/or modify it
|
||||
## under the terms of the GNU General Public License as published by
|
||||
## the Free Software Foundation; either version 3 of the License, or
|
||||
## (at your option) any later version.
|
||||
##
|
||||
## Spot is distributed in the hope that it will be useful, but WITHOUT
|
||||
## ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
|
||||
## or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
|
||||
## License for more details.
|
||||
##
|
||||
## You should have received a copy of the GNU General Public License
|
||||
## along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# The purpose of this script is to fill "$@" with the list of
|
||||
# translator tools we want to benchmark.
|
||||
|
||||
# Add a dummy initial argument to clear "$@" and also in case one of
|
||||
# the tools starts with "-".
|
||||
set dummy
|
||||
|
||||
# Add third-party tools if they are available.
|
||||
test -n "$SPIN" && set "$@" "$SPIN -f %s >%N"
|
||||
test -n "$LTL2BA" && set "$@" "$LTL2BA -f %s >%N"
|
||||
test -n "$LTL3BA" && set "$@" "$LTL3BA -f %s >%N" \
|
||||
"$LTL3BA -M -f %s >%N" \
|
||||
"$LTL3BA -S -f %s >%N" \
|
||||
"$LTL3BA -S -M -f %s >%N"
|
||||
|
||||
# Use -s to output a neverclaim, like the other tools.
|
||||
set "$@" \
|
||||
"$LTL2TGBA --det -s %s >%N" \
|
||||
"$LTL2TGBA --small -s %s >%N"
|
||||
|
||||
# If you want to add your own tool, you can add it here.
|
||||
# See 'man ltlcross' for the list of %-escapes you may use
|
||||
# to specify input formula and output automaton.
|
||||
#
|
||||
# set "$@" "tool options %... > %..."
|
||||
|
||||
# Set the timeout to 5 minutes
|
||||
set "$@" --timeout=300
|
||||
|
||||
# Finaly remove the dummy initial argument
|
||||
shift
|
||||
35
bench/ltl2tgba/tools.sim
Normal file
35
bench/ltl2tgba/tools.sim
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
## -*- coding: utf-8; mode: sh -*-
|
||||
## Copyright (C) 2013 Laboratoire de Recherche et Développement de
|
||||
## l'Epita (LRDE).
|
||||
##
|
||||
## This file is part of Spot, a model checking library.
|
||||
##
|
||||
## Spot is free software; you can redistribute it and/or modify it
|
||||
## under the terms of the GNU General Public License as published by
|
||||
## the Free Software Foundation; either version 3 of the License, or
|
||||
## (at your option) any later version.
|
||||
##
|
||||
## Spot is distributed in the hope that it will be useful, but WITHOUT
|
||||
## ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
|
||||
## or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
|
||||
## License for more details.
|
||||
##
|
||||
## You should have received a copy of the GNU General Public License
|
||||
## along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
## Test simulations algorithms.
|
||||
|
||||
set dummy \
|
||||
"$LTL2TGBA --lbtt -x simul=0 %f >%T" \
|
||||
"$LTL2TGBA --lbtt -x simul=1 %f >%T" \
|
||||
"$LTL2TGBA --lbtt -x simul=2 %f >%T" \
|
||||
"$LTL2TGBA --lbtt -x simul=3 %f >%T" \
|
||||
"$LTL2TGBA --lbtt -x simul=4 %f >%T" \
|
||||
"$LTL2TGBA --lbtt --ba -x simul=0 %f >%T" \
|
||||
"$LTL2TGBA --lbtt --ba -x simul=1 %f >%T" \
|
||||
"$LTL2TGBA --lbtt --ba -x simul=2 %f >%T" \
|
||||
"$LTL2TGBA --lbtt --ba -x simul=3 %f >%T" \
|
||||
"$LTL2TGBA --lbtt --ba -x simul=4 %f >%T" \
|
||||
--timeout=300
|
||||
shift
|
||||
Loading…
Add table
Add a link
Reference in a new issue