add texttest structure for osm scenario (refs #2 #13)

test_server.tsccfg

@@ -0,0 +1,5 @@
+<configuration>
+    <host value=""/>
+    <user value=""/>
+    <password value=""/>
+</configuration>

tests/config_all

@@ -0,0 +1,6 @@
+home_operating_system:posix
+test_data_ignore:.git
+kill_timeout:432000
+
+batch_result_repository:$SCENARIO_BATCH_RESULT
+historical_report_location:$SCENARIO_REPORT

tests/runTSCTests.bat

@@ -1,6 +1,6 @@
 call testEnv.bat %1
-set SIP_HOME=C:\Users\behr_mi\Projekte\simo
+set SIP_HOME=%CD%\..
 set TSC_HOME=%CD%\..
 set TEXTTEST_HOME=%SIP_HOME%\tests
-set TSC_DATA=%SIP_HOME%\projects\tapas
-%TEXTTESTPY% -a tapasVEU.tsc
+set TSC_DATA=%SIP_HOME%
+%TEXTTESTPY% -a tapas_osmVEU.tsc

tests/scenariorunner.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# this script is meant be run from a test directory
+
+import os,subprocess,sys,shutil,types,string
+from glob import glob
+from optparse import OptionParser
+import shutil
+
+if 'SUMO_HOME' in os.environ:
+    sys.path.append(os.path.join(os.environ['SUMO_HOME'], 'tools'))
+else:   
+    sys.exit("please declare environment variable 'SUMO_HOME'")
+
+THIS_DIR = os.path.abspath(os.path.dirname(__file__))
+
+import extractTest
+
+if "SUMO_BINDIR" not in os.environ:
+    if "SUMO_HOME" in os.environ:
+        os.environ["SUMO_BINDIR"] = os.path.join(os.environ["SUMO_HOME"], "bin")
+    if "SUMO" in os.environ:
+        os.environ["SUMO_BINDIR"] = os.path.join(os.environ["SUMO"], "bin")
+
+DEPLOY_SANDBOX = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'DEPLOY_SANDBOX')
+
+def run(process, verbose):
+    for step in process:
+        if callable(step):
+            step()
+        else:
+            if step[0]=='<':
+                os.chdir(step[1:-1])
+                continue
+            cmd = string.Template(step).safe_substitute(os.environ)
+            if verbose:
+                print('calling cmd "%s" in directory "%s"' % (cmd, os.getcwd()))
+            subprocess.call(cmd, shell=True, stdout=sys.stdout, stderr=sys.stderr)
+
+def deploy(path, verbose, toRemove, toDeploy):
+    try: 
+        os.makedirs(path)
+    except: 
+        pass
+    for file in toDeploy:
+        if verbose:
+            print("copying '%s' to '%s'" % (file, path))
+        if os.path.isdir(file):
+            shutil.copytree(file, os.path.join(path, os.path.basename(file)))
+        else:
+            shutil.copy2(file, path)
+        if file in toRemove:
+            if verbose:
+                print("removing '%s'" % (file))
+            os.remove(file)
+
+
+def get_options():
+    optParser = OptionParser()
+    optParser.add_option("-s", "--scenario", 
+            help="path of the scenario to deploy (defaults to current directory)")
+    optParser.add_option("-d", "--dest", 
+            help="path in which to deploy the scenario (defaults to a subdirectory in the current directory with a generated name)")
+    optParser.add_option("-v", "--verbose", action="store_true",
+            default=False, help="tell me what you are doing")
+    optParser.add_option("-a", "--additional", 
+            help="additional argument to pass to the scenario")
+    (options, args) = optParser.parse_args()
+    if args:
+        optParser.error("no arguments allowed. Use options")
+    options.deploy = options.scenario is not None or options.dest is not None
+    if options.deploy:
+        if options.scenario is None:
+            options.scenario = os.getcwd()
+        if options.dest is None:
+            options.dest = extractTest.generateTargetName(THIS_DIR, options.scenario)
+        else:
+            options.dest = os.path.abspath(options.dest)
+    return options
+
+
+
+# there are two ways in which this can be called:
+# 1) without arguments. This assumes texttest has created the appropriate sandbox
+# 2) with arguments for deployment. This means we have to create the sandbox manually and maybe clean up afterwards
+import stddefs
+stddefs.options = get_options();
+if stddefs.options.deploy:
+    # create the sandbox
+    if os.path.isdir(DEPLOY_SANDBOX):
+        shutil.rmtree(DEPLOY_SANDBOX)
+    if stddefs.options.verbose:
+        print("extracting text from '%s'" % stddefs.options.scenario)
+    extractTest.main(extractTest.get_options(
+            ['--skip-configuration',
+                "%s;%s" % (stddefs.options.scenario, DEPLOY_SANDBOX)])) 
+    os.chdir(DEPLOY_SANDBOX)
+    # proceed from the sandbox
+    sys.path.append(DEPLOY_SANDBOX)
+    import scenariodefs
+    run(scenariodefs.buildProcess, stddefs.options.verbose)
+    deploy(stddefs.options.dest, stddefs.options.verbose, scenariodefs.toRemove, scenariodefs.toDeploy)
+else:
+    sys.path.append('.')
+    import scenariodefs
+    run(scenariodefs.buildProcess, stddefs.options.verbose)
+    run(scenariodefs.runProcess, stddefs.options.verbose)

tests/stddefs.py

@@ -0,0 +1,75 @@
+from __future__ import print_function
+import sys
+import os, glob, zipfile, subprocess
+import re
+from datetime import datetime
+sys.path.append(os.path.join(os.environ.get("SUMO_HOME"), 'tools'))
+from sumolib.miscutils import Statistics
+from sumolib.output import parse_fast
+
+def getDefaults(netfile, tlsfile):
+    if tlsfile!="":
+        build = buildProcess + [get_python_tool("tls/tls_csv2SUMO.py") + ' %s %s > %s' % (",".join(sorted(glob.glob("data/*.csv"))), netfile, tlsfile)]
+        toRemove = [netfile, tlsfile]
+    else:
+        build = buildProcess
+        toRemove = [netfile]
+    return build, runProcess, toRemove
+
+def unzip(fname):
+    zip = zipfile.ZipFile(fname)
+    for each in zip.namelist():
+        if not each.endswith('/'):
+            dest = os.path.basename(each)
+            file(dest, 'wb').write(zip.read(each))
+    filelist = zip.namelist()
+    zip.close()
+
+def get_python_tool(rel_path):
+    return '"' + os.environ.get("PYTHON", "python") + '" "' + os.path.join(os.environ["SUMO_HOME"], "tools", rel_path) + '"'
+
+def get_app(name, name_variable):
+    direct = os.environ.get(name_variable)
+    homedir = os.environ.get("SUMO_HOME")
+    if direct:
+        return '"%s"' % direct
+    if homedir:
+        return '"%s"' % os.path.join(homedir, 'bin', name)
+    return name
+
+
+def filterLog(log="data/sumo_log.txt", statsOut="data/stats.txt", statsIn="stats.scenario", tripinfos="data/tripinfos.xml"):
+    collisions = 0
+    timeout = 0
+    simEnd = -1
+    for line in open(log):
+        if "collision" in line:
+            collisions += 1
+        if "waited too long" in line:
+            timeout += 1
+        if line.startswith("Simulation ended at time: "):
+            simEnd = line.split()[-1]
+    if os.path.exists(tripinfos):
+        durationStats = Statistics(' Traveltimes')
+        for trip in parse_fast(tripinfos, 'tripinfo', ['id', 'duration']):
+            durationStats.add(float(trip.duration), trip.id)
+        durationStats = str(durationStats).replace('"','')
+    else:
+        durationStats = ''
+    statLine = "Collisions: %s Timeouts: %s End: %s%s" % (collisions, timeout, simEnd, durationStats)
+    with open(statsOut, 'w') as o:
+        rootLength = len(os.environ["TEXTTEST_SANDBOX_ROOT"]) + 1
+        testNameStart = os.environ["TEXTTEST_SANDBOX"].find("/", rootLength) + 1
+        oldStats = os.path.join(os.environ["TEXTTEST_ROOT"], os.environ["TEXTTEST_SANDBOX"][testNameStart:], statsIn)
+        if os.path.exists(oldStats):
+            for line in open(oldStats):
+                o.write(line)
+            if line.strip() == statLine.strip():
+                o.close()
+                return
+        sumoVersion = subprocess.check_output(get_app('sumo', 'SUMO_BINARY') + " -V", shell=True).splitlines()[0]
+        print("%s %s\n%s" % (datetime.now(), sumoVersion, statLine), file=o)
+
+
+buildProcess = ["%s -c data/build.netccfg" % get_app('netconvert', 'NETCONVERT_BINARY')]
+runProcess = ["%s -c data/run.sumocfg --no-step-log" % get_app('sumo', 'SUMO_BINARY'), filterLog]

tests/tapas_osmVEU/config.tapas_osmVEU

@@ -0,0 +1,46 @@
+import_config_file:../config_all
+kill_timeout:3600
+binary:$TEXTTEST_HOME/scenariorunner.py
+test_data_ignore:.git
+config_module:default
+create_catalogues:true
+
+copy_test_path_merge:data
+copy_test_path:scenariodefs.py
+copy_test_path:template_gen.netccfg
+
+[collate_file]
+log:log.txt
+rectifiedTrips:data/scenario_workdir/*/iteration000/trips/rectified*.csv
+resultingTripFile:data/scenario_workdir/*/*/trips/2*ms.csv
+resultingAllPairsFile:data/scenario_workdir/*/*/trips/all_pairs*.csv
+sumoTrips:data/scenario_workdir/*/iteration000/trips/miv*.trips.xml
+
+[run_dependent_text]
+output:\r{REPLACE \n}
+output:(\\)+{REPLACE /}
+output:[0-9\.]+ms{REPLACE (TIME)}
+output:^Performance:{->}^Vehicles:
+output:^using.*bin/sumo
+output:^function .* finished after .* seconds
+output:^function .* called at .*
+output:Begin time: 
+output:End time: 
+output:Duration:
+output:ended (duration:
+output:answered .* queries and explored
+output:{INTERNAL writedir}{REPLACE <texttest sandbox>}
+output:([A-Z]:)?/.*/bin/netconvert(.exe)?{REPLACE <netconvert>}
+output:([A-Z]:)?/.*/bin/polyconvert(.exe)?{REPLACE <polyconvert>}
+output:([A-Z]:)?/.*/osm_scenario_pre{REPLACE <osm_scenario_pre>}
+output:Simulation.*started with{REPLACE Simulation started with}
+errors:Ran
+errors:Exception AttributeError: AttributeError("'NoneType' object has no attribute 'Error_GetErrorCount'",) in <bound method
+catalogue:.pyc
+catalogue:__pycache__
+log:[0-9\.]+ms{REPLACE (TIME)}
+
+[floating_point_tolerance]
+output:0.0101
+rectifiedTrips:0.0101
+resultingTripFile:0.0000000001

tests/tapas_osmVEU/pre/installTemplateDir/scenariodefs.py

@@ -0,0 +1,10 @@
+import os
+import tscdefs
+
+buildProcess = []
+runProcess = ["<data>",
+              tscdefs.get_python_tool("install_osm_scenario_templates.py")
+               + ' --clean -v -p ' + os.path.join(tscdefs.tscData, "osm_scenario_pre")
+              ]
+toRemove = []
+toDeploy = ['data']

tests/tapas_osmVEU/pre/testsuite.tapas_osmVEU

@@ -0,0 +1,2 @@
+# Test on setting up the basic infrastructure for tapas sumo
+installTemplateDir

tests/tapas_osmVEU/template/data/scenario_templates/mitte_net/__init__.py

@@ -0,0 +1,30 @@
+"""
+@file    __init__.py
+@author  [email protected]
+@date    2015-06-10
+@version $Id: __init__.py 4620 2015-06-27 14:06:36Z behr_mi $
+
+custom script collection for berlin_2010
+
+Copyright (C) 2015-2015 DLR/TS, Germany
+All rights reserved
+"""
+import subprocess
+import assign
+import postprocess
+from sumolib.miscutils import benchmark
+
+assign_trips = assign.run_default
+
+@benchmark
+def post(options, params, conn, routefile):
+    procs = [postprocess.run_pedestrian_sumo(options, routefile),
+             postprocess.run_emission_sumo(options, params, conn, routefile)]
+    err = None
+    for p in procs:
+        if p is not None:
+            retcode = p[1].wait()
+            if retcode:
+                err = subprocess.CalledProcessError(retcode, p[0])
+    if err:
+        raise err