From 1d94ab3293ee0cbbf2df2e496987d472067f9465 Mon Sep 17 00:00:00 2001 From: roel de coninck Date: Sun, 9 Sep 2012 16:05:36 +0200 Subject: [PATCH 1/8] analyse_log improved and discrete files written with create_input_file --- awesim/pymosim.py | 61 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/awesim/pymosim.py b/awesim/pymosim.py index 59d7bac..23fd87b 100644 --- a/awesim/pymosim.py +++ b/awesim/pymosim.py @@ -321,9 +321,23 @@ def analyse_log(log_file): summary['successful'] = True elif line.find('CPU time for integration') > -1 or \ line.find('CPU-time for integration') > -1: - summary['CPU_time'] = line.split(' ')[-2] + summary['cpu_time'] = line.split(' ')[-2] elif line.find('Number of (successful) steps') > -1: - summary['steps_ok'] = line.split(' ')[-1] + summary['successful_steps'] = line.split(' ')[-1] + elif line.find('Number of (model) time events') > -1: + summary['time_events_model'] = line.split(' ')[-1] + elif line.find('Number of (U) time events') > -1: + summary['time_events_U'] = line.split(' ')[-1] + elif line.find('Number of state events') > -1: + summary['state_events'] = line.split(' ')[-1] + elif line.find('Number of step events') > -1: + summary['step_events'] = line.split(' ')[-1] + elif line.find('Minimum integration stepsize') > -1: + summary['step_size_min'] = line.split(' ')[-1] + elif line.find('Maximum integration stepsize') > -1: + summary['step_size_max'] = line.split(' ')[-1] + elif line.find('Maximum integration order') > -1: + summary['int_order_max'] = line.split(' ')[-1] elif line.find('Number of rejected steps') > -1: summary['steps_nok'] = line.split(' ')[-1] elif line.find('Integration started at 0 using integration method:') > -1: @@ -373,26 +387,51 @@ def run_ds(dymosim = '', dsin = '', result = ''): return proc -def create_input_file(data, filename): +def create_input_file(data, filename, discrete=False): """ Create an input file for the TimeTables from the MSL. - The input files are in ascii format - - data has to be an array with time as first column. All columns of this array - will be written in the ascii file. + The input files are in ascii format. + + Parameters + ========== + * data: array with time as first column. All columns of this array + will be written in the ascii file. + * filename: filename (with extension). If this file already exists, it + will be overwritten + * discrete: if True, the data array will be modified to become a discrete + profile. At each timestep, an additional line will be created. See + the documentation of the Modelica.Timetable.mo model for more info. """ + l,w = data.shape + + if discrete: + # create a second, shifted data array. We'll write each row of this + # shifted array after each row of the original one. + data_shifted = data.copy()[:-1,:] + data_shifted[:,0] = data[1:,0] + shape_string = '(' + str(2*l-1) + ',' + str(w) + ')' + else: + shape_string = '(' + str(l) + ',' + str(w) + ')' + + + f = open(filename, 'w') f.write(u'#1\n') - shape_string = str(data.shape) f.write(''.join([u'double data', shape_string, u'# Profiles created by python script: ', sys.argv[0], '\n'])) - for i in range(data.shape[0]): + for i in range(data.shape[0] - 1): f.write('\t'.join([str(v) for v in data[i,:]])) - f.write('\n') - + f.write('\n') + if discrete: + f.write('\t'.join([str(v) for v in data_shifted[i,:]])) + f.write('\n') + + f.write('\t'.join([str(v) for v in data[-1,:]])) + f.write('\n') f.close() + From 03731dcd75efa2f771a8b8c3fd0cc872b63de290 Mon Sep 17 00:00:00 2001 From: roel de coninck Date: Sun, 9 Sep 2012 16:06:39 +0200 Subject: [PATCH 2/8] using pymosim.analyse_log instead of the copy in index_one_sim and more clear error message with memoryerror --- awesim/simdex.py | 70 ++++++++++++++++-------------------------------- 1 file changed, 23 insertions(+), 47 deletions(-) diff --git a/awesim/simdex.py b/awesim/simdex.py index e398529..25d4661 100644 --- a/awesim/simdex.py +++ b/awesim/simdex.py @@ -72,6 +72,7 @@ class Simdex: import pdb from .simulation import Simulation from .result import Result +from .pymosim import analyse_log class Simdex: @@ -284,12 +285,15 @@ def scan(self, folder='', process=None, timecheck=True): process = self.process if folder == '' : - filenames = self.__get_files(os.getcwd(), '.mat') - elif os.path.exists(folder): + folder = os.getcwd() + + try: filenames = self.__get_files(folder, '.mat') - else: - raise IOError('folder does not exist') - + except IOError: + raise IOError('folder %s does not exist' % (folder)) + + if len(filenames) == 0: + raise ValueError("No .mat files found in %s" % (folder)) # Convert to full path filenames to avoid confusion full_path_filenames = [] @@ -312,6 +316,8 @@ def scan(self, folder='', process=None, timecheck=True): # we find a first Dymola file sim = Simulation(full_path_filenames[index]) simulation_file = True + except MemoryError: + print 'WARNING: %s could not be indexed because of a MemoryError.\nThe file is probably too big. It could help to try in a fresh python instance' % (full_path_filenames[index]) except: print '%s is no Dymola file. It is not indexed' % \ (full_path_filenames[index]) @@ -587,46 +593,7 @@ def index_one_par(parameters, parmap, parvalues, par, index, parvalue): def add_meta(simulation, key): """Create a node for the simulation and add data to /Metadata""" - def analyse_log(log_file): - """ - analyse_log(log_file) - log_file = string with path to a dslog.txt file - - Check if the simulation ended successfully, which solver was used and - how much time it took. Optionally, show the number of this and that - Returns a dictionary with the results - - """ - - summary = {'successful':False} - lf = open(log_file, 'r') - lines = lf.readlines() - for line_number, line in enumerate(lines): - if line.find('Integration terminated successfully at T =') > -1: - summary['successful'] = True - elif line.find('CPU time for integration') > -1 or \ - line.find('CPU-time for integration') > -1: - summary['CPU_time'] = line.split(' ')[-2] - elif line.find('Number of (successful) steps') > -1: - summary['steps_ok'] = line.split(' ')[-1] - elif line.find('Number of rejected steps') > -1: - summary['steps_nok'] = line.split(' ')[-1] - elif line.find('Integration started at 0 using integration method:') > -1: - summary['algorithm'] = lines[line_number + 1].strip('\n') - elif line.find('Integration started at T = 0 using integration method') > -1: - summary['algorithm'] = line.split(' ')[-1].strip('\n') - elif line.find('This simulation timed out and was killed') > -1: - summary['successful'] = False - summary['timed_out'] = True - elif line.find('Corresponding result file') > -1: - summary['result file'] = line.split(' ')[-1].strip('\n') - lf.close() - if summary.has_key('steps_nok'): - summary['perc_wrong'] = 100. * float(summary['steps_nok']) / \ - float(summary['steps_ok']) - else: - summary['perc_wrong'] = 0 - return summary + class Meta(tbl.IsDescription): SID = tbl.StringCol(itemsize=16) @@ -634,11 +601,20 @@ class Meta(tbl.IsDescription): log_analysed = tbl.BoolCol() successful = tbl.BoolCol() algorithm = tbl.StringCol(itemsize=16) - CPU_time = tbl.Float32Col() - steps_ok = tbl.Int32Col() + cpu_time = tbl.Float32Col() + successful_steps = tbl.Int32Col() steps_nok = tbl.Int32Col() timed_out = tbl.BoolCol() perc_wrong = tbl.Float32Col() + time_events_model = tbl.Int32Col() + time_events_U = tbl.Int32Col() + state_events = tbl.Int32Col() + step_events = tbl.Int32Col() + step_size_min = tbl.Float32Col() + step_size_max = tbl.Float32Col() + int_order_max = tbl.Int32Col() + + self.openh5() From f2c323fc599ea096a53e20fad1962a1cb037987a Mon Sep 17 00:00:00 2001 From: roel de coninck Date: Sun, 9 Sep 2012 16:08:12 +0200 Subject: [PATCH 3/8] legend in 'best' position --- awesim/result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awesim/result.py b/awesim/result.py index d73e30b..af542be 100644 --- a/awesim/result.py +++ b/awesim/result.py @@ -230,7 +230,7 @@ def create_time4plot(sid): ax.plot(self.val[sid], 'o-', label=label) - leg = ax.legend() + leg = ax.legend(loc='best') lines = ax.get_lines() if plot_type == 'plot_date': ax.set_xlabel('time') From 057bd1abecb52d6e4acdd81bce2a79ae08517ae8 Mon Sep 17 00:00:00 2001 From: roel de coninck Date: Sun, 9 Sep 2012 16:08:25 +0200 Subject: [PATCH 4/8] gitignore --- .gitignore | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 15b9afb..6b916f2 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,31 @@ buildlog.txt /doc/Makefile /doc/make.bat /tests/SpeedInvestigations.xlsx -/doc/Untitled0.ipynb \ No newline at end of file +/doc/Untitled0.ipynb +/tests/SSB_files/SSB_5_HeatingCurve_F24H.mat +/tests/SSB_files/SSB_6_HeatingCurve_Ref - Copy.mat +/tests/SSB_files/TME_SSB_5_34degdata.mat +/tests/SSB_files/TME_SSB_5_DayTime_5000W_R10_3.mat +/tests/SSB_files/TME_SSB_5_DayTime_5000W_R10_4.mat +/tests/SSB_files/TME_SSB_5_DayTime_R5_1.mat +/tests/SSB_files/TME_SSB_5_DayTime_R5_2.mat +/tests/SSB_files/TME_SSB_5_DayTime_R5_3.mat +/tests/SSB_files/TME_SSB_5_DayTime_R5_4.mat +/tests/SSB_files/TME_SSB_5_DayTime_R5_5.mat +/tests/SSB_files/TME_SSB_5_GridLoad_4.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5000W_19.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5000W_20.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5000W_B3000_1.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5000W_B3000_2.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5000W_B3000_3.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5000W_B3000_4.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5000W_B3000_5.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5000W_B3000_6.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5000W_B3000_7.mat +/tests/SSB_files/TME_SSB_5_GridLoad_5000W_B3000_8.mat +/tests/SSB_files/TME_SSB_5_HeatingCurve_4000W_Ref.mat +/tests/SSB_files/TME_SSB_6_DayTime_4.mat +/tests/SSB_files/TME_SSB_6_DayTime_R5_4.mat +/tests/SSB_files/TME_SSB_6_GridLoad_3.mat +/tests/SSB_files/TME_SSB_6_GridLoad_4.mat \ No newline at end of file From 5920c6e9a3ba5c8ff40fa2271639cc626829b90a Mon Sep 17 00:00:00 2001 From: roel de coninck Date: Sun, 9 Sep 2012 16:09:03 +0200 Subject: [PATCH 5/8] Improved demo and 3 overview graphs --- doc/source/Overview.graphml | 429 +++++++++++ doc/source/Simdex_attributes.graphml | 884 ++++++++++++++++++++++ doc/source/Simdex_methods.graphml | 1018 ++++++++++++++++++++++++++ examples/demo_simman.py | 28 +- 4 files changed, 2348 insertions(+), 11 deletions(-) create mode 100644 doc/source/Overview.graphml create mode 100644 doc/source/Simdex_attributes.graphml create mode 100644 doc/source/Simdex_methods.graphml diff --git a/doc/source/Overview.graphml b/doc/source/Overview.graphml new file mode 100644 index 0000000..0d75f7c --- /dev/null +++ b/doc/source/Overview.graphml @@ -0,0 +1,429 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + .hdf5 + + + + + + + + + + + + + + + + + + + + simdex1 + + + + + + + + + + + + + + + + + simdex2 + + + + + + + + + + + + + + + + + sim1 + + + + + + + + + + + + + + + + + sim2 + + + + + + + + + + + + + + + + + sim3 + + + + + + + + + + + + + + + + + .mat + + + + + + + + + + + + + + + + + + .mat + + + + + + + + + + + + + + + + + + .mat + + + + + + + + + + + + + + + + + + process +(optional) + + + + + + + + + + + + + + + + + result + + + + + + + + + + + + + + + + + + + filter() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + get() + + + + + + + + + diff --git a/doc/source/Simdex_attributes.graphml b/doc/source/Simdex_attributes.graphml new file mode 100644 index 0000000..82278cb --- /dev/null +++ b/doc/source/Simdex_attributes.graphml @@ -0,0 +1,884 @@ + + + + + + + + + + + + + + + + + + + + + + + + Optional, when simdex was created through a process + + + + + + + + + + + + + + + + + + + + varmap = {shortname:fullname} for variables + + + + + + + + + + + + + + + + + + + + + varmap = {shortname:fullname} for variables + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + parmap = {shortname:fullname} for parameters + + + + + + + + + + + + + + + + + + + + + parmap = {shortname:fullname} for parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + files = {SID:path} + + + + + + + + + + + + + + + + + + + + + files = {SID:path} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + variablemap +- maps variable presence to simulations +- shape=(v,n) +- array([1, 0, ...1 + 1, 1, ...0 + ... + 0, 0, ... 1]) + + + + + + + + + + + + + + + + + + + + + variablemap +- maps variable presence to simulations +- shape=(v,n) +- array([1, 0, ...1 + 1, 1, ...0 + ... + 0, 0, ... 1]) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + variables = ['var1', 'var2', ... , 'varv'] + + + + + + + + + + + + + + + + + + + + + variables = ['var1', 'var2', ... , 'varv'] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + parametervalues +- maps parameter values to simulations +- shape=(p,n) +- array([x1, 0, ...x2 + x3, x4, ...0 + ... + 0, 0, ... x5]) + + + + + + + + + + + + + + + + + + + + + parametervalues +- maps parameter values to simulations +- shape=(p,n) +- array([x1, 0, ...x2 + x3, x4, ...0 + ... + 0, 0, ... x5]) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + parametermap +- maps parameter presence to simulations +- shape=(p,n) +- array([1, 0, ...1 + 1, 1, ...0 + ... + 0, 0, ... 1]) + + + + + + + + + + + + + + + + + + + + + parametermap +- maps parameter presence to simulations +- shape=(p,n) +- array([1, 0, ...1 + 1, 1, ...0 + ... + 0, 0, ... 1]) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + parameters = ['par1', 'par2', ... , 'parp'] + + + + + + + + + + + + + + + + + + + + + parameters = ['par1', 'par2', ... , 'parp'] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + simulations = ['SID0000', 'SID0001', ... , 'SIDn'] + + + + + + + + + + + + + + + + + + + + + simulations = ['SID0000', 'SID0001', ... , 'SIDn'] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + .hdf5 + + + + + + + + + + + + + + + + + + + + simdex1 + + + + + + + + + + + + + + + + + simdex2 + + + + + + + + + + + + + + + + + shared file with variable trajectories and metadata by different simdexes + + + + + + + + + + + + + + + + + + Main Simdex() attributes + + + + + + + + + + + + + + + + + + process1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/Simdex_methods.graphml b/doc/source/Simdex_methods.graphml new file mode 100644 index 0000000..2c605f9 --- /dev/null +++ b/doc/source/Simdex_methods.graphml @@ -0,0 +1,1018 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + MATPLOTLIB +FIGURE + + + + + + + + + + + + + + + + + + + + + MATPLOTLIB +FIGURE + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + plot(var), scatterplot(var) + + + + + + + + + + + + + + + + + + + + + plot(var), scatterplot(var) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + get(variable or parameter) + + + + + + + + + + + + + + + + + + + + + get(variable or parameter) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + filter(), filter_selection(), +filter_remove(), filter_similar() + + + + + + + + + + + + + + + + + + + + + filter(), filter_selection(), +filter_remove(), filter_similar() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + scan(folder, process2 + + + + + + + + + + + + + + + + + + + + + scan(folder, process2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + index_one_sim(sim2, process1) + + + + + + + + + + + + + + + + + + + + + index_one_sim(sim2, process1) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + index_one_sim(sim1) + + + + + + + + + + + + + + + + + + + + + index_one_sim(sim1) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + .hdf5 + + + + + + + + + + + + + + + + + + + + simdex1 + + + + + + + + + + + + + + + + + simdex2 + + + + + + + + + + + + + + + + + shared file with variable trajectories and metadata by different simdexes + + + + + + + + + + + + + + + + + + Main Simdex() methods + + + + + + + + + + + + + + + + + + process1 + + + + + + + + + + + + + + + + + sim1 + + + + + + + + + + + + + + + + + sim2 + + + + + + + + + + + + + + + + + sim3 + + + + + + + + + + + + + + + + + .mat + + + + + + + + + + + + + + + + + + .mat + + + + + + + + + + + + + + + + + + .mat + + + + + + + + + + + + + + + + + + sim1 + + + + + + + + + + + + + + + + + sim2 + + + + + + + + + + + + + + + + + process2 + + + + + + + + + + + + + + + + + result1 + + + + + + + + + + + + + + + + + result2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/demo_simman.py b/examples/demo_simman.py index 422736f..8090e03 100644 --- a/examples/demo_simman.py +++ b/examples/demo_simman.py @@ -11,7 +11,9 @@ import numpy as np import os -from simman import Simulation, Simdex, Process, load_simdex +from awesim import Simulation, Simdex, Process, load_simdex + +path = os.path.join(os.path.split(os.getcwd())[0], 'tests') # first, test the package #runfile(r'C:\Workspace\Python\SimulationManagement\test_simman.py', @@ -19,7 +21,7 @@ # a Simulation is a python object for 1 simulation result file -sim=Simulation('LinkedCapacities_A') #with our without .mat extension +sim=Simulation(os.path.join(path, 'LinkedCapacities_A')) #with our without .mat extension sim.separate() # optional, makes attributes for parameters and variables sim.parameters @@ -41,9 +43,9 @@ # there are different ways to create a simdex s1 = Simdex() # an empty simdex object -s1.scan() # scan current folder and add all found simulation files to the simdex +s1.scan(folder=path) # scan current folder and add all found simulation files to the simdex -s2 = Simdex('SubfolderWithCrappyFiles') +s2 = Simdex(os.path.join(path, 'SubfolderWithCrappyFiles'), h5='simdex2.h5') # create a simdex directly from a folder print s1 @@ -68,16 +70,18 @@ # Next step: using a process -p1 = Process(variables={'T2':'c2.T', 'dt2':'c[2].der(T)'}) -s1 = Simdex(os.getcwd(), process=p1) +p1 = Process(variables={'T1':'c1.T', 'T2':'c2.T', 'dt2':'c[2].der(T)'}) +s1 = Simdex(path, process=p1) + +s1.get('T1').values() mothers=['c1', 'c2'] parameters={'cap1':'c1.C', 'res':'r.R'} sub_pars={'cap':'C'} variables={} -sub_vars={'Qflow':'heatPort.Q_flow'} -pp=['Qflow10 = 10 * Qflow'] +sub_vars={'T':'T', 'Qflow':'heatPort.Q_flow'} +pp=['Qflow_kW = Qflow / 1000.'] J2kWh = 1e-6/3.6 vars_to_integrate = {'Qflow':J2kWh} @@ -85,6 +89,7 @@ sub_pars=sub_pars, variables=variables, sub_vars=sub_vars, pp=pp, integrate=vars_to_integrate) +s1 = Simdex(path, process=p) # remove a simulation from the simdex manually s3 = s1.filter_remove(['SID0000']) @@ -114,8 +119,9 @@ s4 = s1.filter(fltr2) # plotting -s3.plot('c1.T') -s3.scatterplot('c1.T', 'c2.T') +print s1.vardic +s3.plot('c1_T') +s3.scatterplot('c1_T', 'c2_T') # saving a specific simdex s2 = s1.filter(fltr) @@ -124,4 +130,4 @@ s = load_simdex('simdex2') -s.plot('r.heatPort_b.Q_flow') \ No newline at end of file +s.plot('c1_T') \ No newline at end of file From 66741a2d8e0c2f7131dd28afb7c6eeacbb94a1ac Mon Sep 17 00:00:00 2001 From: roel de coninck Date: Mon, 10 Sep 2012 15:24:20 +0200 Subject: [PATCH 6/8] Bugfix for indexing a simulation with variables beyond the simdex.variables (same for paramters) --- awesim/simdex.py | 69 ++++++++++++++++++++++---------------------- tests/test_awesim.py | 19 +++++++++++- 2 files changed, 53 insertions(+), 35 deletions(-) diff --git a/awesim/simdex.py b/awesim/simdex.py index 25d4661..03762df 100644 --- a/awesim/simdex.py +++ b/awesim/simdex.py @@ -533,24 +533,24 @@ def index_one_var(variables, varmap, var, index): """ - if var==variables[index]: - # var is the first element in variables, update varmap - varmap[index,-1] = 1 - pos = index+1 - else: - try: - # search for it in variables, but only in the part AFTER index + + try: + if var==variables[index]: + # var is the first element in variables, update varmap + varmap[index,-1] = 1 + pos = index+1 + else:# search for it in variables, but only in the part AFTER index pos=variables[index:].index(var)+index varmap[pos,-1] = 1 - except(ValueError): - # this variable was not found. Add it in the right position - # keeping the list in sorted order - pos = bisect.bisect_left(variables, var, lo=index) - variables.insert(pos,var) - # make new row in variablemap and add '1' in the last column - varmap = np.insert(varmap, pos, 0, axis=0) - varmap[pos,-1] = 1 - pos+=1 + except(ValueError, IndexError): + # this variable was not found. Add it in the right position + # keeping the list in sorted order + pos = bisect.bisect_left(variables, var, lo=index) + variables.insert(pos,var) + # make new row in variablemap and add '1' in the last column + varmap = np.insert(varmap, pos, 0, axis=0) + varmap[pos,-1] = 1 + pos+=1 return variables, varmap, pos # internal function to enhance readibility @@ -565,28 +565,29 @@ def index_one_par(parameters, parmap, parvalues, par, index, parvalue): Returns the new parameters, parmap, parvalues and index """ - if par==parameters[index]: - # par is the first element in parameters, update parmap, parvalues - parmap[index,-1] = 1 - parvalues[index, -1] = parvalue - pos = index+1 - else: - try: + + try: + if par==parameters[index]: + # par is the first element in parameters, update parmap, parvalues + parmap[index,-1] = 1 + parvalues[index, -1] = parvalue + pos = index+1 + else: # search for it in parameters, but only in the part AFTER index pos=parameters[index:].index(par)+index parmap[pos,-1] = 1 parvalues[pos, -1] = parvalue - except(ValueError): - # this parameter was not found. Add it in the right position - # keeping the list in sorted order - pos = bisect.bisect_left(parameters, par, lo=index) - parameters.insert(pos,par) - # make new row in parametermap and add '1' in the last column - parmap = np.insert(parmap, pos, 0, axis=0) - parmap[pos,-1] = 1 - parvalues = np.insert(parvalues, pos, 0, axis=0) - parvalues[pos,-1] = parvalue - pos+=1 + except(ValueError, IndexError): + # this parameter was not found. Add it in the right position + # keeping the list in sorted order + pos = bisect.bisect_left(parameters, par, lo=index) + parameters.insert(pos,par) + # make new row in parametermap and add '1' in the last column + parmap = np.insert(parmap, pos, 0, axis=0) + parmap[pos,-1] = 1 + parvalues = np.insert(parvalues, pos, 0, axis=0) + parvalues[pos,-1] = parvalue + pos+=1 return parameters, parmap, parvalues, pos diff --git a/tests/test_awesim.py b/tests/test_awesim.py index c8de719..b548ed1 100644 --- a/tests/test_awesim.py +++ b/tests/test_awesim.py @@ -446,7 +446,7 @@ def setUp(self): # # It doesn't matter if N exceeds the numer of questions # NOTE: this may affect interactive debugging tools - N = 1000 + N = 100 f = StringIO("y\n" * N) sys.stdin = f @@ -494,6 +494,23 @@ def test_init(self): filenames.sort() self.assertEqual(self.sims, filenames) self.simdex.h5.close() + + + def test_init_different_order(self): + """ + Tests if a Simdex object is created correctly when the files are + in a different order, resulting in new variables to be added after all + previously indexed variables are found + + """ + s2=Simdex() + sim1 = Simulation('LinkedCapacities.mat') + sim2 = Simulation('Array.mat') + s2.index_one_sim(sim1) + s2.index_one_sim(sim2) + + self.assertEqual(sorted(s2.get_filenames()), + sorted(['LinkedCapacities.mat', 'Array.mat'])) def test_init_with_process(self): """ From 0899d28c5cf37e7b9c3f53a37372ed017df1ff86 Mon Sep 17 00:00:00 2001 From: roel de coninck Date: Mon, 10 Sep 2012 15:24:51 +0200 Subject: [PATCH 7/8] Add a 'compress' option to create_input_file --- awesim/pymosim.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/awesim/pymosim.py b/awesim/pymosim.py index 23fd87b..aa11cd7 100644 --- a/awesim/pymosim.py +++ b/awesim/pymosim.py @@ -9,9 +9,10 @@ from subprocess import Popen import subprocess import sys -from simman import Simulation, Simdex from shutil import copyfile import copy +import numpy as np + def set_solver(solver, dsin = '', copy_to = None): """ @@ -387,7 +388,7 @@ def run_ds(dymosim = '', dsin = '', result = ''): return proc -def create_input_file(data, filename, discrete=False): +def create_input_file(data, filename, discrete=False, compress=True): """ Create an input file for the TimeTables from the MSL. The input files are in ascii format. @@ -401,10 +402,22 @@ def create_input_file(data, filename, discrete=False): * discrete: if True, the data array will be modified to become a discrete profile. At each timestep, an additional line will be created. See the documentation of the Modelica.Timetable.mo model for more info. + * compress: if True, all reduntant lines will be removed from data. """ - l,w = data.shape + if compress: + # rows with only zeros are removed, UNLESS they come after a row + # containing any value. + + row_contains_data = data[:,1:].any(axis=1) + row_contains_data_rolled = np.roll(row_contains_data, 1) + row_contains_data[0] = True + row_contains_data[-1] = True + keep = np.logical_or(row_contains_data, row_contains_data_rolled) + data = data[keep] + l,w = data.shape + if discrete: # create a second, shifted data array. We'll write each row of this # shifted array after each row of the original one. From 4029dbc5f78a3ee62a35d33c8dc8f1c6ab9e55eb Mon Sep 17 00:00:00 2001 From: roel de coninck Date: Wed, 12 Sep 2012 23:23:20 +0200 Subject: [PATCH 8/8] added analyse_cputime() to Simulation class --- awesim/simulation.py | 91 +++++++++++++++++++++++++++++- tests/TestSet2/reswithCPUtime.mat | Bin 0 -> 13079 bytes tests/test_awesim.py | 27 ++++++++- 3 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 tests/TestSet2/reswithCPUtime.mat diff --git a/awesim/simulation.py b/awesim/simulation.py index 8e7fd13..75747c3 100644 --- a/awesim/simulation.py +++ b/awesim/simulation.py @@ -25,6 +25,7 @@ class Simulation: """ import numpy as np +from scipy.integrate import cumtrapz import os import scipy.io import re @@ -562,4 +563,92 @@ def aggregate_by_time(signal, period=86400, interval=900, label='left'): result.update(d) result.pop('aggregate_by_time') - return result \ No newline at end of file + return result + + def analyse_cputime(self, variables=None, interval=900, screendump=30): + """Analyse the relation between cputime and some variables. + + It is required that the time array of the simulation contains an + entry for every multiple of the interval. + + Parameters: + ----------- + * variables: dictionary with name/array for which the correlation + is to be found. If the variablename ends with Int (case sensitive) + it is supposed to be cumulative + if variables = None, all variables found in the simulation are tested. + * interval: the desired interval for resampling, integer in seconds + * screendump: integer, number of lines to print. A negative number + prints all the results + + + Output: + ------- + dictionary with same keys as variables, and the correlation as values + """ + + res = {} + + #pdb.set_trace() + time = self.get_value('Time') + try: + cputime = self.get_value('CPUtime') + except(ValueError): + raise ValueError("This simulation has no trajectory for CPUtime") + + x = np.arange(time[0], time[-1], interval) + index = np.searchsorted(time, x) + try: + cpu_smpl = cputime[index] + except(IndexError): + print "====================== ERROR =====================" + print "This IndexError can be caused by a wrong interval." + print "Your interval setting was %s seconds" % (interval) + print "Make sure the simulation has this output interval" + raise + + cpu_diff = np.diff(cpu_smpl) + + if variables is None: + self.separate() + variables = self.extract({v:v for v in self.variables}) + + for varname, array in variables.items(): + if varname.endswith('Int') or varname == 'CPUtime': + # we suppose the array is cumulative + var_cum = array + else: + var_cum = cumtrapz(array, time) + + try: + var_smpl = var_cum[index] + except(IndexError): + print "====================== ERROR =====================" + print "This IndexError can be caused by a wrong interval." + print "Your interval setting was %s seconds." % (interval) + print "Make sure the simulation has this output interval." + raise + + var_diff = np.diff(var_smpl) + + res[varname] = np.min(np.corrcoef(var_diff, cpu_diff)) + + good_results = {k:v for k,v in res.items() if not np.isnan(v)} + + keys_sorted = sorted(good_results.items(), key=lambda x: np.abs(x[1]), reverse=True) + if screendump < 0: + nlines = len(keys_sorted) + else: + nlines = screendump + + if nlines > 0: + print 'The highest correlations were found for :' + for k,v in keys_sorted[:nlines]: + print '\t', k, '.'*(50-len(k)), ' ==> ', '%.2f' % (v) + + #print 'The lowest correlations were found for :' + #for k,v in keys_sorted[:10]: + # print '\t', k, ' '*(40-len(k)), ' ==> ', '%.2f' % (v) + + + return res \ No newline at end of file diff --git a/tests/TestSet2/reswithCPUtime.mat b/tests/TestSet2/reswithCPUtime.mat new file mode 100644 index 0000000000000000000000000000000000000000..afdf00e2d4bbff553322008530b0bc25bad35b18 GIT binary patch literal 13079 zcmeHNYiu0V6}}Wmd6xzXF;H@oP~il7V+T=+i}5l9F{hA5~~8WmNEKmxI)3dqYtBs`)%REkOiNF-9zKR|vEB+_$c_U`PA z$MZ0&%0GOi@ywZfzw^G2Io=xyA?Lt#AzawL4p)D^B$_7a@1|R=6p@W?s)~eW@2AvY zl!|#uRf|$nqm&qw((Q2F2(VRfW#PhhE?lapFp`xOz$9tvJ(|kQB#3OkvNhLzUrN^u zE0v9d^dRmWl}g&S2pC7iVsG;rW1=O9x|o+O%|P{p2oZD|B2w5iM1dLY+2tXEf)Jyj za;707-5iVvb8}u($64NzH5DBdHjMHE6Ua4=rPTFBz4=0%uVNiIaBR}doH8ZSaB zl@rQv5aBW`rs&KNt+K&Jm})xN80xV=L}=V+M$DvJc7r~`ER46P=q1@I7orvl7qLw) zSYuJ)gMJqQ4v@7%G{i-yF(EEOlO&T_No1IoXjn0pNLL0fwsG3HR$cbghsiZPAg%!mJwY*yCvO20q*dL4t(=7-QZhtT$sSJZj%wQ@+ zv?wZy-xEPHTm)_nHmYfRn2t~r`w9`s>`+rleT9gmGF;1}eI-TY6n1-kwK*|DACn_7 zMMEpAg%l)hL>L;JJcuwC1tG%25R6DFBRmWZ8HKW2%=>tn%=W&%tZZcl)g-#Z zu2kfDmbF{kb=EtiTCB*7VZn5#fG<5p+jY&9 zEqR<#S+ZZlUP>iHQ>a$9Oc}nlWYyBBHO6ROQ*2=KY3iVy$vQd z#}mwSN-{AR&&_1;<&~X6b}58*-Uf*U^@@7dc;X&N^g_+hsiAZ_NT{adhC61HY$I|} z6-!W$)D31rK4Eq{x$G2@An$=R6c_^)mO?piiSX3~ISzxP40F@K z{zfSZCbC{GnUQ6az)Z)NTkINDgAJtKcW~W`znQz7?L|h?V9!w*XE4i+%-y^kMNHa$ zEA65R6IBzYha{^q>>g@ki>r@<=+F_ZWYym~ty;*+5DSSm z&Y9iSN?)}wf-Uq_3t55N@x<7Il*r9TaORx!_2lMc1SNz0ung5<`=E`dA=9q@)7;>W zr1^2z9V5KpiR)-Hv!qi>9}{60HOFMh^0P{syz?uAFAqaVoX#y;8JbQ57)vwYED)lI zr)xff9(RNiEsWs2+e1C=>1?6fn-1Gwb>eKaS5a9lum``w^s^0}pU$3mhZ1vhqo1G7 zp+WO>ws1Pn7YF8x1D@P!5pg;EuL6IA$^Un8KK}1Qw8SA*(%_Mhv+TcwoPmbVU#ru2 z!{-a?H1;FS`#Ig*kNC6e9A72N__v4%O z$G$H?5MsGJzdp@0+RODr4ANX5PUDYWdBgtmI01L0F(AAl&Go@|Nb?`hB8~cS8ui2e zkLvnjKhhsI(KsIeS%nS9BYm+=xir@Y`w`E_<1nQ8c%z0@rvS?)SfGS`Y!U(Z9e)cAAPltzNU%BeBk-T{mJJm z(l}Q!e{mc(^ffN7qun-1$V4Bx&Ke?Qhv2-JNE7n&)#Ro9YoYHDnWzZlSZz&xFx&=L z6C%~e^0&YP`P8pBkKBG6A$!v#y&G`AqtT=(c$^^VYw-R5T`sx;UB%tia0UKyAKD2r zK*%oPiKQp29jiX5!KT!Yuk)8bDO}&(-baq@6l{0kOK523n@5iH4jv~1r_W3Zzg>K? znqB!pO*nIQ|Iv{z`VlvEDi{au&gA$0HS?b}lfoa{9Zw3)}{EOv* z@@9i`v^||US$r=@|$?;Dw}#goE6hR*AGvSd=Y;TLbcaO_lV>U-7c-R)E5hApuC2cK z)RnciMK6P%A2)Da?|b8C+xOBx2kt%ibY|?aityge%c@=buddzv_JQ;bw=7tPe);!? z&c2Sn1>nf(fuBA!l{xuoMHt=lOwXlv{~@!p<8}0>jeCW5_4WM4!Q~%&_GKpq@OSS; zYbrM#&HQ=Aq#&+Y@xp@q(y32+{+`~qtHZ{bW8Hm6j|RwtDWflZEpy;hMflnHh1Fk? cb88R3dl>xEXXAdYuId|lH92m{!C>6~0P*he9{>OV literal 0 HcmV?d00001 diff --git a/tests/test_awesim.py b/tests/test_awesim.py index b548ed1..925e036 100644 --- a/tests/test_awesim.py +++ b/tests/test_awesim.py @@ -7,6 +7,7 @@ @author: RDC """ +import pdb import numpy as np import unittest from os import getcwd, path, remove @@ -333,8 +334,13 @@ def test_separate_twice(self): self.assertEqual(variables, sim.variables) def test_extract(self): - """to be implemented""" - pass + """check if extract() gives the same arrays as get_value()""" + + sim = Simulation('LinkedCapacities') + sim.separate() + extracted = sim.extract({v:v for v in sim.variables}) + for k,v in extracted.items(): + np.testing.assert_equal(v, sim.get_value(k)) def test_get_objects(self): """Test if get_objects works with empty mother model""" @@ -433,6 +439,23 @@ def test_postprocess_integration(self): self.assertIsNotNone(result_pp['Time_Int']) + def test_analyse_cputime_raises(self): + """check if the analysis of cpu-time raises an error when there is no CPUtime""" + + sim = Simulation('LinkedCapacities') + sim.separate() + extracted = sim.extract({v:v for v in sim.variables}) + self.assertRaises(ValueError, sim.analyse_cputime, extracted) + + def test_analyse_cputime(self): + """check if the analysis of cpu-time raises an error when there is no CPUtime""" + + #pdb.set_trace() + sim = Simulation('./TestSet2/reswithCPUtime') + sim.separate() + extracted = sim.extract({v:v for v in sim.variables}) + corr = sim.analyse_cputime(interval=1000) + self.assertListEqual(sorted(extracted.keys()), sorted(corr.keys())) class SimdexTest(unittest.TestCase): """