OpenM++ runtime library (libopenm)
metaLoader.h
Go to the documentation of this file.
1
5// Copyright (c) 2013-2015 OpenM++
6// This code is licensed under the MIT license (see LICENSE.txt for details)
7
8#ifndef META_LOADER_H
9#define META_LOADER_H
10
11#include <algorithm>
12using namespace std;
13
15#include "msg.h"
16
17namespace openm
18{
21 {
23 static constexpr const char * subValueCount = "OpenM.SubValues";
24
26 static constexpr const char * runName = "OpenM.RunName";
27
29 static constexpr const char * runId = "OpenM.RunId";
30
32 static constexpr const char * runCreated = "OpenM.RunCreated";
33
35 static constexpr const char * restartRunId = "OpenM.RestartRunId";
36
38 static constexpr const char * setId = "OpenM.SetId";
39
41 static constexpr const char * setName = "OpenM.SetName";
42
44 static constexpr const char * baseRunName = "OpenM.BaseRunName";
45
47 static constexpr const char * baseRunId = "OpenM.BaseRunId";
48
50 static constexpr const char * baseRunDigest = "OpenM.BaseRunDigest";
51
53 static constexpr const char * taskId = "OpenM.TaskId";
54
56 static constexpr const char * taskName = "OpenM.TaskName";
57
59 static constexpr const char * taskRunName = "OpenM.TaskRunName";
60
62 static constexpr const char * taskRunId = "OpenM.TaskRunId";
63
65 static constexpr const char * taskWait = "OpenM.TaskWait";
66
68 static constexpr const char * profile = "OpenM.Profile";
69
71 static constexpr const char * threadCount = "OpenM.Threads";
72
74 static constexpr const char * notOnRoot = "OpenM.NotOnRoot";
75
77 static constexpr const char * dbConnStr = "OpenM.Database";
78
80 static constexpr const char * dbSqlite = "OpenM.Sqlite";
81
83 static constexpr const char * dbFromBin = "OpenM.SqliteFromBin";
84
86 static constexpr const char * useSparse = "OpenM.SparseOutput";
87
89 static constexpr const char * sparseNull = "OpenM.SparseNullValue";
90
92 static constexpr const char * doubleFormat = "OpenM.DoubleFormat";
93
95 static constexpr const char * paramDir = "OpenM.ParamDir";
96
98 static constexpr const char * useIdCsv = "OpenM.IdCsv";
99
101 static constexpr const char * useIdParamValue = "OpenM.IdParameterValue";
102
104 static constexpr const char * progressPercent = "OpenM.ProgressPercent";
105
107 static constexpr const char * progressStep = "OpenM.ProgressStep";
108
110 static constexpr const char * parameterPrefix = "Parameter";
111
113 static constexpr const char * subFromPrefix = "SubFrom";
114
116 static constexpr const char * subValuesPrefix = "SubValues";
117
119 static constexpr const char * tableSuppress = "Tables.Suppress";
120
122 static constexpr const char * tableRetain = "Tables.Retain";
123
125 static constexpr const char * microdataToDb = "Microdata.ToDb";
126
128 static constexpr const char * microdataToCsv = "Microdata.ToCsv";
129
131 static constexpr const char * microdataToTrace = "Microdata.ToTrace";
132
134 static constexpr const char * microdataCsvDir = "Microdata.CsvDir";
135
137 static constexpr const char* microdataAll = "Microdata.All";
138
140 static constexpr const char * microdataInternal = "Microdata.UseInternal";
141
143 static constexpr const char * microdataEvents = "Microdata.Events";
144
146 static constexpr const char * microdataEventColumn = "Microdata.CsvEventColumn";
147
149 static constexpr const char* microdataPrefix = "Microdata";
150
152 // static constexpr const char* microdataFilterPrefix = "MicrodataFilter";
153
155 static constexpr const char * importAll = "Import.All";
156
158 static constexpr const char * importPrefix = "Import";
159
161 static constexpr const char * importRunDigestPrefix = "ImportRunDigest";
162
164 static constexpr const char * importRunIdPrefix = "ImportRunId";
165
167 static constexpr const char * importRunNamePrefix = "ImportRunName";
168
170 static constexpr const char * importModelDigestPrefix = "ImportModelDigest";
171
173 static constexpr const char * importModelIdPrefix = "ImportModelId";
174
176 static constexpr const char * importExprPrefix = "ImportExpr";
177
179 static constexpr const char * importDbPrefix = "ImportDb";
180
182 static constexpr const char * runDescrSuffix = "RunDescription";
183
185 static constexpr const char * runNotePathSuffix = "RunNotesPath";
186
188 static constexpr const char * traceToConsole = "OpenM.TraceToConsole";
189
191 static constexpr const char * traceToFile = "OpenM.TraceToFile";
192
194 static constexpr const char * traceFilePath = "OpenM.TraceFilePath";
195
197 static constexpr const char * traceToStamped = "OpenM.TraceToStampedFile";
198
200 static constexpr const char * traceUseTs = "OpenM.TraceUseTimeStamp";
201
203 static constexpr const char * traceUsePid = "OpenM.TraceUsePidStamp";
204
206 static constexpr const char * traceNoMsgTime = "OpenM.TraceNoMsgTime";
207
209 static constexpr const char * traceRank = "OpenM.TraceRank";
210
212 static constexpr const char * logRank = "OpenM.LogRank";
213
215 static constexpr const char * messageLang = "OpenM.MessageLanguage";
216
218 static constexpr const char * dbSubValue = "db";
219
221 static constexpr const char * iotaSubValue = "iota";
222
224 static constexpr const char * csvSubValue = "csv";
225
227 static constexpr const char * defaultValue = "default";
228
230 static constexpr const char * allValue = "All";
231 };
232
235 {
237 static constexpr const char * iniFile = "ini";
238
240 static constexpr const char * dbSqlite = "db";
241
243 static constexpr const char * setName = "s";
244
246 static constexpr const char * paramDir = "p";
247 };
248
251 {
252 public:
254 virtual ~MetaLoader(void) noexcept = 0;
255
258
261
263 const ArgReader & argOpts(void) const { return argStore; }
264
266 const MetaHolder * meta(void) const { return metaStore.get(); }
267
269 const RunOptions modelRunOptions(int i_subCount, int i_subId) const;
270
272 static const ArgReader getRunOptions(int argc, char ** argv);
273
275 int parameterSubCount(int i_paramId) const
276 {
277 return binary_search(paramIdSubArr.cbegin(), paramIdSubArr.cend(), i_paramId) ? subValueCount : 1;
278 }
279
281 int parameterIdByName(const char * i_name) const;
282
284 int tableIdByName(const char * i_name) const;
285
287 bool isSuppressed(const char * i_name) const { return isSuppressed(tableIdByName(i_name)); }
288
290 bool isSuppressed(int i_tableId) const
291 {
292 return binary_search(tableIdSuppressArr.cbegin(), tableIdSuppressArr.cend(), i_tableId);
293 }
294
295 protected:
296 int modelId; // model id in database
297 unique_ptr<MetaHolder> metaStore; // metadata tables
298 vector<int> paramIdSubArr; // ids of parameters where sub-values count same as model run sub-values count
299 vector<int> tableIdSuppressArr; // id's of tables to suppress from calculation and output
300 vector<int> entityIdxArr; // microdata attributes indices in EntityNameSizeArr
301 bool isCsvEventColumn; // if true then write even names into entity microdata CSV
302 vector<bool> entityUseEvents; // microdata event filter: if this array[event id] is true then use this event
303
305 MetaLoader(const ArgReader & i_argStore) :
306 subValueCount(0),
307 threadCount(1),
308 modelId(0),
309 isCsvEventColumn(false),
310 argStore(i_argStore)
311 { }
312
314 const RunOptions & modelRunOptions(void) const { return baseRunOpts; }
315
317 void setRunOptions(const RunOptions & i_opts) { baseRunOpts = i_opts; }
318
319 // read metadata tables from db, except of run_option table
320 static int readMetaTables(IDbExec * i_dbExec, MetaHolder * io_metaStore, const char * i_name, const char * i_digest);
321
327 void loadMessages(IDbExec * i_dbExec);
328
330 void mergeOptions(IDbExec * i_dbExec);
331
332 // enum to indicate what kind of sub id's selected: single, default, range or list
333 enum class KindSubIds : int
334 {
335 none = 0, // SubValues option not defined
336 single = 1, // single sub id
337 defaultId = 2, // "default" single sub id
338 range =3, // range of sub id's [first id, last id]
339 list = 4 // list of sub id's
340 };
341
342 // enum to indicate where to get sub-values of input parameter: database input workset, csv file or generate iota
343 enum class FromSub : int
344 {
345 defaultValue = 0, // SubFrom option not defined
346 db = 1, // sub values from database input workset
347 csv = 2, // sub values from csv file
348 iota = 3 // generate iota sub values
349 };
350
351 // parameter sub-values options:
352 // SubFrom: get sub-values from "db", "iota" or "csv"
353 // SubValues: can be list of id's: 1,2,3,4 or range: [1,4] or mask: x0F or single id: 7 or default id: "default"
355 {
356 int subCount = 1; // number of sub-values
357 KindSubIds kind = KindSubIds::none; // is it a single sub id, range or list of id's
358 vector<int> subIds; // list of sub id's to use, if not default id
359 FromSub from = FromSub::defaultValue; // get sub-values from: "db", "iota" or "csv"
360 };
361
362 map<int, ParamSubOpts> subOptsMap; // map parameter id to sub-values options
363
364 // enum to indicate how to find source model run for import
365 enum class ImportKind : int
366 {
367 none = 0, // no import
368 modelName = 1, // use model name: last run id of model with that name
369 modelId = 2, // use model id: last run id of that model id
370 modelDigest = 3, // use model digest: last run id of that model digest
371 runName = 4, // use run name: last run id with that name
372 runId = 5, // use run id
373 runDigest = 6 // use run digest
374 };
375
376 // import options for each model run
377 struct ImportOpts {
378 ImportKind kind = ImportKind::none;
379 int modelId = 0; // source model id
380 string modelDigest; // source model digest
381 int runId = 0; // source run id
382 string runName; // source run name
383 string runDigest; // source run digest
384 string connectionStr; // if not empty then db connection string
385 };
386 map<string, ImportOpts> importOptsMap; // import options for each model name
387
388 // import source for each model run: run id and digest, model id and digest
390 int runId = 0; // source run id
391 string runDigest; // source run digest
392 int modelId = 0; // source model id
393 string modelDigest; // source model digest
394 unique_ptr<MetaHolder> meta; // metadata rows: source model, parameters and output tables to import from
395 unique_ptr<IDbExec> dbExec; // database connection, if null then use "main" database connection
396 };
397
398 // import options for parameter
400 string expr; // output table expression name to import as parameter value
401 };
402 map<int, ParamImportOpts> paramImportOptsMap; // parameter id mapped to import options
403
404 // model run language-specific options: map language id to run description and notes
405 map<int, pair<string, string> > langOptsMap;
406
407 private:
408 RunOptions baseRunOpts; // basic model run options
409 ArgReader argStore; // arguments as key-value string pairs with case-neutral search
410
411 // merge parameter name arguments with profile_option table, ie "Parameter.Age" or "SubValues.Age" argument
412 void mergeParameterProfile(
413 const string & i_profileName, const char * i_prefix, const IProfileOptionTable * i_profileOpt, const vector<ParamDicRow> & i_paramRs
414 );
415
417 void parseParamSubOpts(void);
418
420 void parseImportOptions(void);
421
423 void parseSuppressOptions(void);
424
426 void parseLangOptions(void);
427
429 void parseEntityOptions(void);
430
432 tuple<bool, string, const EntityDicRow *> parseEntityNameOption(const string & i_key, const string & i_metadataPrefix) const;
433
434 private:
435 MetaLoader(const MetaLoader & i_metaLoader) = delete;
436 MetaLoader & operator=(const MetaLoader & i_metaLoader) = delete;
437 };
438}
439
440#endif // META_LOADER_H
OpenM++ arguments reader: parse command line and ini-file arguments.
database connection wrapper to execute sql commands.
Definition: dbExec.h:21
model metadata loader: read and broadcast metadata and run options.
Definition: metaLoader.h:251
int threadCount
max number of modeling threads
Definition: metaLoader.h:260
int parameterSubCount(int i_paramId) const
return sub-values count by parameter id
Definition: metaLoader.h:275
bool isSuppressed(const char *i_name) const
check by name if output table suppressed
Definition: metaLoader.h:287
int subValueCount
total number of sub-values
Definition: metaLoader.h:257
MetaLoader(const ArgReader &i_argStore)
create metadata loader.
Definition: metaLoader.h:305
const ArgReader & argOpts(void) const
arguments from command line and ini-file
Definition: metaLoader.h:263
int parameterIdByName(const char *i_name) const
return index of parameter by name
Definition: metaLoader.cpp:116
void setRunOptions(const RunOptions &i_opts)
set basic model run options
Definition: metaLoader.h:317
const RunOptions & modelRunOptions(void) const
return basic model run options
Definition: metaLoader.h:314
int tableIdByName(const char *i_name) const
return id of output table by name
Definition: metaLoader.cpp:127
void loadMessages(IDbExec *i_dbExec)
read model messages from database.
Definition: metaLoader.cpp:201
static const ArgReader getRunOptions(int argc, char **argv)
initialize run options from command line and ini-file
Definition: metaLoader.cpp:138
virtual ~MetaLoader(void) noexcept=0
last cleanup
Definition: metaLoader.cpp:113
void mergeOptions(IDbExec *i_dbExec)
merge command line and ini-file arguments with profile_option table values.
Definition: metaLoader.cpp:289
bool isSuppressed(int i_tableId) const
check by id if output table suppressed
Definition: metaLoader.h:290
const MetaHolder * meta(void) const
model metadata tables
Definition: metaLoader.h:266
OpenM++: message passing library public interface.
openM++ namespace
Definition: log.h:32
arguments reader to get runtime arguments from command line and ini-file.
Definition: argReader.h:56
profile_option table public interface.
Definition: dbMetaTable.h:867
Holder for model metadata core part (not include model text).
Definition: metaHolder.h:20
Definition: metaLoader.h:377
Definition: metaLoader.h:399
Definition: metaLoader.h:355
Definition: metaLoader.h:389
basic model run options
Definition: omModel.h:26
keys for model run options
Definition: metaLoader.h:21
static constexpr const char * traceToFile
trace log to file
Definition: metaLoader.h:191
static constexpr const char * subValuesPrefix
options started with "SubValues." used specify sub-values of input parameter or group of parameters,...
Definition: metaLoader.h:116
static constexpr const char * tableSuppress
exclude output tables or tables group from run results, ex: -Tables.Suppress AgeTable,...
Definition: metaLoader.h:119
static constexpr const char * useSparse
OBSOLETE, for backward compatibility only.
Definition: metaLoader.h:86
static constexpr const char * parameterPrefix
options started with "Parameter." treated as value of model scalar input parameter,...
Definition: metaLoader.h:110
static constexpr const char * useIdCsv
if true then parameter(s) csv file(s) contain enum id's, default: enum code
Definition: metaLoader.h:98
static constexpr const char * sparseNull
OBSOLETE, for backward compatibility only.
Definition: metaLoader.h:89
static constexpr const char * traceUsePid
trace use pid-stamp in log "stamped" file name
Definition: metaLoader.h:203
static constexpr const char * importRunIdPrefix
options started with "ImportRunId." used to specify run id to import parameters from,...
Definition: metaLoader.h:164
static constexpr const char * threadCount
number of modeling threads
Definition: metaLoader.h:71
static constexpr const char * doubleFormat
convert to string format for float, double, long double
Definition: metaLoader.h:92
static constexpr const char * setId
working set id to get input parameters
Definition: metaLoader.h:38
static constexpr const char * microdataEvents
filter entity events, ex: -Microdata.Events Birth,Union or -Microdata.Events All
Definition: metaLoader.h:143
static constexpr const char * iotaSubValue
sub-value of parameter created as integer from 0 to sub-value count
Definition: metaLoader.h:221
static constexpr const char * paramDir
dir/to/read/input/parameter.csv
Definition: metaLoader.h:95
static constexpr const char * messageLang
language to display output messages
Definition: metaLoader.h:215
static constexpr const char * logRank
use process rank as log message prefix
Definition: metaLoader.h:212
static constexpr const char * baseRunName
base run name to get input parameters
Definition: metaLoader.h:44
static constexpr const char * microdataEventColumn
if true then write event names in CSV file, ex: -Microdata.CsvEventColumn true
Definition: metaLoader.h:146
static constexpr const char * restartRunId
model run id to restart model run
Definition: metaLoader.h:35
static constexpr const char * traceToStamped
trace log to "stamped" file
Definition: metaLoader.h:197
static constexpr const char * microdataToDb
store entity microdata in database, ex: -Microdata.ToDb true
Definition: metaLoader.h:125
static constexpr const char * importPrefix
options started with "Import." used to specify parameters import from upstream model run,...
Definition: metaLoader.h:158
static constexpr const char * dbConnStr
database connection string
Definition: metaLoader.h:77
static constexpr const char * dbFromBin
SQLite database file is: directory/of/model/exe/ModelName.sqlite.
Definition: metaLoader.h:83
static constexpr const char * traceRank
trace use process rank as log message prefix
Definition: metaLoader.h:209
static constexpr const char * runNotePathSuffix
options ended with ".RunNotesPath" used to specify path to run notes file, ex: -EN....
Definition: metaLoader.h:185
static constexpr const char * importAll
options started with "MicrodataFilter." used to specify which entity filters, ex: -MicrodataFilter....
Definition: metaLoader.h:155
static constexpr const char * subValueCount
number of sub-values
Definition: metaLoader.h:23
static constexpr const char * csvSubValue
all parameter sub-values must be in parameter.csv file
Definition: metaLoader.h:224
static constexpr const char * importModelIdPrefix
options started with "ImportId." used to specify model id to import parameters from last run of that ...
Definition: metaLoader.h:173
static constexpr const char * traceUseTs
trace use time-stamp in log "stamped" file name
Definition: metaLoader.h:200
static constexpr const char * microdataCsvDir
write entity microdata into CSV file, ex: -Microdata.CsvDir csv/output/dir
Definition: metaLoader.h:134
static constexpr const char * importDbPrefix
options started with "ImportDb." used to specify database connection string to import parameters from...
Definition: metaLoader.h:179
static constexpr const char * importRunNamePrefix
options started with "ImportRunName." used to specify run name to import parameters from,...
Definition: metaLoader.h:167
static constexpr const char * baseRunDigest
model run digest to get input parameters
Definition: metaLoader.h:50
static constexpr const char * runName
model run name in database
Definition: metaLoader.h:26
static constexpr const char * microdataAll
store all entities and all non-internal attributes, ex: -Microdata.All true
Definition: metaLoader.h:137
static constexpr const char * microdataToTrace
write entity microdata into trace output, ex: -Microdata.ToTrace true
Definition: metaLoader.h:131
static constexpr const char * importRunDigestPrefix
options started with "ImportRunDigest." used to specify run name to import parameters from,...
Definition: metaLoader.h:161
static constexpr const char * useIdParamValue
if true then parameter value is enum id, default: enum code
Definition: metaLoader.h:101
static constexpr const char * importModelDigestPrefix
options started with "ImportDigest." used to specify model digest to import parameters from last run ...
Definition: metaLoader.h:170
static constexpr const char * progressPercent
if positive then used for simulation progress reporting, ex: every 10%
Definition: metaLoader.h:104
static constexpr const char * allValue
all value for any option
Definition: metaLoader.h:230
static constexpr const char * runDescrSuffix
options ended with ".RunDescription" used to specify run decsription, ex: -EN.RunDescription "run mod...
Definition: metaLoader.h:182
static constexpr const char * taskRunName
modeling task run name in database
Definition: metaLoader.h:59
static constexpr const char * taskName
modeling task name
Definition: metaLoader.h:56
static constexpr const char * traceToConsole
trace log to console
Definition: metaLoader.h:188
static constexpr const char * runCreated
read-only option, current model run create timestamp, ex: 2018_11_10_22_47_46_076
Definition: metaLoader.h:32
static constexpr const char * microdataInternal
allow to use internal attributes, ex: -Microdata.UseInternal true
Definition: metaLoader.h:140
static constexpr const char * baseRunId
model run id to get input parameters
Definition: metaLoader.h:47
static constexpr const char * dbSqlite
SQLite database file path.
Definition: metaLoader.h:80
static constexpr const char * taskWait
modeling task under external supervision
Definition: metaLoader.h:65
static constexpr const char * dbSubValue
sub-value of parameter must be in the input workset
Definition: metaLoader.h:218
static constexpr const char * notOnRoot
if true then do not run modeling threads at root process
Definition: metaLoader.h:74
static constexpr const char * defaultValue
default value for any option
Definition: metaLoader.h:227
static constexpr const char * subFromPrefix
options started with "SubFrom." used to specify where to get sub-values of input parameter or group o...
Definition: metaLoader.h:113
static constexpr const char * runId
read-only option, current model run id
Definition: metaLoader.h:29
static constexpr const char * microdataPrefix
options started with "Microdata." used to specify which entity attributes to store,...
Definition: metaLoader.h:149
static constexpr const char * traceFilePath
trace log file path
Definition: metaLoader.h:194
static constexpr const char * microdataToCsv
write entity microdata into CSV file, ex: -Microdata.ToCsv true
Definition: metaLoader.h:128
static constexpr const char * tableRetain
include only specified output tables or tables group into run results, ex: -Tables....
Definition: metaLoader.h:122
static constexpr const char * importExprPrefix
options started with "ImportExpr." used to specify expression name to import from output table,...
Definition: metaLoader.h:176
static constexpr const char * traceNoMsgTime
do not prefix trace log messages with date-time
Definition: metaLoader.h:206
static constexpr const char * profile
profile name to get run options
Definition: metaLoader.h:68
static constexpr const char * setName
working set name to get input parameters
Definition: metaLoader.h:41
static constexpr const char * taskId
modeling task id
Definition: metaLoader.h:53
static constexpr const char * taskRunId
read-only option, modeling task run id
Definition: metaLoader.h:62
static constexpr const char * progressStep
if positive then used for simulation progress reporting, ex: every 1000 cases or every 0....
Definition: metaLoader.h:107
keys for model run options (short form)
Definition: metaLoader.h:235
static constexpr const char * dbSqlite
short name for SQLite database file path: -db fileName.sqlite
Definition: metaLoader.h:240
static constexpr const char * setName
short name for: -s working set name to get input parameters
Definition: metaLoader.h:243
static constexpr const char * iniFile
short name for ini file path: -ini fileName.ini
Definition: metaLoader.h:237
static constexpr const char * paramDir
short name for: -p dir/to/read/input/parameter.csv
Definition: metaLoader.h:246