controller for root process: create new model run(s), send input parameters to children and receieve output tables. More...

#include <runControllerImpl.h>

Inheritance diagram for openm::RootController:

[legend]

Collaboration diagram for openm::RootController:

[legend]

Public Member Functions
	RootController (int i_processCount, const ArgReader &i_argStore, IDbExec i_dbExec, IMsgExec i_msgExec)
	create new root run controller

virtual	~RootController (void) noexcept
	last cleanup

virtual int	nextRun (void) override
	create new run and input parameters in database. More...

virtual void	readParameter (const char i_name, int i_subId, const type_info &i_type, size_t i_size, void io_valueArr) override
	read input parameter values. More...

virtual void	writeAccumulators (const RunOptions &i_runOpts, bool i_isLastTable, const char *i_name, size_t i_size, forward_list< unique_ptr< double[]> > &io_accValues) override
	write output table accumulators. More...

virtual void	shutdownRun (int i_runId) override
	model run shutdown: save results and update run status. More...

virtual void	shutdownWaitAll (void) override
	model process shutdown: wait for all child to be completed and do final cleanup. More...

virtual void	shutdownOnExit (ModelStatus i_status) override
	model process shutdown if exiting without completion (ie: exit on error). More...

virtual bool	childExchange (void) override
	exchange between root and child processes and threads. More...

Public Member Functions inherited from openm::RunController
virtual	~RunController (void) noexcept=0
	last cleanup

int	parameterIdByName (const char *i_name) const override
	return index of parameter by name More...

int	parameterSubCount (int i_paramId) const override
	number of parameter sub-values for current model run More...

int	parameterSelfSubCount (int i_paramId) const override
	number of parameter sub-values for current process More...

int	parameterSubValueIndex (int i_paramId, int i_subId) const override
	return index of parameter sub-value in the storage array of sub-values More...

bool	isUseSubValue (int i_subId) const override
	return true if sub-value used by current process More...

RunStateHolder &	runStateStore (void)
	return holder of all sub-values modeling run states

virtual int	nextRun (void)=0
	create new run and input parameters in database. More...

virtual void	shutdownRun (int i_runId)=0
	model run shutdown: save results and update run status. More...

virtual void	shutdownWaitAll (void)=0
	model process shutdown: cleanup resources. More...

virtual void	shutdownOnExit (ModelStatus i_status)=0
	model process shutdown if exiting without completion (ie: exit on error). More...

virtual bool	childExchange (void)=0
	exchange between root and child processes and threads. More...

virtual void	writeAccumulators (const RunOptions &i_runOpts, bool i_isLastTable, const char *i_name, size_t i_size, forward_list< unique_ptr< double[]> > &io_accValues)=0
	write output table accumulators or send data to root process. More...

bool	isOptionExist (const char *i_key) const noexcept override
	return true if run option found by i_key in run_option table for the current run id. More...

string	strOption (const char *i_key, const string &i_default="") const noexcept override
	return string value of run option by i_key or default value if not found.

bool	boolOption (const char *i_key) const noexcept override
	return boolean value of run option by i_key or false if not found or value is not "yes", "1", "true" or empty "" string. More...

int	boolOptionToInt (const char *i_key) const noexcept override
	search for boolean value of run option by i_key and return one of: return 1 if key found and value is one of: "yes", "1", "true" or empty value, return 0 if key found and value is one of: "no", "0", "false", return -1 if key not found, return -2 otherwise. More...

int	intOption (const char *i_key, int i_default) const noexcept override
	return int value of run option by i_key or default if not found or can not be converted to int. More...

long long	longOption (const char *i_key, long long i_default) const noexcept override
	return long value of run option by i_key or default if not found or can not be converted to long. More...

double	doubleOption (const char *i_key, double i_default) const noexcept override
	return double value of run option by i_key or default if not found or can not be converted to double. More...

vector< pair< string, string > >	allOptions (void) const noexcept override
	return a copy of all run options as [key, value] pairs, ordered by key. More...

void	pushDbMicrodata (int i_runId, int i_entityKind, uint64_t i_microdataKey, const void *i_entityThis)
	push microdata database rows into the buffer. More...

void	writeCsvMicrodata (int i_entityKind, uint64_t i_microdataKey, int i_eventId, bool i_isSameEntity, const void *i_entityThis, string &io_line)
	write microdata into the CSV file. More...

const string	csvHeaderMicrodata (int i_entityKind) const
	return microdata entity csv file header

void	makeCsvLineMicrodata (const EntityItem &i_entityItem, uint64_t i_microdataKey, int i_eventId, bool i_isSameEntity, const void *i_entityThis, string &io_line) const
	make attributes csv line by converting attribute values into string

tuple< bool, const RunController::EntityItem & >	findEntityItem (int i_entityKind) const
	check if any microdata write required for this entity kind

bool	isEntityEventInFilter (int i_entityKind, uint64_t i_microdataKey, int i_eventId) const
	retrun true if entity event id is matching events filter or if there no filter and all events are allowed

Public Member Functions inherited from openm::MetaLoader
virtual	~MetaLoader (void) noexcept=0
	last cleanup

const ArgReader &	argOpts (void) const
	arguments from command line and ini-file

const MetaHolder *	meta (void) const
	model metadata tables

const RunOptions	modelRunOptions (int i_subCount, int i_subId) const
	return basic model run options

int	parameterSubCount (int i_paramId) const
	return sub-values count by parameter id

int	parameterIdByName (const char *i_name) const
	return index of parameter by name More...

int	tableIdByName (const char *i_name) const
	return id of output table by name

bool	isSuppressed (const char *i_name) const
	check by name if output table suppressed

bool	isSuppressed (int i_tableId) const
	check by id if output table suppressed

Public Member Functions inherited from openm::IRunBase
virtual int	parameterIdByName (const char *i_name) const =0
	return id of parameter by name More...

virtual int	parameterSubCount (int i_paramId) const =0
	number of parameter sub-values More...

virtual int	parameterSelfSubCount (int i_paramId) const =0
	number of parameter sub-values for current process More...

virtual int	parameterSubValueIndex (int i_paramId, int i_subId) const =0
	return index of parameter sub-value in the storage array of sub-values More...

virtual bool	isUseSubValue (int i_subId) const =0
	return true if sub-value used by current process More...

virtual void	readParameter (const char i_name, int i_subId, const std::type_info &i_type, size_t i_size, void io_valueArr)=0
	read model parameter

Public Member Functions inherited from openm::IRunOptions
virtual bool	isOptionExist (const char *i_key) const noexcept=0
	return true if run option found by i_key in run_option table for the current run id. More...

virtual std::string	strOption (const char *i_key, const std::string &i_default="") const noexcept=0
	return string value of run option by i_key or default value if not found.

virtual bool	boolOption (const char *i_key) const noexcept=0
	return boolean value of run option by i_key or false if not found or value is not "yes", "1", "true" or empty "" string. More...

virtual int	boolOptionToInt (const char *i_key) const noexcept=0
	search for boolean value of run option by i_key and return one of: return 1 if key found and value is one of: "yes", "1", "true" or empty value, return 0 if key found and value is one of: "no", "0", "false", return -1 if key not found, return -2 otherwise. More...

virtual int	intOption (const char *i_key, int i_default) const noexcept=0
	return int value of run option by i_key or default if not found or can not be converted to int. More...

virtual long long	longOption (const char *i_key, long long i_default) const noexcept=0
	return long value of run option by i_key or default if not found or can not be converted to long. More...

virtual double	doubleOption (const char *i_key, double i_default) const noexcept=0
	return double value of run option by i_key or default if not found or can not be converted to double. More...

virtual std::vector< std::pair< std::string, std::string > >	allOptions (void) const noexcept=0
	return a copy of all run options as [key, value] pairs, ordered by key. More...

Protected Member Functions
virtual void	init (void) override
	initialize root modeling process. More...

int	currentRunId (void) const noexcept override
	return run id of root process model run, if root process is doing any model runs More...

Protected Member Functions inherited from openm::RunController
	RunController (const ArgReader &i_argStore)
	create run controller

virtual void	init (void)=0
	get number of sub-values, read and broadcast metadata. More...

virtual int	currentRunId (void) const noexcept=0
	get run id of the current model run. More...

int	createTaskRun (int i_taskId, IDbExec *i_dbExec)
	create task run entry in database

int	findTask (IDbExec *i_dbExec)
	find modeling task, if specified

tuple< int, int, ModelStatus >	createNewRun (int i_taskRunId, bool i_isWaitTaskRun, IDbExec *i_dbExec)
	create new run, create input parameters and run options for input working sets

void	doShutdownOnExit (ModelStatus i_status, int i_runId, int i_taskRunId, IDbExec *i_dbExec)
	impelementation of model process shutdown if exiting without completion.

void	doShutdownRun (int i_runId, int i_taskRunId, IDbExec *i_dbExec)
	implementation of model run shutdown.

void	doShutdownAll (int i_taskRunId, IDbExec *i_dbExec)
	implementation model process shutdown. More...

void	doWriteAccumulators (int i_runId, IDbExec i_dbExec, const RunOptions &i_runOpts, const char i_name, size_t i_size, forward_list< unique_ptr< double[]> > &io_accValues) const
	write output table accumulators if table is not suppressed.

void	updateRestartSubValueId (int i_runId, IDbExec *i_dbExec, int i_subRestart) const
	update sub-value index to restart the run

void	updateRunState (IDbExec *i_dbExec, const map< pair< int, int >, RunState > i_updated) const
	merge updated sub-values run statue into database

map< int, list< unique_ptr< uint8_t[]> > >	pullDbMicrodata (bool i_isNow=false)
	pull microdata database rows from the buffer. More...

size_t	doDbMicrodata (IDbExec *i_dbExec, int i_entityId, IRowsFirstNext &i_entityMdRows)
	write entity microdata rows into database and return inserted rows count More...

size_t	doDbMicrodataSql (IDbExec *i_dbExec, const map< int, list< unique_ptr< uint8_t[]> > > &i_entityMdRows)
	write microdata into database using sql insert literal and return inserted rows count More...

void	openCsvMicrodata (void)
	create microdata CSV files for new model run.

Protected Member Functions inherited from openm::MetaLoader
	MetaLoader (const ArgReader &i_argStore)
	create metadata loader.

const RunOptions &	modelRunOptions (void) const
	return basic model run options

void	setRunOptions (const RunOptions &i_opts)
	set basic model run options

void	loadMessages (IDbExec *i_dbExec)
	read model messages from database. More...

void	mergeOptions (IDbExec *i_dbExec)
	merge command line and ini-file arguments with profile_option table values. More...

Additional Inherited Members
Static Public Member Functions inherited from openm::RunController
static RunController *	create (const ArgReader &i_argOpts, bool i_isMpiUsed, IDbExec i_dbExec, IMsgExec i_msgExec)
	create run controller, load metadata tables and broadcast it to all modeling processes.

Static Public Member Functions inherited from openm::MetaLoader
static const ArgReader	getRunOptions (int argc, char **argv)
	initialize run options from command line and ini-file More...

Public Attributes inherited from openm::RunController
int	subFirstId
	sub-value staring index for current modeling process

int	selfSubCount
	number of sub-values for current process

int	processCount
	number of modeling processes: MPI world size

int	processRank
	modeling processes rank: MPI rank

Public Attributes inherited from openm::MetaLoader
int	subValueCount
	total number of sub-values

int	threadCount
	max number of modeling threads

Protected Types inherited from openm::MetaLoader
enum class	KindSubIds : int { none = 0 , single = 1 , defaultId = 2 , range =3 , list = 4 }

enum class	FromSub : int { defaultValue = 0 , db = 1 , csv = 2 , iota = 3 }

enum class	ImportKind : int { none = 0 , modelName = 1 , modelId = 2 , modelDigest = 3 , runName = 4 , runId = 5 , runDigest = 6 }

Static Protected Member Functions inherited from openm::MetaLoader
static int	readMetaTables (IDbExec i_dbExec, MetaHolder io_metaStore, const char i_name, const char i_digest)

Protected Attributes inherited from openm::RunController
map< int, EntityItem >	entityMap

vector< int >	entityIds

Protected Attributes inherited from openm::MetaLoader
int	modelId

unique_ptr< MetaHolder >	metaStore

vector< int >	paramIdSubArr

vector< int >	tableIdSuppressArr

vector< int >	entityIdxArr

bool	isCsvEventColumn

vector< bool >	entityUseEvents

map< int, ParamSubOpts >	subOptsMap

map< string, ImportOpts >	importOptsMap

map< int, ParamImportOpts >	paramImportOptsMap

map< int, pair< string, string > >	langOptsMap

Detailed Description

controller for root process: create new model run(s), send input parameters to children and receieve output tables.

Member Function Documentation

◆ childExchange()

bool RootController::childExchange ( void )

overridevirtual

exchange between root and child processes and threads.

start new run, send new input, receive accumulators of output tables, send and receive status update. return true if any: data received, run completed, run started, status update received.

Implements openm::RunController.

◆ currentRunId()

int openm::RootController::currentRunId ( void ) const

inlineoverrideprotectedvirtualnoexcept

return run id of root process model run, if root process is doing any model runs

Implements openm::RunController.

◆ init()

void RootController::init ( void )

overrideprotectedvirtual

initialize root modeling process.

(a) get number of sub-values, processes, threads, groups
(b) merge command line and ini-file options with db profile table
(c) initialize task run if required
(d) load metadata tables from database and broadcast it to child processes

(a) get number of sub-values, processes, threads, groups

Number of sub-values by default = 1 and it can be specified using command-line argument, ini-file or profile table in database

Number of modeling threads by default = 1 and sub-values run sequentially in single thread. If more threads specified (i.e. by command-line argument) then sub-values run in parallel.

For example:
model.exe -OpenM.SubValues 8
model.exe -OpenM.SubValues 8 -OpenM.Threads 4
mpiexec -n 2 model.exe -OpenM.SubValues 31 -OpenM.Threads 7

Number of modeling processes = MPI world size

Processes may be combined into groups for parallel run of input data sets and size of the group is such to calculate all sub-values by the group.

Size of modeling group = number of sub-values / number of threads
group size always >= 1 therefore group always include at least one process
Number of groups = number of processes / group size
by default is only one group, which include all modeling processes
Root process may or may not be be used for modeling
if (group size * group count) == (process count) then "root is active"
else root process dedicated for data exchange and modeling done only by child processes
if run option "NotOnRoot" is true then no modeling done at root process

Number of sub-values per process = total number of sub-values / group size
if total number of sub-values % number of processes != 0 then remainder calculated at group last process

(b) merge command line and ini-file options with db profile table

model run options can be specified as (in order of priority):

command line arguments, i.e.:
model.exe -Parameter.Population 1234
through ini-file:
model.exe -s modelOne.ini
as rows of profile_option table:
profile name can be command line argument or ini-file entry, i.e.:
model.exe -OpenM.Profile TestProfile
some options have hard coded default values which used if nothing above is specified

run options which name starts with "Parameter." are treated as value of scalar input parameter (see below).
if there is no input parameter with such name then exception raised.
for example, if command line is:
model.exe -Parameter.Population 1234
and model does not have "Population" parameter then execution aborted.

(c) initialize task run

modeling task can be specified by name or by task id:

model.exe -OpenM.SubValues 16 -OpenM.TaskName someTask
model.exe -OpenM.SubValues 16 -OpenM.Taskid 1
model.exe -OpenM.SubValues 16 -OpenM.TaskName someTask -OpenM.TaskWait true

if task specified then model would run for each input working set of parameters
if task "wait" is true then modeling task is run under external supervision:

model must wait until task status set as "completed" by other external process
external process can append new inputs into task_set table and model will continue to run

Implements openm::RunController.

◆ nextRun()

int RootController::nextRun ( void )

overridevirtual

create new run and input parameters in database.

next run for root process: create new run and input parameters in database and support data exchange.

(a) find id of source working set for input parameters
(b) create new "run" in database (if required)
(c) prepare model input parameters
(d) prepare to receive accumulators data from to child processes

(a) find id of source working set for input parameters

use following to find input parameters set id:

if task id or task name specified then find task id in task_lst
if set id specified as run option then use such set id
if set name specified as run option then find set id by name
else use min(set id) as default set of model parameters

(b) create new "run" in database

root process creates "new run" in database:

insert new row with new run_id key into run_lst
insert run options into run_option table under run_id
save run options in database

(c) prepare model input parameters

it creates a copy of input paramters from source working set under destination run_id

search for input parameter value in following order:

use value of parameter specified as command line or ini-file argument or from profile_option table
use parameter csv file if parameters csv directory specified, it can be any of: .csv .tsv .id.csv .id.tsv file
use value of parameter from working set of model parameters
if working set based on model run then search by base run id to get parameter value
else raise an exception

any scalar parameter value can be overriden by model run option with "Parameter" prefix
for example, command line:
model.exe -Parameter.Population 1234
means input parameter with name "Population" will be =1234
in that case working set value of "Population" input parameter ignored
because command line options have higher priority than database values.

Implements openm::RunController.

◆ readParameter()

void RootController::readParameter	(	const char *	i_name,
		int	i_subId,
		const type_info &	i_type,
		size_t	i_size,
		void *	io_valueArr
	)

overridevirtual

read input parameter values.

Parameters

[in]	i_name	parameter name
[in]	i_subId	parameter sub-value index
[in]	i_type	parameter value type
[in]	i_size	parameter size (number of parameter values)
[in,out]	io_valueArr	array to return parameter values, size must be =i_size

◆ shutdownOnExit()

void RootController::shutdownOnExit ( ModelStatus i_status )

overridevirtual

model process shutdown if exiting without completion (ie: exit on error).

Implements openm::RunController.

◆ shutdownRun()

void RootController::shutdownRun ( int i_runId )

overridevirtual

model run shutdown: save results and update run status.

Implements openm::RunController.

◆ shutdownWaitAll()

void RootController::shutdownWaitAll ( void )

overridevirtual

model process shutdown: wait for all child to be completed and do final cleanup.

Implements openm::RunController.

◆ writeAccumulators()

void RootController::writeAccumulators	(	const RunOptions &	i_runOpts,
		bool	i_isLastTable,
		const char *	i_name,
		size_t	i_size,
		forward_list< unique_ptr< double[]> > &	io_accValues
	)

overridevirtual

write output table accumulators.

Parameters

[in]	i_runOpts	model run options
[in]	i_isLastTable	if true then it is last output table to write
[in]	i_name	output table name
[in]	i_size	number of cells for each accumulator
[in,out]	io_accValues	accumulator values

Implements openm::RunController.

The documentation for this class was generated from the following files:

model/runControllerImpl.h
model/rootController.cpp

Public Member Functions

Protected Member Functions

Additional Inherited Members

Detailed Description

Member Function Documentation

◆ childExchange()

◆ currentRunId()

◆ init()

(a) get number of sub-values, processes, threads, groups

(b) merge command line and ini-file options with db profile table

(c) initialize task run

◆ nextRun()

(a) find id of source working set for input parameters

(b) create new "run" in database

(c) prepare model input parameters

◆ readParameter()

◆ shutdownOnExit()

◆ shutdownRun()

◆ shutdownWaitAll()

◆ writeAccumulators()