diff --git a/Makefile b/Makefile index 27edae41..a8df1846 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,7 @@ SIM_SERV_DIRS = \ ${TRICK_HOME}/trick_source/sim_services/MemoryManager \ ${TRICK_HOME}/trick_source/sim_services/Message \ ${TRICK_HOME}/trick_source/sim_services/MonteCarlo \ + ${TRICK_HOME}/trick_source/sim_services/MonteCarloGeneration \ ${TRICK_HOME}/trick_source/sim_services/RealtimeInjector \ ${TRICK_HOME}/trick_source/sim_services/RealtimeSync \ ${TRICK_HOME}/trick_source/sim_services/ScheduledJobQueue \ diff --git a/docs/documentation/Documentation-Home.md b/docs/documentation/Documentation-Home.md index f2eed8ac..3c9e5183 100644 --- a/docs/documentation/Documentation-Home.md +++ b/docs/documentation/Documentation-Home.md @@ -36,6 +36,7 @@ The user guide contains information pertinent to Trick users. These pages will h 01. [Realtime Sleep Timer](simulation_capabilities/Realtime-Timer) 01. [Realtime Injector](simulation_capabilities/Realtime-Injector) 01. [Monte Carlo](simulation_capabilities/UserGuide-Monte-Carlo) + 02. [Monte Carlo Generation](miscellaneous_trick_tools/MonteCarloGeneration) 01. [Master Slave](simulation_capabilities/Master-Slave) 01. [Data Record](simulation_capabilities/Data-Record) 01. [Checkpoints](simulation_capabilities/Checkpoints) diff --git a/docs/documentation/install_guide/Install-Guide.md b/docs/documentation/install_guide/Install-Guide.md index dd92121a..93aad9ca 100644 --- a/docs/documentation/install_guide/Install-Guide.md +++ b/docs/documentation/install_guide/Install-Guide.md @@ -215,7 +215,7 @@ xcode-select --install brew install python java xquartz swig@3 maven udunits openmotif ``` -IMPORTANT: Make sure to follow the instructions for adding java to your path provided by brew. If you missed them, you can see them again by using `brew info java`. +IMPORTANT: Make sure to follow the instructions for adding java and swig to your `PATH` provided by brew. If you missed them, you can see them again by using `brew info java` and `brew info swig@3`. Remember, you may need to restart your terminal for these `PATH` changes to take effect. 5. Download and un-compress the latest pre-built clang+llvm from llvm-project github. Go to https://github.com/llvm/llvm-project/releases and download the latest version llvm that matches your Xcode version from the release assets. For example, if your Xcode version is 14 then you will want the latest 14.x.x release of llvm. 13.0.1 is the latest as of the writing of this guide, the link I used is below: @@ -243,7 +243,7 @@ e.g. OPTIONAL: Trick uses google test (gtest) version 1.8 for unit testing. To install gtest: ``` -brew install wget +brew install cmake wget wget https://github.com/google/googletest/archive/release-1.8.0.tar.gz tar xzvf release-1.8.0.tar.gz cd googletest-release-1.8.0/googletest diff --git a/docs/documentation/miscellaneous_trick_tools/MCG_verification_2020.pdf b/docs/documentation/miscellaneous_trick_tools/MCG_verification_2020.pdf new file mode 100644 index 00000000..e53844d2 Binary files /dev/null and b/docs/documentation/miscellaneous_trick_tools/MCG_verification_2020.pdf differ diff --git a/docs/documentation/miscellaneous_trick_tools/MonteCarloGeneration.md b/docs/documentation/miscellaneous_trick_tools/MonteCarloGeneration.md new file mode 100644 index 00000000..90139ec8 --- /dev/null +++ b/docs/documentation/miscellaneous_trick_tools/MonteCarloGeneration.md @@ -0,0 +1,705 @@ +# MonteCarloGeneration Model + +# Revision History +| Version | Date | Author | Purpose | +| :--- |:---| :--- | :--- | +| 1 | April 2020 | Gary Turner | Initial Version | +| 2 | March 2021 | Gary Turner | Added Verification | +| 3 | October 2022 | Isaac Reaves | Converted to Markdown | + +# 1 Introduction + +The MonteCarlo Model is used to disperse the values assigned to variables at the start of a simulation. Dispersing the initial +conditions and configurations for the simulation allows for robust testing and statistical analysis of the probability of +undesirable outcomes, and measuring the confidence levels associated with achieving desirable outcomes. + +Conventionally, most of the time we think about dispersing variables, we think about apply some sort of statistical +distribution to the value. Most often, that is a normal or uniform distribution, but there may be situations in which other +distributions are desired. In particular, this model provides an extensible framework allowing for any type of distribution to +be applied to any variable. + +For extensive analysis of safety-critical scenarios, where it is necessary to demonstrate high probability of success with high +confidence, traditional MonteCarlo analyses require often many thousands of runs. For long duration simulations, it may +not be feasible to run the number of simulations necessary to reach the high confidence of high success probability that is +necessary to meet requirements. Typically, failure cases occur out near the edges of state-space, but most of the runs will be +“right down the middle”; using conventional MonteCarlo techniques, most of these runs are completely unnecessary. With +a Sequential-MonteCarlo configuration, a small number of runs can be executed, allowing for identification of problem +areas, and a focussing of the distribution on those areas of state-space, thereby reducing the overall number of runs while +adding complexity to the setup. While this model does not (at this time) provide a Sequential-MonteCarlo capability, the +organization of the model has been designed to support external tools seeking to sequentially modify the distributions being +applied to the dispersed variables, and generate new dispersion sets. + +# 2 Requirements + +1. The model shall provide common statistical distribution capabilities, including: + 1. Uniform distribution between specified values + 1. as a floating-point value + 1. as an integer value + 1. Normal distribution, specified by mean and standard deviation + 1. Truncated Normal Distribution, including + 1. symmetric and asymmetric truncations + 1. it shall be possible to specify truncations by: + 1. some number of standard deviations from the mean, + 1. a numerical difference from the mean, and + 1. an upper and lower limit +1. The model shall provide an extensible framework suitable for supporting other statistical distributions +1. The model shall provide the ability to assign a common value to all runs: + 1. This value could be a fixed, user-defined value + 1. This value could be a random assignment, generated once and then applied to all runs +1. The model shall provide the capability to read values from a pre-generated file instead of generating its own values +1. The model shall provide the ability to randomly select from a discrete data set, including: + 1. enumerations, + 1. character-strings, + 1. boolean values, and + 1. numerical values +1. The model shall provide the capability to compute follow-on variables, the values of which are a function of one or more dispersed variables with values generated using any of the methods in requirements 1-5. +1. The model shall provide a record of the generated distributions, allowing for repeated execution of the same scenario using exactly the same conditions. +1. The model shall provide summary data of the dispersions which have been applied, including: + 1. number of dispersions + 1. types of dispersions + 1. correlations between variables + +# 3 Model Specification + +## 3.1 Code Structure + +The model can be broken down into its constituent classes; there are two principle components to the model – the variables, +and the management of the variables. + +### 3.1.1 Variable Management (MonteCarloMaster) + +MonteCarloMaster is the manager of the MonteCarlo variables. This class controls how many sets of dispersed variables +are to be generated; for each set, it has the responsibility for +* instructing each variable to generate its own dispersed value +* collecting those values and writing them to an external file + +### 3.1.2 Dispersed Variables (MonteCarloVariable) + +MonteCarloVariable is an abstract class that forms the basis for all dispersed variables. The following classes inherit from +MonteCarloVariable: +* MonteCarloVariableFile will extract a value for its variable from a specified text file. Typically, a data file will +comprise some number of rows and some number of columns of data. Each column of data represents the possible +values for one variable. Each row of data represents a correlated set of data to be applied to several variables; each +data-set generation will be taken from one line of data. Typically, each subsequent data-set will be generated from the +next line of data; however, this is not required. +* In some situations, it is desirable to have the next line of data to be used for any given data set be somewhat randomly +chosen. This has the disadvantageous effect of having some data sets being used more than others, but it supports +better cross-population when multiple data files are being used. + * For example, if file1 contained 2 data sets and file2 contained 4 data sets, then a sequential sweep through +these file would set up a repeating pattern with line 1 of file2 always being paired with line 1 of file1. For +example, in 8 runs, we would get this pattern of line numbers from each run: + * (1,1), (2,2), (1,3), (2,4), (1,1), (2,2), (1,3), (2,4) + * If the first file was allowed to skip a line, the pattern can produce a more comprehensive combination of +data: + * (1,1), (1,2), (2,3), (1,4), (2,1), (2,2), (2,3), (1,4) +* MonteCarloVariableFixed provides fixed-values to a variable for all generated data-sets. The values can be +represented as a double, int, or STL-string. +* MonteCarloVariableRandom is the base class for all variables being assigned a random value. The values can be +represented as a double, int, or STL-string. There are several subclasses: + * MonteCarloVariableRandomNormal provides a variable with a value dispersed according to a normal +distribution specified by its mean and standard deviation. + * MonteCarloVariableRandomUniformInt provides a variable with a value dispersed according to a uniform +distribution specified by its upper and lower bounds. This class represents a discrete distribution, providing an +integer value. + * MonteCarloVariableRandomUniform provides a variable with a value dispersed according to a uniform +distribution specified by its upper and lower bounds. This class represents a continuous distribution. + * MonteCarloVariableRandomStringSet represents a discrete variable, drawn from a set of STL-strings. The +class inherits from MonteCarloVariableRandomUniform; this distribution generates a continuous value in [0,1) +and scales and casts that to an integer index in {0, …, size-1} where size is the number of available strings +from which to choose. + + Note – an astute reader may question why the discrete MonteCarloVariableRandomStringSet inherits from +the continuous MonteCarloVariableRandomUniform rather than from the discrete +MonteCarloVariableRandomUniformInt. The rationale is based on the population of the vector of +selectable strings in this class. It is desirable to have this vector be available for population outside the +construction of the class, so at construction time the size of this vector is not known. However, the +construction of the MonteCarloVariableRandomUniformInt requires specifying the lower and upper +bounds, which would be 0 and size-1 respectively. Because size is not known at construction, this cannot +be specified. Conversely, constructing a MonteCarloVariableRandomUniform with bounds at [0,1) still +allows for scaling to the eventual size of the strings vector. + +* MonteCarloVariableSemiFixed utilizes a two-step process. First, a seed-variable has its value generated, then that +value is copied to this variable. The seed-variable could be a “throw-away” variable, used only to seed this value, or it +could be an instance of another dispersed variable. Once the value has been copied to this instance, it is retained in this +instance for all data sets. The seed-variable will continue to generate a new value for each data set, but they will not be +seen by this variable after that first set. + + The seed-variable can be any type of MonteCarloVariable, but note that not all types of MonteCarloVariable actually +make sense to use in this context. Most of the usable types are specialized types of MonteCarloVariableRandom. + + However, restricting the seed-variable in such a way would limit the extensibility of the model. All +MonteCarloVariableRandom types use the C++ \ library for data generation. Limiting the +MonteCarloVariableSemiFixed type to be seeded only by something using the \ library violates the concept of +free-extensibility. Consequently, the assigned value may be extracted from any MonteCarloVariable type. The only +constraint is that the command generated by the seed-variable includes an “=” symbol; everything to the right of that +symbol will be assigned to this variable. + +* MonteCarloPythonLineExec provides a line of executable Python code that can be used to compute the value of this +variable. So rather than generating an assignment statement, e.g. + +``` +var_x = 5 +``` + +when the MonteCarloMaster processes an instance of this class, it will use a character string to generate an +instruction statement, e.g. + +``` +var_x = math.sin(2 * math.pi * object.circle_fraction) +``` + +(in this case, the character string would be “math.sin(2 * math.pi * object.circle_fraction)” and +object.circle_fraction could be a previously-dispersed variable). + + A particularly useful application of this capability is in generating systematic data sweeps across a domain, as +opposed to random distributions within a domain. These are commonly implemented as a for-loop, but we can use +the MonteCarloPythonLineExec to generate them internally. The first data assignment made in each file is to a +run-number, which can be used as an index. The example shown below will generate a sweep across the domain +[20,45) in steps of 2.5. + +``` +object.sweep_variable = (monte_carlo.master.monte_run_number % 10) * 2.5 + 20 +``` + + * MonteCarloPythonFileExec is used when simple assignments and one-line instructions are insufficient, such as +when one generated-value that feeds into an analytical algorithm to generate multiple other values. With this class, +the execution of the Python file generated by MonteCarloMaster will hit a call to execute a file as specified by this +class. This is an oddity among the bank of MonteCarloVariable implementations. In all other implementations, +the identifying variable_name is used to identify the variable whose value is to be assigned (or computed). With +the MonteCarloPythonFileExec implementation, the variable_name is hijacked to provide the name of the file to +be executed. + +## 3.2 Mathematical Formulation + +No mathematical formulation. The random number generators use the C++ \ library. + +# 4 User's Guide + +## 4.1 What to expect + +This role played by this model can be easily misunderstood, so let’s start there. +**This model generates Python files containing assignments to variables.** + +That’s it!! It does not manage MonteCarlo runs. It does not execute any simulations. When it runs, it creates the requested +number of Python files and exits. + +This design is deliberate; we want the model to generate the instruction sets that will allow execution of a set of dispersed +configurations. At that point, the simulation should cease, returning control to the user to distribute the execution of those +configurations according to whatever distribution mechanism they desire. This could be: + +* something really simple, like a wild-card, \ `MONTE_RUN_test/RUN*/monte_input.py` +* a batch-script, +* a set of batch-scripts launching subsets onto different machines, +* a load-management service, like SLURM +* any other mechanism tailored to the user’s currently available computing resources + +The intention is that the model runs very early in the simulation sequence. If the model is inactive (as when running a regular, non-MonteCarlo run), it will take no action. But when this model is activated, the user should expect the simulation to terminate before it starts on any propagation. + +**When a simulation executes with this model active, the only result of the simulation will be the generation of files containing the assignments to the dispersed variables. The simulation should be expected to terminate at t=0.** + +## 4.1.1 Trick Users + +The model is currently configured for users of the Trick simulation engine. The functionality of the model is almost exclusively independent of the chosen simulation engine, with the exceptions being the shutdown sequence, and the application of units information in the variables. + +Found at the end of the `MonteCarloMaster::execute()` method, the following code: + +```c++ +exec_terminate_with_return(0, __FILE__, __LINE__,message.c_str()); +``` + +is a Trick instruction set to end the simulation. + +Found at the end of `MonteCarloVariable::insert_units()`, the following code: + +```c++ +// TODO: Pick a unit-conversion mechanism +// Right now, the only one available is Trick: +trick_units( pos_equ+1); +``` + +provides the call to + +```c++ +MonteCarloVariable::trick_units( + size_t insertion_pt) +{ + command.insert(insertion_pt, " trick.attach_units(\"" + units + "\","); + command.append(")"); +``` + +which appends Trick instructions to interpret the generated value as being represented in the specified units. + +The rest of the User’s Guide will use examples of configurations for Trick-simulation input files + +## 4.1.2 Non-Trick Users + +To configure the model for simulation engines other than Trick, the Trick-specific content identified above should be replaced with equivalent content that will result in: +* the shutdown of the simulation, and +* the conversion of units from the type specified in the distribution specification to the type native to the variable to which the generated value is to be assigned. + +While the rest of the User’s Guide will use examples of configurations for Trick-simulation input files, understand that these are mostly just C++ or Python code setting the values in this model to make it work as desired. Similar assignments would be required for any other simulation engine. + +## 4.2 MonteCarlo Manager (MonteCarloMaster) + +### 4.2.1 Instantiation + +The instantiation of MonteCarloMaster would typically be done directly in the S_module. The construction of this instance takes a single argument, a STL-string describing its own location within the simulation data-structure. + +The MonteCarloMaster class has a single public-interface method call, `MonteCarloMaster::execute()`. This method has 2 gate-keeping flags that must be set (the reason for there being 2 will be explained later): +* `active` +* `generate_dispersions` + +If either of these flags is false (for reference, `active` is constructed as false and `generate_dispersions` is constructed as true) then this method returns with no action. If both are true, then the model will generate the dispersions, write those dispersions to the external files, and shut down the simulation. + +An example S-module + +```c++ +class MonteCarloSimObject : public Trick::SimObject +{ + public: + MonteCarloMaster master; // <--- master is instantiated + MonteCarloSimObject(std::string location) + : + master(location) // <--- master is constructed with this STL-string + { + P_MONTECARLO ("initialization") master.execute(); // <--- the only function +call + } +}; +MonteCarloSimObject monte_carlo("monte_carlo.master"); // <--- location of “master” + is passed as an + argument +``` + +### 4.2.2 Configuration + +The configuration of the MonteCarloMaster is something to be handled as a user-input to the simulation without requiring re-compilation; as such, it is typically handled in a Python input file. There are two sections for configuration: +* modifications to the regular input file, and +* new file-input or other external monte-carlo initiation mechanism + +#### 4.2.2.1 Modifications to the regular input file + +A regular input file sets up a particular scenario for a nominal run. To add monte-carlo capabilities to this input file, the +following code should be inserted somewhere in the file: + +```python +if monte_carlo.master.active: + # insert non-mc-variable MC configurations like logging + if monte_carlo.master.generate_dispersions: + exec(open(“Modified_data/monte_variables.py").read()) +``` + +Let’s break this down, because it explains the reason for having 2 flags: + +| `generate_dispersions` | `active` | Result | +| :--- |:---| :--- | +| false | false | Regular (non-monte-carlo) run | +| false | true | Run scenario with monte-carlo configuration and pre-generated dispersions | +| true | false | Regular (non-monte-carlo) runs | +| true | true | Generate dispersions for this scenario, but do not run the scenario | + + 1. If the master is inactive, this content is passed over and the input file runs just as it would without this content + 2. Having the master `active` flag set to true instructs the simulation that the execution is intended to be part of a monte-carlo analysis. Now there are 2 types of executions that fit this intent: + * The generation of the dispersion files + * The execution of this run with the application of previously-generated dispersions + +Any code to be executed for case (a) must go inside the `generate_dispersions` gate. Any code to be executed for +case (b) goes inside the `active` gate, but outside the `generate_dispersions` gate. + +You may wonder why this distinction is made. In many cases, it is desirable to make the execution for monte-carlo +analysis subtly different to that for regular analysis. One commonly used distinction is logging of data; the logging +requirement may differ between a regular run and one as part of a monte-carlo analysis (typically, monte-carlo runs +execute with reduced logging). By providing a configuration area for a monte-carlo run, we can support these +distinctions. +Note – any code to be executed for only non-monte-carlo runs can be put in an else: block. For example, this code +will set up one set of logging for a monte-carlo run, and another for a non-monte-carlo run of the same scenario: +```python +if monte_carlo.master.active: + exec(open(“Log_data/log_for_monte_carlo.py”).read()) + if monte_carlo.master.generate_dispersions: + exec(open(“Modified_data/monte_variables.py").read()) +else: + exec(open(“Log_data/log_for_regular.py”).read()) +``` + 3. If the `generate_dispersions` flag is also set to true, the `MonteCarloMaster::execute()` method will execute, +generating the dispersion files and shutting down the simulation. + +#### 4.2.2.2 Initiating MonteCarlo + +Somewhere outside this file, the `active` and generate_dispersion flags must be set. This can be performed either in a separate input file or via a command-line argument. Unless the command-line argument capability is already supported, by far the easiest mechanism is to create a new input file that subsequently reads the existing input file: + +``` +monte_carlo.master.activate("RUN_1") +exec(open("RUN_1/input.py").read()) +``` + +The activate method takes a single string argument, representing the name of the run. This must be exactly the same name as the directory containing the original input file, “RUN_1” in the example above. This argument is used in 2 places (\ in these descriptions refers to the content of the argument string): + +* In the creation of a `MONTE_` directory. This directory will contain some number of sub-directories identified as, for example, RUN_01, RUN_02, RUN_03, etc. each of which will contain one of the generated dispersion files. +* In the instructions written into the generated dispersion files to execute the content of the input file found in ``. + +#### 4.2.2.3 Additional Configurations + +There are additional configurations instructing the MonteCarloMaster on the generation of the new dispersion files. Depending on the use-case, these could either be embedded within the `if monte_carlo.master.generate_dispersions:` block of the original input file, or in the secondary input file (or command-line arguments if configured to do so). + +* Number of runs is controlled with a single statement, e.g. + + ```monte_carlo.master.set_num_runs(10)``` + +* Generation of meta-data. The meta-data provides a summary of which variables are being dispersed, the type of dispersion applied to each, the random seeds being used, and correlation between different variables. This is written out to a file called MonteCarlo_Meta_data_output in the MONTE_* directory. + + ```monte_carlo.master.generate_meta_data = True``` + +* Changing the name of the automatically-generated monte-directory. By default, this takes the form “MONTE_\” as assigned in the MonteCarloMaster::activate(...) method. The monte_dir variable is public and can be reset after activation and before the `MonteCarloMaster::execute()` method runs. This is particularly useful if it is desired to compare two distribution sets for the same run. + + ```monte_carlo.master.monte_dir = “MONTE_RUN_1_vers2”``` + +* Changing the input file name. It is expected that most applications of this model will run with a typical organization of a Trick simulation. Consequently, the original input file is probably named input.py, and this is the default setting for the input_file_name variable. However, to support other cases, this variable is public and can be changed at any time between construction and the execution of the `MonteCarloMaster::execute()` method. + + ```monte_carlo.master.input_file_name = “modified_input.py”``` + +* Padding the filenames of the generated files. By default, the generated RUN directories in the generated MONTE_* directory will have their numerical component padded according to the number of runs. When: + * between 1 - 10 runs are generated, the directories will be named RUN_0, RUN_1, … + * between 11-100 runs are generated, the directories will be named RUN_00, RUN_01, … + * between 101-1000 runs are generated, the directories will be named RUN_000, RUN_001, … + * etc. + + Specification of a minimum padding width is supported. For example, it might be desired to create 3 runs with names RUN_00000, RUN_00001, and RUN_00002, in which case the minimum-padding should be specified as 5 characters + + ```monte_carlo.master.minimum_padding = 5``` + +* Changing the run-name. For convenience, the run-name is provided as an argument in the MonteCarloMaster::activate(...) method. The run_name variable is public, and can be reset after activation and before the `MonteCarloMaster::execute()` method runs. Because this setting determines which run is to be launched from the dispersion files, resetting run_name has limited application – effectively limited to correcting an error, which could typically be more easily corrected directly. + + ```monte_carlo.master.run_name = “RUN_2”``` + +## 4.3 MonteCarlo Variables (MonteCarloVariable) + +The instantiation of the MonteCarloVariable instances is typically handled as a user-input to the simulation without requiring re-compilation. As such, these are usually implemented in Python input files. This is not a requirement, and these instances can be compiled as part of the simulation build. Both cases are presented. + +### 4.3.1 Instantiation and Registration + +For each variable to be dispersed, an instance of a MonteCarloVariable must be created, and that instance registered with the MonteCarloMaster instance: + +1. Identify the type of dispersion desired +2. Select the appropriate type of MonteCarloVariable to provide that dispersion. +3. Create the new instance using its constructor. +4. Register it with the MonteCarloMaster using the `MonteCarloMaster::add_variable( MonteVarloVariable&)` method + +#### 4.3.1.1 Python input file implementation for Trick: + +When the individual instances are registered with the master, it only records the address of those instances. A user may create completely new variable names for each dispersion, or use a generic name as illustrated in the example below. Because these are typically created within a Python function, it is important to add the thisown=False instruction on each creation to prevent its destruction when the function returns. + +```python +mc_var = trick.MonteCarloVariableRandomUniform( "object.x_uniform", 0, 10, 20) +mc_var.thisown = False +monte_carlo.master.add_variable(mc_var) +mc_var = trick.MonteCarloVariableRandomNormal( "object.x_normal", 0, 0, 5) +mc_var.thisown = False +monte_carlo.master.add_variable(mc_var) +``` + +#### 4.3.1.2 C++ implementation in its own class: + +In this case, the instances do have to be uniquely named. + +Note that the registering of the variables could be done in the class constructor rather than in an additional method (process_variables), thereby eliminating the need to store the reference to MonteCarloMaster. In this case, the `generate_dispersions` flag is completely redundant because the variables are already registered by the time the input file is executed. Realize, however, that doing so does carry the overhead of registering those variables with the MonteCarloMaster every time the simulation starts up. This can a viable solution when there are only a few MonteCarloVariable instances, but is generally not recommended; using an independent method (process_variables) allows restricting the registering of the variables to be executed only when generating new dispersions. + +```c++ +class MonteCarloVarSet { + private: + MonteCarloMaster & master; + public: + MonteCarloVariableRandomUniform x_uniform; + MonteCarloVariableRandomNormal x_normal; + ... + MonteCarloVarSet( MonteCarloMaster & master_) + : + master(master_), + x_uniform("object.x_uniform", 0, 10, 20), + x_normal ("object.x_normal", 0, 0, 5), + ... + { }; + + void process_variables() { + master.add_variable(x_uniform); + master.add_variable(x_normal); + ... + } +}; +``` + +#### 4.3.1.3 C++ implementation within a Trick S-module: + +Instantiating the variables into the same S-module as the master is also a viable design pattern. However, this can lead to a very long S-module so is typically only recommended when there are few variables. As with the C++ implementation in a class, the variables can be registered with the master in the constructor rather than in an additional method, with the same caveats presented earlier. + +```c++ +class MonteCarloSimObject : public Trick::SimObject +{ + public: + MonteCarloMaster master; + MonteCarloVariableRandomUniform x_uniform; + MonteCarloVariableRandomNormal x_normal; + ... + MonteCarloSimObject(std::string location) + : + master(location), + x_uniform("object.x_uniform", 0, 10, 20), + x_normal ("object.x_normal", 0, 0, 5), + ... +{ }; + void process_variables() { + master.add_variable(x_uniform); + master.add_variable(x_normal); + ... +}; + { + P_MONTECARLO ("initialization") master.execute(); +} }; +MonteCarloSimObject monte_carlo("monte_carlo.master"); +``` + +### 4.3.2 input-file Access + +If using a (compiled) C++ implementation with the registration conducted at construction, the `generate_dispersions` flag is not used in the input file. + +```python +if monte_carlo.master.active: + if monte_carlo.master.generate_dispersions: + exec(open(“Modified_data/monte_variables.py").read()) +``` + +(where monte_variables.py is the file containing the mc_var = … content described earlier) + +```python +if monte_carlo.master.active: + if monte_carlo.master.generate_dispersions: + monte_carlo_variables.process_variables() +``` + +If using a (compiled) C++ implementation with a method to process the registration, that method call must be contained inside the `generate_dispersions` gate in the input file: +``` +if monte_carlo.master.active: + # add only those lines such as logging configuration +``` + +### 4.3.3 Configuration + +For all variable-types, the variable_name is provided as the first argument to the constructor. This variable name must include the full address from the top level of the simulation. After this argument, each variable type differs in its construction arguments and subsequent configuration options. + +#### 4.3.3.1 MonteCarloVariable + +MonteCarloVariable is an abstract class; its instantiable implementations are presented below. There is one important configuration for general application to these implementations, the setting of units. In a typical simulation, a variable has an inherent unit-type; these are often SI units, but may be based on another system. Those native units may be different to those in which the distribution is described. In this case, assigning the generated numerical value to the variable without heed to the units mismatch would result in significant error. + +```set_units(std::string units)``` + +This method specifies that the numerical value being generated is to be interpreted in the specified units. + +Notes +* if it is known that the variable’s native units and the dispersion units match (including the case of a dimensionless value), this method is not needed. +* This method is not applicable to all types of MonteCarloVariable; use with MonteCarloVariableRandomBool and MonteCarloPython* is considered undefined behavior. + +#### 4.3.3.2 MonteCarloVariableFile + +The construction arguments are: + +1. variable name +2. filename containing the data +3. column number containing data for this variable +4. (optional) first column number. This defaults to 1, but some users may want to zero-index their column numbers, in which case it can be set to 0. + +There is no additional configuration beyond the constructor + +There is no additional configuration beyond the constructor. + +#### 4.3.3.3 MonteCarloVariableFixed + +The construction arguments are: +1. variable name +2. value to be assigned + +Additional configuration for this model includes the specification of the maximum number of lines to skip between runs. +`max_skip`. This public variable has a default value of 0 – meaning that the next run will be drawn from the next line of data, but this can be adjusted. + +#### 4.3.3.4 MonteCarloVariableRandomBool + +The construction arguments are: +1. variable name +2. seed for random generator +There is no additional configuration beyond the constructor. + +#### 4.3.3.5 MonteCarloVariableRandomNormal + +The construction arguments are: +1. variable name +2. seed for random generator, defaults to 0 +3. mean of distribution, defaults to 0 +4. standard-deviation of distribution, defaults to 1. + +The normal distribution may be truncated, and there are several configuration settings associated with truncation. Note that for all of these truncation options, if the lower truncation bound is set to be larger than the upper truncation bound, the generation of the dispersed value will fail and the simulation will terminate without generation of files. If the upper andlower bound are set to be equal, the result will be a forced assignment to that value. + +`TruncationType` + +This is an enumerated type, supporting the specification of the truncation limits in one of three ways: +* `StandardDeviation`: The distribution will be truncated at the specified number(s) of standard deviations away from the mean. +* `Relative`: The distribution will be truncated at the specified value(s) relative to the mean value. +* `Absolute`: The distribution will be truncated at the specified value(s). + +`max_num_tries` + +The truncation is performed by repeatedly generating a number from the unbounded distribution until one is found that lies within the truncation limits. This max_num_tries value determines how many attempts may be made before the algorithm concedes. It defaults to 10,000. If a value has not been found within the specified number of tries, an error message is sent and the value is calculated according to the following rules: +* For a distribution truncated at only one end, the truncation limit is used +* For a distribution truncated at both ends, the midpoint value between the two truncation limits is used. + +`truncate( double limit, TruncationType)` + +This method provides a symmetric truncation, with the numerical value provided by limit being interpreted as a number of standard-deviations either side of the mean, a relative numerical value from the mean, or an absolute value. + +The value limit should be positive. If a negative value is provided, it will be negated to a positive value. + +The use of TruncationType Absolute and this method requires a brief clarification because this may result in an asymmetric distribution. In this case, the distribution will be truncated to lie between (-limit, limit) which will be asymmteric for all cases in which the mean is non-zero. + +`truncate( double min, double max, TruncationType)` + +This method provides a more general truncation, with the numerical value provided by min and max being interpreted as a number of standard-deviations away from the mean, a relative numerical value from the mean, or an absolute value. + +Unlike the previous method, the numerical arguments (min and max) may be positive or negative, and care must be taken especially when specifying min with TruncationType StandardDeviation or Relative. Realize that a positive value of min will result in a lower bound with value above that of the mean; min does not mean “distance to the left of the mean”, it means the smallest acceptable value relative to the mean. + +`truncate_low( double limit, TruncationType)` + +This method provides a one-sided truncation. All generated values will be above the limit specification. + +`truncate_high( double limit, TruncationType)` + +This method provides a one-sided truncation. All generated values will be below the limit specification. + +`untruncate()` + +This method removes previously configured truncation limits. + +#### 4.3.3.6 MonteCarloVariableRandomStringSet + +The construction arguments are: +1. variable name +2. seed for random generator + +This type of MonteCarloVariable contains a STL-vector of STL-strings containing the possible values that can be assigned by this generator. This vector is NOT populated at construction time and must be configured. + +`add_string(std::string new_string)` + +This method adds the specified string (`new_string`) to the vector of available strings + +#### 4.3.3.7 MonteCarloVariableRandomUniform + +The construction arguments are: +1. variable name +2. seed for random generator, defaults to 0 +3. lower-bound of distribution, default to 0 +4. upper-bound for distribution, defaults to 1 + +There is no additional configuration beyond the constructor + +#### 4.3.3.8 MonteCarloVariableRandomUniformInt + +The construction arguments are: +1. variable name +2. seed for random generator, defaults to 0 +3. lower-bound of distribution, default to 0 +4. upper-bound for distribution, defaults to 1 + +There is no additional configuration beyond the constructor + +#### 4.3.3.9 MonteCarloVariableSemiFixed + +The construction arguments are: +1. variable name +2. reference to the MonteCarloVariable whose generated value is to be used as the fixed value. + +There is no additional configuration beyond the constructor. + +#### 4.3.3.10 MonteCarloPythonLineExec + +The construction arguments are: +1. variable name +2. an STL-string providing the Python instruction for the computing of the value to be assigned to the specified variable. + +There is no additional configuration beyond the constructor. + +#### 4.3.3.11 MonteCarloPythonFileExec +The construction argument is: +1. name of the file to be executed from the generated input file. + +There is no additional configuration beyond the constructor. + +## 4.4 Information on the Generated Files + +This section is for informational purposes only to describe the contents of the automatically-generated dispersion files. Users do not need to take action on any content in here. + +The generated files can be broken down into 3 parts: +* Configuration for the input file. These two lines set the flags such that when this file is executed, the content of the original input file will configure the run for a monte-carlo analysis but without re-generating the dispersion files. + +```python +monte_carlo.master.active = True +monte_carlo.master.generate_dispersions = False +``` + +* Execution of the original input file. This line opens the original input file so that when this file is executed, the original input file is also executed automatically. + +```python +exec(open('RUN_1/input.py').read()) +``` + +* Assignment to simulation variables. This section always starts with the assignment to the run-number, which is also found in the name of the run, so RUN_0 gets a 0, RUN_1 gets a 1, etc. This value can be used, for example, to generate data sweeps as described in section MonteCarloPythonLineExec above. + +```python +monte_carlo.master.monte_run_number = 0 +object.test_variable1 = 5 +object.test_variable1 = 1.23456789 +... +``` + +## 4.5 Extension + +The model is designed to be extensible and while we have tried to cover the most commonly used applications, complete anticipation of all use-case needs is impossible. The most likely candidate for extension is in the area of additional distributions. In this case: +* A new distribution should be defined in its own class +* That class shall inherit from MonteCarloVariable or, if it involves a random generation using a distribution found in the C++ `` library, from MonteCarloVariableRandom. + * Populate the command variable inherited from MonteCarloVariable. This is the STL string representing the content that the MonteCarloMaster will place into the generated dispersion files. + * Call the `insert_units()` method inherited from MonteCarloVariable + * Set the `command_generated` flag to true if the command has been successfully generated. + +## 4.6 Running generated runs within an HPC framework + +Modern HPC (High Performance Computing) labs typically have one or more tools for managing the execution of jobs across multiple computers. There are several linux-based scheduling tools, but this section focuses on running the generated runs using a SLURM (Simple Linux Utility for Resource Management) array job. Consider this script using a simulation built with gcc 4.8 and a user-configured run named `RUN_example` which has already executed once with the Monte-Carlo Generation model enabled to generate 100 runs on disk: + +```bash +#SBATCH --array=0-99 + +# This is an example sbatch script demonstrating running an array job in SLURM. +# SLURM is an HPC (High-Performance-Computing) scheduling tool installed in +# many modern super-compute clusters that manages execution of a massive +# number of user-jobs. When a script like this is associated with an array +# job, this script is executed once per enumerated value in the array. After +# the Monte Carlo Generation Model executes, the resulting RUNs can be queued +# for SLURM execution using a script like this. Alternatively, sbatch --wrap +# can be used. See the SLURM documentation for more in-depth information. +# +# Slurm: https://slurm.schedmd.com/documentation.html + +# $SLURM_ARRAY_TASK_ID is automatically provided by slurm, and will be an +# integer between 0-99 per the "SBATCH --array" flag specified at the top of +# this script +echo "SLURM has provided us with array job integer: $SLURM_ARRAY_TASK_ID" +# Convert this integer to a zero-padded string matching the RUN naming +# convention associated with thi +RUN_NUM=`printf %02d $SLURM_ARRAY_TASK_ID` +# Execute the single trick simulation run associated with RUN_NUM +echo "Running RUN_$RUN_NUM ..." +./S_main_Linux_4.8_x86_64.exe MONTE_RUN_example/RUN_${RUN_NUM}/monte_input.py +``` + +The above script can be executed within a SLURM environment by running `sbatch `. This single command will create 100 independent array jobs in SLURM, allowing the scheduler to execute them as resources permit. Be extra careful with the zero-padding logic in the script above. The monte-carlo generation model will create zero-padded `RUN` names suitable for the number of runs requested to be generated by the user. The `%02d` part of the script above specifies 2-digit zero-padding which is suitable for 100 runs. Be sure to match this logic with the zero-padding as appropriate for your use-case. + +For more information on SLURM, refer to the project documentation: https://slurm.schedmd.com/documentation.html + +# 5 Verification + +The verification of the model is provided by tests defined in `test/SIM_mc_generation`. This sim was originally developed by by JSC/EG NASA in the 2020 timeframe. The verification section of the original documentation is omitted from this markdown file because it heavily leverages formatting that markdown cannot support. It can be viewed [here](MCG_verification_2020.pdf) diff --git a/docs/documentation/miscellaneous_trick_tools/TrickOps.md b/docs/documentation/miscellaneous_trick_tools/TrickOps.md index 38f549f4..c84ebb51 100644 --- a/docs/documentation/miscellaneous_trick_tools/TrickOps.md +++ b/docs/documentation/miscellaneous_trick_tools/TrickOps.md @@ -12,10 +12,11 @@ * [Other Useful Examples](#other-useful-examples) * [The TrickOps Design](#regarding-the-design-why-do-i-have-to-write-my-own-script) * [Tips & Best Practices](#tips--best-practices) +* [MonteCarloGenerationHelper](#montecarlogenerationhelper---trickops-helper-class-for-montecarlogeneratesm-users) # TrickOps -TrickOps is shorthand for "Trick Operations", and is a `python3` framework that provides an easy-to-use interface for common testing and workflow actions that Trick simulation developers and users often run repeatedly. Good software developer workflows typically have a script or set of scripts that the developer can run to answer the question "have I broken anything?". The purpose of TrickOps is to provide the logic central to managing these tests while allowing each project to define how and and what they wish to test. Don't reinvent the wheel, use TrickOps! +TrickOps is shorthand for "Trick Operations". TricOps is a `python3` framework that provides an easy-to-use interface for common testing and workflow actions that Trick simulation developers and users often run repeatedly. Good software developer workflows typically have a script or set of scripts that the developer can run to answer the question "have I broken anything?". The purpose of TrickOps is to provide the logic central to managing these tests while allowing each project to define how and and what they wish to test. Don't reinvent the wheel, use TrickOps! TrickOps is *not* a GUI, it's a set of python modules that you can `import` that let you build a testing framework for your Trick-based project with just a few lines of python code. @@ -54,40 +55,42 @@ Simple and readable, this config file is parsed by `PyYAML` and adheres to all n ```yaml globals: - env: <-- optional literal string executed before all tests, e.g. env setup - parallel_safety: <-- strict won't allow multiple input files per RUN dir - + env: <-- optional literal string executed before all tests, ex: ". env.sh" SIM_abc: <-- required unique name for sim of interest, must start with SIM path: <-- required SIM path relative to project top level description: <-- optional description for this sim labels: <-- optional list of labels for this sim, can be used to get sims - model_x by label within the framework, or for any other project-defined - verification purpose - build_command: <-- optional literal cmd executed for SIM_build, defaults to trick-CP + build_args: <-- optional literal args passed to trick-CP during sim build binary: <-- optional name of sim binary, defaults to S_main_{cpu}.exe size: <-- optional estimated size of successful build output file in bytes + phase: <-- optional phase to be used for ordering builds if needed + parallel_safety: <-- strict won't allow multiple input files per RUN dir. + Defaults to "loose" if not specified runs: <-- optional dict of runs to be executed for this sim, where the RUN_1/input.py --foo: dict keys are the literal arguments passed to the sim binary RUN_2/input.py: and the dict values are other run-specific optional dictionaries - ... described as follows ... + RUN_[10-20]/input.py: described in indented sections below. Zero-padded integer ranges + can specify a set of runs with continuous numbering using + [-] notation returns: <---- optional exit code of this run upon completion (0-255). Defaults to 0 compare: <---- optional list of vs. comparison strings to be - - a vs. b compared after this run is complete. This is extensible in that - - d vs. e all non-list values are ignored and assumed to be used to define - - ... an alternate comparison method in derived classes + - a vs. b compared after this run is complete. Zero-padded integer ranges + - d vs. e are supported as long as they match the pattern in the parent run. + - ... All non-list values are ignored and assumed to be used to define + - ... an alternate comparison method in a class extending this one analyze: <-- optional arbitrary string to execute as job in bash shell from project top level, for project-specific post-run analysis - valgrind: <-- optional dict describing how to execute runs within valgrind - flags: <-- string of all flags passed to valgrind for all runs - runs: <-- list of literal arguments passed to the sim binary through - - RUN_1... valgrind - + phase: <-- optional phase to be used for ordering runs if needed + valgrind: <-- optional string of flags passed to valgrind for this run. + If missing or empty, this run will not use valgrind non_sim_extension_example: will: be ignored by TrickWorkflow parsing for derived classes to implement as they wish ``` -Almost everything in this file is optional, but there must be at least one top-level key that starts with `SIM` and it must contain a valid `path: ` with respect to the top level directory of your project. Here, `SIM_abc` represents "any sim" and the name is up to the user, but it *must* begin with `SIM` since `TrickWorkflow` purposefully ignores any top-level key not beginning with `SIM` in order to allow for extensibility of the YAML file for non-sim tests specific to a project. +Almost everything in this file is optional, but there must be at least one top-level key that starts with `SIM` and it must contain a valid `path: ` with respect to the top level directory of your project. Here, `SIM_abc` represents "any sim" and the name is up to the user, but it *must* begin with `SIM` since `TrickWorkflow` purposefully ignores any top-level key not beginning with `SIM` and any key found under the `SIM` key not matching any named parameter above. This design allows for extensibility of the YAML file for non-sim tests specific to a project. There is *no limit* to number of `SIM`s, `runs:`, `compare:` lists, `valgrind` `runs:` list, etc. This file is intended to contain every Sim and and every sim's run, and every run's comparison and so on that your project cares about. Remember, this file represents the *pool* of tests, not necessarily what *must* be tested every time your scripts which use it run. @@ -101,19 +104,21 @@ cd trick/share/trick/trickops/ ``` When running, you should see output that looks like this: -![ExampleWorkflow In Action](trickops_example.png) +![ExampleWorkflow In Action](images/trickops_example.png) -When running, you'll notice that tests occur in two phases. First, sims build in parallel up to three at a time. Then when all builds complete, sims run in parallel up to three at a time. Progress bars show how far along each build and sim run is at any given time. The terminal window will accept scroll wheel and arrow input to view current builds/runs that are longer than the terminal height. +When running this example script, you'll notice that tests occur in two phases. First, sims build in parallel up to three at a time. Then when all builds complete, sims run in parallel up to three at a time. Progress bars show how far along each build and sim run is at any given time. The terminal window will accept scroll wheel and arrow input to view current builds/runs that are longer than the terminal height. Before the script finishes, it reports a summary of what was done, providing a list of which sims and runs were successful and which were not. -Looking inside the script, the code at top of the script creates a yaml file containing a large portion of the sims and runs that ship with trick and writes it to `/tmp/config.yml`. This config file will be input to the framework. At the bottom of the script is where the magic happens, this is where the TrickOps modules are used: +Looking inside the script, the code at top of the script creates a yaml file containing a large portion of the sims and runs that ship with trick and writes it to `/tmp/config.yml`. This config file is then used as input to the `TrickWorkflow` framework. At the bottom of the script is where the magic happens, this is where the TrickOps modules are used: ```python from TrickWorkflow import * class ExampleWorkflow(TrickWorkflow): def __init__( self, quiet, trick_top_level='/tmp/trick'): - # Real projects already have trick somewhere, but for this test, just clone it + # Real projects already have trick somewhere, but for this example, just clone & build it if not os.path.exists(trick_top_level): os.system('cd %s && git clone https://github.com/nasa/trick' % (os.path.dirname(trick_top_level))) + if not os.path.exists(os.path.join(trick_top_level, 'lib64/libtrick.a')): + os.system('cd %s && ./configure && make' % (trick_top_level)) # Base Class initialize, this creates internal management structures TrickWorkflow.__init__(self, project_top_level=trick_top_level, log_dir='/tmp/', trick_dir=trick_top_level, config_file="/tmp/config.yml", cpus=3, quiet=quiet) @@ -135,9 +140,11 @@ Let's look at a few key parts of the example script. Here, we create a new class from TrickWorkflow import * class ExampleWorkflow(TrickWorkflow): def __init__( self, quiet, trick_top_level='/tmp/trick'): - # Real projects already have trick somewhere, but for this test, just clone it + # Real projects already have trick somewhere, but for this example, just clone & build it if not os.path.exists(trick_top_level): os.system('cd %s && git clone https://github.com/nasa/trick' % (os.path.dirname(trick_top_level))) + if not os.path.exists(os.path.join(trick_top_level, 'lib64/libtrick.a')): + os.system('cd %s && ./configure && make' % (trick_top_level)) ``` Our new class `ExampleWorkflow.py` can be initialized however we wish as long as it provides the necessary arguments to it's Base class initializer. In this example, `__init__` takes two parameters: `trick_top_level` which defaults to `/tmp/trick`, and `quiet` which will be `False` unless `quiet` is found in the command-line args to this script. The magic happens on the very next line where we call the base-class `TrickWorkflow` initializer which accepts four required parameters: @@ -149,15 +156,15 @@ The required parameters are described as follows: * `project_top_level` is the absolute path to the highest-level directory of your project. The "top level" is up to the user to define, but usually this is the top level of your repository and at minimum must be a directory from which all sims, runs, and other files used in your testing are recursively reachable. * `log_dir` is a path to a user-chosen directory where all logging for all tests will go. This path will be created for you if it doesn't already exist. * `trick_dir` is an absolute path to the top level directory for the instance of trick used for your project. For projects that use trick as a `git` `submodule`, this is usually `/trick` -* `config_file` is the path to a YAML config file describing the sims, runs, etc. for your project. It's recommended this file be tracked in your SCM tool but that is not required. More information on the syntax expected in this file in the **The YAML File** section below. +* `config_file` is the path to a YAML config file describing the sims, runs, etc. for your project. It's recommended this file be tracked in your SCM tool but that is not required. More information on the syntax expected in this file in the **The YAML File** section above. The optional parameters are described as follows: * `cpus` tells the framework how many CPUs to use on sim builds. This translates directly to `MAKEFLAGS` and is separate from the maximum number of simultaneous sim builds. * `quiet` tells the framework to suppress progress bars and other verbose output. It's a good idea to use `quiet=True` if your scripts are going to be run in a continuous integration (CI) testing framework such as GitHub Actions, GitLab CI, or Jenkins, because it suppresses all `curses` logic during job execution which itself expects `stdin` to exist. -When `TrickWorkflow` initializes, it reads the `config_file` and verifies the information given matches the expected convention. If a non-fatal error is encountered, a message detailing the error is printed to `stdout` and the internal timestamped log file under `log_dir`. A fatal error will `raise RuntimeError`. +When `TrickWorkflow` initializes, it reads the `config_file` and verifies the information given matches the expected convention. If a non-fatal error is encountered, a message detailing the error is printed to `stdout` and the internal timestamped log file under `log_dir`. A fatal error will `raise RuntimeError`. Classes which inherit from `TrickWorkflow` may also access `self.parsing_errors` and `self.config_errors` which are lists of errors encountered from parsing the YAML file and errors encountered from processing the YAML file respectively. -Moving on to the next important lines of code in our `ExampleWorkflow.py` script. The `def run(self):` line declares a function whose return code on run is passed back to the calling shell via `sys.exit()`. This is where we use the functions given to us by inherting from `TrickWorkflow`: +Moving on to the next few important lines of code in our `ExampleWorkflow.py` script. The `def run(self):` line declares a function whose return code on run is passed back to the calling shell via `sys.exit()`. This is where we use the functions given to us by inherting from `TrickWorkflow`: ```python @@ -177,7 +184,7 @@ The last three lines simply print a detailed report of what was executed and man return (builds_status or runs_status or self.config_errors) ``` -The `ExampleWorkflow.py` uses sims/runs provided by trick to exercise *some* of the functionality provided by TrickOps. This script does not have any comparisons, post-run analyses, or valgrind runs defined in the YAML file, so there is no execution of those tests in this example. +The `ExampleWorkflow.py` script uses sims/runs provided by trick to exercise *some* of the functionality provided by TrickOps. This script does not have any comparisons, post-run analyses, or valgrind runs defined in the YAML file, so there is no execution of those tests in this example. ## `compare:` - File vs. File Comparisons @@ -192,8 +199,8 @@ SIM_ball: RUN_foo/input.py: RUN_test/input.py: compare: - - path/to/SIM_/ball/RUN_test/log_a.csv vs. regression/SIM_ball/log_a.csv - - path/to/SIM_/ball/RUN_test/log_b.trk vs. regression/SIM_ball/log_b.trk + - path/to/SIM_ball/RUN_test/log_a.csv vs. regression/SIM_ball/log_a.csv + - path/to/SIM_ball/RUN_test/log_b.trk vs. regression/SIM_ball/log_b.trk ``` In this example, `SIM_ball`'s run `RUN_foo/input.py` doesn't have any comparisons, but `RUN_test/input.py` contains two comparisons, each of which compares data generated by the execution of `RUN_test/input.py` to a stored off version of the file under the `regression/` directory relative to the top level of the project. The comparisons themselves can be executed in your python script via the `compare()` function in multiple ways. For example: @@ -237,10 +244,98 @@ if not failure: If an error is encountered, like `koviz` or a given directory cannot be found, `None` is returned in the first index of the tuple, and the error information is returned in the second index of the tuple for `get_koviz_report_job()`. The `get_koviz_report_jobs()` function just wraps the singular call and returns a tuple of `( list_of_jobs, list_of_any_failures )`. Note that `koviz` accepts entire directories as input, not specific paths to files. Keep this in mind when you organize how regression data is stored and how logged data is generated by your runs. + ## `analyze:` - Post-Run Analysis The optional `analyze:` section of a `run:` is intended to be a catch-all for "post-run analysis". The string given will be transformed into a `Job()` instance that can be retrieved and executed via `execute_jobs()` just like any other test. All analyze jobs are assumed to return 0 on success, non-zero on failure. One example use case for this would be creating a `jupytr` notebook that contains an analysis of a particular run. +## Defining sets of runs using [integer-integer] range notation + +The `yaml` file for your project can grow quite large if your sims have a lot of runs. This is especially the case for users of monte-carlo, which may generate hundreds or thousands of runs that you may want to execute as part of your TrickOps script. In order to support these use cases without requiring the user to specify all of these runs individually, TrickOps supports a zero-padded `[integer-integer]` range notation in the `run:` and `compare:` fields. Consider this example `yaml` file: + +```yaml +SIM_many_runs: + path: sims/SIM_many_runs + runs: + RUN_[000-100]/monte_input.py: + returns: 0 + compare: + sims/SIM_many_runs/RUN_[000-100]/log_common.csv vs. baseline/sims/SIM_many_runs/log_common.csv + sims/SIM_many_runs/RUN_[000-100]/log_verif.csv vs. baseline/sims/SIM_many_runs/RUN_[000-100]/log_verif.csv +``` +In this example, `SIM_many_runs` has 101 runs. Instead of specifying each individual run (`RUN_000/`, `RUN_001`, etc), in the `yaml` file, the `[000-100]` notation is used to specify a set of runs. All sub-fields of the run apply to that same set. For example, the default value of `0` is used for `returns:`, which also applies to all 101 runs. The `compare:` subsection supports the same range notation, as long as the same range is used in the `run:` named field. Each of the 101 runs shown above has two comparisons. The first `compare:` line defines a common file to be compared against all 101 runs. The second `compare:` line defines run-specific comparisons using the same `[integer-integer]` sequence. Note that when using these range notations zero-padding must be consistent, the values (inclusive) must be non-negative, and the square bracket notation must be used with the format `[minimum-maximum]`. + + +## `phase:` - An optional mechanism to order builds, runs, and analyses + +The `yaml` file supports an optional parameter `phase: ` at the sim and run level which allows the user to easily order sim builds, runs, and/or analyses, to suit their specific project constraints. If not specified, all sims, runs, and analyses, have a `phase` value of `0` by default. Consider this example `yaml` file with three sims: + +```yaml +SIM_car: + path: sims/SIM_car + +SIM_monte: + path: sims/SIM_monte + runs: + RUN_nominal/input.py --monte-carlo: # Generates the runs below + phase: -1 + MONTE_RUN_nominal/RUN_000/monte_input.py: # Generated run + MONTE_RUN_nominal/RUN_001/monte_input.py: # Generated run + MONTE_RUN_nominal/RUN_002/monte_input.py: # Generated run + MONTE_RUN_nominal/RUN_003/monte_input.py: # Generated run + MONTE_RUN_nominal/RUN_004/monte_input.py: # Generated run + +# A sim with constraints that make the build finnicky, and we can't change the code +SIM_external: + path: sims/SIM_external + phase: -1 + runs: + RUN_test/input.py: + returns: 0 +``` +Here we have three sims: `SIM_car`, `SIM_monte`, and `SIM_external`. `SIM_car` and `SIM_monte` have the default `phase` of `0` and `SIM_external` has been assigned `phase: -1` explicitly. If using non-zero phases, jobs can be optionally filtered by them when calling helper functions like `self.get_jobs(kind, phase)`. Some examples: +```python + build_jobs = self.get_jobs(kind='build') # Get all build jobs regardless of phase + build_jobs = self.get_jobs(kind='build', phase=0) # Get all build jobs with (default) phase 0 + build_jobs = self.get_jobs(kind='build', phase=-1) # Get all build jobs with phase -1 + build_jobs = self.get_jobs(kind='build', phase=[0, 1, 3]) # Get all build jobs with phase 0, 1, or 3 + build_jobs = self.get_jobs(kind='build', phase=range(-10,11)) # Get all build jobs with phases between -10 and 10 +``` +This can be done for runs and analyses in the same manner: +```python + run_jobs = self.get_jobs(kind='run') # Get all run jobs regardless of phase + run_jobs = self.get_jobs(kind='run', phase=0) # Get all run jobs with (default) phase 0 + # Get all run jobs with all phases less than zero + run_jobs = self.get_jobs(kind='run', phase=range(TrickWorkflow.allowed_phase_range['min'],0)) + # Get all analysis jobs with all phases zero or greater + an_jobs = self.get_jobs(kind='analysis', phase=range(0, TrickWorkflow.allowed_phase_range['max'+1])) +``` +Note that since analysis jobs are directly tied to a single named run, they inherit the `phase` value of their run as specfied in the `yaml` file. In other words, do not add a `phase:` section indented under any `analyze:` section in your `yaml` file. + +It's worth emphasizing that the specfiication of a non-zero `phase` in the `yaml` file, by itself, does not affect the order in which actions are taken. **It is on the user of TrickOps to use this information to order jobs appropriately**. Here's an example in code of what that might look for the example use-case described by the `yaml` file in this section: + +```python + first_build_jobs = self.get_jobs(kind='build', phase=-1) # Get all build jobs with phase -1 (SIM_external) + second_build_jobs = self.get_jobs(kind='build', phase=0) # Get all build jobs with phase 0 (SIM_car & SIM_monte) + first_run_jobs = self.get_jobs(kind='run', phase=-1) # Get all run jobs with phase -1 (RUN_nominal/input.py --monte-carlo) + second_run_jobs = self.get_jobs(kind='run', phase=0) # Get all run jobs with phase 0 (All generated runs & RUN_test/input.py) + + # SIM_external must build before SIM_car and SIM_monte, for project-specific reasons + builds_status1 = self.execute_jobs(first_build_jobs, max_concurrent=3, header='Executing 1st phase sim builds.') + # SIM_car and SIM_monte can build at the same time with no issue + builds_status2 = self.execute_jobs(second_build_jobs, max_concurrent=3, header='Executing 2nd phase sim builds.') + # SIM_monte's 'RUN_nominal/input.py --monte-carlo' generates runs + runs_status1 = self.execute_jobs(first_run_jobs, max_concurrent=3, header='Executing 1st phase sim runs.') + # SIM_monte's 'MONTE_RUN_nominal/RUN*/monte_input.py' are the generated runs, they must execute after the generation is complete + runs_status2 = self.execute_jobs(second_run_jobs, max_concurrent=3, header='Executing 2nd phase sim runs.') +``` +Astute observers may have noticed that `SIM_external`'s `RUN_test/input.py` technically has no order dependencies and could execute in either the first or second run job set without issue. + +A couple important points on the motivation for this capability: +* Run phasing was primarly developed to support testing monte-carlo and checkpoint sim scenarios, where output from a set of scenarios (like generated runs or dumped checkpoints) becomes the input to another set of sim scenarios. +* Sim phasing exists primarly to support testing scenarios where sims are poorly architectured or immutable, making them unable to be built independently. + + ## Where does the output of my tests go? All output goes to a single directory `log_dir`, which is a required input to the `TrickWorkflow.__init__()` function. Sim builds, runs, comparisons, koviz reports etc. are all put in a single directory with unique names. This is purposeful for two reasons: @@ -291,9 +386,45 @@ This is purposeful -- handling every project-specific constraint is impossible. * If `TrickWorkflow` encounters non-fatal errors while validating the content of the given YAML config file, it will set the internal member `self.config_erros` to be `True`. If you want your script to return non-zero on any non-fatal error, add this return code to your final script `sys.exit()`. * Treat the YAML file like your project owns it. You can store project-specific information and retrieve that information in your scripts by accessing the `self.config` dictionary. Anything not recognized by the internal validation of the YAML file is ignored, but that information is still provided to the user. For example, if you wanted to store a list of POCS in your YAML file so that your script could print a helpful message on error, simply add a new entry `project_pocs: email1, email2...` and then access that information via `self.config['project_pocs']` in your script. +## `MonteCarloGenerationHelper` - TrickOps Helper Class for `MonteCarloGenerate.sm` users + +TrickOps provides the `MonteCarloGenerationHelper` python module as an interface between a sim using the `MonteCarloGenerate.sm` (MCG) sim module and a typical Trick-based workflow. This module allows MCG users to easily generate monte-carlo runs and execute them locally or alternatively through an HPC job scheduler like SLURM. Below is an example usage of the module. This example assumes: +1. The using script inherits from or otherwise leverages `TrickWorkflow`, giving it access to `self.execute_jobs()` +2. `SIM_A` is already built and configured with the `MonteCarloGenerate.sm` sim module +3. `RUN_mc/input.py` is configured with to generate runs when executed, specifically that `monte_carlo.mc_master.generate_dispersions == monte_carlo.mc_master.active == True` in the input file. + +```python +# Instantiate an MCG helper instance, providing the sim and input file for generation +mgh = MonteCarloGenerationHelper(sim_path="path/to/SIM_A", input_path="RUN_mc/input.py") +# Get the generation SingleRun() instance +gj = mgh.get_generation_job() +# Execute the generation Job to generate RUNS +ret = self.execute_jobs([gj]) + +if ret == 0: # Successful generation + # Get a SLURM sbatch array job for all generated runs found in monte_dir + # SLURM is an HPC (High-Performance-Computing) scheduling tool installed on + # many modern super-compute clusters that manages execution of a massive + # number of jobs. See the official documentation for more information + # Slurm: https://slurm.schedmd.com/documentation.html + sbj = mgh.get_sbatch_job(monte_dir="path/to/MONTE_RUN_mc") + # Execute the sbatch job, which queues all runs in SLURM for execution + # Use hpc_passthrough_args ='--wait' to block until all runs complete + ret = self.execute_jobs([sbj]) + + # Instead of using SLURM, generated runs can be executed locally through + # TrickOps calls on the host where this script runs. First get a list of + # run jobs + run_jobs = mgh.get_generated_run_jobs(monte_dir="path/to/MONTE_RUN_mc") + # Then execute all generated SingleRun instances, up to 10 at once + ret = self.execute_jobs(run_jobs, max_concurrent=10) +``` + +Note that the number of runs to-be-generated is configured somewhere in the `input.py` code and this module cannot robustly know that information for any particular use-case. This is why `monte_dir` is a required input to several functions - this directory is processed by the module to understand how many runs were generated. + ## More Information -A lot of time was spent adding `python` docstrings to the `TrickWorkflow.py` and `WorkflowCommon.py` modules. This README does not cover all functionality, so please see the in-code documentation for more detailed information on the framework. +A lot of time was spent adding `python` docstrings to the modules in the `trickops/` directory and tests under the `trickops/tests/`. This README does not cover all functionality, so please see the in-code documentation and unit tests for more detailed information on the framework capabilities. -[Continue to Software Requirements](../software_requirements_specification/SRS) \ No newline at end of file +[Continue to Software Requirements](../software_requirements_specification/SRS) diff --git a/docs/documentation/miscellaneous_trick_tools/trickops_example.png b/docs/documentation/miscellaneous_trick_tools/images/trickops_example.png similarity index 100% rename from docs/documentation/miscellaneous_trick_tools/trickops_example.png rename to docs/documentation/miscellaneous_trick_tools/images/trickops_example.png diff --git a/docs/documentation/simulation_capabilities/Data-Record.md b/docs/documentation/simulation_capabilities/Data-Record.md index c7ab2391..f742acbd 100644 --- a/docs/documentation/simulation_capabilities/Data-Record.md +++ b/docs/documentation/simulation_capabilities/Data-Record.md @@ -45,6 +45,7 @@ For example: drg.add_variable("ball.obj.state.output.position[0]") drg.add_variable("ball.obj.state.output.position[1]") ``` +In this example `position` is an array of floating point numbers. **DO NOT ATTEMPT TO DATA RECORD C OR C++ STRINGS. THIS HAS BEEN OBSERVED TO CREATE MEMORY ISSUES AND TRICK DOES NOT CURRENTLY PROVIDE ERROR CHECKING FOR THIS UNSUPPORTED USE CASE** An optional alias may also be specified in the method as drg.add_variable("" [, ""]). If an alias is present as a second argument, the alias name will be used in the data recording file instead of the actual variable name. diff --git a/include/trick/files_to_ICG.hh b/include/trick/files_to_ICG.hh index c725117d..524e5241 100644 --- a/include/trick/files_to_ICG.hh +++ b/include/trick/files_to_ICG.hh @@ -18,6 +18,17 @@ #include "trick/JSONVariableServer.hh" #include "trick/JSONVariableServerThread.hh" #include "trick/Master.hh" +#include "trick/mc_master.hh" +#include "trick/mc_python_code.hh" +#include "trick/mc_variable_file.hh" +#include "trick/mc_variable_fixed.hh" +#include "trick/mc_variable.hh" +#include "trick/mc_variable_random_bool.hh" +#include "trick/mc_variable_random.hh" +#include "trick/mc_variable_random_normal.hh" +#include "trick/mc_variable_random_string.hh" +#include "trick/mc_variable_random_uniform.hh" +#include "trick/mc_variable_semi_fixed.hh" #include "trick/Slave.hh" #include "trick/MSSocket.hh" #include "trick/MSSharedMem.hh" @@ -33,14 +44,12 @@ #include "trick/RealtimeSync.hh" #include "trick/ITimer.hh" #include "trick/VariableServer.hh" - #include "trick/regula_falsi.h" #include "trick/Integrator.hh" #include "trick/IntegAlgorithms.hh" #include "trick/IntegLoopScheduler.hh" #include "trick/IntegLoopManager.hh" #include "trick/IntegLoopSimObject.hh" - #include "trick/ABM_Integrator.hh" #include "trick/Euler_Cromer_Integrator.hh" #include "trick/Euler_Integrator.hh" @@ -51,7 +60,6 @@ #include "trick/RKF45_Integrator.hh" #include "trick/RKF78_Integrator.hh" #include "trick/RKG4_Integrator.hh" - #include "trick/SimTime.hh" /* from the er7_utils directory */ diff --git a/include/trick/mc_master.hh b/include/trick/mc_master.hh new file mode 100644 index 00000000..588d1bc3 --- /dev/null +++ b/include/trick/mc_master.hh @@ -0,0 +1,124 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: (Provides the front-end interface to the monte-carlo model) + +LIBRARY DEPENDENCY: + ((../src/mc_master.cc)) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_MASTER_HH +#define CML_MONTE_CARLO_MASTER_HH + +#include +#include +#include // std::pair + +#include "mc_variable.hh" +#include "mc_variable_file.hh" +#include "mc_python_code.hh" +#include "mc_variable_random.hh" +#include "mc_variable_fixed.hh" +#include "mc_variable_semi_fixed.hh" + +/***************************************************************************** +MonteCarloMaster +Purpose:() +*****************************************************************************/ +class MonteCarloMaster +{ + public: + bool active; /* (--) + The main active flag determining whether an input file should be + processed for a monte-carlo run. This flag is used to manage the + configuration of the scenario, including things like which variables + to log.*/ + bool generate_dispersions; /* (--) + This flag controls whether the variables should be loaded and + dispersed. It has no effect if the active flag is off. + False: Configure the run for MC; this + configuration typically uses a previously-generated + monte_input.py file; it does not read in MC variables and + does not generate new monte_input files. + True: Use this execution of the sim to read in the MC variables and + generating the monte-input files. After doing so, the + execution will terminate. + The sim can then be re-run using one of the new monte_input files. + Default: true. This is set to false in the monte_input.py files to + allow the base input file to be processed from within the + monte-input file without regenerting the monte-input files.*/ + std::string run_name; /* (--) + The name of the scenario, used in generating the "MONTE_" + directory, which contains all of the runs.*/ + std::string monte_dir; /* (--) + The name of the MONTE directory relative to the SIM directory + where runs will be generated*/ + std::string input_file_name; /* (--) + The name of the original file used as the main input file. + Default: input.py.*/ + bool generate_meta_data; /* (--) + Flag indicating whether to generate the meta-data output file.*/ + bool generate_summary; /* (--) + Flag indicating whether to generate the dispersion summary file. */ + int minimum_padding; /* (--) + The minimum width of the run-number field, e.g. RUN_1 vs RUN_00001; + The run-number field will be sized to the minimum of this value or the + width necessary to accommodate the highest number. + Defaults to 0. */ + size_t monte_run_number; /* (--) + A unique identifying number for each run.*/ + + protected: + bool input_files_prepared; /* (--) + Internal flag indicating that the input files have been generated and + waiting for execution. Effectively blocks further modifications to + variables after this flag has been set.*/ + + std::string location; /* (--) + The location in the main sim by which this instance may be addressed + in an input file. */ + + std::list variables; /* (--) + A STL-list of pointers to instances of all the base-class + MonteVarVariable instances. Note that this has to be a list of pointers + rather than instances because the actual instances are polymorphic; + making a copy would restrict them to be actual MonteCarloVariable + instances and we need the polymorphic capabilities for generating the + monte_input command.*/ + + unsigned int num_runs; /* (--) + The number of runs to execute for this scenario.*/ + + private: + std::list< std::pair< std::string, + MonteCarloVariableFile *> > file_list; /* (--) + A list of filenames being read as part of the MonteVarFile variables + being managed by this class. */ + + + public: + MonteCarloMaster(std::string location); + virtual ~MonteCarloMaster(){}; + + void activate( std::string run_name); + bool prepare_input_files(); + void add_variable( MonteCarloVariable & variable); + MonteCarloVariable * find_variable( std::string var_name); + void remove_variable( std::string var_name); + void set_num_runs( unsigned int num_runs); + void execute(); + void collate_meta_data(); + + private: + static bool seed_sort( std::pair< unsigned int, std::string> left, + std::pair< unsigned int, std::string> right) + { + return left.first < right.first; + } + + // and undefined: + MonteCarloMaster (const MonteCarloMaster&); + MonteCarloMaster& operator = (const MonteCarloMaster&); +}; +#endif diff --git a/include/trick/mc_python_code.hh b/include/trick/mc_python_code.hh new file mode 100644 index 00000000..4beaf9be --- /dev/null +++ b/include/trick/mc_python_code.hh @@ -0,0 +1,115 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a simple python executable code instruction) + +LIMITATION: (This implementation is intended to provide a fixed operation + that generates one variable as a function of one or more + variables -- such as random variables -- that were previously + generated by the MonteCarlo process. + E.g. y = x + 2 + It does not provide randomization of the operation itself; in + this example the value y will always be generated as x+2, using + the value x which may be variable. However, it will never be + generated as x+3. + To implement randomization of the operation itself, use the + MonteCarloVariableRandomStringSet class instead.) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_PYTHON_CODE_HH +#define CML_MONTE_CARLO_PYTHON_CODE_HH + +#include "mc_variable.hh" + +class MonteCarloPythonLineExec : public MonteCarloVariable +{ + public: + std::string instruction_set; /* (--) + The right-hand-side of an equation that gets inserted into the + monte-input file and looks like: + = */ + protected: + bool instruction_is_command; /* (--) + Indicates whether to implement a command that looks like: + - variable=instruction vs + variable representing the standalone command, in which case + variable_name and instruction_set are identical. */ + + public: + // 2 constructors: + MonteCarloPythonLineExec(const std::string & var_name, + const std::string & instruction) + : + MonteCarloVariable( var_name), + instruction_set(instruction), + instruction_is_command(false) + { + include_in_summary = false; + type = MonteCarloVariable::Calculated; + } + // other constructor + MonteCarloPythonLineExec( const std::string & instruction) + : + MonteCarloVariable( instruction), + instruction_set(instruction), + instruction_is_command(true) + { + include_in_summary = false; + type = MonteCarloVariable::Execute; + } + virtual ~MonteCarloPythonLineExec(){}; + + void generate_assignment() + { + if (instruction_is_command) { + command = "\n" + instruction_set; + } + else { + command = "\n" + variable_name + " = " + instruction_set; + } + } + private: // and undefined: + MonteCarloPythonLineExec( const MonteCarloPythonLineExec & ); + MonteCarloPythonLineExec& operator = (const MonteCarloPythonLineExec&); +}; + + +/***************************************************************************** +MonteCarloPythonFileExec +Purpose:(Provides a filename for execution to support more extensive + calculations than are possible with the simple one-liner commands + provided by MonteCarloPythonLineExec) +Assumptions: The file identified by filename is expected to be a Python file +Limitations: The file is not tested prior to execution +Other notes: + This class inherits from MonteCarloVariable to simplify the inclusion of + its "command" into the monte_input files. However, it does not populate + a specific variable; its command string is an executive statement, unlike + other MonteCarloVariable instances, which have a command string that + looks like "variable = ..." + This class's "variable_name" is instead re-purposed as a filename +*****************************************************************************/ +class MonteCarloPythonFileExec : public MonteCarloVariable +{ + public: + MonteCarloPythonFileExec(const std::string & filename) + : + MonteCarloVariable( filename) + { + include_in_summary = false; + type = MonteCarloVariable::Execute; + } + virtual ~MonteCarloPythonFileExec(){}; + + void generate_assignment() + { + command = + "\nexec(open('" + variable_name + "').read())"; + } + private: // and undefined: + MonteCarloPythonFileExec( const MonteCarloPythonFileExec & ); + MonteCarloPythonFileExec& operator = (const MonteCarloPythonFileExec&); +}; + +#endif diff --git a/include/trick/mc_variable.hh b/include/trick/mc_variable.hh new file mode 100644 index 00000000..84ba2ed7 --- /dev/null +++ b/include/trick/mc_variable.hh @@ -0,0 +1,83 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Base class for the MonteCarloVariable type) + +LIBRARY DEPENDENCY: + ((../src/mc_variable.cc)) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_VARIABLE_HH +#define CML_MONTE_CARLO_VARIABLE_HH + +#include + +class MonteCarloVariable +{ + public: + enum MonteCarloVariableType + { + Undefined = 0, + Calculated, + Constant, + Execute, + Prescribed, + Random + }; + + std::string units; /* (--) + optional setting in the case where the specified values are in units + different from the native units of the variable. + These are the units associated with the specified-value.*/ + bool include_in_summary; /* (--) + Flag telling MonteCarloMaster whether to include this variable in the + dispersion summary file. The default depends on the type of variable but + is generally true. */ + + protected: + std::string variable_name; /* (--) + The name of the sim-variable being assigned by this instance. */ + std::string assignment; /* (--) + The value assigned to the variable. Used in MonteCarloMaster to generate + the dispersion summary file. */ + std::string command; /* (--) + the command that gets pushed to the monte_input input file.*/ + MonteCarloVariableType type; /* (--) + Broad categorization of types of MonteCarloVariable. This is set in the + constructor of the specific classes derived from MonteCarloVariable and + provides information to the MonteCarloMaster about what general type of + variable it is dealing with.*/ + + + public: + MonteCarloVariable( const std::string & var_name); + virtual ~MonteCarloVariable() {}; + + + virtual void generate_assignment() = 0; + virtual void shutdown() {}; // deliberately empty + + // These getters are intended to be used by the MonteCarloMaster class in + // preparing the input files and summary data files. They may also be used + // in the user interface, but -- especially get_assignment and get_command -- + // have limited use there. + const std::string & get_command() const {return command;} + const std::string & get_variable_name() const {return variable_name;} + const std::string & get_assignment() const {return assignment;} + MonteCarloVariableType get_type() const {return type;} + virtual unsigned int get_seed() const {return 0;} + + protected: + void insert_units(); + void trick_units(size_t); + void assign_double(double value); + void assign_int(int value); + void generate_command(); + + private: // and undefined: + MonteCarloVariable( const MonteCarloVariable &); + MonteCarloVariable& operator = (const MonteCarloVariable&); +}; + +#endif diff --git a/include/trick/mc_variable_file.hh b/include/trick/mc_variable_file.hh new file mode 100644 index 00000000..7b1df70b --- /dev/null +++ b/include/trick/mc_variable_file.hh @@ -0,0 +1,87 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a file-lookup assignment + +LIBRARY DEPENDENCY: + (../src/mc_variable_file.cc) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_VARIABLE_FILE_HH +#define CML_MONTE_CARLO_VARIABLE_FILE_HH + +#include // ifstream +#include // default_random_engine, uniform_int_distribution +#include + +#include "trick/mc_variable.hh" + +/***************************************************************************** +MonteCarloVariableFile +Purpose:( + Grabs a value from a data file. + The data value is located within the file at some column number and + row number. + The column number is specified for the variable. + The row -- or line -- number changes from run to run. + The file is expected to provide consistent presentation of data from + row to row.) +*****************************************************************************/ +class MonteCarloVariableFile : public MonteCarloVariable +{ + public: + size_t max_skip; /* (--) + The maximum number of lines to skip in a file. This defaults to 0, + indicating that the file will be read sequentially.*/ + bool is_dependent; /* (--) + A flag indicating that this instance is dependent upon another instance + for reading the file "filename". Only one instance will read each + filename. Default: false. */ + protected: + std::mt19937 rand_gen; /* (--) + A random number generator used in the case that the lines are not read + sequentially. This will generate a value from 0 to max_skip indicating + how many lines should be skipped.*/ + std::string filename; /* (--) + The name of the file containing the data for this variable. Assigned + at construction.*/ + size_t column_number;/* (--) + The column number indicating from where in the data file the data for + this variable should be extracted.*/ + size_t first_column_number; /* (--) + Usually used to distinguish between whether the first column should be + identified with a 0 or 1, but extensible to other integers as well. + Default: 1. */ + std::list< MonteCarloVariableFile *> dependents; /* (--) + A list of MonteVarVariableFile instances that use the same file as + this one. This list is only populated if this instance was the first + registered to use this file.*/ + std::ifstream file; /* (--) + Input file stream being the file containing the data.*/ + + public: + MonteCarloVariableFile( const std::string & var_name, + const std::string & filename, + size_t column_number_, + size_t first_column_number = 1); + virtual ~MonteCarloVariableFile(){}; + void initialize_file(); + void generate_assignment(); + void register_dependent( MonteCarloVariableFile *); + virtual void shutdown() {file.close();} + + + bool has_dependents() {return (dependents.size() > 1);} + size_t get_column_number() {return column_number;} + size_t get_first_column_number() {return first_column_number;} + const std::string & get_filename() {return filename;} + const std::list< MonteCarloVariableFile *> & get_dependents() {return dependents;} + protected: + void process_line(); + static bool sort_by_col_num(MonteCarloVariableFile *, MonteCarloVariableFile *); + private: // and undefined: + MonteCarloVariableFile(const MonteCarloVariableFile&); + MonteCarloVariableFile& operator = ( const MonteCarloVariableFile&); +}; +#endif diff --git a/include/trick/mc_variable_fixed.hh b/include/trick/mc_variable_fixed.hh new file mode 100644 index 00000000..31ac366a --- /dev/null +++ b/include/trick/mc_variable_fixed.hh @@ -0,0 +1,70 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a class template to support assignment of + a fixed value to a variable based on its type.) + +LIMITATION: (Because these types are typically instantiated + from the Python input processor via SWIG, use of + templates is problematic. Consequently, it would + require a whole different setup to handle integers + differently than floats. Instead, integers and floats + will both be treated using the "double" data type. + Values that cannot be assigned to a double (like bool or strings) + cannot be represented by an instance of this class.) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_VARIABLE_FIXED_HH +#define CML_MONTE_CARLO_VARIABLE_FIXED_HH + +#include "mc_variable.hh" +#include // ostringstream + +class MonteCarloVariableFixed : public MonteCarloVariable +{ + protected: + int var_type; /* (--) 0: double + 1: integer + 2: string;*/ + public: + + MonteCarloVariableFixed(const std::string & var_name, + double assignment_) + : + MonteCarloVariable( var_name), + var_type(0) + { + type = MonteCarloVariable::Constant; + assign_double(assignment_); + } + + MonteCarloVariableFixed(const std::string & var_name, + int assignment_) + : + MonteCarloVariable( var_name), + var_type(1) + { + type = MonteCarloVariable::Constant; + assign_int(assignment_); + } + MonteCarloVariableFixed(const std::string & var_name, + const std::string & assignment_) + : + MonteCarloVariable( var_name), + var_type(2) + { + include_in_summary = false; + assignment = assignment_; + type = MonteCarloVariable::Constant; + generate_command(); + } + virtual ~MonteCarloVariableFixed(){}; + + void generate_assignment(){}; // to make this class instantiable + + private: // and undefined: + MonteCarloVariableFixed( const MonteCarloVariableFixed & ); + MonteCarloVariableFixed& operator = (const MonteCarloVariableFixed&); +}; +#endif diff --git a/include/trick/mc_variable_random.hh b/include/trick/mc_variable_random.hh new file mode 100644 index 00000000..649b4ea1 --- /dev/null +++ b/include/trick/mc_variable_random.hh @@ -0,0 +1,48 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a class to support assignment of + a random value to a variable based on its type.) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_VARIABLE_RANDOM_HH +#define CML_MONTE_CARLO_VARIABLE_RANDOM_HH + +#include "mc_variable.hh" + +#include +#include + +/***************************************************************************** +MonteCarloVariableRandom +Purpose:(An intermediate interface class that supports generation of random + variables of multiple types. Currently, double, integer, bool, and + string assignments are supported; others may be added later.) +NOTE - deliberately not using templates here because of the difficulty of having + SWIG use templates; Monte-Carlo variables are typically created and + populated via an input-process so the SWIG interface is critical and the + use of templates a non-starter. +*****************************************************************************/ +class MonteCarloVariableRandom : public MonteCarloVariable +{ + protected: + std::mt19937 random_generator; /* (--) + the basic random-generator used by all different types of random number + generators in the library. */ + unsigned int seed_m; /* (--) + the value used to seed the generator.*/ + public: + MonteCarloVariableRandom(const std::string & var_name, unsigned int seed = 0): MonteCarloVariable(var_name),random_generator(seed),seed_m(seed) //changed seed member variable to seed_m + { + type = MonteCarloVariable::Random; + } + virtual ~MonteCarloVariableRandom(){}; + unsigned int get_seed() const {return seed_m;} // override but SWIG cannot process the + // override keyword + + private: // and undefined: + MonteCarloVariableRandom( const MonteCarloVariableRandom & ); + MonteCarloVariableRandom& operator = (const MonteCarloVariableRandom&); +}; +#endif diff --git a/include/trick/mc_variable_random_bool.hh b/include/trick/mc_variable_random_bool.hh new file mode 100644 index 00000000..73d9a6d5 --- /dev/null +++ b/include/trick/mc_variable_random_bool.hh @@ -0,0 +1,42 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Uses the RandomString generator to generate either a "True" or + "False" string for assignment through the SWIG interface. + Note that SWIG uses the Python uppercase True/False rather than + C++ lowercase true/false identifiers. + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_VARIABLE_RANDOM_BOOL_HH +#define CML_MONTE_CARLO_VARIABLE_RANDOM_BOOL_HH + +#include "mc_variable_random_string.hh" + +/***************************************************************************** +MonteCarloVariableRandomBool +Purpose:(Generates either a True or False string for assignment) +*****************************************************************************/ +class MonteCarloVariableRandomBool : public MonteCarloVariableRandomStringSet +{ + public: + MonteCarloVariableRandomBool( const std::string & var_name, + unsigned int seed) + : + MonteCarloVariableRandomStringSet( var_name, seed) + { + add_string("False"); + add_string("True"); + include_in_summary = true; // String variables are excluded by default + // because they may contain commas, which would + // cause trouble in the comma-delimited summary + // file. However, this is a special case in which + // the possible strings are "True" and "False". + } + virtual ~MonteCarloVariableRandomBool(){}; + private: // and undefined: + MonteCarloVariableRandomBool(const MonteCarloVariableRandomBool&); + MonteCarloVariableRandomBool& operator = ( + const MonteCarloVariableRandomBool&); +}; +#endif diff --git a/include/trick/mc_variable_random_normal.hh b/include/trick/mc_variable_random_normal.hh new file mode 100644 index 00000000..2353ecf4 --- /dev/null +++ b/include/trick/mc_variable_random_normal.hh @@ -0,0 +1,57 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a class to support generation and assignment + of a random value distributed normally.) + +LIBRARY DEPENDENCY: + (../src/mc_variable_random_normal.cc) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_VARIABLE_RANDOM_NORMAL_HH +#define CML_MONTE_CARLO_VARIABLE_RANDOM_NORMAL_HH + +#include + +#include "mc_variable_random.hh" + +class MonteCarloVariableRandomNormal : public MonteCarloVariableRandom +{ + public: + enum TruncationType + { + StandardDeviation, + Relative, + Absolute + }; + size_t max_num_tries; + + protected: + #ifndef SWIG + std::normal_distribution distribution; + #endif + double min_value; + double max_value; + bool truncated_low; + bool truncated_high; + + public: + MonteCarloVariableRandomNormal( const std::string & var_name, + unsigned int seed = 0, + double mean = 0, + double stdev = 1); + + virtual ~MonteCarloVariableRandomNormal(){}; + virtual void generate_assignment(); + void truncate(double limit, TruncationType type = StandardDeviation); + void truncate(double min, double max, TruncationType type = StandardDeviation); + void truncate_low(double limit, TruncationType type = StandardDeviation); + void truncate_high(double limit, TruncationType type = StandardDeviation); + void untruncate(); + private: // and undefined: + MonteCarloVariableRandomNormal(const MonteCarloVariableRandomNormal&); + MonteCarloVariableRandomNormal& operator = ( + const MonteCarloVariableRandomNormal&); +}; +#endif diff --git a/include/trick/mc_variable_random_string.hh b/include/trick/mc_variable_random_string.hh new file mode 100644 index 00000000..602e8eea --- /dev/null +++ b/include/trick/mc_variable_random_string.hh @@ -0,0 +1,52 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a class to randomly pick one of a set of + character strings. These strings could represent actual string + variables, or enumerated types, or commands, or any other concept + that can be expressed as an assignment.) + +ASSUMPTIONS: (The content of the selected string will be assigned as written. + Consequently, if the value is being assigned to an actual string + or char variable, the contents of the string should be enclosed + in quotes. + E.g. the string might look like: "'actual_string'" so that the + assignment would look like + variable = 'actual_string' + This is to support the use of a string to represent a non-string + variable such as an enumeration or command: + E.g. the string might look like "x * 3 + 2" to achieve: + variable = x * 3 + 2 + ) + + +LIBRARY DEPENDENCY: + (../src/mc_variable_random_string.cc) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_VARIABLE_RANDOM_STRING_HH +#define CML_MONTE_CARLO_VARIABLE_RANDOM_STRING_HH + +#include "mc_variable_random_uniform.hh" + +#include +#include +#include + +class MonteCarloVariableRandomStringSet : public MonteCarloVariableRandomUniform +{ + public: + std::vector< std::string> values; + MonteCarloVariableRandomStringSet( const std::string & var_name, + unsigned int seed); + + virtual ~MonteCarloVariableRandomStringSet(){}; + virtual void generate_assignment(); + void add_string(std::string); + private: // and undefined: + MonteCarloVariableRandomStringSet(const MonteCarloVariableRandomStringSet&); + MonteCarloVariableRandomStringSet& operator = ( + const MonteCarloVariableRandomStringSet&); +}; +#endif diff --git a/include/trick/mc_variable_random_uniform.hh b/include/trick/mc_variable_random_uniform.hh new file mode 100644 index 00000000..e563ab34 --- /dev/null +++ b/include/trick/mc_variable_random_uniform.hh @@ -0,0 +1,67 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a class to support generation and assignment + of a random value distributed uniformally. + Provides float and integer distributions) + +LIBRARY DEPENDENCY: + (../src/mc_variable_random_uniform.cc) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_VARIABLE_RANDOM_UNIFORM_HH +#define CML_MONTE_CARLO_VARIABLE_RANDOM_UNIFORM_HH + +#include "mc_variable_random.hh" + +#include + +/***************************************************************************** +MonteCarloVariableRandomUniform +Purpose:() +*****************************************************************************/ +class MonteCarloVariableRandomUniform : public MonteCarloVariableRandom +{ + protected: + #ifndef SWIG + std::uniform_real_distribution distribution; + #endif + + public: + MonteCarloVariableRandomUniform( const std::string & var_name, + unsigned int seed = 0, + double lower_bound = 0.0, + double upper_bound = 1.0); + virtual ~MonteCarloVariableRandomUniform(){}; + virtual void generate_assignment(); + private: // and undefined: + MonteCarloVariableRandomUniform( const MonteCarloVariableRandomUniform & ); + MonteCarloVariableRandomUniform& operator = ( + const MonteCarloVariableRandomUniform&); +}; + +/***************************************************************************** +MonteCarloVariableRandomUniformInt +Purpose:() +*****************************************************************************/ +class MonteCarloVariableRandomUniformInt : public MonteCarloVariableRandom +{ + protected: + #ifndef SWIG + std::uniform_int_distribution distribution; + #endif + + public: + MonteCarloVariableRandomUniformInt( const std::string & var_name, + unsigned int seed = 0, + double lower_bound = 0, + double upper_bound = 1); + virtual ~MonteCarloVariableRandomUniformInt(){}; + virtual void generate_assignment(); + private: // and undefined: + MonteCarloVariableRandomUniformInt(const MonteCarloVariableRandomUniformInt&); + MonteCarloVariableRandomUniformInt& operator = ( + const MonteCarloVariableRandomUniformInt&); +}; +#endif diff --git a/include/trick/mc_variable_semi_fixed.hh b/include/trick/mc_variable_semi_fixed.hh new file mode 100644 index 00000000..cde702e5 --- /dev/null +++ b/include/trick/mc_variable_semi_fixed.hh @@ -0,0 +1,81 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( + Implementation of a semi-fixed monte-carlo variable. + The value of a MonteCarloVariableFixed instance is assigned at + construction time and held at that value for all runs. + The value of a MonteCarloVariableSemiFixed instance is assigned + from another MonteCarloVariable generated value for the first + input file generated, and held at that value for all runs. So the + assignment to a Semi-fixed can change each time the input files are + generated, but it is the same for all input files in any given + generation.) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#ifndef CML_MONTE_CARLO_VARIABLE_SEMI_FIXED_HH +#define CML_MONTE_CARLO_VARIABLE_SEMI_FIXED_HH + +#include "mc_variable_random.hh" +#include "trick/message_proto.h" +#include "trick/message_proto.h" + +// TODO Turner 2019/11 +// The reference to a MonteCarloVariable might be difficult to +// obtain because these are typically created on-the-fly and +// the handle to the created instance is lost. Might be +// nice to provide the seed-variable-name instead of the reference +// to the seed-variable itself, and have the MonteCarloMaster find +// the MonteCarloVariable associated with that name. +// But that's non-trivial and not necessarily desirable, so it is +// left unimplemented. + +class MonteCarloVariableSemiFixed : public MonteCarloVariable +{ + protected: + const MonteCarloVariable & seed_variable; /* (--) + A reference to another MonteCarloVariable; the value of *this* + variable is taken from the value of this seed-variable during + preparation of the first monte-input file. */ + bool command_generated; /* (--) + flag indicating the fixed command has been generated.*/ + public: + MonteCarloVariableSemiFixed(const std::string & var_name, + const MonteCarloVariable & seed_) + : + MonteCarloVariable( var_name), + seed_variable(seed_), + command_generated(false) + { + type = MonteCarloVariable::Constant; + } + virtual ~MonteCarloVariableSemiFixed(){}; + + void generate_assignment() { + if (!command_generated) { + // parse the command from seed_variable to get the assignment. + std::string seed_command = seed_variable.get_command(); + size_t pos_equ = seed_command.find("="); + if (pos_equ == std::string::npos) { + std::string message = + std::string("File: ") + __FILE__ + + ", Line: " + std::to_string(__LINE__) + std::string(", Invalid " + "sequencing\nFor variable ") + variable_name.c_str() + + std::string(" the necessary pre-dispersion to obtain the\n " + "random value for assignment has not been completed.\nCannot " + "generate the assignment for this variable.\n"); + message_publish(MSG_ERROR, message.c_str()); + return; + } + assignment = seed_command.substr(pos_equ+1); + generate_command(); + insert_units(); + command_generated = true; + } + } + private: // and undefined: + MonteCarloVariableSemiFixed( const MonteCarloVariableSemiFixed & ); + MonteCarloVariableSemiFixed& operator = (const MonteCarloVariableSemiFixed&); +}; +#endif diff --git a/share/trick/sim_objects/MonteCarloGenerate.sm b/share/trick/sim_objects/MonteCarloGenerate.sm new file mode 100644 index 00000000..9fa92326 --- /dev/null +++ b/share/trick/sim_objects/MonteCarloGenerate.sm @@ -0,0 +1,73 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: Provides a one-stop shop for all MonteCarlo operations that Trick + cannot support, including: + - assignment of values to non-floats + - assignment of variables to other dispersed variables + - computation of variables as a function of one or more dispersed + variables. +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2018) (Antares) (initial creation for CML))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +***********************************************************************/ +#ifndef SIM_OBJECT_MONTE_CARLO_Generation +#define SIM_OBJECT_MONTE_CARLO_Generation +##include "trick/mc_master.hh" +##include "trick/mc_python_code.hh" +##include "trick/mc_variable_random_uniform.hh" +##include "trick/mc_variable_random_normal.hh" +##include "trick/mc_variable_random_string.hh" +##include "trick/mc_variable_random_bool.hh" +##include "trick/mc_variable_file.hh" +##include "trick/mc_variable_fixed.hh" +##include "trick/mc_variable_semi_fixed.hh" +##include "trick/message_type.h" +##include "trick/message_proto.h" +##include "trick/exec_proto.h" +#include "sim_objects/default_trick_sys.sm" +##include "trick/MonteCarlo.hh" + +class MonteCarloGeneratorSimObject : public Trick::SimObject +{ + protected: + Trick::MonteCarlo * mc; + public: + MonteCarloMaster mc_master; + MonteCarloGeneratorSimObject(std::string location, Trick::MonteCarlo * mc_) + : + mc(mc_), + mc_master(location) + { + // pre-initialization assignments: + P0 ("initialization") generate_dispersions(); + } + void generate_dispersions() + { + if (!mc_master.active || !mc_master.generate_dispersions) return; + const std::vector variables = mc->get_variables(); + if (!variables.empty()) { + for (auto it :variables) { + std::string message = std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Monte Carlo Variable added using wrong " + "method\n" + "The variable " + it->name + " was added " + "improperly. Details:\n" + it->describe_variable(); + message_publish(MSG_ERROR, message.c_str()); + } + std::string message = std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", MonteCarloGeneratorSimObject is active but " + "variable(s) have been added using the older MonteCarloSimObject method. " + "Only one method can be used per sim. You must replace all trick_mc " + "based variables with monte_carlo sim object equivalents. See the trick " + "documentation for support.\n"; + message_publish(MSG_ERROR, message.c_str()); + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + } + mc_master.execute(); + } + private: + // copy constructor, operator= blocked: + MonteCarloGeneratorSimObject (const MonteCarloGeneratorSimObject&); + MonteCarloGeneratorSimObject & operator = + ( const MonteCarloGeneratorSimObject&); +}; +MonteCarloGeneratorSimObject monte_carlo("monte_carlo.mc_master", &trick_mc.mc); +#endif diff --git a/share/trick/sim_objects/default_trick_sys.sm b/share/trick/sim_objects/default_trick_sys.sm index cf4a1f0d..bacb1dde 100644 --- a/share/trick/sim_objects/default_trick_sys.sm +++ b/share/trick/sim_objects/default_trick_sys.sm @@ -175,7 +175,7 @@ class MonteCarloSimObject : public Trick::SimObject { exec_register_scheduler(&mc) ; {TRK} P0 ("default_data") mc.process_sim_args() ; - {TRK} P0 ("initialization") mc.execute_monte() ; + {TRK} P1 ("initialization") mc.execute_monte() ; {TRK} ("shutdown") mc.shutdown() ; } } diff --git a/share/trick/trickops/.yaml_requirements.yml b/share/trick/trickops/.yaml_requirements.yml new file mode 100644 index 00000000..3be57db5 --- /dev/null +++ b/share/trick/trickops/.yaml_requirements.yml @@ -0,0 +1,94 @@ +# This file represents the required types, ranges, and values +# as applicable expected for keys found while parsing TrickWorkflow +# Yaml files. This is used by TrickWorkflowYamlVerifier to verify +# user-supplied content. The information is used in the following +# way: +# content: +# required: 0|1 <-- if 1, content must not be empty (None) +# default: <-- if required:0 and content not given, use this value +# overridable: 0|1 <-- if 0, check all given keys below +# type: <-- isinstance(content, type) +# contains: <-- contains in content +# min: <-- content > min +# max: <-- content < max +# This file should not be modified except in the case of TrickOps +# development. + +# Default values of content if not specified in structures below +# Keys with empty values end up as None. Structures below override +# these values. +content: + required: 0 + default: + overridable: 0 + type: + contains: + min: + max: + +# Expectations for everything under a top-level global key: +globals: + env: + default: '' + type: str + +# Expectations for everything under a top-level SIM key: +sim: + name: # Comes from SIM key itself + required: 1 + type: str + path: + required: 1 + type: str + description: + type: str + labels: + type: list + build_args: + type: str + binary: + default: S_main_{cpu}.exe + type: str + size: + default: 2200000 + type: int + min: 1 + phase: + default: 0 + type: int + min: -1000 + max: 1000 + parallel_safety: + default: loose + type: str + contains: + - strict + - loose + runs: + type: dict # Subsection validation articulated below + +# Expectations for every key under a sim's run: subsection +run: + input: # Comes from run key itself + required: 1 + type: str + returns: + type: int + default: 0 + min: 0 + max: 255 + compare: + overridable: 1 + type: list + analyze: + overridable: 1 + type: str + phase: + default: 0 + type: int + min: -1000 + max: 1000 + valgrind: + type: str + + diff --git a/share/trick/trickops/ExampleWorkflow.py b/share/trick/trickops/ExampleWorkflow.py index 118d7c28..0bcec229 100755 --- a/share/trick/trickops/ExampleWorkflow.py +++ b/share/trick/trickops/ExampleWorkflow.py @@ -124,9 +124,11 @@ f.close() from TrickWorkflow import * class ExampleWorkflow(TrickWorkflow): def __init__( self, quiet, trick_top_level='/tmp/trick'): - # Real projects already have trick somewhere, but for this test, just clone it + # Real projects already have trick somewhere, but for this example, just clone & build it if not os.path.exists(trick_top_level): os.system('cd %s && git clone https://github.com/nasa/trick' % (os.path.dirname(trick_top_level))) + if not os.path.exists(os.path.join(trick_top_level, 'lib64/libtrick.a')): + os.system('cd %s && ./configure && make' % (trick_top_level)) # Base Class initialize, this creates internal management structures TrickWorkflow.__init__(self, project_top_level=trick_top_level, log_dir='/tmp/', trick_dir=trick_top_level, config_file="/tmp/config.yml", cpus=3, quiet=quiet) diff --git a/share/trick/trickops/MonteCarloGenerationHelper.py b/share/trick/trickops/MonteCarloGenerationHelper.py new file mode 100644 index 00000000..10d2c62a --- /dev/null +++ b/share/trick/trickops/MonteCarloGenerationHelper.py @@ -0,0 +1,305 @@ +""" +Module to be used in conjunction with the MonteCarloGenerate (MCG) sim module +provided by Trick and optionally TrickOps. This module allows MCG users to +easily generate monte-carlo runs and execute them locally or through an HPC job +scheduler like SLURM. Below is an example usage of the module assuming: + 1. The using script inherits from TrickWorkflow, giving access to execute_jobs() + 2. SIM_A/RUN_mc/input.py is configured with MonteCarloGenerate.sm sim module + to generate runs when executed + +# Instantiate an MCG helper instance, providing the sim and input file for generation +mgh = MonteCarloGenerationHelper(sim_path="path/to/SIM_A", input_path="RUN_mc/input.py") +# Get the generation SingleRun() instance +gj = mgh.get_generation_job() +# Execute the generation Job to generate RUNS +ret = self.execute_jobs([gj]) + +if ret == 0: # Successful generation + # Get a SLURM sbatch array job for all generated runs + sbj = mgh.get_sbatch_job(monte_dir="path/to/MONTE_RUN_mc") + # Execute the sbatch job, which queues all runs in SLURM for execution + # Use hpc_passthrough_args ='--wait' to block until all runs complete + ret = self.execute_jobs([sbj]) + + # Instead of using SLURM, generated runs can be executed locally through + # TrickOps calls. First get a list of run jobs + run_jobs = mgh.get_generated_run_jobs(monte_dir="path/to/MONTE_RUN_mc") + # Execute all generated SingleRun instances, up to 10 at once + ret = self.execute_jobs(run_jobs, max_concurrent=10) +""" + +import sys, os + +import send_hs +import argparse, glob + +import subprocess, errno +import pprint, logging +import WorkflowCommon, TrickWorkflow + +# This global is the result of hours of frustration and debugging. See comment at the top of +# TrickWorkflow.py for details +this_trick = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../..')) + +class MonteCarloGenerationHelper(): + """ + Helper class for generating runs via Trick's built-in MonteCarloGenerate sim module. + """ + def __init__(self, sim_path, input_path, S_main_name='S_main*.exe', env=None, sim_args_gen=None): + """ + Initialize this instance. + + >>> mgh = MonteCarloGenerationHelper(sim_path=os.path.join(this_trick, "test/SIM_mc_generation"), input_path="RUN_nominal/input_a.py") + + Parameters + ---------- + sim_path : str + Path to the directory containing the built simulation + input_path : str + Path to input file used for monte-carlo generation, relative to sim_path + S_main_name : str + Literal string of S_main binary. Defaults to (lazy) S_main*.exe + env : str + Optional literal string to execute before everything, typically for sourcing + a project environment file. + sim_args_gen : str + Optional literal string to pass to the simulation after the input_path during + generation. + """ + self.sim_path = os.path.abspath(sim_path) + self.input_path = input_path + self.S_main_name = S_main_name + self.sim_args_gen = '' if not sim_args_gen else sim_args_gen + self.env = '' if not env else env + + self.generated_input_files = [] # Will be list of successfully generated runs + # Sanity check inputs + if not isinstance(self.sim_args_gen, str): + msg = (f"ERROR: Given sim_args_gen ({sim_args_gen}) has invalid type {type(self.sim_args_gen)}." + f" It must be a string.") + raise TypeError(msg) + if not isinstance(self.env, str): + msg = (f"ERROR: Given env ({env}) has invalid type {type(self.env)}." + f" It must be a string.") + raise TypeError(msg) + if not os.path.isfile(os.path.join(self.sim_path, self.input_path)): + msg = (f"ERROR: input_path {os.path.join(self.sim_path, self.input_path)} " + f"doesn't exist.") + raise RuntimeError(msg) + # Construct job used to generate runs + cmd = "%s cd %s && " % (self.env, self.sim_path) + cmd += "./%s " % (self.S_main_name) + cmd += "%s " % self.input_path + if self.sim_args_gen: + cmd += "%s" % self.sim_args_gen + self.generation_job = TrickWorkflow.SingleRun( + name=f'Monte Carlo Generation for: {self.sim_path}/{self.input_path}', + command=cmd, log_file=os.path.join(self.sim_path, os.path.dirname(self.input_path), + 'MCG_generation_output.txt'), + expected_exit_status=0, use_var_server=False) + + def get_generation_job(self): + """ + Return the SingleRun (Job) instance corresponding to an MCG generation execution + as configured inside this instance. This SingleRun can be executed via + WorkflowCommon.execute_jobs() to generate runs. + + >>> mgh = MonteCarloGenerationHelper(sim_path=os.path.join(this_trick, "test/SIM_mc_generation"), input_path="RUN_nominal/input_a.py") + >>> job = mgh.get_generation_job() + """ + return self.generation_job + + def get_generated_input_files(self, monte_dir): + """ + Return a sorted list of absolute paths to the generated input files. + + Parameters + ---------- + monte_dir : str + Path to monte_dir in which input_files were generated. This is a required input + since this class has no way to know where the user configured the sim to locate + the generated runs. + + Returns + ---------- + list of strings (absolute paths) + Sorted list of absolute paths to input files found under monte_dir + + Raises + ------ + RuntimeError + If an error in finding input files occurs. + """ + if os.path.isdir(monte_dir) == False: + msg = (f"Given monte_dir {monte_dir} doesn't exist! Cannot get run list.") + raise RuntimeError(msg) + just_input_file = ("monte_"+os.path.basename(self.input_path)) + monte_dir_path_full = os.path.abspath(monte_dir) + run_list = [x for x in os.listdir(monte_dir_path_full) if \ + (x.startswith("RUN_") and os.path.isdir(os.path.join(monte_dir_path_full,x)))] + if len(run_list) == 0: + msg = (f"Error: {monte_dir} doesn't have any runs!") + raise RuntimeError(msg) + self.generated_input_files = [os.path.join(monte_dir, x, just_input_file) for x in run_list + if os.path.isfile(os.path.join( monte_dir_path_full, x, just_input_file ))] + if len(self.generated_input_files) == 0: + msg = (f"Error: {monte_dir}'s RUN directories don't have any input files of expected " + f"name: {just_input_file}. Make sure input_path is correct and that MCG is configured " + "appropriately.") + raise RuntimeError(msg) + self.generated_input_files.sort() + # error checking regarding missing files or not enough runs + if len(run_list) != len(self.generated_input_files): + msg = ("WARNING in get_generated_input_files(): There's a mismatch between the number " + f"of MONTE*/RUN* directories and the number of input files ({just_input_file}) " + "in those directories. Returning only the found subset") + print(msg) + + return(list(self.generated_input_files)) + + def get_zero_padding(self, monte_dir=None): + """ + Returns an integer representing the highest zero-padding contained in + monte_dir, or in self.generated_input_files list if monte_dir is None + + Parameters + ---------- + monte_dir : str + Path to monte_dir in which input_files were generated + + Returns + ---------- + int + Integer representing zero-padding length. Ready for use in printf-style + "%0d" use-cases. + + Raises + ------ + RuntimeError + If zero-padding information cannot be determined + """ + gif = [] + if monte_dir: + gif = self.get_generated_input_files(monte_dir) + else: + gif = self.generated_input_files + + if not gif or len(gif) < 1: + msg = ("Cannot find zero-padding information because self.generated_input_files " + "is empty! Have you run get_generated_input_files()?") + raise RuntimeError(msg) + # This exception may be unreachable, but for extra safety... + try: + padding = len(os.path.dirname(gif[-1]).split('_')[-1]) + except Exception as e: + msg = ("Encountered unhandled exception attempting to determine zero-padding " + f"information. Error:\n{e}") + raise RuntimeError(msg) + return padding + + def get_sbatch_job(self, monte_dir, sim_args=None, hpc_passthrough_args=None): + """ + Return a Job() whose command is an sbatch array job for running generated runs + found in monte_dir, with the given hpc_passthrough_args if specified. This single + Job() when executed will submit the entire set of monte-carlo runs in monte_dir + to a SLURM job scheduling system as an array job. This function simply creates + the Job(), it does not execute it. + + Parameters + ---------- + monte_dir : str + Path to monte_dir in which input_files were generated. This is a required input + since this class has no way to know where the user configured the sim to locate + the generated runs. + + hpc_passthrough_args : str + Literal string of args to be passed through to the HPC scheduling system SLURM + + sim_args : str + Literal string of args to be passed through to the simulation binary + + Returns + ---------- + Job() + Job instance with sbatch command configured for array submission. + """ + generated_runs = self.get_generated_input_files(monte_dir) + zero_padding = self.get_zero_padding() + just_input_file = os.path.basename(generated_runs[0]) + sbatch_cmd = self.env + " sbatch " + + # Build the default --array option, this is overridden later if the user gave it + # Homogeneous array assumption, last zero-padded num in last run in array + array = ("--array 0-%s " % os.path.dirname(generated_runs[-1]).split('_')[-1]) + # A couple sbatch options are special: if the user has specified --array , use + # theirs, if not then generate the --array field for the mc_num_runs given. If --wait + # is given, we won't be able to post-process, so store that information off. Easiest + # way to get this passthrough information is to use arparse to parse the + # hpc_passthrough_args list, leaving what's left in rest list to be passed through + sbatch_parser = argparse.ArgumentParser() + sbatch_parser.add_argument('-a', '--array') + sbatch_parser.add_argument('-W', '--wait', action="store_true") + if hpc_passthrough_args: + subargs, rest = sbatch_parser.parse_known_args(hpc_passthrough_args.split()) + if subargs.wait: # Since --wait not in rest, re-add it to passthrough args + rest.append('--wait') + if subargs.array: # If custom array given by user + # Custom array definition + array = "--array %s " % subargs.array + sbatch_cmd += ' '.join(rest) + ' ' + sbatch_cmd += array + sbatch_cmd += "--wrap 'RUN_NUM=`printf %0" + str(zero_padding) + 'd $SLURM_ARRAY_TASK_ID`; ' + sbatch_cmd += "%s cd %s && " % (self.env, self.sim_path) + sbatch_cmd += "./%s %s/RUN_${RUN_NUM}/%s" % (self.S_main_name, monte_dir, just_input_file) + if sim_args: + sbatch_cmd += " " + sim_args + " " + + sbatch_cmd += (" 2> %s/RUN_${RUN_NUM}/stderr 1> %s/RUN_${RUN_NUM}/stdout '" % + (monte_dir, monte_dir)) + job = WorkflowCommon.Job(name=(f'Running sbatch array job for ' + f'{os.path.basename(self.sim_path)}' + f' {os.path.basename(monte_dir)}'), + command=sbatch_cmd, log_file=f'{os.path.join(monte_dir)}/sbatch_out.txt') + return (job) + + #TODO Implement a Portable Batch System (PBS) job getter, and support for other HPC + # schedulers go here + + def get_generated_run_jobs(self, monte_dir, sim_args=None): + """ + Return a list of SingleRun() instances, configured for each of the RUNs in + monte_dir. Each run's output goes to the generated RUN location in a file + called stdouterr, containing both stdout and stderr + + Parameters + ---------- + monte_dir : str + Path to monte_dir in which input_files were generated. This must either be a + path relative to the sim dir or an absolute path. + + sim_args : str + Literal string of args to be passed through to the sim binary + """ + jobs = [] + generated_runs = self.get_generated_input_files(monte_dir) + for run in generated_runs: + cmd = "%s cd %s && " % (self.env, self.sim_path) + cmd += "./%s %s" % (self.S_main_name, run) + if sim_args: + cmd += " " + sim_args + " " + jobs.append( TrickWorkflow.SingleRun( + name=f'Executing generated run {os.path.basename(os.path.dirname(run))}', + command=cmd, log_file=os.path.join(os.path.dirname(run), + 'stdouterr'), expected_exit_status=0) + ) + return jobs + + def is_sim_built(self): + """ + Return True if self.S_main_name exists in self.sim_path. False otherwise + """ + if glob.glob(os.path.join(self.sim_path, self.S_main_name)): + return True + else: + return False + diff --git a/share/trick/trickops/TrickWorkflow.py b/share/trick/trickops/TrickWorkflow.py index d67f5419..28d2b9f4 100644 --- a/share/trick/trickops/TrickWorkflow.py +++ b/share/trick/trickops/TrickWorkflow.py @@ -8,7 +8,7 @@ Requires: python3 and requirements.txt containing: psutil # For child process acquisition """ import os, sys, threading, socket, abc, time, re, copy, subprocess, hashlib, inspect -import yaml # Provided by PyYAML +from TrickWorkflowYamlVerifier import * # TODO revisit this import - Jordan from WorkflowCommon import * import pprint @@ -59,7 +59,7 @@ class TrickWorkflow(WorkflowCommon): # Job statuses are stored internally and can be queried after they've already # executed, for example, after execute_jobs() finishes: for b in builds: - print("Build job %s completed with status %d " % (b.name, b.get_status() is job.Status.SUCCESS)) + print("Build job %s completed with status %d " % (b.name, b.get_status())) print("and ran command %s " % (b._command)) # All jobs store their status internally regardless of whether they have @@ -76,6 +76,83 @@ class TrickWorkflow(WorkflowCommon): # Note that runs, comparisons, and analysis management classes and/or jobs also # support the report() method """ + # These are static so Run and Sim classes can access them when needed + # TODO: It would be nice to consolidate and read this from .yaml_requirements.txt + # -Jordan 12/2022 + allowed_phase_range = {'min': -1000, 'max': 1000} + all_possible_phases = range(allowed_phase_range['min'], allowed_phase_range['max']+1) + + def listify_phase(phase=None): + """ + Given a phase of different potential types, return a list of integers within + the allowed_phase_range. + + >>> TrickWorkflow.listify_phase(0) + [0] + >>> TrickWorkflow.listify_phase(range(0,5)) + [0, 1, 2, 3, 4] + + Parameters + ---------- + phase : int, list, range, or None + Phase or phases requested by user + + Returns + ------- + list of integers which must be equal to or a subset of all_possible_phases + + Raises + ------ + RuntimeError + If input is not an int, list of ints, range, or None, or if given values + are not within TrickWorkflow.all_possible_phases + """ + # Handle type of phase given + phases = [] # To be populated with all valid phases requested by user + if (phase is None): + phases = list(TrickWorkflow.all_possible_phases) + elif isinstance(phase, int) and phase in TrickWorkflow.all_possible_phases: + phases = [phase] + elif ( (isinstance(phase, list) or isinstance(phase, range)) and + all(isinstance(p, int) for p in phase) and + all(p in TrickWorkflow.all_possible_phases for p in phase)): + phases = list(phase) # Cast to list to cover range case + else: + msg =("ERROR: Given phase %s in listify_given_phase() must be an int," + " list of ints, or range() within the bounds [%s, %s]" % (phase, + TrickWorkflow.allowed_phase_range['min'], + TrickWorkflow.allowed_phase_range['max'])) + raise RuntimeError(msg) + return(phases) + + def _find_range_string(string): + """ + Given a string, return the range in square bracket notation if found + This does not check for validity of the string, it only finds it. + + >>> TrickWorkflow._find_range_string('SET_hi/RUN_[000-999]/input.py') + '[000-999]' + + Returns: str or None + str: string of range notation found + None: if not found + + Raises + ------ + RuntimeError + If more than one range string found + """ + pattern = "\[\d+-\d+\]" + if (len(re.findall(pattern, string))) > 1: + msg = ("ERROR: [min-max] pattern found more than once in %s. Only one instance is" + " supported." % (string)) + raise RuntimeError(msg) + m = re.search(pattern, string) + if m: + return m.group(0) + else: + return None + def __init__(self, project_top_level, log_dir, trick_dir, config_file, cpus=3, quiet=False): """ Initialize this instance. @@ -100,20 +177,114 @@ class TrickWorkflow(WorkflowCommon): available """ super().__init__(project_top_level=project_top_level, log_dir=log_dir, quiet=quiet) - # If not found in the config file, these defaults are used - self.defaults = {'cpus': 3, 'name': None, 'description': None, - 'build_command': 'trick-CP', 'size': 2200000, 'env': None, 'binary': 'S_main_{cpu}.exe'} - self.config_errors = False + self.config_errors = [] # Contains errors in setup of management classes self.compare_errors = False # True if comparison errors were found self.sims = [] # list of Sim() instances, filled out from config file self.config_file = config_file # path to yml config self.cpus = cpus # Number of CPUs this workflow should use when running - # 'loose' or 'strict' controls if multiple input files per RUN dir are allowed in YAML config file - self.parallel_safety = 'loose' - self.config = self._read_config(self.config_file) # Contains resulting Dict parsed by yaml.load + self.yaml_verifier = TrickWorkflowYamlVerifier(self.config_file) + # If not found in the config file, these defaults are used self.trick_dir = trick_dir self.trick_host_cpu = self.get_trick_host_cpu() - self._validate_config() + self.env = '' + self.config = self.yaml_verifier.verify() + self.parsing_errors = self.yaml_verifier.parsing_errors # All errors found during parsing + for e in self.parsing_errors: + tprint(e, 'DARK_RED') + self._populate_sims() + + def _populate_sims(self): + """ + Given a verified self.config, populate the self.sims list of Sim() instances + Note that all the checking of expected dict key/values has already taken place + during yaml verification, so we can safely access all dict keys here. + Furthermore, any sim or run without required key/values have already been purged + from self.config so we can create all management classes at the sim and run + levels without worry. + """ + def cprint(msg, color): + self.config_errors.append(msg) + tprint(msg, color) + self.env = self.config['globals']['env'] + all_sim_paths = [] # Keep a list of all paths for uniqueness check + for s in self.config: + if not str(s).startswith('SIM'): # Ignore everything not starting with SIM + continue + if (not os.path.exists(os.path.join(self.project_top_level, self.config[s]['path'])) ): + cprint("ERROR: %s's 'path' %s not found. Continuing but skipping this entire entry from %s." + % (s, self.config[s]['path'], self.config_file), 'DARK_RED') + continue + if self.config[s]['path'] in all_sim_paths: + cprint("ERROR: %s's 'path' is not unique in %s. Continuing but ignoring this sim." + % (s, self.config_file), 'DARK_RED') + continue + # Add the CPU format string (won't fail if there is no CPU placeholder) + self.config[s]['binary'] = self.config[s]['binary'].format(cpu=self.trick_host_cpu) + # Add the full path to the build command + trick_CP=os.path.join(self.trick_dir, "bin/trick-CP") + if self.config[s]['build_args']: + trick_CP+=(' ' + self.config[s]['build_args']) + thisSim = TrickWorkflow.Sim(name=s, sim_dir=self.config[s]['path'], + description=self.config[s]['description'], labels=self.config[s]['labels'], + prebuild_cmd=self.env, build_cmd=trick_CP, + cpus=self.cpus, size=self.config[s]['size'], phase=self.config[s]['phase'], + log_dir=self.log_dir) + all_sim_paths.append(self.config[s]['path']) + + all_run_paths = [] # Keep a list of all paths for uniqueness check + for r in self.config[s]['runs']: + just_RUN = r.split()[0] # Drop arguments after RUN../...py + just_RUN_dir = os.path.dirname(just_RUN) # Drop path before RUN../..py + # TODO: This check will break generated runs, do we even need to check this at all? - Jordan 2022 + #if not os.path.exists(os.path.join(self.project_top_level, self.config[s]['path'], just_RUN)): + # cprint("ERROR: %s's 'run' path %s not found. Continuing but skipping this run " + # "from %s." % (s, just_RUN, self.config_file), 'DARK_RED') + # continue + if just_RUN_dir in all_run_paths and self.config[s]['parallel_safety'] == 'strict': + cprint("ERROR: %s's run directory %s is not unique in %s. With setting " + "parallel_safety: strict, you cannot have the same RUN directory listed " + "more than once per sim. Continuing but skipping this run." % + (s, r, self.config_file), 'DARK_RED') + continue + + thisRun = TrickWorkflow.Run(sim_dir=self.config[s]['path'], input_file=r, + binary= self.config[s]['binary'], prerun_cmd=self.env, + returns=self.config[s]['runs'][r]['returns'], + valgrind_flags=self.config[s]['runs'][r]['valgrind'], + phase=self.config[s]['runs'][r]['phase'], log_dir=self.log_dir) + + # The check for list allows all other non-list types in the yaml file, + # allowing groups to define their own comparison methodology + if isinstance(self.config[s]['runs'][r]['compare'], list): + for cmp in self.config[s]['runs'][r]['compare']: + lhs, rhs = [ s.strip() for s in cmp.split(' vs.') ] + thisRun.add_comparison(test_data=lhs, baseline_data=rhs) + + if self.config[s]['runs'][r]['analyze'] is not None: + thisRun.add_analysis(cmd=self.config[s]['runs'][r]['analyze']) + + theseRuns = [] + try: + theseRuns = thisRun.multiply() # If [from-to] notation used, expand runs + except RuntimeError as e: + msg = ("ERROR: Unable to multiply run %s in sim %s. Ignoring this run. " + "Check for bad [min-max] syntax in run keys in %s and try again.\n %s" + % (r, s, self.config_file, e) ) + cprint(msg, 'DARK_RED') + + for r in theseRuns: + thisSim.add_run(r) # Add Run/Runs to this Sim() + + all_run_paths.append(just_RUN_dir) # Keep track of all runs to check parallel safety + + self.sims.append(thisSim) # Add Sim() to internal list + if len(self.sims) < 1: # At minimum, one valid SIM structure must exist + msg = ("ERROR: After validating config file, there is insufficient information to continue." + " Check the syntax in config file %s and try again." + % (self.config_file) ) + cprint(msg, 'DARK_RED') + self._cleanup() + raise RuntimeError(msg) def create_test_suite(self): """ @@ -214,6 +385,7 @@ class TrickWorkflow(WorkflowCommon): return self.sims sims_found = [] ls = [] + if type(labels) == str: ls = [labels] elif type(labels) == list: @@ -225,321 +397,6 @@ class TrickWorkflow(WorkflowCommon): sims_found.append(sim) return sims_found - def _read_config(self, config_file): - """ - Read the yaml file into a dict and return it - - - Parameters - ---------- - config_file : str - path to YAML config file to be read - - Returns - ------- - dict or None - dictionary representation of YAML content as parsed by yaml.safe_load() - or None if parsing failed - """ - try: - with open(config_file) as file: - y = yaml.safe_load(file) - return y - except Exception as e: - tprint("Unable to parse config file: %s\nERROR: %s" % (config_file,e), 'DARK_RED') - return None - - def _validate_config(self): - """ - Sanity check what we've read from the yml config file and create internal Sim() - objects which populate the self.sims [] list. Makes sure some values - exist and paths are valid locations where applicable. The self.config dict - may be modified to add missing entries as part of this process. If errors - in the config file are detected, self.config_errors is set to True - - The yaml format expected by this framework is described as follows: - - globals: - env: <-- optional literal string executed before all tests, e.g. env setup - parallel_safety: <-- strict won't allow multiple input files per RUN dir - SIM_abc: <-- required unique name for sim of interest, must start with SIM - path: <-- required SIM path relative to project top level - description: <-- optional description for this sim - labels: <-- optional list of labels for this sim, can be used to get sims - - model_x by label within the framework, or for any other project-defined - - verification purpose - build_command: <-- optional literal cmd executed for SIM_build, defaults to trick-CP - binary: <-- optional name of sim binary, defaults to S_main_{cpu}.exe - size: <-- optional estimated size of successful build output file in bytes - runs: <-- optional dict of runs to be executed for this sim, where the - RUN_1/input.py --foo: dict keys are the literal arguments passed to the sim binary - RUN_2/input.py: and the dict values are other run-specific optional dictionaries - ... described as follows ... - returns: <---- optional exit code of this run upon completion (0-255). Defaults - to 0 - compare: <---- optional list of vs. comparison strings to be - - a vs. b compared after this run is complete. This is extensible in that - - d vs. e all non-list values are ignored and assumed to be used to define - - ... an alternate comparison method in a class extending this one - analyze: <-- optional arbitrary string to execute as job in bash shell from - project top level, for project-specific post-run analysis - valgrind: <-- optional dict describing how to execute runs within valgrind - flags: <-- string of all flags passed to valgrind for all runs - runs: <-- list of literal arguments passed to the sim binary through - valgrind - non_sim_extension_example: - will: be ignored by TrickWorkflow parsing for derived classes to implement as they wish - - Any top-level key not matching the SIM naming pattern is ignored purposefully to provide - users of this framework to extend the same YAML file for other non-trick tests - - Raises - ------ - RuntimeError - If self.config has insufficient content - """ - # All error messages in config validation will trigger self.config_errors to be True - def cprint(msg, color): - self.config_errors = True - tprint(msg, color) - - # Utility method for ensuring a variable is the expected type. Returns True if type - # matches expected type, False otherwise. If fatal=True, raise RuntimeError. - def type_expected(var, expected_type, fatal=False, extra_msg=''): - if type(var) != expected_type: - prepend = "FATAL: " if fatal else "ERROR: " - msg =(prepend + "Entry resembling %s expected to be %s, but got %s instead. Please" - " look for errors in %s. " % - ( str(var), expected_type, type(var), self.config_file) + extra_msg) - cprint(msg, 'DARK_RED') - if fatal: - raise RuntimeError(msg) - else: - return False - return True - - c = copy.deepcopy(self.config) # Make a copy for extra saftey - if not c: # If entire config is empty - msg =("ERROR: Config file %s could not be loaded. Make sure file exists and is valid YAML syntax." - " Cannot continue." % (self.config_file)) - cprint(msg, 'DARK_RED') - self._cleanup() - raise RuntimeError(msg) - # Check global parameters - if 'globals' not in c or not c['globals']: # If globals section is missing or None - self.env = '' - self.parallel_safety = 'loose' - else: - if 'env' not in c['globals'] or not c['globals']['env']: # If section is missing or None - self.env = '' - else: - self.env = c['globals']['env'] - # If env section is missing or None - if 'parallel_safety' not in c['globals'] or not c['globals']['parallel_safety']: - self.parallel_safety = 'loose' - elif c['globals']['parallel_safety'] != 'loose' and c['globals']['parallel_safety'] != 'strict': - cprint( "ERROR: parallel_safety value of %s in config file %s is unsupported. Defaulting to" - " 'loose' and continuing..." % (c['globals']['parallel_safety'], self.config_file), - 'DARK_RED') - self.parallel_safety = 'loose' - else: - self.parallel_safety = c['globals']['parallel_safety'] - if not type_expected(c, expected_type=dict, fatal=True, extra_msg='Cannot continue.'): - pass # Unreachable, type_expected will raise - c.pop('globals', None) # Remove to iterate on the rest of the non-global content - all_sim_paths = [] # Keep a list of all paths for uniqueness check - # Check sub-parameters of SIM entries - for s in c: - if not c[s]: # If the structure is completely empty, skip it - cprint("ERROR: %s is empty!. Continuing but skipping this entire entry from %s." - % (s, self.config_file), 'DARK_RED') - self.config.pop(s) - continue - # If the structure doesn't start with SIM, ignore it and move on - if not s.lower().startswith('sim'): - continue - # If what's stored under SIM_..: is not itself a dict - if not type_expected(c[s], expected_type=dict, extra_msg="SIM definitions must start with SIM, end" - " with :, and contain the path: key-value pair. Skipping over %s." % c[s]): - self.config.pop(s) - continue - # If optional entries missing, or None, set defaults - if 'description' not in c[s] or not c[s]['description']: - self.config[s]['description'] = self.defaults['description'] - if 'build_command' not in c[s] or not c[s]['build_command']: - self.config[s]['build_command'] = self.defaults['build_command'] - if 'size' not in c[s] or not c[s]['size']: - self.config[s]['size'] = self.defaults['size'] - # SIM dir path check - if ('path' not in c[s] or not c[s]['path'] or not - os.path.exists(os.path.join(self.project_top_level, c[s]['path'])) ): - cprint("ERROR: %s's 'path' not found. Continuing but skipping this entire entry from %s." - % (s, self.config_file), 'DARK_RED') - self.config.pop(s) - continue - # SIM dir uniquness check - if c[s]['path'] in all_sim_paths: - cprint("ERROR: %s's 'path' is not unique in %s. Continuing but skipping this sim." - % (s, self.config_file), 'DARK_RED') - self.config.pop(s) - continue - # Ensure labels is a list of strings - if ('labels' not in c[s] or not c[s]['labels'] or not type_expected( - c[s]['labels'], expected_type=list, extra_msg='Ignoring labels.')): - self.config[s]['labels'] = [] - else: - sanitized_labels =( [l for l in c[s]['labels'] if type_expected( - l, expected_type=str, extra_msg='Ignoring label "%s".' % l) ] ) - self.config[s]['labels'] = sanitized_labels - # Check for binary argument - if 'binary' not in c[s] or not c[s]['binary']: - self.config[s]['binary'] = self.defaults['binary'] - - # Add the CPU format string (won't fail if there is no CPU placeholder) - self.config[s]['binary'] = self.config[s]['binary'].format(cpu=self.trick_host_cpu) - - # Add the full path to the build command - self.config[s]['build_command'] = os.path.join(self.trick_dir, "bin", self.config[s]['build_command']) - - # Create internal object to be populated with runs, valgrind runs, etc - thisSim = TrickWorkflow.Sim(name=s, sim_dir=self.config[s]['path'], - description=self.config[s]['description'], labels=self.config[s]['labels'], - prebuild_cmd=self.env, build_cmd=self.config[s]['build_command'], - cpus=self.cpus, size=self.config[s]['size'], log_dir=self.log_dir) - - all_sim_paths.append(c[s]['path']) - # RUN sanity checks - if 'runs' in c[s]: # If it's there... - if not type_expected(c[s]['runs'], expected_type=dict, - extra_msg='Skipping entire run: section in %s' % c[s]): - self.config[s].pop('runs') - else: # If it's there and a valid list, check paths - all_run_paths = [] # Keep a list of all paths for uniqueness check - for r in c[s]['runs']: - if not type_expected(r, expected_type=str, extra_msg='Skipping this run.'): - continue - just_RUN = r.split()[0] - just_RUN_dir = os.path.dirname(just_RUN) - if not os.path.exists(os.path.join(self.project_top_level, c[s]['path'], just_RUN)): - cprint("ERROR: %s's 'run' path %s not found. Continuing but skipping this run " - "from %s." % (s, just_RUN, self.config_file), 'DARK_RED') - self.config[s]['runs'].pop(r) - continue - if just_RUN_dir in all_run_paths and self.parallel_safety == 'strict': - cprint("ERROR: %s's run directory %s is not unique in %s. With global setting " - "parallel_safety: strict, you cannot have the same RUN directory listed " - "more than once per sim. Continuing but skipping this run." % - (s, r, self.config_file), 'DARK_RED') - self.config[s]['runs'].pop(r) - continue - # if the value of : has nothing specified under it, fill out defaults - if not c[s]['runs'][r]: - self.config[s]['runs'][r] = {} - self.config[s]['runs'][r]['returns'] = 0 - self.config[s]['runs'][r]['compare'] = None - self.config[s]['runs'][r]['analyze'] = None - # Have to update the dict we're iterating b/c we check more content a dozen lines down - c[s]['runs'][r] = dict(self.config[s]['runs'][r]) - elif 'returns' not in c[s]['runs'][r]: - self.config[s]['runs'][r]['returns'] = 0 - - elif (not type_expected(c[s]['runs'][r]['returns'], expected_type=int, - extra_msg='Continuing but ignoring %s %s "returns:" value "%s"' % - (s, r, c[s]['runs'][r]['returns']))): - self.config[s]['runs'][r]['returns'] = 0 # Default to zero - elif (c[s]['runs'][r]['returns'] < 0 or c[s]['runs'][r]['returns'] > 255): - cprint("ERROR: %s's run '%s' has invalid 'returns' value (must be 0-255). " - "Continuing but assuming this run is expected to return 0 in %s." % (s, r, self.config_file), - 'DARK_RED') - self.config[s]['runs'][r]['returns'] = 0 # Default to zero - # Create internal object to be added to thisSim - thisRun = TrickWorkflow.Run(sim_dir=self.config[s]['path'], input=r, - binary= self.config[s]['binary'], prerun_cmd=self.env, - returns=self.config[s]['runs'][r]['returns'], - valgrind_flags=None, log_dir=self.log_dir) - # Handle 'compare' option, if given, if not, assume 0 - if 'compare' not in c[s]['runs'][r]: - self.config[s]['runs'][r]['compare'] = None - elif type(c[s]['runs'][r]['compare']) != list: - pass # Deliberately leave open for workflows to extend how comparisons are defined - else: # If it's a list, make sure it fits the 'path vs. path' format - for cmp in c[s]['runs'][r]['compare']: - if not type_expected(cmp, expected_type=str, - extra_msg='Continuing but ignoring comparison %s' % cmp): - continue - if ' vs. ' not in cmp: - cprint("ERROR: %s's run %s comparison '%s' does not match expected pattern. Must be of " - "form: 'path/to/log1 vs. path/to/log2'. Continuing but ignoring this comparison in %s." - % (s, r, cmp, self.config_file), 'DARK_RED') - self.config[s]['runs'][r]['compare'].remove(cmp) - continue - lhs, rhs = [ s.strip() for s in cmp.split(' vs.') ] - thisRun.add_comparison(test_data=lhs, baseline_data=rhs) - # Handle 'analysze' option, if given, if not, assume 0 - if 'analyze' not in c[s]['runs'][r]: - self.config[s]['runs'][r]['analyze'] = None - elif type(c[s]['runs'][r]['analyze']) != str: - pass # Deliberately leave open for workflows to extend how analyze is defined - else: - thisRun.add_analysis(cmd=self.config[s]['runs'][r]['analyze']) - all_run_paths.append(just_RUN_dir) - thisSim.add_run(thisRun) - # SIM's valgrind RUN path checks - if 'valgrind' in c[s]: # If it's there... - if not type_expected(c[s]['valgrind'], expected_type=dict, - extra_msg='Skipping entire valgrind: section in %s' % c[s]): - self.config[s].pop('valgrind') - else: # If it's there and a valid dict - if self.this_os == 'darwin': - cprint("ERROR: Valgrind entry for %s is not valid for platform: %s in " - "config file %s. Continuing but skipping this valgrind: section..." - % (s, self.this_os, self.config_file), 'DARK_RED') - self.config[s].pop('valgrind') - else: - if 'flags' not in c[s]['valgrind']: - self.config[s]['valgrind']['flags'] = '' - if 'runs' in c[s]['valgrind']: # If it's there... - if not type_expected(c[s]['valgrind']['runs'], expected_type=list, - extra_msg='Skipping this valgrind runs: section for %s' % c[s]): - self.config[s].pop('valgrind') - else: - for r in c[s]['valgrind']['runs']: # If it's there and a valid list, check paths - if not type_expected(r, expected_type=str, extra_msg='Skipping this valgrind run.'): - continue - just_RUN = r.split()[0] - if not os.path.exists(os.path.join(self.project_top_level, c[s]['path'], just_RUN)): - cprint("ERROR: %s's valgrind 'run' path %s not found. Continuing but skipping " - "this run from %s." % (s, just_RUN, self.config_file), 'DARK_RED') - self.config[s]['valgrind']['runs'].remove(r) - else: - # Create internal object to be added to thisSim - vRun = TrickWorkflow.Run(sim_dir=self.config[s]['path'], input=r, - binary= self.config[s]['binary'], - prerun_cmd=self.env, valgrind_flags=self.config[s]['valgrind']['flags'], - log_dir=self.log_dir) - thisSim.add_run(vRun) - # Done building up thisSim, store it off for later - self.sims.append(thisSim) - if len(self.sims) < 1: # At minimum, one valid SIM structure must exist - msg = ("ERROR: After validating config file, there is insufficient information to continue." - " Check the syntax in config file %s and try again." - % (self.config_file) ) - cprint(msg, 'DARK_RED') - self._cleanup() - raise RuntimeError(msg) - self._validate_config_custom(copy.deepcopy(self.config)) - - def _validate_config_custom(self, config): - """ - Customization of config file validation. Intended to be extended in derived class. - If changes are needed to the config file, derived classes should edit self.config, not config. - - Parameters - ---------- - config : dict - deep copy of self.config for reading - """ - pass def report(self, indent=''): """ @@ -604,7 +461,7 @@ class TrickWorkflow(WorkflowCommon): """ return any([sim.compare() for sim in self.sims]) - def get_jobs(self, kind): + def get_jobs(self, kind, phase=None): """ Return a list of Jobs() from the self.sims structure of the kind given @@ -620,26 +477,37 @@ class TrickWorkflow(WorkflowCommon): ---------- kind : str Kind of jobs to return from internal self.sims structure: 'build', 'run', 'valgrind', or 'analysis' + phase : int, list of ints, or None + Optional filter to return only jobs of phase or phases given. Analysis jobs + inherit the phase of the Run() it is associated with Returns ------- list - List of jobs of given kind + List of jobs of given kind (and phase if specified) """ + # Transform given phase into a list + phases = TrickWorkflow.listify_phase(phase) + jobs = [] if kind == 'build' or kind == 'builds': - jobs = [ sim.get_build_job() for sim in self.sims ] + jobs = [ sim.get_build_job() for sim in self.sims if sim.phase in phases ] elif kind == 'run' or kind == 'runs': for sim in self.sims: - jobs += sim.get_run_jobs(kind='normal') + jobs += sim.get_run_jobs(kind='normal', phase=phases) elif kind == 'valgrind' or kind == 'valgrinds': for sim in self.sims: - jobs += sim.get_run_jobs(kind='valgrind') + jobs += sim.get_run_jobs(kind='valgrind', phase=phases) elif kind == 'analysis' or kind == 'analyses' or kind == 'analyze': for sim in self.sims: - jobs += sim.get_analysis_jobs() + jobs += sim.get_analysis_jobs(phase=phases) else: raise TypeError('get_jobs() only accepts kinds: build, run, valgrind, analysis') + # If these jobs are of type SingleRun and self.quiet is True, tell the jobs to + # skip the variable server connection logic + for job in jobs: + if self.quiet and isinstance(job, SingleRun): + job.set_use_var_server(False) return (jobs) def get_comparisons(self): @@ -771,7 +639,7 @@ class TrickWorkflow(WorkflowCommon): stored in the TrickWorkflow.sims list. """ def __init__(self, name, sim_dir, description=None, labels=[], prebuild_cmd='', - build_cmd='trick-CP', cpus=3, size=2200000, log_dir='/tmp'): + build_cmd='trick-CP', cpus=3, size=2200000, phase=0, log_dir='/tmp'): """ Initialize this instance. @@ -793,6 +661,9 @@ class TrickWorkflow(WorkflowCommon): Optional Number of CPUs to give via MAKEFLAGS for sim build size : int Optional estimated size of successful build output file, for progress bars + phase : int + Optional ordering phase, typically used to order sim builds when parallel + execution is not robust log_dir: str Directory in which log files will be written """ @@ -804,6 +675,7 @@ class TrickWorkflow(WorkflowCommon): self.build_cmd = build_cmd # Build command for sim self.cpus = cpus # Number of CPUs to use in build self.size = size # Estimated size of successful build output in bytes + self.phase = phase # Phase associated with this sim build self.log_dir = log_dir # Directory for which log file should be written self.build_job = None # Contains Build Job instance self.runs = [] # List of normal Run instances @@ -833,9 +705,10 @@ class TrickWorkflow(WorkflowCommon): size=self.size ) return (self.build_job) - def get_run_jobs( self, kind='normal'): + def get_run_jobs( self, kind='normal', phase=None): """ - Collect all SingleRun() instances and return them for all sims + Collect all SingleRun() instances and return them for all sims subject to the filtering + paramters kind and phase >>> s = TrickWorkflow.Sim(name='alloc', sim_dir=os.path.join(this_trick, 'test/SIM_alloc_test')) >>> s.get_run_jobs() # A sim has no runs by default @@ -845,33 +718,48 @@ class TrickWorkflow(WorkflowCommon): ------- kind : str 'normal' for normal runs, 'valgrind' for valgrind runs + phase : int, list of ints, or None + Get only jobs of the given phase integer or list of integers + If None (default), return all jobs Returns ------- list - List of SingleRun() job instances + List of SingleRun() job instances associated with kind and phase """ + # Transform given phase into a list + phases = TrickWorkflow.listify_phase(phase) + all_jobs = [] if (kind == 'valgrind'): - return ([r.get_run_job() for r in self.valgrind_runs]) + all_jobs = [r.get_run_job() for r in self.valgrind_runs if r.phase in phases] else: - return ([r.get_run_job() for r in self.runs]) + all_jobs = [r.get_run_job() for r in self.runs if r.phase in phases] + return (all_jobs) - def get_analysis_jobs( self): + def get_analysis_jobs( self, phase=None): """ - Collect all Job() instances for all analysis across all sim runs + Collect all Job() instances for all analysis across all sim runs and valgrind runs >>> s = TrickWorkflow.Sim(name='alloc', sim_dir=os.path.join(this_trick, 'test/SIM_alloc_test')) >>> s.get_analysis_jobs() # A sim has no analysis jobs by default [] + Parameters + ------- + phase : int, list of ints, or None + Get only jobs of the given phase integer or list of integers + If None (default), return all jobs + Returns ------- list() List of Job() instances for all analyses """ - return ([r.analysis for r in self.runs if r.analysis]) + # Transform given phase into a list + phases = TrickWorkflow.listify_phase(phase) + return ([r.analysis for r in (self.runs + self.valgrind_runs) if (r.analysis and r.phase in phases) ]) - def get_run(self, input): + def get_run(self, input_file): """ Get a Run() instance by unique full input name. This is the full string intended to be passed to the binary. @@ -895,11 +783,11 @@ class TrickWorkflow(WorkflowCommon): TypeError If input is not a str """ - if type(input) != str: + if type(input_file) != str: raise TypeError('get_run() only accepts the unique key representing the entire input to' ' the sim binary. Ex: "RUN_test/input.py --flags-too"') for run in self.runs: - if run.input == input: + if run.input_file == input_file: return run return None @@ -925,13 +813,50 @@ class TrickWorkflow(WorkflowCommon): """ return self.valgrind_runs + def get_phase(self): + """ + Get integer phase member + + >>> s = TrickWorkflow.Sim(name='alloc', sim_dir=os.path.join(this_trick, 'test/SIM_alloc_test')) + >>> s.get_phase() # Default is zero + 0 + + """ + return self.phase + + def set_phase( self, phase): + """ + Set the phase member variable. Phase is an integer in the range(TrickWorkflow.allowed_phase_range['min'], + TrickWorkflow.allowed_phase_range['max']) which can be used to order sim builds when a workflow cannot + support sims building in parallel. + + >>> s = TrickWorkflow.Sim(name='alloc', sim_dir=os.path.join(this_trick, 'test/SIM_alloc_test')) + >>> s.set_phase(1) + >>> s.set_phase(-1001) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + RuntimeError: ERROR: set_phase() for SIM alloc must be an integer between ... + + Parameters + ------- + phase : int + phase to change to + """ + if (not isinstance(phase, int) or phase < TrickWorkflow.allowed_phase_range['min'] or + phase > TrickWorkflow.allowed_phase_range['max']): + msg =("ERROR: set_phase() for SIM %s must be an integer between [%s, %s]" + % (self.name, TrickWorkflow.allowed_phase_range['min'], TrickWorkflow.allowed_phase_range['max'])) + raise RuntimeError(msg) + else: + self.phase = phase + def add_run( self, run): """ Append a new Run() instance to the internal run lists. Appends to valgrind list if run.valgrind_flags is not None, appends to normal run list otherwise >>> s = TrickWorkflow.Sim(name='alloc', sim_dir=os.path.join(this_trick, 'test/SIM_alloc_test')) - >>> r = TrickWorkflow.Run(sim_dir=os.path.join(this_trick, 'test/SIM_alloc_test'), input='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') + >>> r = TrickWorkflow.Run(sim_dir=os.path.join(this_trick, 'test/SIM_alloc_test'), input_file='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') >>> s.add_run(r) Parameters @@ -944,25 +869,27 @@ class TrickWorkflow(WorkflowCommon): else: self.runs.append(run) - def pop_run( self, input): + def pop_run( self, input_file): """ - Remove a run by its unique self.input value + Remove a run by its unique self.input_file value >>> tw = TrickWorkflow(project_top_level=this_trick, log_dir='/tmp/', trick_dir=this_trick, config_file=os.path.join(this_trick,"share/trick/trickops/tests/trick_sims.yml")) - >>> s = tw.get_sim('SIM_alloc_test') + >>> s = tw.get_sim('SIM_demo_inputfile') + >>> len(s.runs) + 2 >>> r = s.pop_run('RUN_test/input.py') - >>> r.input + >>> r.input_file 'RUN_test/input.py' >>> len(s.runs) - 0 + 1 Returns ------- Run() - Instance in this sim's runs list matching self.input + Instance in this sim's runs list matching self.input_file """ for i, run in enumerate(self.runs): - if run.input == input: + if run.input_file == input_file: return self.runs.pop(i) def compare( self): @@ -1014,20 +941,20 @@ class TrickWorkflow(WorkflowCommon): Management class for run content read from yml config file. Each key in the runs: sub-dict will become a single instance of this management class """ - def __init__(self, sim_dir, input, binary, prerun_cmd = '', returns=0, valgrind_flags=None, - log_dir='/tmp/'): + def __init__(self, sim_dir, input_file, binary, prerun_cmd = '', returns=0, valgrind_flags=None, + phase=0, log_dir='/tmp/'): """ Initialize this instance. - >>> r = TrickWorkflow.Run(sim_dir=os.path.join(this_trick, 'test/SIM_alloc_test'), input='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') - >>> r.input + >>> r = TrickWorkflow.Run(sim_dir=os.path.join(this_trick, 'test/SIM_alloc_test'), input_file='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') + >>> r.input_file 'RUN_test/input.py' Parameters ---------- sim_dir : str path to sim directory from top level of repository for this run - input : str + input_file : str literal argument to sim binary representing how this run is initiated including command-line arguments ex: RUN_test/input.py, or RUN_test/input.py --my-arg @@ -1037,16 +964,20 @@ class TrickWorkflow(WorkflowCommon): Optional string to execute immediately before sim run, e.g. env sourcing valgrind_flags : str If not None, use these flag and valgrind for this run + phase : int + Optional ordering phase, typically used to order sim runs when parallel + execution is not robust log_dir : str Directory in which log files will be written """ self.sim_dir = sim_dir # Path to sim directory wrt to top level of project for this run self.prerun_cmd = prerun_cmd # Optional string to execute in shell immediately before running (env) - self.input = input # Full RUN.../input.py --any-flags --as-well, relative to sim_dir + self.input_file = input_file # Full RUN.../input.py --any-flags --as-well, relative to sim_dir self.returns = returns # Expected exit code on success for this run self.valgrind_flags = valgrind_flags # If not None, this run is to be run in valgrind w/ these flags + self.phase = phase # Phase associated with this run self.log_dir = log_dir # Dir where all logged output will go - self.just_input = self.input.split(' ')[0] # Strip flags if any + self.just_input = self.input_file.split(' ')[0] # Strip flags if any # Derive Just the "RUN_something" part of run_dir_path self.just_run_dir = os.path.dirname(self.just_input) # Derive Path to run directory wrt to top level of project @@ -1061,7 +992,7 @@ class TrickWorkflow(WorkflowCommon): """ Given two data directories, add a new Comparison() instance to self.comparisons list - >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') + >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') >>> r.add_comparison('share/trick/trickops/tests/baselinedata/log_a.csv','share/trick/trickops/tests/testdata/log_a.csv') Parameters @@ -1078,7 +1009,7 @@ class TrickWorkflow(WorkflowCommon): """ Given an analysis command, add it as a post-run analysis for this run - >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') + >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') >>> r.add_analysis('echo analysis goes here') Parameters @@ -1088,17 +1019,58 @@ class TrickWorkflow(WorkflowCommon): """ if self.analysis: tprint("WARNING: Overwriting analysis definition for %s's %s" % (self.sim_dir, - self.input), 'DARK_YELLOW') + self.input_file), 'DARK_YELLOW') logfile = (os.path.join(self.log_dir, unixify_string(self.sim_dir)) - +'_'+ unixify_string(self.input) + '_analysis.txt') + +'_'+ unixify_string(self.input_file) + '_analysis.txt') self.analysis = Job(name=textwrap.shorten(cmd, width=90), command=self.prerun_cmd + " " +cmd, log_file=logfile, expected_exit_status=0) + def get_phase(self): + """ + Get integer phase member + + >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') + >>> r.get_phase() # Default is zero + 0 + + """ + return self.phase + + def set_phase( self, phase): + """ + Set the phase member variable. Phase is an integer in the range(TrickWorkflow.allowed_phase_range['min'], + TrickWorkflow.allowed_phase_range['max']) which can be used to order sim runs when a workflow cannot + support running sims in parallel. + + >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') + >>> r.set_phase(1) + >>> r.set_phase(-1001) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + RuntimeError: ERROR: set_phase() for test/SIM_alloc_test/RUN_test/input.py must be an integer between ... + + Parameters + ------- + phase : int + phase to change to + """ + # TODO: This is very similar to Sim.set_phase() in functionality but I didn't want to create + # a base class just to support this single instance of reducing code duplication + # -Jordan 12/2022 + if (not isinstance(phase, int) or phase < TrickWorkflow.allowed_phase_range['min'] or + phase > TrickWorkflow.allowed_phase_range['max']): + msg =("ERROR: set_phase() for %s/%s must be an integer between [%s, %s]" + % (self.sim_dir, self.input_file, TrickWorkflow.allowed_phase_range['min'], + TrickWorkflow.allowed_phase_range['max'])) + raise RuntimeError(msg) + else: + self.phase = phase + def compare( self): """ Execute all internal comparisons for this run - >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') + >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') >>> r.compare() # No comparisons means success a.k.a 0 False @@ -1121,7 +1093,7 @@ class TrickWorkflow(WorkflowCommon): """ tprint(indent + " %-20s %s" % (self.run_job._translate_status() if self.run_job else printer.colorstr('NOT RUN', 'DARK_YELLOW'), - self.input)) + self.input_file)) if self.comparisons: tprint(indent + " Run Comparisons:") for comparison in self.comparisons: @@ -1134,7 +1106,7 @@ class TrickWorkflow(WorkflowCommon): """ Create if necessary and Return the SingleRun() job instance - >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') + >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py', binary='S_main_Linux_x86_64.exe') >>> j = r.get_run_job() Returns @@ -1152,17 +1124,121 @@ class TrickWorkflow(WorkflowCommon): if self.valgrind_flags: cmd += ( "valgrind %s --log-file=%s " % (self.valgrind_flags, (os.path.join(self.log_dir, sim_name) +'_valgrind_' - + unixify_string(self.input) + '.valgrind') )) + + unixify_string(self.input_file) + '.valgrind') )) logfile += '_valgrind' name += 'Valgrind ' - logfile += "_" + unixify_string(self.input) + '.txt' - cmd += (" ./%s %s" % (self.binary, self.input)) - name += self.sim_dir + ' ' + self.input + logfile += "_" + unixify_string(self.input_file) + '.txt' + cmd += (" ./%s %s" % (self.binary, self.input_file)) + name += self.sim_dir + ' ' + self.input_file self.run_job = SingleRun(name=name, command=(cmd), expected_exit_status=self.returns, log_file=logfile) return (self.run_job) + def _get_range_list(self, pattern): + """ + Given a string matching the following pattern: + [-] - One set of enclosed in square braces + < - First integer must be less than second integer + & > 0 - Both integers must be positive + [001-999] - Leading zeros must be included and consistent + Return a list(range(int1, int2+1)) with leading zeros maintained in + string format + + >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_[01-04]/input.py', binary='S_main_Linux_x86_64.exe') + >>> rl =r._get_range_list('[0-4]') + >>> len(rl) + 5 + + Returns + ------- + list() + List of zero-padded integer range as strings + Raises + ------ + RuntimeError + If pattern is unrecognized or contains errors + """ + must_exist = ['[', ']', '-'] + if any([char not in pattern for char in must_exist]): + msg = ("ERROR: Pattern %s doesn't match expected syntax of \"[min-max]\"" % (pattern)) + raise RuntimeError(msg) + min, max = pattern.strip('[').strip(']').split('-') + if len(min) != len(max): + msg = ("ERROR: Pattern %s has inconsistent leading zeros." % (pattern)) + raise RuntimeError(msg) + leading_zeros = int(len(min)) + try: + min = int(min) + except ValueError as e: + msg = ("ERROR: Pattern %s minimum cannot be converted to integer. \n%s" % (pattern, e)) + raise RuntimeError(msg) + try: + max = int(max) + except ValueError as e: + msg = ("ERROR: Pattern %s maximum cannot be converted to integer. \n%s" % (pattern, e)) + raise RuntimeError(msg) + if min >= (max): + msg = ("ERROR: Pattern %s minimum must be less than maximum." % pattern) + raise RuntimeError(msg) + range_list = [] + for num in range(min, max+1): + range_list.append(str(num).zfill(leading_zeros)) + return (list(range_list)) + + + def multiply(self): + """ + If [from-to] notation used in input_file, return individual instances + of Run() for the expanded set. Otherwise, return self. The pattern + must meet the following criteria for input_file and optionally comparisons: + [-] - One set of enclosed in square braces + < - First integer must be less than second integer + & > 0 - Both integers must be positive + [001-999] - Leading zeros must be included and consistent + + >>> r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_[01-04]/input.py', binary='S_main_Linux_x86_64.exe') + >>> runs = r.multiply() + >>> len(runs) + 4 + + Returns + ------- + list of Run() + Resultant list of Run() instances after expansion + Raises + ------ + RuntimeError + If expansion of pattern is not possible + + TODO: Note that this approach requires a 1:1 mapping between the [min-max] notation + specified in the self.input_file and notation in that same run's comparisons. This + approach does not support range expansion for an input_file with no range pattern + but which contains a compare range pattern. If this is attempted by the user in + yaml file, the literal [min-max] is used instead, resulting in (missing) errors. + This use case could be supported by changing this method and potentially + Comparison.pattern_replace() so that [min-max] is expanded in comparisons differently + if self.input file does not contain the range notation. This is left unimplemented + for now since not only does this add significant complexity but it also would mean + two separate behaviors for the user with the same notation in yaml, which feels + like bad mojo. -Jordan 1/2023 + """ + rs = TrickWorkflow._find_range_string(self.input_file) + if rs is None: + return [self] + else: + range_list = self._get_range_list(rs) + multiplied_runs = [] + for i in range_list: + # replace the range string notation with the single run equivalent + acopy = copy.deepcopy(self) + acopy.input_file = self.input_file.replace(rs, i) + # Add comparisons back as multiplied set + for c in acopy.comparisons: + c.pattern_replace(expecting_pattern=rs, replace_with=i) + multiplied_runs.append(acopy) + return multiplied_runs + class Comparison(object): """ Management class for a logged data comparison @@ -1283,60 +1359,78 @@ class TrickWorkflow(WorkflowCommon): string += printer.colorstr(" (missing)", 'DARK_RED') tprint(string) -class SimulationJob(Job): + def pattern_replace(self, expecting_pattern, replace_with): + """ + Replaces expecting_pattern value in self.test_data and self.baseline_data + with replace_with value if found. Mostly intended to be used via + Run.multiply() + + >>> c = TrickWorkflow.Comparison('testdata/RUN_[00-03]/log_a.csv','baseline/RUN_[00-03]/log_a.csv') + >>> c.pattern_replace(expecting_pattern='[00-03]', replace_with='01') + >>> c.test_data + 'testdata/RUN_01/log_a.csv' + >>> c.baseline_data + 'baseline/RUN_01/log_a.csv' + + Raises + ------ + RuntimeError + If expected_pattern doesn't match found pattern in test_data or baseline_data + """ + rsb = TrickWorkflow._find_range_string(self.baseline_data) + rst = TrickWorkflow._find_range_string(self.test_data) + if rsb is None and rst is None: # If no patterns, do nothing + return + if (rsb and rsb != expecting_pattern) or (rst and rst != expecting_pattern): + msg = ("ERROR: [min-max] pattern from run (%s) must match pattern in run's" + " comparisons test (%s) and baseline (%s) sections, if specified." + % (expecting_pattern, rst, rsb)) + raise RuntimeError(msg) + if rsb: + self.baseline_data = self.baseline_data.replace(expecting_pattern, replace_with) + if rst: + self.test_data = self.test_data.replace(expecting_pattern, replace_with) + + +class SingleRun(Job): """ - A Job which is a Trick simulation. + A single trick simulation run Job. SingleRun's can optionally connect to the + trick variable server to get progress bar information. """ - - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def _create_variables(self): + def __init__(self, name, command, log_file, expected_exit_status=0, use_var_server=True): """ - Create and return a list of Variables to periodically sample - via the sim's variable server. + Initialize this instance. - Returns - ------- - [variable_server.Variable] - A list of variables to periodically sample. + Parameters + ---------- + name : str + The name of this job. + command : str + The command to execute when start() is called. + log_file : str + The file to which to write log information. """ - pass + self._use_var_server = use_var_server + self._connected = False + super().__init__(name=name, command=command, log_file=log_file, + expected_exit_status=expected_exit_status) - @abc.abstractmethod - def _connected_string(self): - """ - Get a string displaying status information. This method will - be only called after this instance has successfull connected - to the sim. + def set_use_var_server(self, value): + if (not isinstance(value, bool)): + msg =("ERROR: SingleRun.set_use_var_server() Requires a True/False value") + raise RuntimeError(msg) + else: + self._use_var_server = value - Returns - ------- - str - A string containing status information. - """ - pass - - @abc.abstractmethod - def _connected_bar(self): - """ - Get a progress bar representing the current progress. This - method will be only called after this instance has successfull - connected to the sim. - - Returns - ------- - str - A progress bar representing the current progress. - """ - pass + def get_use_var_server(self): + return (self._use_var_server) def start(self): """ Start this Simulation job. Attempts a connection to the sim variable server in another thread after calling the base class Job() start() method """ - super(SimulationJob, self).start() + super(SingleRun, self).start() self._connected = False # Finding a sim via PID can take several seconds. @@ -1380,20 +1474,25 @@ class SimulationJob(Job): return except socket.timeout: pass + # If a SingleRun job terminates before the thread can connect to the + # variable server, Trick's variable_server module throws an IOError + # with the message: The remote endpoint has closed the connection + except IOError as e: + pass - thread = threading.Thread(target=connect, - name='Looking for ' + self.name) - thread.daemon = True - thread.start() + if self._use_var_server: + thread = threading.Thread(target=connect, name='Looking for ' + self.name) + thread.daemon = True + thread.start() def get_status_string_line_count(self): - return super(SimulationJob, self).get_status_string_line_count() + 1 + return super(SingleRun, self).get_status_string_line_count() + 1 def _not_started_string(self): - return super(SimulationJob, self)._not_started_string() + '\n' + return super(SingleRun, self)._not_started_string() + '\n' def _running_string(self): - elapsed_time = super(SimulationJob, self)._running_string() + elapsed_time = super(SingleRun, self)._running_string() if self._connected: return (elapsed_time + self._connected_string() + '\n' + @@ -1403,13 +1502,13 @@ class SimulationJob(Job): create_progress_bar(0, 'Connecting')) def _success_string(self): - text = super(SimulationJob, self)._success_string() + text = super(SingleRun, self)._success_string() if self._connected: text += self._connected_string() return text + '\n' + self._success_progress_bar def _failed_string(self): - text = super(SimulationJob, self)._failed_string() + text = super(SingleRun, self)._failed_string() if self._connected: text += self._connected_string() return text + '\n' + self._failed_progress_bar @@ -1419,7 +1518,7 @@ class SimulationJob(Job): self._variable_server.close() except: pass - super(SimulationJob, self).die() + super(SingleRun, self).die() def __del__(self): try: @@ -1427,11 +1526,6 @@ class SimulationJob(Job): except: pass -class SingleRun(SimulationJob): - """ - A regular (not Monte Carlo) sim. - """ - def _create_variables(self): self._tics = variable_server.Variable( 'trick_sys.sched.time_tics', type_=float) @@ -1469,7 +1563,7 @@ class SingleRun(SimulationJob): Returns ------- str - A string for displaying the ratio of sime time to real time. + A string for displaying the ratio of sim time to real time. """ elapsed_time = ( (self._stop_time if self._stop_time else time.time()) @@ -1477,43 +1571,3 @@ class SingleRun(SimulationJob): return 'Average Speed: {0:4.1f} X'.format( self._tics.value / self._tics_per_sec.value / elapsed_time) -class MonteCarlo(SimulationJob): - """ - A Monte Carlo simulation. - - TODO: This is not currently tested or supported by the TrickWorkflow - management layer. However, this class has been used in a project in - the 2017-2020 timeframe and should still be functional. Reason for - not currently supporting it is mostly because the biggest users of - Trick monte-carlo have already moved away from it's internal master/ - slave architecture so there isn't much of a need. - """ - - def _create_variables(self): - self._total_runs = variable_server.Variable( - 'trick_mc.mc.actual_num_runs', type_=int) - self._finished_runs = variable_server.Variable( - 'trick_mc.mc.num_results', type_=int) - self._num_slaves = variable_server.Variable( - 'trick_mc.mc.num_slaves', type_=int) - return self._total_runs, self._finished_runs, self._num_slaves - - def _connected_string(self): - return ' {0} {1}'.format( - self._slave_count(), self._run_status()) - - def _connected_bar(self): - if self._total_runs.value > 0: - progress = (float(self._finished_runs.value) - / self._total_runs.value) - else: - progress = 0.0 - return create_progress_bar( - progress, '{0:.0f}%'.format(100 * progress)) - - def _slave_count(self): - return 'Slaves: {0:5d}'.format(self._num_slaves.value) - - def _run_status(self): - return 'Completed Runs: {0:>14}'.format('{0}/{1}'.format( - self._finished_runs.value, self._total_runs.value)) diff --git a/share/trick/trickops/TrickWorkflowYamlVerifier.py b/share/trick/trickops/TrickWorkflowYamlVerifier.py new file mode 100644 index 00000000..8e1b5626 --- /dev/null +++ b/share/trick/trickops/TrickWorkflowYamlVerifier.py @@ -0,0 +1,376 @@ +import os, sys, copy +import yaml # Provided by PyYAML +#from WorkflowCommon import tprint +from pydoc import locate +from exceptions import RequiredParamException, IncorrectYAMLException + +this_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)))) + +# This global is the result of hours of frustration and debugging. See comment at the top of +# TrickWorkflow.py for details +this_trick = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../..')) + +class TrickWorkflowYamlVerifier(): + """ + This class accepts a project YAML config file which contains a list of all + possible sims, runs, analyses, comparisons, etc and verifies that the content + meets the constraints that the TrickWorkflow expects. Specifically, self.config + is santitized to ensure that it is a dictionary matching the format below + + globals: + env: <-- optional literal string executed before all tests, e.g. env setup + SIM_abc: <-- required unique name for sim of interest, must start with SIM + path: <-- required SIM path relative to project top level + description: <-- optional description for this sim + labels: <-- optional list of labels for this sim, can be used to get sims + - model_x by label within the framework, or for any other project-defined + - verification purpose + build_args: <-- optional literal args passed to trick-CP during sim build + binary: <-- optional name of sim binary, defaults to S_main_{cpu}.exe + size: <-- optional estimated size of successful build output file in bytes + phase: <-- optional phase to be used for ordering builds if needed + parallel_safety: <-- strict won't allow multiple input files per RUN dir + runs: <-- optional dict of runs to be executed for this sim, where the + RUN_1/input.py --foo: dict keys are the literal arguments passed to the sim binary + RUN_2/input.py: and the dict values are other run-specific optional dictionaries + RUN_[10-20]/input.py: described in indented sections below. Zero-padded integer ranges + can specify a set of runs with continuous numbering using + [-] notation + returns: <---- optional exit code of this run upon completion (0-255). Defaults + to 0 + compare: <---- optional list of vs. comparison strings to be + - a vs. b compared after this run is complete. This is extensible in that + - d vs. e all non-list values are ignored and assumed to be used to define + - ... an alternate comparison method in a class extending this one + analyze: <-- optional arbitrary string to execute as job in bash shell from + project top level, for project-specific post-run analysis + phase: <-- optional phase to be used for ordering runs if needed + valgrind: <-- optional string of flags passed to valgrind for this run. + If missing or empty, this run will not use valgrind + non_sim_extension_example: + will: be ignored by TrickWorkflow parsing for derived classes to implement as they wish + + Any key matching the above naming schema is checked for type and value validity + Any key not matching the above naming schema is ignored purposefully to provide + users of this framework the ability use the YAML file for project-specific use-cases + """ + def __init__(self, config_file, quiet=False): + """ + Initialize this instance. + + >>> twyv = TrickWorkflowYamlVerifier(config_file=os.path.join(this_trick,"share/trick/trickops/tests/trick_sims.yml")) + >>> twyv.parsing_errors + [] + >>> len(twyv.config.keys()) + 57 + + """ + self.config_file=config_file + self.parsing_errors = [] # List of errors encountered during parsing + self.config = self._read_config(self.config_file) # Contains resulting Dict parsed by yaml.load + self.original_config= copy.deepcopy(self.config) # Keep a copy of original dict as parsed + # Contains requirements the format of key/value pairs inside self.config_file + self.yaml_requirements = self._read_config(os.path.join(this_dir,".yaml_requirements.yml")) + defaults = self.yaml_requirements.pop('content') # Only used to populate default values, can remove + self._populate_yaml_requirements(self.yaml_requirements, defaults) + + def _populate_yaml_requirements(self, thedict, defaults): + """ + Recursively traverse thedict, and overlay thedict values on top + of defaults dict values. This ensures all non-dict values exist + as defined in the content: section of .yaml_requirements.yml without + having to specify them individually in that file + """ + for k,v in thedict.items(): + if isinstance(v, dict): + subdicts = False + for j,u in v.items(): + if isinstance(u, dict): + subdicts = True + if subdicts: # If there are subdicts, recurse + self._populate_yaml_requirements(v, defaults) + else: # No subdicts, overlay values over defaults + thedict[k] = TrickWorkflowYamlVerifier.DictVerifier().merge( + defaults,thedict[k]) + + def _read_config(self, config_file): + """ + Read the yaml file into a dict and return it + + Parameters + ---------- + config_file : str + path to YAML config file to be read + + Returns + ------- + dict + dictionary representation of YAML content as parsed by yaml.safe_load() + + Raises + ------ + RuntimeError + If config_file cannot be read + """ + try: + with open(config_file) as file: + y = yaml.safe_load(file) + return y + except Exception as e: + msg = ("Unable to parse config file: %s\nERROR: %s\nSee" + " the TrickOps documentation for YAML file format expectations" + % (config_file,e)) + raise RuntimeError(msg) + + def get_parsing_errors(self): + """ + Return the list of errors encountered while parsing self.original_config + + Returns + ------- + list + List of error messages encountered during parsing + """ + return self.parsing_errors + + def verify(self): + """ + Verify the content of self.config. This is done by recursively checking content + in self.config against self.yaml_requirements using subclasses defined below. + Parsing errors are added as strings to the self.parsing_errors list + """ + if not self.config: # If entire config is empty + msg =("ERROR: Config file %s appears to be empty. Make sure file exists and is" + " valid YAML syntax." % (self.config_file)) + self.parsing_errors.append(msg) + raise RuntimeError(msg) + if not isinstance(self.config, dict): # If parser did not produce a dict + msg =("ERROR: Config file %s doesn't match expected dictionary format. " + " See the TrickOps documentation for YAML file format expectations." % + (self.config_file)) + self.parsing_errors.append(msg) + raise RuntimeError(msg) + + if 'globals' not in self.config: + self.config['globals'] = {'env': ''} + gv = self.GlobalVerifier( + self.config['globals'], self.yaml_requirements['globals']) + gv.verify() + self.config['globals'] = gv.get_verified_globals_dict() + self.parsing_errors += gv.get_parsing_errors() + for sim in list(self.config.keys()): + if not str(sim).startswith('SIM'): # Ignore everything not starting with SIM + continue + else: + sv = self.SimVerifier(sim_name=sim, sim_dict=self.config[sim], + req_dict=self.yaml_requirements) + try: + # Replace the entire dict for this sim with it's verified equivalent + sv.verify() + self.config[sim] = sv.get_verified_sim_dict() + # If not all required params given, remove this sim entry + except RequiredParamException as e: + del self.config[sim] + finally: + self.parsing_errors += sv.get_parsing_errors() + # Return a copy of sanitized config + return(dict(self.config)) + + # These inner classes provide helper methods for merging default expected content + # in dictionaries produced by YAML files with the actual content produced by + # YAML files. The merged content is then verified for correctness according to + # expected types, ranges, and other constraints. + class DictVerifier(): + def __init__(self): + self.errors = [] # collects all errors found in the YAML file parsing + def get_parsing_errors(self): + return(self.errors) + def merge(self, default_dict, authoritative_dict): + """ + Merges the content of authoritative_dict into default_dict, using values + from authoritative_dict over values in default_dict + """ + try: + merged ={**default_dict, **authoritative_dict} + except Exception as e: + msg=( "Unable to merge given dictionary with default dictionary." + "\n Given: %s" + "\n Default: %s" + "\nThis is usually caused by incorrect format in the YAML file. " + "See the TrickOps documentation for YAML file format expectations." + % (authoritative_dict, default_dict)) + raise IncorrectYAMLException(msg) + return (merged) + def defaults_only(self, thedict): + """ + Given a dict of yaml_requirements format, produce a dict containing + only {key: } + """ + newdict = {} + for k,v in thedict.items(): + newdict[k] = thedict[k]['default'] + return newdict + # Utility methods for ensuring given content matches expected constraints. + # These return True if check passes, False otherwise. Errors ar logged to + # self.errors + def expect_type(self, param, expected_type, where): + if param is None and expected_type is None: + return True + if not isinstance(param, expected_type): + self.errors.append(where + + "value \"%s\" expected to be type:%s, but got type:%s instead. Ignoring." % + ( param, expected_type, type(param))) + return False + return True + def expect_less_than_or_eq(self, param, maximum, where): + if (not param <= maximum): + self.errors.append(where + + "value \"%s\" expected to be <= %s. Ignoring." % + ( param, maximum)) + return False + return True + def expect_more_than_or_eq(self, param, minimum, where): + if (not param >= minimum): + self.errors.append(where + + "value \"%s\" expected to be >= %s. Ignoring." % + ( param, minimum)) + return False + return True + def expect_contains(self, param, contained_in, where): + if (param not in contained_in): + self.errors.append(where + + "value \"%s\" expected to be in subset %s. Ignoring." % + ( param, contained_in)) + return False + return True + def verify(self, user_dict, req_dict, identifier): + """ + Verifies the content of of user_dict matches the requirements of req_dict. + The content of req_dict comes from the .yaml_requirements file + The merge with defaults during __init__ guarantees all keys exist. + """ + for k,v in req_dict.items(): + before_num_errors = len(self.errors) + if v['required']: # Must be given by user + if not self.expect_type(user_dict[k], locate(v['type']), + where=("In %s, required param \"%s\" " % (identifier, k))): + raise RequiredParamException("%s must exist and be of type %s in %s" + % (k, locate(v['type']), identifier)) + elif v['default'] is None and user_dict[k] is None: # Can be given but have None type + # TODO: we may be able to set user_dict[k] = locate(v['type']) here + # which would make runs: and labels: lists, but need to see if that + # has downstream effects. If this works we can remove manual checking + # for None in the derived verifier classes + pass + # Could be another type used by derived class, so don't check anything + elif v['overridable'] == 1: + pass + else: # Check optional params (majority) + # Checking possible values only makes sense if we got the right type + if self.expect_type(user_dict[k], locate(v['type']), + where=("In %s, param \"%s\" " % (identifier, k))): + if v['min'] is not None: + self.expect_more_than_or_eq(int(user_dict[k]), int(v['min']), + where=("In %s, param \"%s\" " % (identifier, k))) + if v['max'] is not None: + self.expect_less_than_or_eq(int(user_dict[k]), int(v['max']), + where=("In %s, param \"%s\" " % (identifier, k))) + if v['contains'] is not None: + self.expect_contains(user_dict[k], v['contains'], + where=("In %s, param \"%s\" " % (identifier, k))) + # If we encountered any errors, set param to default value + if len(self.errors) > before_num_errors: + user_dict[k] = v['default'] + + class GlobalVerifier(DictVerifier): + def __init__(self, globals_dict, globals_req_dict): + super().__init__() + self.globals_req_dict = globals_req_dict + self.defaults = self.defaults_only(globals_req_dict) + self.globals_dict = self.merge(self.defaults, globals_dict) + def verify(self): + """ + Verifies the content of of self.globals_dict. The merge with defaults + during __init__ guarantees all keys exist. + """ + TrickWorkflowYamlVerifier.DictVerifier.verify(self, user_dict=self.globals_dict, + req_dict=self.globals_req_dict, identifier='globals') + def get_verified_globals_dict(self): + return (dict(self.globals_dict)) # Return a copy + + class SimVerifier(DictVerifier): + def __init__(self, sim_name, sim_dict, req_dict): + super().__init__() + sim_dict = {} if not isinstance(sim_dict, dict) else sim_dict + self.req_dict = req_dict # TODO every derived class needs this, move it to the base class + self.defaults = self.defaults_only(self.req_dict['sim']) + self.defaults['name'] = sim_name + # TODO what if SIM: dict is empty? + self.sim_dict = self.merge(self.defaults, sim_dict) + def verify(self): + TrickWorkflowYamlVerifier.DictVerifier.verify(self, user_dict=self.sim_dict, + req_dict=self.req_dict['sim'], identifier=self.sim_dict['name']) + if self.sim_dict['labels'] is None: + self.sim_dict['labels'] = [] # If not given, make it an empty list + # Handle the edge case where labels is a list, but a list of not just strings + if any([ not isinstance(label, str) for label in self.sim_dict['labels']]): + self.errors.append("In %s, labels \"%s\" expected to be list of strings. Ignoring." % + ( self.sim_dict['name'], self.sim_dict['labels'])) + self.sim_dict['labels'] = [] + if self.sim_dict['runs'] is None: + self.sim_dict['runs'] = {} # If not given, make it an empty dict + for run in list(self.sim_dict['runs']): + rv = (TrickWorkflowYamlVerifier.RunVerifier( run, + self.sim_dict['runs'][run], self.req_dict)) + try: + rv.verify() + self.sim_dict['runs'][run] = rv.get_verified_run_dict() + # If not all required params given, remove this run entry. I believe this + # to be unreachable code, but added anyhow for safety -Jordan 12/2022 + except RequiredParamException as e: + del self.sim_dict['runs'][run] + finally: + # Pass errors up to sim level + self.errors += rv.get_parsing_errors() + + def get_verified_sim_dict(self): + return (dict(self.sim_dict)) # Return a copy + + class RunVerifier(DictVerifier): + def __init__(self, run_name, run_dict, req_dict): + super().__init__() + self.req_dict = req_dict + self.defaults = self.defaults_only(self.req_dict['run']) + self.defaults['input'] = run_name + # If run_dict is None, use defaults, otherwise merge content into defaults + self.run_dict = (self.merge(self.defaults, run_dict) if run_dict else dict(self.defaults)) + def verify(self): + """ + Verifies the content of of self.run_dict. The merge with defaults + during __init__ guarantees all keys exist. + """ + # Verify the run + TrickWorkflowYamlVerifier.DictVerifier.verify(self, user_dict=self.run_dict, + req_dict=self.req_dict['run'], identifier=self.run_dict['input']) + # Verify the analyze: section of the run, setting to None if invalid + if (self.run_dict['analyze'] is not None and + not self.expect_type(self.run_dict['analyze'], str, + where=("In %s, analyze section " % (self.run_dict['input'])))): + self.run_dict['analyze'] = None + if self.run_dict['compare'] is None: + self.run_dict['compare'] = [] # If not given, make it an empty list + # Verify the compare list of the run, removing any that aren't correct + # The check for list allows all other non-list types in the yaml file, + # allowing groups to define their own comparison methodology + if isinstance(self.run_dict['compare'], list): + for compare in list(self.run_dict['compare']): + if (not self.expect_type(compare, str, + where=("In %s, compare section " % (self.run_dict['input']))) or + not self.expect_contains(" vs. ", compare, + where=("In %s, compare section " % (self.run_dict['input']))) + ): + self.run_dict['compare'].remove(compare) + + def get_verified_run_dict(self): + return (dict(self.run_dict)) # Return a copy + diff --git a/share/trick/trickops/WorkflowCommon.py b/share/trick/trickops/WorkflowCommon.py index c49bbfd7..0846313b 100644 --- a/share/trick/trickops/WorkflowCommon.py +++ b/share/trick/trickops/WorkflowCommon.py @@ -250,13 +250,15 @@ class Job(object): """ Start this job. """ - logging.debug('Executing command: ' + self._command) - self._start_time = time.time() - self._log_file = open(self.log_file, 'w') - self._process = subprocess.Popen( - self._command, stdout=self._log_file, stderr=self._log_file, - stdin=open(os.devnull, 'r'), shell=True, preexec_fn=os.setsid, - close_fds=True) + # Guard against multiple starts + if self.get_status != self.Status.RUNNING: + logging.debug('Executing command: ' + self._command) + self._start_time = time.time() + self._log_file = open(self.log_file, 'w') + self._process = subprocess.Popen( + self._command, stdout=self._log_file, stderr=self._log_file, + stdin=open(os.devnull, 'r'), shell=True, preexec_fn=os.setsid, + close_fds=True) def get_status(self): """ @@ -592,8 +594,8 @@ class WorkflowCommon: """ if not os.environ.get('TERM') and not self.quiet: tprint( - 'The TERM environment variable must be set when the command\n' - 'line option --quiet is not used. This is usually set by one\n' + 'The TERM environment variable must be set when\n' + 'TrickWorkflow.quiet is False. This is usually set by one\n' "of the shell's configuration files (.profile, .cshrc, etc).\n" 'However, if this was executed via a non-interactive,\n' "non-login shell (for instance: ssh ''), it\n" diff --git a/share/trick/trickops/exceptions.py b/share/trick/trickops/exceptions.py new file mode 100644 index 00000000..355b8f61 --- /dev/null +++ b/share/trick/trickops/exceptions.py @@ -0,0 +1,7 @@ +# define Python user-defined exceptions +class RequiredParamException(Exception): + "Raised when a parameter from a YAML file must exist but doesn't" + pass +class IncorrectYAMLException(Exception): + "Raised when a YAML file does not meet expected format" + pass diff --git a/share/trick/trickops/requirements.txt b/share/trick/trickops/requirements.txt index 6a4f9f2d..c74562aa 100644 --- a/share/trick/trickops/requirements.txt +++ b/share/trick/trickops/requirements.txt @@ -1,2 +1,2 @@ -PyYAML +PyYAML # Needed by TrickWorkflowYamlVerifier.py psutil diff --git a/share/trick/trickops/send_hs.py b/share/trick/trickops/send_hs.py new file mode 100644 index 00000000..8404769f --- /dev/null +++ b/share/trick/trickops/send_hs.py @@ -0,0 +1,67 @@ +import re, os +import pdb + +class send_hs(object): + """ + Reads a file containing the send_hs output and returns a send_hs + object containing the values from that output + """ + def __init__(self, hs_file): + self.hs_file = hs_file + self.actual_init_time = None + self.actual_elapsed_time = None + self.start_time = None + self.stop_time = None + self.elapsed_time = None + self.actual_cpu_time_used = None + self.sim_cpu_time = None + self.init_cpu_time = None + self.parse() + def parse(self): + f = open(self.hs_file, 'r') + lines = f.readlines() + for line in lines: + self.actual_init_time = self.attempt_hs_match('ACTUAL INIT TIME',self.actual_init_time, line) + self.actual_elapsed_time = self.attempt_hs_match('ACTUAL ELAPSED TIME',self.actual_elapsed_time, line) + self.start_time = self.attempt_hs_match('SIMULATION START TIME',self.start_time, line) + self.stop_time = self.attempt_hs_match('SIMULATION STOP TIME',self.stop_time, line) + self.elapsed_time = self.attempt_hs_match('SIMULATION ELAPSED TIME',self.elapsed_time, line) + self.actual_cpu_time_used = self.attempt_hs_match('ACTUAL CPU TIME USED',self.actual_cpu_time_used, line) + self.sim_cpu_time = self.attempt_hs_match('SIMULATION / CPU TIME',self.sim_cpu_time, line) + self.init_cpu_time = self.attempt_hs_match('INITIALIZATION CPU TIME',self.init_cpu_time, line) + # TODO add capture of blade and DIAGNOSTIC: Reached termination time as success criteria + + def attempt_hs_match(self, name, var, text): + """ + name: pattern to match (e.g. SIMULATION START TIME) + var: variable to assign value if match found + text: text to search for pattern + returns: var if not found, found value if found + """ + m = re.match(name+': +([-]?[0-9]*\.?[0-9]+)', text.strip()) + if m: + return(float(m.group(1))) + return(var) + + def get(self,name): + """ + Get a value by the name that appears in the send_hs message + """ + if 'ACTUAL INIT TIME' in name: + return self.actual_init_time + if 'ACTUAL ELAPSED TIME' in name: + return self.actual_elapsed_time + if 'SIMULATION START TIME' in name: + return self.start_time + if 'SIMULATION STOP TIME' in name: + return self.stop_time + if 'SIMULATION ELAPSED TIME' in name: + return self.elapsed_time + if 'ACTUAL CPU TIME USED' in name: + return self.actual_cpu_time_used + if 'SIMULATION / CPU TIME' in name: + return self.sim_cpu_time + if 'INITIALIZATION CPU TIME' in name: + return self.init_cpu_time + else: + return None diff --git a/share/trick/trickops/tests/empty1.yml b/share/trick/trickops/tests/empty1.yml new file mode 100644 index 00000000..6ead163f --- /dev/null +++ b/share/trick/trickops/tests/empty1.yml @@ -0,0 +1,4 @@ +# A YAML file which doesn't specify a single valid SIM +SIM_ball_L1: + path: 30 + diff --git a/share/trick/trickops/tests/errors_nonfatal.yml b/share/trick/trickops/tests/errors_nonfatal.yml index 41775640..be09d3ce 100644 --- a/share/trick/trickops/tests/errors_nonfatal.yml +++ b/share/trick/trickops/tests/errors_nonfatal.yml @@ -7,25 +7,24 @@ globals: extension_example: should: be ignored by this framework -# This sim exists, but has duplicate run entries which is an error +# This sim exists, but has duplicate run entries which means the last +# one is the only one respected SIM_ball_L1: path: trick_sims/Ball/SIM_ball_L1 - size: 6000 + size: 0 + phase: 1001 # Out of bounds runs: RUN_test/input.py: RUN_test/input.py: - valgrind: - runs: - RUN_test/input.py: # Wrong type, dict - - RUN_test/input.py: # Should be list of str not list of dict + valgrind: -# This sim exists, but its valgrind section is empty and its runs have problems +# This sim exists, but runs have problems SIM_alloc_test: path: test/SIM_alloc_test - valgrind: runs: RUN_buddy/input.py: # Doesn't exist RUN_test/input.py: # Does exist + phase: -1001 # Out of bounds extension: foobar # Should be retained in self.config but not used compare: - RUN_test/log.trk: # Should be list of str not list of dict @@ -42,8 +41,6 @@ SIM_events: runs: - RUN_test/input.py: # List of dicts, should be ignored - RUN_test/unit_test.py: # List of dicts, should be ignored - valgrind: # Empty so should be ignored - runs: # Sim exists, but runs have bad return values SIM_threads: diff --git a/share/trick/trickops/tests/run_tests.py b/share/trick/trickops/tests/run_tests.py index 02b94b2d..35f17813 100755 --- a/share/trick/trickops/tests/run_tests.py +++ b/share/trick/trickops/tests/run_tests.py @@ -33,7 +33,8 @@ def run_tests(args): ut_results = runner.run(overall_suite) # Run all doc tests by eating our own dogfood - doctest_files = ['TrickWorkflow.py', 'WorkflowCommon.py'] + doctest_files = ['TrickWorkflow.py', 'WorkflowCommon.py', 'TrickWorkflowYamlVerifier.py', + 'MonteCarloGenerationHelper.py'] wc = WorkflowCommon(this_dir, quiet=True) jobs = [] log_prepend = '_doctest_log.txt' diff --git a/share/trick/trickops/tests/test.py b/share/trick/trickops/tests/test.py index f0fc3baa..8568bcc8 100644 --- a/share/trick/trickops/tests/test.py +++ b/share/trick/trickops/tests/test.py @@ -6,20 +6,26 @@ import os, sys, pdb import unittest import ut_WorkflowCommon +import ut_TrickWorkflowYamlVerifier import ut_TrickWorkflow +import ut_MonteCarloGenerationHelper # Define load_tests function for dynamic loading using Nose2 def load_tests(*args): passed_args = locals() suite = unittest.TestSuite() + suite.addTests(ut_TrickWorkflowYamlVerifier.suite()) suite.addTests(ut_TrickWorkflow.suite()) suite.addTests(ut_WorkflowCommon.suite()) + suite.addTests(ut_MonteCarloGenerationHelper.suite()) return suite # Local module level execution only if __name__ == '__main__': suites = unittest.TestSuite() + suites.addTests(ut_TrickWorkflowYamlVerifier.suite()) suites.addTests(ut_TrickWorkflow.suite()) suites.addTests(ut_WorkflowCommon.suite()) + suites.addTests(ut_MonteCarloGenerationHelper.suite()) unittest.TextTestRunner(verbosity=2).run(suites) diff --git a/share/trick/trickops/tests/trick_sims.yml b/share/trick/trickops/tests/trick_sims.yml index 74aa27a7..f87f5ed2 100644 --- a/share/trick/trickops/tests/trick_sims.yml +++ b/share/trick/trickops/tests/trick_sims.yml @@ -5,19 +5,18 @@ SIM_ball_L1: path: trick_sims/Ball/SIM_ball_L1 size: 6000 + phase: 0 runs: RUN_test/input.py: analyze: echo hi compare: - share/trick/trickops/tests/testdata/log_a.csv vs. share/trick/trickops/tests/baselinedata/log_a.csv - valgrind: - flags: -v - runs: - - RUN_test/input.py SIM_alloc_test: path: test/SIM_alloc_test runs: RUN_test/input.py: + analyze: echo hi there + valgrind: -v SIM_default_member_initializer: path: test/SIM_default_member_initializer SIM_demo_inputfile: @@ -34,7 +33,9 @@ SIM_demo_sdefine: - unit_test runs: RUN_test/input.py: + phase: 0 RUN_test/unit_test.py: + phase: 1 SIM_dynamic_sim_object: path: test/SIM_dynamic_sim_object runs: @@ -83,7 +84,7 @@ SIM_segments: RUN_test/input.py: SIM_stls: binary: 'T_main_{cpu}_test.exe' - build_command: "trick-CP -t" + build_args: "-t" path: test/SIM_stls labels: - unit_test @@ -157,21 +158,35 @@ SIM_threads: - unit_test runs: RUN_test/sched.py: + phase: 1 + analyze: echo phase 1 analysis RUN_test/amf.py: + phase: 2 + analyze: echo phase 2 analysis RUN_test/async.py: + phase: 3 + analyze: echo phase 3 analysis RUN_test/unit_test.py: + phase: 4 + analyze: echo phase 4 analysis SIM_threads_simple: path: test/SIM_threads_simple runs: RUN_test/input.py: + phase: 1 + analyze: echo phase 1 analysis RUN_test/sched.py: + phase: 2 + analyze: echo phase 2 analysis RUN_test/async.py: + phase: 3 SIM_trickcomm: path: test/SIM_trickcomm runs: RUN_test/input.py: SIM_ball_L2: path: trick_sims/Ball/SIM_ball_L2 + runs: SIM_ball_L3: path: trick_sims/Ball/SIM_ball_L3 SIM_amoeba: @@ -187,6 +202,7 @@ SIM_cannon_jet: SIM_cannon_numeric: path: trick_sims/Cannon/SIM_cannon_numeric SIM_monte: + phase: -1 path: trick_sims/Cannon/SIM_monte SIM_ode_ball: path: trick_sims/ODE/SIM_ode_ball @@ -213,10 +229,12 @@ SIM_sat2d: SIM_satellite: path: trick_sims/SIM_satellite SIM_sun: + phase: 72 path: trick_sims/SIM_sun SIM_wheelbot: path: trick_sims/SIM_wheelbot SIM_ball_L1_er7_utils: + phase: -88 path: trick_source/er7_utils/sims/SIM_ball_L1 SIM_grav: path: trick_source/er7_utils/sims/SIM_grav diff --git a/share/trick/trickops/tests/type_errors.yml b/share/trick/trickops/tests/type_errors.yml new file mode 100644 index 00000000..b80e1686 --- /dev/null +++ b/share/trick/trickops/tests/type_errors.yml @@ -0,0 +1,60 @@ +# This is a file containing trick sims from this repository to be used +# for unit testing the trickops module. Specifically this file contains +# values for parameters with the wrong type +SIM_type_errors1: + path: path/to/SIM + build_args: 1 # Should be string not int + parallel_safety: 2 # Should be string not int + description: 10 # Should be a string not int + binary: 20 # Should be a string not int + size: 'string' # Should be an int not string + phase: 'string' # Should be an int not string + labels: 'hi there' # Should be a list not string + runs: + RUN_test1/input.py: + returns: 256 # Should be between 0-255 + analyze: 35 # Should be a string not int + compare: + - 40 # Should be a string not int + RUN_test2/input.py: + returns: -1 # Should be between 0-255 + valgrind: 79 # Should be a string not int + + +SIM_type_errors2: + path: path/to/SIM + build_args: + - this shouldnt + - be a list + binary: + - this shouldnt + - be a list + build_args: + - this shouldnt + - be a list + parallel_safety: + - this shouldnt + - be a list + description: + - this shouldnt + - be a list + binary: + - this shouldnt + - be a list + size: + - this shouldnt + - be a list + phase: + - this shouldnt + - be a list + +SIM_range_errors: + path: path/to/SIM + parallel_safety: 'bad_value' + size: -10 + phase: -1000 + +# Bad path must be tested separate, since it halts processing +# of other parameters immediately +SIM_bad_path: + path: 30 # Should be a str not int diff --git a/share/trick/trickops/tests/ut_MonteCarloGenerationHelper.py b/share/trick/trickops/tests/ut_MonteCarloGenerationHelper.py new file mode 100644 index 00000000..ee248002 --- /dev/null +++ b/share/trick/trickops/tests/ut_MonteCarloGenerationHelper.py @@ -0,0 +1,204 @@ +import os, sys, glob +import unittest, shutil +import pdb +from testconfig import this_trick, tests_dir +import TrickWorkflow +from MonteCarloGenerationHelper import * + +def suite(): + """Create test suite from test cases here and return""" + suites = [] + suites.append(unittest.TestLoader().loadTestsFromTestCase(MCGNominalTestCase)) + suites.append(unittest.TestLoader().loadTestsFromTestCase(MCGAllArgsTestCase)) + suites.append(unittest.TestLoader().loadTestsFromTestCase(MCGInvalidGenerationTestCase)) + suites.append(unittest.TestLoader().loadTestsFromTestCase(MCGInvalidInputsTestCase)) + return (suites) + +class MCGNominalTestCase(unittest.TestCase): + + def setUp(self): + # Nominal no-error when parsing the trick-sims config file scenario + self.instance = MonteCarloGenerationHelper(sim_path=os.path.join(this_trick, 'test/SIM_mc_generation'), + input_path='RUN_nominal/input_a.py') + # Create expected generated output directories and files this tests needs. Note this + # matches what SIM_mc_generation produces when run and should work whether output + # from that sim's tests are lingering in the workspace or not + self.monte_dir = os.path.join(this_trick, 'test/SIM_mc_generation', 'MONTE_RUN_nominal') + os.makedirs(os.path.join(self.monte_dir, 'RUN_000'), exist_ok=True) + os.makedirs(os.path.join(self.monte_dir, 'RUN_001'), exist_ok=True) + for dir in ['RUN_000', 'RUN_001']: + if not os.path.isfile(os.path.join(self.monte_dir, dir, 'monte_input_a.py')): + with open(os.path.join(self.monte_dir, dir, 'monte_input_a.py'), 'w') as fp: + pass + + def tearDown(self): + if self.instance: + del self.instance + self.instance = None + + def test_init(self): + self.assertEqual(self.instance.S_main_name, "S_main*.exe") + self.assertEqual(self.instance.sim_path, os.path.join(this_trick,"test/SIM_mc_generation")) + self.assertEqual(self.instance.input_path, "RUN_nominal/input_a.py") + self.assertEqual(self.instance.sim_args_gen, '') + self.assertEqual(self.instance.env, '') + self.assertEqual(self.instance.generated_input_files, []) + self.assertTrue(('trick/test/SIM_mc_generation && ./S_main*.exe RUN_nominal/input_a.py') + in self.instance.generation_job._command) + self.assertTrue(os.path.join(self.instance.sim_path, os.path.dirname(self.instance.input_path), + 'MCG_generation_output.txt') in self.instance.generation_job.log_file) + + def test_get_generation_job(self): + j = self.instance.get_generation_job() + self.assertTrue(isinstance(j, TrickWorkflow.SingleRun)) + + def test_get_generated_input_files(self): + gif = self.instance.get_generated_input_files(os.path.join(self.instance.sim_path, 'MONTE_RUN_nominal')) + self.assertTrue('trick/test/SIM_mc_generation/MONTE_RUN_nominal/RUN_000/monte_input_a.py' in gif[0]) + self.assertTrue('trick/test/SIM_mc_generation/MONTE_RUN_nominal/RUN_001/monte_input_a.py' in gif[1]) + # Given the relative path from sim + os.chdir(self.instance.sim_path) + gif = self.instance.get_generated_input_files('MONTE_RUN_nominal') + + # Given invalid path (relative to cwd()) + with self.assertRaises(RuntimeError): + gif = self.instance.get_generated_input_files('fake/path/MONTE_RUN_nominal') + + def test_get_zero_padding(self): + zp = self.instance.get_zero_padding(monte_dir=self.monte_dir) + self.assertEqual(zp, 3) + # Without any args it should use interal self.generated_input_files + zp = self.instance.get_zero_padding() + self.assertEqual(zp, 3) + + def test_get_sbatch_job(self): + sbj = self.instance.get_sbatch_job(self.monte_dir) + self.assertTrue('--array 0-001 ' in sbj._command) + self.assertTrue(self.instance.S_main_name in sbj._command) + self.assertTrue(self.monte_dir in sbj._command) + self.assertTrue('RUN_${RUN_NUM}' in sbj._command) + + def test_get_sbatch_job_with_passthrough_args(self): + sbj = self.instance.get_sbatch_job(self.monte_dir, hpc_passthrough_args="--array 50-80 --wait") + self.assertTrue('--array 50-80 ' in sbj._command) + self.assertTrue('--wait ' in sbj._command) + sbj = self.instance.get_sbatch_job(self.monte_dir, hpc_passthrough_args="--array 1-100:10") + self.assertTrue('--array 1-100:10 ' in sbj._command) + self.assertTrue('--wait ' not in sbj._command) + sbj = self.instance.get_sbatch_job(self.monte_dir, hpc_passthrough_args="--begin 16:00") + self.assertTrue('--begin 16:00 ' in sbj._command) + self.assertTrue('--array 0-001 ' in sbj._command) + sbj = self.instance.get_sbatch_job(self.monte_dir, sim_args="--logging=verbose") + self.assertTrue('--logging=verbose' in sbj._command) + + def test_get_generated_run_jobs(self): + jobs = self.instance.get_generated_run_jobs(self.monte_dir) + self.assertEqual(len(jobs), 2) + self.assertTrue('MONTE_RUN_nominal/RUN_000/monte_input_a.py' in jobs[0]._command) + self.assertTrue('MONTE_RUN_nominal/RUN_001/monte_input_a.py' in jobs[1]._command) + jobs = self.instance.get_generated_run_jobs(self.monte_dir, sim_args='--logging=off') + self.assertTrue('--logging=off' in jobs[0]._command) + self.assertTrue('--logging=off' in jobs[1]._command) + +class MCGAllArgsTestCase(unittest.TestCase): + + def setUp(self): + # Nominal no-error when parsing the trick-sims config file scenario + self.instance = MonteCarloGenerationHelper(sim_path=os.path.join(this_trick, 'test/SIM_mc_generation'), + input_path='RUN_nominal/input_a.py', env='source bashrc;', sim_args_gen='--monte-carlo --runs=32') + + def tearDown(self): + if self.instance: + del self.instance + self.instance = None + + def test_init(self): + self.assertTrue( self.instance.generation_job._command.startswith('source bashrc; ')) + self.assertTrue( self.instance.generation_job._command.endswith('--monte-carlo --runs=32')) + +class MCGInvalidGenerationTestCase(unittest.TestCase): + + def setUp(self): + # Nominal no-error when parsing the trick-sims config file scenario + self.instance = MonteCarloGenerationHelper(sim_path=os.path.join(this_trick, 'test/SIM_mc_generation'), + input_path='RUN_nominal/input_a.py') + # Create incorrect generated output directories and files this tests needs. + # This simulates a generation error and tests that the MonteCarloGenerationHelper class can + # detect it + monte_dir = os.path.join(this_trick, 'test/SIM_mc_generation', 'MONTE_RUN_missing_input_files') + os.makedirs(os.path.join(monte_dir, 'RUN_000'), exist_ok=True) + os.makedirs(os.path.join(monte_dir, 'RUN_001'), exist_ok=True) + monte_dir = os.path.join(this_trick, 'test/SIM_mc_generation', 'MONTE_RUN_mismatch_input_files') + os.makedirs(os.path.join(monte_dir, 'RUN_000'), exist_ok=True) + os.makedirs(os.path.join(monte_dir, 'RUN_001'), exist_ok=True) + with open(os.path.join(monte_dir, 'RUN_000', 'monte_input_a.py'), 'w') as fp: + pass + monte_dir = os.path.join(this_trick, 'test/SIM_mc_generation', 'MONTE_RUN_incorrect_input_files') + os.makedirs(os.path.join(monte_dir, 'RUN_000'), exist_ok=True) + os.makedirs(os.path.join(monte_dir, 'RUN_001'), exist_ok=True) + for dir in ['RUN_000', 'RUN_001']: + if not os.path.isfile(os.path.join(monte_dir, dir, 'input_a.py')): + with open(os.path.join(monte_dir, dir, 'input_a.py'), 'w') as fp: + pass + monte_dir = os.path.join(this_trick, 'test/SIM_mc_generation', 'MONTE_RUN_completely_bonkers') + os.makedirs(os.path.join(monte_dir, 'RUN_makes_no'), exist_ok=True) + os.makedirs(os.path.join(monte_dir, 'RUN_sense'), exist_ok=True) + for dir in ['RUN_makes_no', 'RUN_sense']: + if not os.path.isfile(os.path.join(monte_dir, dir, 'input_a.py')): + with open(os.path.join(monte_dir, dir, 'input_a.py'), 'w') as fp: + pass + + def tearDown(self): + if self.instance: + del self.instance + self.instance = None + # Remove the fake directory tree created in this test + shutil.rmtree(os.path.join(this_trick, 'test/SIM_mc_generation', 'MONTE_RUN_missing_input_files')) + shutil.rmtree(os.path.join(this_trick, 'test/SIM_mc_generation', 'MONTE_RUN_mismatch_input_files')) + shutil.rmtree(os.path.join(this_trick, 'test/SIM_mc_generation', 'MONTE_RUN_incorrect_input_files')) + shutil.rmtree(os.path.join(this_trick, 'test/SIM_mc_generation', 'MONTE_RUN_completely_bonkers')) + + def test_get_generated_input_files(self): + # Just a warning + gif = self.instance.get_generated_input_files( + os.path.join(self.instance.sim_path, 'MONTE_RUN_mismatch_input_files')) + with self.assertRaises(RuntimeError): + gif = self.instance.get_generated_input_files( + os.path.join(self.instance.sim_path, 'MONTE_RUN_missing_input_files')) + with self.assertRaises(RuntimeError): + gif = self.instance.get_generated_input_files( + os.path.join(self.instance.sim_path, 'MONTE_RUN_incorrect_input_files')) + with self.assertRaises(RuntimeError): + gif = self.instance.get_generated_input_files( + os.path.join(self.instance.sim_path, 'MONTE_RUN_completely_bonkers')) + + def test_get_zero_padding(self): + # Without monte_dir and get_generated_input files never having been run, this will fail + with self.assertRaises(RuntimeError): + zp = self.instance.get_zero_padding() + # Nothing in monte_dir, will fail + with self.assertRaises(RuntimeError): + zp = self.instance.get_zero_padding(monte_dir=os.path.join(this_trick, 'test/SIM_mc_generation', 'MONTE_RUN_missing_input_files')) + +class MCGInvalidInputsTestCase(unittest.TestCase): + + def setUp(self): + # Invalid sim_path + with self.assertRaises(RuntimeError): + self.instance = MonteCarloGenerationHelper(sim_path=os.path.join(this_trick, 'test/SIM_notexist'), + input_path='RUN_nominal/input_a.py') + # Invalid input_path + with self.assertRaises(RuntimeError): + self.instance = MonteCarloGenerationHelper(sim_path=os.path.join(this_trick, 'test/SIM_mc_generation'), + input_path='RUN_nominal/input_x.py',) + # Invalid sim_args_gen + with self.assertRaises(TypeError): + self.instance = MonteCarloGenerationHelper(sim_path=os.path.join(this_trick, 'test/SIM_mc_generation'), + input_path='RUN_nominal/input_a.py', sim_args_gen=3) + # Invalid env + with self.assertRaises(TypeError): + self.instance = MonteCarloGenerationHelper(sim_path=os.path.join(this_trick, 'test/SIM_mc_generation'), + input_path='RUN_nominal/input_a.py', env=3) + + def test_init(self): + pass diff --git a/share/trick/trickops/tests/ut_TrickWorkflow.py b/share/trick/trickops/tests/ut_TrickWorkflow.py index aef72b5e..8f08a0fd 100644 --- a/share/trick/trickops/tests/ut_TrickWorkflow.py +++ b/share/trick/trickops/tests/ut_TrickWorkflow.py @@ -5,11 +5,36 @@ from testconfig import this_trick, tests_dir from TrickWorkflow import * def suite(): - """Create test suite from TrickWorkflowTestCase unit test class and return""" - return unittest.TestLoader().loadTestsFromTestCase(TrickWorkflowTestCase) + """Create test suite from test cases here and return""" + suites = [] + suites.append(unittest.TestLoader().loadTestsFromTestCase(TrickWorkflowTestCase)) + suites.append(unittest.TestLoader().loadTestsFromTestCase(TrickWorkflowSingleRunTestCase)) + return (suites) + +class TrickWorkflowSingleRunTestCase(unittest.TestCase): + def setUp(self): + self.instance= SingleRun(name='testname', command='echo hi', + log_file='/tmp/WorkflowCommonTestCase_hi.txt') + + def tearDown(self): + del self.instance + self.instance = None + + def test_SingleRun_Nominal(self): + self.assertEqual(self.instance.name, 'testname') + self.assertEqual(self.instance._command, 'echo hi') + self.assertEqual(self.instance.log_file, '/tmp/WorkflowCommonTestCase_hi.txt') + self.assertEqual(self.instance._log_file, None) + self.assertEqual(self.instance._expected_exit_status, 0) + self.assertEqual(self.instance.get_use_var_server(), True) + # Test the setter for var server interaction + self.instance.set_use_var_server(False) + self.assertEqual(self.instance.get_use_var_server(), False) class TrickWorkflowTestCase(unittest.TestCase): + # TODO: not all jobs even use this setUp, probably should split the ones + # out that need to load a different file or no file at all (static tests) def setUp(self): # Nominal no-error when parsing the trick-sims config file scenario self.instance = TrickWorkflow(project_top_level=this_trick, log_dir='/tmp/', @@ -22,9 +47,9 @@ class TrickWorkflowTestCase(unittest.TestCase): del self.instance self.instance = None - def setUpWithEmptyConfig(self): + def setUpWithEmptyConfig(self, file="empty.yml"): self.instance = TrickWorkflow(project_top_level=this_trick, log_dir='/tmp/', - trick_dir=this_trick, config_file=os.path.join(tests_dir,"empty.yml"), + trick_dir=this_trick, config_file=os.path.join(tests_dir,file), quiet=True) def setUpWithErrorConfig(self, file): @@ -33,21 +58,43 @@ class TrickWorkflowTestCase(unittest.TestCase): trick_dir=this_trick, config_file=os.path.join(tests_dir,file), quiet=True) + def test_static_members(self): + self.assertEqual(TrickWorkflow.all_possible_phases, + range(TrickWorkflow.allowed_phase_range['min'], TrickWorkflow.allowed_phase_range['max']+1) ) + self.assertEqual(TrickWorkflow.listify_phase(3), [3]) # Int becomes list of int + self.assertEqual(TrickWorkflow.listify_phase([-3, 3, 9]), [-3, 3, 9]) # Valid list passes through + self.assertEqual(TrickWorkflow.listify_phase(), list(TrickWorkflow.all_possible_phases)) + self.assertEqual(TrickWorkflow.listify_phase(None), list(TrickWorkflow.all_possible_phases)) + self.assertEqual(TrickWorkflow.listify_phase(range(0,5)), [0, 1, 2, 3, 4]) + with self.assertRaises(RuntimeError): + p = TrickWorkflow.listify_phase('hey') # Bad type + with self.assertRaises(RuntimeError): + p = TrickWorkflow.listify_phase([-3, 3, 'hi']) # Bad type + with self.assertRaises(RuntimeError): + p = TrickWorkflow.listify_phase([-30000, 3, '30000']) # Out of range + with self.assertRaises(RuntimeError): + p = TrickWorkflow.listify_phase([-30000, 3, 'hello']) # Out of range and bad type + def test_init_nominal(self): self.assertEqual(self.instance.cpus, 3) - self.assertEqual(self.instance.parallel_safety, 'loose') - self.assertEqual(self.instance.config_errors, False) + self.assertEqual(self.instance.config_errors, []) self.instance.report() build_jobs = self.instance.get_jobs('build') self.assertEqual(len(build_jobs), 56) self.assertEqual(len(self.instance.sims), 56) run_jobs = self.instance.get_jobs('run') - self.assertEqual(len(run_jobs), 37 ) + self.assertEqual(len(run_jobs), 36) + val_run_jobs = self.instance.get_jobs('valgrind') + self.assertEqual(len(val_run_jobs), 1) def test_init_empty_so_raises(self): with self.assertRaises(RuntimeError): self.setUpWithEmptyConfig() + def test_init_empty_after_parsing_so_raises(self): + with self.assertRaises(RuntimeError): + self.setUpWithEmptyConfig("empty1.yml") + def test_init_bad_yaml_so_raises(self): with self.assertRaises(RuntimeError): self.setUpWithErrorConfig("errors_fatal1.yml") @@ -56,17 +103,18 @@ class TrickWorkflowTestCase(unittest.TestCase): def test_init_errors_but_no_raise(self): self.setUpWithErrorConfig("errors_nonfatal.yml") - self.assertTrue(self.instance.config_errors) - self.assertEqual(self.instance.parallel_safety , 'loose') + self.assertEqual(len(self.instance.parsing_errors), 11) + self.assertEqual(len(self.instance.config_errors), 2) self.assertEqual(len(self.instance.sims), 4) self.assertEqual(len(self.instance.get_sim('SIM_ball_L1').get_runs()), 1) + #import pprint; pprint.pprint(self.instance.parsing_errors) self.assertEqual(len(self.instance.get_sim('SIM_ball_L1').get_valgrind_runs()), 0) - self.assertEqual(len(self.instance.get_sim('SIM_alloc_test').get_runs()), 1) + self.assertEqual(self.instance.get_sim('SIM_ball_L1').get_phase(), 0) + self.assertEqual(len(self.instance.get_sim('SIM_alloc_test').get_runs()), 2) self.assertEqual(len(self.instance.get_sim('SIM_alloc_test').get_valgrind_runs()), 0) self.assertEqual(self.instance.config['SIM_alloc_test']['runs']['RUN_test/input.py']['extension'], 'foobar') self.assertEqual(len(self.instance.get_sim('SIM_events').get_runs()), 0) - self.assertTrue('fine1' in self.instance.get_sim('SIM_events').labels) - self.assertTrue('fine2' in self.instance.get_sim('SIM_events').labels) + self.assertEqual(self.instance.get_sim('SIM_events').labels, []) self.assertEqual(self.instance.get_sim('SIM_threads').get_run('RUN_test/sched.py').returns, 0) self.assertEqual(self.instance.get_sim('SIM_threads').get_run('RUN_test/amf.py').returns, 0) self.assertEqual(self.instance.get_sim('SIM_threads').get_run('RUN_test/async.py').returns, 0) @@ -75,8 +123,7 @@ class TrickWorkflowTestCase(unittest.TestCase): self.assertTrue(self.instance.get_sim('SIM_L1_ball') is None) self.assertTrue(self.instance.get_sim('SIM_foobar') is None) self.assertTrue(self.instance.get_sim('SIM_parachute') is None) - - self.assertTrue(self.instance.config['extension_example']) + self.assertEqual(self.instance.config['extension_example'], {'should': 'be ignored by this framework'}) self.instance.report() def test_get_sim_nominal(self): @@ -146,15 +193,24 @@ class TrickWorkflowTestCase(unittest.TestCase): self.assertTrue(ucd[0][0] is not None) self.assertTrue(ucd[0][1] is not None) - def test_get_koviz_report_jobs_nominal(self): + def test_get_koviz_report_jobs(self): + # Since instrumenting "is koviz on your PATH?" is difficult, this test is + # designed to check both error (not on PATH) and non-error (on PATH) cases krj = self.instance.get_koviz_report_jobs() - self.assertTrue(isinstance(krj[0][0], Job)) - self.assertTrue(not krj[1]) + if krj[1] == []: # No errors implies we have koviz on PATH + self.assertTrue(isinstance(krj[0], list)) + self.assertTrue(isinstance(krj[0][0], Job)) + self.assertTrue(not krj[1]) + else: # Otherwise koviz not on PATH + self.assertTrue('koviz is not found' in krj[1][0]) + self.assertTrue(isinstance(krj[0], list)) + self.assertEqual(len(krj[0]), 0) def test_get_koviz_report_job_missing_dir(self): krj = self.instance.get_koviz_report_job('share/trick/trickops/tests/testdata_noexist', 'share/trick/trickops/tests/baselinedata') self.assertTrue(krj[0] is None) + # Loose check but works even if koviz not on PATH self.assertTrue('ERROR' in krj[1]) def test_status_summary_nominal(self): @@ -165,18 +221,49 @@ class TrickWorkflowTestCase(unittest.TestCase): def test_get_and_pop_run(self): sim = self.instance.get_sim('SIM_ball_L1') run = sim.get_run('RUN_test/input.py') - self.assertEqual(run.input, 'RUN_test/input.py') + self.assertEqual(run.input_file, 'RUN_test/input.py') run = sim.pop_run('RUN_test/input.py') - self.assertEqual(run.input, 'RUN_test/input.py') + self.assertEqual(run.input_file, 'RUN_test/input.py') self.assertEqual(len(sim.get_runs()), 0) - def test_check_run_jobs(self): + def test_get_run_jobs_kind(self): sim = self.instance.get_sim('SIM_ball_L1') normal_run_jobs = sim.get_run_jobs() self.assertTrue('valgrind' not in normal_run_jobs[0]._command) + sim = self.instance.get_sim('SIM_alloc_test') valgrind_run_jobs = sim.get_run_jobs(kind='valgrind') self.assertTrue('valgrind' in valgrind_run_jobs[0]._command) + def test_get_run_jobs_phase(self): + sim = self.instance.get_sim('SIM_demo_sdefine') + run_jobs = sim.get_run_jobs() # Ask for everything + self.assertEqual(len(run_jobs), 2) + run_jobs_phase0_int = sim.get_run_jobs(phase=0) # Ask for only phase 0 as int + self.assertEqual(len(run_jobs_phase0_int), 1) + run_jobs_phase0_list = sim.get_run_jobs(phase=[0]) # Ask for only phase 0 as list + self.assertEqual(run_jobs_phase0_int, run_jobs_phase0_list) # Should be the same + self.assertEqual(len(run_jobs_phase0_list), 1) + run_jobs_phase1_int = sim.get_run_jobs(phase=1) # Ask for only phase 1 as int + self.assertEqual(len(run_jobs_phase1_int), 1) + run_jobs_phase1_list = sim.get_run_jobs(phase=[1]) # Ask for only phase 1 as list + self.assertEqual(run_jobs_phase1_int, run_jobs_phase1_list) # Should be the same + self.assertEqual(len(run_jobs_phase1_list), 1) + run_jobs_both_phases_list = sim.get_run_jobs(phase=[0,1]) # Ask for phase 0-1 as list + self.assertEqual(len(run_jobs_both_phases_list), 2) + self.assertEqual(run_jobs_both_phases_list, run_jobs) # These should be equivalent + run_jobs_one_phase_not_exist = sim.get_run_jobs(phase=[0,1,2]) # Ask for phase 0-2 as list, 2 doesn't exist + self.assertEqual(len(run_jobs_one_phase_not_exist), 2) # We only get what exists + # Ask for everything explicitly, this is far slower than phase=None but should produce equivalent list + run_jobs_all_possible_explicit = sim.get_run_jobs(phase=TrickWorkflow.all_possible_phases) + self.assertEqual(run_jobs_all_possible_explicit , run_jobs) # These should be equivalent + # Error cases + with self.assertRaises(RuntimeError): + run_jobs_invalid_input = sim.get_run_jobs(phase=['abc',1]) # strings aren't permitted + with self.assertRaises(RuntimeError): + run_jobs_invalid_input = sim.get_run_jobs(phase=-10000) # out of range as int + with self.assertRaises(RuntimeError): + run_jobs_invalid_input = sim.get_run_jobs(phase=[-10000]) # out of range as list + def test_compare(self): sim = self.instance.get_sim('SIM_ball_L1') # Sim level comparison (test_data.csv vs. baseline_data.csv) will fail @@ -189,29 +276,93 @@ class TrickWorkflowTestCase(unittest.TestCase): self.assertEqual(run.comparisons[0]._translate_status(), '\x1b[31mFAIL\x1b[0m') def test_get_jobs_nominal(self): - # Test all the permissive permutations + # Test all the kinds permutations builds = self.instance.get_jobs('build') self.assertEqual(len(builds), 56) builds = self.instance.get_jobs('builds') self.assertEqual(len(builds), 56) runs = self.instance.get_jobs('run') - self.assertEqual(len(runs), 37) + self.assertEqual(len(runs), 36) runs = self.instance.get_jobs('runs') - self.assertEqual(len(runs), 37) + self.assertEqual(len(runs), 36) vg = self.instance.get_jobs('valgrind') self.assertEqual(len(vg), 1) vg = self.instance.get_jobs('valgrinds') self.assertEqual(len(vg), 1) a = self.instance.get_jobs('analysis') - self.assertEqual(len(a), 1) + self.assertEqual(len(a), 8) a = self.instance.get_jobs('analyses') - self.assertEqual(len(a), 1) + self.assertEqual(len(a), 8) a = self.instance.get_jobs('analyze') - self.assertEqual(len(a), 1) + self.assertEqual(len(a), 8) + + def test_get_jobs_builds_with_phases(self): + # All builds all phases + builds = self.instance.get_jobs('build', phase=None) + self.assertEqual(len(builds), 56) + builds = self.instance.get_jobs('build', phase=TrickWorkflow.all_possible_phases) + self.assertEqual(len(builds), 56) + # Builds only specific phases + builds = self.instance.get_jobs('build', phase=970) + self.assertEqual(len(builds), 0) + builds = self.instance.get_jobs('build', phase=0) + self.assertEqual(len(builds), 53) + builds = self.instance.get_jobs('build', phase=[0]) + self.assertEqual(len(builds), 53) + builds = self.instance.get_jobs('build', phase=-1) + self.assertEqual(len(builds), 1) + builds = self.instance.get_jobs('build', phase=72) + self.assertEqual(len(builds), 1) + builds = self.instance.get_jobs('build', phase=-88) + self.assertEqual(len(builds), 1) + builds = self.instance.get_jobs('build', phase=[-88, 72]) + self.assertEqual(len(builds), 2) + + def test_get_jobs_runs_with_phases(self): + # All runs all phases + vruns = self.instance.get_jobs('valgrind', phase=None) + self.assertEqual(len(vruns), 1) + vruns = self.instance.get_jobs('valgrind', phase=7) + self.assertEqual(len(vruns), 0) + runs = self.instance.get_jobs('run', phase=None) + self.assertEqual(len(runs), 36) + runs = self.instance.get_jobs('run', phase=TrickWorkflow.all_possible_phases) + self.assertEqual(len(runs), 36) + # Runs specific phases + runs = self.instance.get_jobs('run', phase=[8, 19]) + self.assertEqual(len(runs), 0) + runs = self.instance.get_jobs('run', phase=1) + self.assertEqual(len(runs), 3) + runs = self.instance.get_jobs('run', phase=2) + self.assertEqual(len(runs), 2) + runs = self.instance.get_jobs('run', phase=3) + self.assertEqual(len(runs), 2) + runs = self.instance.get_jobs('run', phase=4) + self.assertEqual(len(runs), 1) + + def test_get_jobs_analysis_with_phases(self): + # All analysis all phases + an = self.instance.get_jobs('analysis', phase=None) + self.assertEqual(len(an), 8) + an = self.instance.get_jobs('analysis', phase=TrickWorkflow.all_possible_phases) + self.assertEqual(len(an), 8) + # Analysis specific phases + an = self.instance.get_jobs('analysis', phase=[8, 19]) + self.assertEqual(len(an), 0) + an = self.instance.get_jobs('analysis', phase=1) + self.assertEqual(len(an), 2) + an = self.instance.get_jobs('analysis', phase=2) + self.assertEqual(len(an), 2) + an = self.instance.get_jobs('analysis', phase=3) + self.assertEqual(len(an), 1) def test_get_jobs_raises(self): with self.assertRaises(TypeError): jobs = self.instance.get_jobs(kind='bucees') + with self.assertRaises(RuntimeError): + jobs = self.instance.get_jobs(kind='build', phase='abx') + with self.assertRaises(RuntimeError): + jobs = self.instance.get_jobs(kind='run', phase=[-10000, 60000]) def test_get_comparisons_nominal(self): c = self.instance.get_comparisons() @@ -219,14 +370,14 @@ class TrickWorkflowTestCase(unittest.TestCase): self.assertEqual(c[0]._translate_status(), '\x1b[33mNOT RUN\x1b[0m') def test_add_comparison(self): - sim = self.instance.get_sim('SIM_alloc_test') + sim = self.instance.get_sim('SIM_demo_inputfile') run = sim.get_run('RUN_test/input.py') - run.add_comparison('share/trick/trickops/tests/baselinedata/log_a.csv', - 'share/trick/trickops/tests/testdata/log_a.csv') + run.add_comparison('share/trick/trickops/tests/testdata/log_a.csv', + 'share/trick/trickops/tests/baselinedata/log_a.csv') self.assertTrue(len(run.comparisons) == 1) def test_add_analysis_nominal(self): - sim = self.instance.get_sim('SIM_alloc_test') + sim = self.instance.get_sim('SIM_demo_inputfile') run = sim.get_run('RUN_test/input.py') run.add_analysis('echo analysis goes here') self.assertTrue( 'echo analysis goes here' in run.analysis._command) @@ -238,12 +389,13 @@ class TrickWorkflowTestCase(unittest.TestCase): self.assertTrue( 'echo overwriting analysis' in run.analysis._command) def test_run_init(self): - r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input='RUN_test/input.py --someflag', + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py --someflag', binary='S_main_Linux_x86_64.exe') self.assertEqual(r.sim_dir, 'test/SIM_alloc_test') self.assertEqual(r.prerun_cmd, '') - self.assertTrue(r.input == 'RUN_test/input.py --someflag') + self.assertTrue(r.input_file == 'RUN_test/input.py --someflag') self.assertEqual(r.returns, 0) + self.assertEqual(r.phase, 0) self.assertTrue(r.valgrind_flags is None) self.assertEqual(r.log_dir, '/tmp/') self.assertEqual(r.just_input,'RUN_test/input.py') @@ -263,7 +415,7 @@ class TrickWorkflowTestCase(unittest.TestCase): self.assertTrue(c.error is None) def test_run_compare_pass(self): - r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input='RUN_test/input.py --someflag', + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py --someflag', binary='S_main_Linux_x86_64.exe') # Use same data to get a pass test_data = 'share/trick/trickops/tests/baselinedata/log_a.csv' @@ -272,7 +424,7 @@ class TrickWorkflowTestCase(unittest.TestCase): self.assertEqual(r.compare(), 0) def test_run_compare_fail(self): - r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input='RUN_test/input.py --someflag', + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py --someflag', binary='S_main_Linux_x86_64.exe') # Use same data to get a pass test_data = 'share/trick/trickops/tests/testdata/log_a.csv' @@ -295,9 +447,8 @@ class TrickWorkflowTestCase(unittest.TestCase): os.makedirs(just_RUN_root, exist_ok=True) Path(os.path.join(SIM_root,run)).touch() yml_content=textwrap.dedent(""" - globals: - parallel_safety: """ + parallel_safety + """ SIM_fake: + parallel_safety: """ + parallel_safety + """ path: """ + SIM_root_rel + """ runs: """) @@ -356,3 +507,165 @@ class TrickWorkflowTestCase(unittest.TestCase): self.assertEqual(len(self.instance.get_sims()), 1) self.assertEqual(len(self.instance.get_sim('SIM_fake').get_runs()), 1) self.teardown_deep_directory_structure() + + def test_sim_init_default_args(self): + s = TrickWorkflow.Sim(name='mySim', sim_dir='sims/SIM_fake') + self.assertEqual(s.name, 'mySim') + self.assertEqual(s.sim_dir, 'sims/SIM_fake') + self.assertEqual(s.description, None) + self.assertEqual(s.build_cmd, 'trick-CP') + self.assertEqual(s.cpus, 3) + self.assertEqual(s.size, 2200000) + self.assertEqual(s.labels, []) + self.assertEqual(s.phase, 0) + self.assertEqual(s.log_dir, '/tmp') + self.assertEqual(s.build_job, None) + self.assertEqual(s.runs, []) + self.assertEqual(s.valgrind_runs, []) + self.assertTrue(isinstance(s.printer, ColorStr)) + job = s.get_build_job() + self.assertEqual(s.build_job, job ) # First get stores it locally + self.assertTrue('cd sims/SIM_fake && export MAKEFLAGS=-j3 && trick-CP' in job._command ) + runs = s.get_run_jobs() + self.assertEqual(runs, []) # No runs have been added + + def test_sim_init_all_args(self): + s = TrickWorkflow.Sim(name='yourSim', sim_dir='sims/SIM_foo', description='desc', + labels=['label1', 'label2'], prebuild_cmd='source env/env.sh; ', + build_cmd='trick-CP --flag', cpus=2, size=10000, phase=2, log_dir='~/logs') + self.assertEqual(s.name, 'yourSim') + self.assertEqual(s.sim_dir, 'sims/SIM_foo') + self.assertEqual(s.description, 'desc') + self.assertEqual(s.build_cmd, 'trick-CP --flag') + self.assertEqual(s.cpus, 2) + self.assertEqual(s.size, 10000) + self.assertEqual(s.labels, ['label1', 'label2']) + self.assertEqual(s.phase, 2) + self.assertEqual(s.log_dir, '~/logs') + job = s.get_build_job() + self.assertTrue('cd sims/SIM_foo && export MAKEFLAGS=-j2 && trick-CP --flag' in job._command ) + self.assertTrue('source env/env.sh;' in job._command ) + #import pdb; pdb.set_trace() + + def test_phase_getters_setters(self): + s = TrickWorkflow.Sim(name='mySim', sim_dir='sims/SIM_fake') + s.set_phase(99) # OK + self.assertEqual(s.get_phase(), 99) + s.set_phase(999) # OK + self.assertEqual(s.get_phase(), 999) + s.set_phase(TrickWorkflow.allowed_phase_range['max']) + self.assertEqual(s.get_phase(), TrickWorkflow.allowed_phase_range['max']) # OK boundary + with self.assertRaises(RuntimeError): + s.set_phase(TrickWorkflow.allowed_phase_range['max']+1) # Over boundary + with self.assertRaises(RuntimeError): + s.set_phase(TrickWorkflow.allowed_phase_range['min']-1) # Under boundary + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py', + binary='S_main_Linux_x86_64.exe') + r.set_phase(99) # OK + self.assertEqual(r.get_phase(), 99) + r.set_phase(999) # OK + self.assertEqual(r.get_phase(), 999) + with self.assertRaises(RuntimeError): + r.set_phase(TrickWorkflow.allowed_phase_range['max']+1) # Over boundary + with self.assertRaises(RuntimeError): + r.set_phase(TrickWorkflow.allowed_phase_range['min']-1) # Under boundary + + def test_run__find_range_string(self): + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py', + binary='S_main_Linux_x86_64.exe') + self.assertEqual(TrickWorkflow._find_range_string("[01-09]"), "[01-09]") + self.assertEqual(TrickWorkflow._find_range_string("SET_foo/RUN_[0-9]/input.py"), "[0-9]") + self.assertEqual(TrickWorkflow._find_range_string("SET_foo/RUN_[01-09]/input.py"), "[01-09]") + self.assertEqual(TrickWorkflow._find_range_string("SET_foo/RUN_[001-999]/input.py"), "[001-999]") + self.assertEqual(TrickWorkflow._find_range_string("SET_foo/RUN_[0000-9999]/input.py"), "[0000-9999]") + self.assertEqual(TrickWorkflow._find_range_string("SET_[01-09]/RUN_hi/input.py"), "[01-09]" ) + self.assertEqual(TrickWorkflow._find_range_string("[01-09]/RUN_hello/input.py"), "[01-09]") + self.assertEqual(TrickWorkflow._find_range_string("SET_foo/RUN_bar/input.py"), None) + self.assertEqual(TrickWorkflow._find_range_string("SET_foo/RUN_[001-009/input.py"), None) + self.assertEqual(TrickWorkflow._find_range_string("SET_foo/RUN_(01-09)/input.py"), None) + with self.assertRaises(RuntimeError): + TrickWorkflow._find_range_string("SET_[00-03]/RUN_[01-09]/input.py") + + def test_run__get_range_list(self): + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_test/input.py', + binary='S_main_Linux_x86_64.exe') + myRange = r._get_range_list("[01-09]") + self.assertEqual(myRange, ["01", "02", "03", "04", "05", "06", "07", "08", "09"]) + myRange = r._get_range_list("[001-009]") + self.assertEqual(myRange, ["001", "002", "003", "004", "005", "006", "007", "008", "009"]) + with self.assertRaises(RuntimeError): + myRange = r._get_range_list("[009-001]") # Min not less than max + with self.assertRaises(RuntimeError): + myRange = r._get_range_list("[09-001]") # Inconsistent leading zeros + with self.assertRaises(RuntimeError): + myRange = r._get_range_list("[abc-009]") # Min Can't be converted to int + with self.assertRaises(RuntimeError): + myRange = r._get_range_list("[01-zy]") # Max Can't be converted to int + with self.assertRaises(RuntimeError): + myRange = r._get_range_list("01-04") # Wrong syntax + with self.assertRaises(RuntimeError): + myRange = r._get_range_list("[01-04") # Wrong syntax + with self.assertRaises(RuntimeError): + myRange = r._get_range_list("[01/04") # Wrong syntax + myRange = r._get_range_list("[[01-04]]") # Extra brackets are ignored + self.assertEqual(myRange, ["01", "02", "03", "04"]) + + def test_run__multiply(self): + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_[00-05]/input.py', + binary='S_main_Linux_x86_64.exe') + runs = r.multiply() + self.assertEqual(len(runs), 6) # Expect 6 copies + + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_[0000-9999]/input.py', + binary='S_main_Linux_x86_64.exe') + runs = r.multiply() + self.assertEqual(len(runs), 10000) # Expect 10000 copies + + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_[05-01]/input.py', + binary='S_main_Linux_x86_64.exe') + with self.assertRaises(RuntimeError): + runs = r.multiply() # Reverse order in syntax in input_file + + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='SET_[01-02]/RUN_[05-01]/input.py', + binary='S_main_Linux_x86_64.exe') + with self.assertRaises(RuntimeError): + runs = r.multiply() # Invalid syntax, more than one pattern found + + def test_run__multiply_with_double_pattern_comparisons(self): + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_[00-05]/input.py', + binary='S_main_Linux_x86_64.exe') + # Add a fake comparison with correct [min-max] notation + r.add_comparison('testdata/RUN_[00-05]/log_a.csv', 'baselinedata/RUN_[00-05]/log_a.csv') + runs = r.multiply() + # Ensure the run's comparisons patterns are replaced with the expected value + self.assertEqual(len(runs), 6) # Expect 6 copies + self.assertEqual(runs[0].comparisons[0].test_data, 'testdata/RUN_00/log_a.csv') + self.assertEqual(runs[0].comparisons[0].baseline_data, 'baselinedata/RUN_00/log_a.csv') + self.assertEqual(runs[5].comparisons[0].test_data, 'testdata/RUN_05/log_a.csv') + self.assertEqual(runs[5].comparisons[0].baseline_data, 'baselinedata/RUN_05/log_a.csv') + + def test_run__multiply_with_single_pattern_comparisons(self): + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_[00-05]/input.py', + binary='S_main_Linux_x86_64.exe') + # Compare many to one + r.add_comparison('testdata/RUN_[00-05]/log_common.csv', 'baselinedata/RUN_common/log_common.csv') + r.add_comparison('testdata/RUN_[00-05]/log_a.csv', 'baselinedata/RUN_[00-05]/log_a.csv') + runs = r.multiply() + # Ensure the run's comparisons patterns are replaced with the expected value + self.assertEqual(len(runs), 6) # Expect 6 copies + self.assertEqual(runs[0].comparisons[0].test_data, 'testdata/RUN_00/log_common.csv') + self.assertEqual(runs[0].comparisons[0].baseline_data, 'baselinedata/RUN_common/log_common.csv') + self.assertEqual(runs[0].comparisons[1].test_data, 'testdata/RUN_00/log_a.csv') + self.assertEqual(runs[0].comparisons[1].baseline_data, 'baselinedata/RUN_00/log_a.csv') + self.assertEqual(runs[5].comparisons[0].test_data, 'testdata/RUN_05/log_common.csv') + self.assertEqual(runs[5].comparisons[0].baseline_data, 'baselinedata/RUN_common/log_common.csv') + self.assertEqual(runs[5].comparisons[1].test_data, 'testdata/RUN_05/log_a.csv') + self.assertEqual(runs[5].comparisons[1].baseline_data, 'baselinedata/RUN_05/log_a.csv') + + def test_run__multiply_with_mismatched_patterns(self): + r = TrickWorkflow.Run(sim_dir='test/SIM_alloc_test', input_file='RUN_[00-05]/input.py', + binary='S_main_Linux_x86_64.exe') + # Compare many to one + r.add_comparison('testdata/RUN_[00-06]/log_common.csv', 'baselinedata/RUN_common/log_common.csv') + with self.assertRaises(RuntimeError): + runs = r.multiply() diff --git a/share/trick/trickops/tests/ut_TrickWorkflowYamlVerifier.py b/share/trick/trickops/tests/ut_TrickWorkflowYamlVerifier.py new file mode 100644 index 00000000..29145da2 --- /dev/null +++ b/share/trick/trickops/tests/ut_TrickWorkflowYamlVerifier.py @@ -0,0 +1,73 @@ +import os, sys +import unittest +import pdb +from testconfig import this_trick, tests_dir +from TrickWorkflowYamlVerifier import * + +def suite(): + """Create test suite from TrickWorkflowTestCase unit test class and return""" + return unittest.TestLoader().loadTestsFromTestCase(TrickWorkflowYamlVerifierTestCase) + +class TrickWorkflowYamlVerifierTestCase(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def _verify(self, config_file): + """ + Given a config_file, create a TrickWorkflowYamlVerifier, assert expectations, + and return the instance for further examination + """ + twyv = TrickWorkflowYamlVerifier(config_file=config_file) + twyv.verify() + sim_keys = [sim for sim in twyv.config.keys() if sim.startswith('SIM')] + self.assertTrue('globals' in twyv.config) + self.assertTrue('env' in twyv.config['globals']) + for sk in sim_keys: + self.assertTrue( isinstance( twyv.config[sk]['name' ], str)) + self.assertTrue( isinstance( twyv.config[sk]['binary' ], str)) + self.assertTrue( isinstance( twyv.config[sk]['build_args'], str) or + twyv.config[sk]['build_args'] == None ) + self.assertTrue( isinstance( twyv.config[sk]['name' ], str)) + self.assertTrue( isinstance( twyv.config[sk]['parallel_safety'], str)) + self.assertTrue( twyv.config[sk]['description' ] is None) + self.assertTrue( isinstance( twyv.config[sk]['runs'], dict)) + for run in twyv.config[sk]['runs']: + self.assertTrue( isinstance( twyv.config[sk]['runs'][run]['input'], str)) + self.assertTrue( isinstance( twyv.config[sk]['runs'][run]['returns'], int)) + self.assertTrue( isinstance( twyv.config[sk]['runs'][run]['valgrind'], str) or + (twyv.config[sk]['runs'][run]['valgrind'] == None) ) + self.assertTrue( isinstance( twyv.config[sk]['runs'][run]['analyze'], str) or + (twyv.config[sk]['runs'][run]['analyze'] == None) ) + self.assertTrue( isinstance( twyv.config[sk]['runs'][run]['phase'], int)) + self.assertTrue( isinstance( twyv.config[sk]['runs'][run]['compare'], list)) + self.assertTrue( isinstance( twyv.config[sk]['phase' ], int)) + self.assertTrue( isinstance( twyv.config[sk]['path' ], str)) + return twyv + + def test_type_errors_in_config(self): + twyv = self._verify(config_file=os.path.join(tests_dir,"type_errors.yml")) + self.assertEqual(len(twyv.get_parsing_errors()), 21) + + def test_no_SIM_dict_keys_in_config(self): + twyv = TrickWorkflowYamlVerifier(config_file=os.path.join(tests_dir,"errors_fatal2.yml")) + with self.assertRaises(RuntimeError): + twyv.verify() + self.assertEqual(len(twyv.get_parsing_errors()), 1) + + def test_empty_config(self): + twyv = TrickWorkflowYamlVerifier(config_file=os.path.join(tests_dir,"empty.yml")) + with self.assertRaises(RuntimeError): + twyv.verify() + self.assertEqual(len(twyv.get_parsing_errors()), 1) + + def test_nominal_config(self): + twyv = self._verify(config_file=os.path.join(tests_dir,"trick_sims.yml")) + sim_keys = [sim for sim in twyv.config.keys() if sim.startswith('SIM')] + non_sim_keys = [entry for entry in twyv.config.keys() if not entry.startswith('SIM')] + self.assertEqual(len(non_sim_keys), 2) # Expect 2 non-SIM.*: dict key + self.assertEqual(len(sim_keys), 56) # Expect 56 SIM.*: dict keys + self.assertEqual(len(twyv.get_parsing_errors()), 0) diff --git a/test/.gitignore b/test/.gitignore index 1e4698c4..f3d79c4a 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -6,7 +6,7 @@ send_hs varserver_log log_* chkpnt_* -MONTE_RUN_* +MONTE_* .S_library* .icg_no_found CP_out @@ -25,4 +25,5 @@ trick.zip jitlib build S_sie.json -*.ckpnt \ No newline at end of file +*.ckpnt +MonteCarlo_Meta_data_output diff --git a/test/SIM_mc_generation/FAIL_IO_error/input.py b/test/SIM_mc_generation/FAIL_IO_error/input.py new file mode 100644 index 00000000..62982da8 --- /dev/null +++ b/test/SIM_mc_generation/FAIL_IO_error/input.py @@ -0,0 +1,18 @@ +monte_carlo.mc_master.activate("FAIL_IO_error") + +print('*********************************************************************') +print('this message is expected:') +print(' DIAGNOSTIC: Fatal Error I/O error') +print(' Unable to open file Modified_data/nonexistent_file.txt for reading.') +print(' Required for variable test.x_file_lookup[0].') +print('*********************************************************************') + + +# try to open a non-existant file +# code coverage for 'mc_variable_file.cc', lines 71-76. +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/nonexistent_file.txt", + 3) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/FAIL_config_error/input.py b/test/SIM_mc_generation/FAIL_config_error/input.py new file mode 100644 index 00000000..32508da0 --- /dev/null +++ b/test/SIM_mc_generation/FAIL_config_error/input.py @@ -0,0 +1,19 @@ +monte_carlo.mc_master.activate("FAIL_config_error") + +print('************************************************************************************') +print('this message is expected:') +print(' DIAGNOSTIC: Fatal Error Configuration Error') +print(' In configuring the file for variable test.x_file_lookup[0], it was identified that') +print(' it was specified to draw data from column 4, but that the first') +print(' column was identified as having index 7.') +print('************************************************************************************') + + +# generate the error +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 4, + 7) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/FAIL_duplicate_variable/input.py b/test/SIM_mc_generation/FAIL_duplicate_variable/input.py new file mode 100644 index 00000000..97aabdaf --- /dev/null +++ b/test/SIM_mc_generation/FAIL_duplicate_variable/input.py @@ -0,0 +1,17 @@ +monte_carlo.mc_master.activate("FAIL_duplicate_variable") + +print('************************************************************') +print('this message is expected:') +print(' DIAGNOSTIC: Fatal Error Duplicated variable.') +print(' Attempted to add two settings for variable test.x_uniform.') +print(' Terminating to allow resolution of which setting to use.') +print('************************************************************') + + +mc_var = trick.MonteCarloVariableRandomUniform( "test.x_uniform", 0, 10, 20) +monte_carlo.mc_master.add_variable(mc_var) +# Add the variable twice to trigger the "Duplicated variable" fail in +# MonteCarloMaster::add_variable +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/FAIL_illegal_config/input.py b/test/SIM_mc_generation/FAIL_illegal_config/input.py new file mode 100644 index 00000000..eb263cd6 --- /dev/null +++ b/test/SIM_mc_generation/FAIL_illegal_config/input.py @@ -0,0 +1,18 @@ +monte_carlo.mc_master.activate("FAIL_illegal_config") +monte_carlo.mc_master.set_num_runs(1) + +print('**********************************************************************************\n' + +'this message is expected:\n' + +' DIAGNOSTIC: Fatal Error Illegal configuration\n' + +'For variable test.x_normal the specified minimum allowable value (6) >= the specified maximum allowable value (3).\n' + +'One or both of the limits must be changed to generate a random value.\n' + +'**********************************************************************************') + + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal", 2, 10, 2) +# flip low and high values to generate the desired error +mc_var.truncate_low(6, trick.MonteCarloVariableRandomNormal.Absolute) +mc_var.truncate_high(3, trick.MonteCarloVariableRandomNormal.Absolute) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/FAIL_invalid_config/input.py b/test/SIM_mc_generation/FAIL_invalid_config/input.py new file mode 100644 index 00000000..640e89f6 --- /dev/null +++ b/test/SIM_mc_generation/FAIL_invalid_config/input.py @@ -0,0 +1,24 @@ +monte_carlo.mc_master.activate("FAIL_invalid_config") + +print('******************************************************************************************') +print('this message is expected:') +print(' DIAGNOSTIC: Fatal Error Invalid configuration') +print(' Error in attempting to make test.x_file_lookup[1] be dependent on test.x_file_lookup[0].') +print(' test.x_file_lookup[1] cannot be marked as dependent when it has dependencies of its own.') +print(' The dependency hierarchy can only be one level deep.') +print('******************************************************************************************') + + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 3) +monte_carlo.mc_master.add_variable(mc_var) + +mc_var2 = trick.MonteCarloVariableFile( "test.x_file_lookup[1]", + "Modified_data/datafile.txt", + 3) +# the next command is the source of the error! +mc_var2.register_dependent(mc_var) +monte_carlo.mc_master.add_variable(mc_var2) + +trick.stop(1) diff --git a/test/SIM_mc_generation/FAIL_invalid_data_file/input.py b/test/SIM_mc_generation/FAIL_invalid_data_file/input.py new file mode 100644 index 00000000..db137118 --- /dev/null +++ b/test/SIM_mc_generation/FAIL_invalid_data_file/input.py @@ -0,0 +1,17 @@ +monte_carlo.mc_master.activate("FAIL_invalid_data_file") + +print('*****************************************************************************') +print('this message is expected:') +print(' DIAGNOSTIC: Fatal Error Invalid data file') +print(' Data file Modified_data/empty_file.txt contains no recognized lines of data') +print(' Required for variable test.x_file_lookup[0].') +print('*****************************************************************************') + + +# try to open an empty file +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/empty_file.txt", + 3) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/FAIL_malformed_data_file/input.py b/test/SIM_mc_generation/FAIL_malformed_data_file/input.py new file mode 100644 index 00000000..ade700c9 --- /dev/null +++ b/test/SIM_mc_generation/FAIL_malformed_data_file/input.py @@ -0,0 +1,20 @@ +monte_carlo.mc_master.activate("FAIL_malformed_data_file") +monte_carlo.mc_master.set_num_runs(1) + +print('**************************************************************************************************') +print('this message is expected:') +print(' DIAGNOSTIC: Fatal Error Malformed data file') +print(' Data file for variable test.x_file_lookup[0] includes this line:') +print(' 0 1 2 3 4') +print(' Which has only 5 values.') +print(' Variable test.x_file_lookup[0] uses the value from position 9, which does not exist in this line') +print('**************************************************************************************************') + + +# generate the error +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 9) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/IO_FAIL/input.py b/test/SIM_mc_generation/IO_FAIL/input.py new file mode 100644 index 00000000..cae646a6 --- /dev/null +++ b/test/SIM_mc_generation/IO_FAIL/input.py @@ -0,0 +1,27 @@ +import os, shutil + +# remove write permission to the 'MONTE_RUN' directory +os.chmod("MONTE_IO_FAIL", 0o555) + +monte_carlo.mc_master.activate("IO_FAIL") +monte_carlo.mc_master.generate_meta_data = True +monte_carlo.mc_master.set_num_runs(1) + +print('*********************************************************************************') +print('these messages are expected:') +print(' Error I/O error') +print(' Unable to open the variable summary files for writing.') +print(' Dispersion summary will not be generated.') +print('') +print(' DIAGNOSTIC: Fatal Error I/O error') +print(' Unable to open file MONTE_FAIL_IO_error2/RUN_0/monte_input.py for writing.') +print('*********************************************************************************') + +# this simulation attempts to create good data but with the target +# directory write-protected, it can't generate the required input files. +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 3) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/IO_RUN_ERROR1/input.py b/test/SIM_mc_generation/IO_RUN_ERROR1/input.py new file mode 100644 index 00000000..fc44a2dd --- /dev/null +++ b/test/SIM_mc_generation/IO_RUN_ERROR1/input.py @@ -0,0 +1,32 @@ +import os + +# remove write permission to the 'MONTE_RUN' directory +os.chmod("MONTE_IO_RUN_ERROR1", 0o555) + +monte_carlo.mc_master.activate("IO_RUN_ERROR1") +monte_carlo.mc_master.generate_meta_data = True + +print('***********************************************************************************') +print('these messages are expected:') +print(' Error I/O error') +print(' Unable to open the variable summary files for writing.') +print(' Dispersion summary will not be generated.') +print('') +print(' Warning I/O error') +print(' Unable to open file MONTE_ERROR_IO_error/MonteCarlo_Meta_data_output for writing.') +print(' Aborting generation of meta-data.') +print('***********************************************************************************') + +# this simulation attempts to create good data but with the target +# directory write-protected, it can't generate the (optional) summary files. +# NOTE - we avoid the terminal failure of not being able to generate the input +# files by having num_runs = 0, so none are attempted. +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 3) +monte_carlo.mc_master.add_variable(mc_var) + +trick.add_read(0,""" +os.chmod('MONTE_RUN_ERROR_IO_error', 0o755) +""") +trick.stop(1) diff --git a/test/SIM_mc_generation/IO_RUN_ERROR2/input.py b/test/SIM_mc_generation/IO_RUN_ERROR2/input.py new file mode 100644 index 00000000..5ceb02df --- /dev/null +++ b/test/SIM_mc_generation/IO_RUN_ERROR2/input.py @@ -0,0 +1,25 @@ +import os + +# remove write permission to the 'RUN_0' directory +os.chmod("MONTE_IO_RUN_ERROR2/RUN_0", 0o500) + +monte_carlo.mc_master.activate("IO_RUN_ERROR2") +monte_carlo.mc_master.generate_meta_data = True +monte_carlo.mc_master.set_num_runs(1) + +print('***********************************************************************************') +print('this message is expected:\n'+ + ' Error Output failure\n'+ + ' Failed to record summary data for run 0.') +print('***********************************************************************************') + +# this simulation attempts to create good data but with the target +# directory write-protected, it can't generate the (optional) summary files. +# NOTE - we avoid the terminal failure of not being able to generate the input +# files by having num_runs = 0, so none are attempted. +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 3) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/Log_data/log_nominal.py b/test/SIM_mc_generation/Log_data/log_nominal.py new file mode 100755 index 00000000..1cdcb2ef --- /dev/null +++ b/test/SIM_mc_generation/Log_data/log_nominal.py @@ -0,0 +1,20 @@ +dr_group = trick.sim_services.DRAscii("test_data") +dr_group.set_cycle(1) +dr_group.freq = trick.sim_services.DR_Always +trick.add_data_record_group(dr_group, trick.DR_Buffer) + +dr_group.add_variable( "test.x_uniform") +dr_group.add_variable( "test.x_normal") +for ii in range(5): + dr_group.add_variable( "test.x_normal_trunc[%d]" %ii) +dr_group.add_variable( "test.x_normal_length") +dr_group.add_variable( "test.x_integer") +dr_group.add_variable( "test.x_line_command") +for ii in range(3): + dr_group.add_variable( "test.x_file_command[%d]" %ii) +dr_group.add_variable( "test.x_boolean") +dr_group.add_variable( "test.x_file_lookup") +dr_group.add_variable( "test.x_fixed_value_double") +dr_group.add_variable( "test.x_fixed_value_int") +dr_group.add_variable( "test.x_semi_fixed_value") +dr_group.add_variable( "test.x_sdefine_routine_called") diff --git a/test/SIM_mc_generation/Log_data/log_variable_file.py b/test/SIM_mc_generation/Log_data/log_variable_file.py new file mode 100644 index 00000000..fc6d940a --- /dev/null +++ b/test/SIM_mc_generation/Log_data/log_variable_file.py @@ -0,0 +1,6 @@ +dr_group = trick.sim_services.DRAscii("test_data") +dr_group.set_cycle(1) +dr_group.freq = trick.sim_services.DR_Always +trick.add_data_record_group(dr_group, trick.DR_Buffer) + +dr_group.add_variable( "test.x_sdefine_routine_called") diff --git a/test/SIM_mc_generation/MCGTrickOps/.gitignore b/test/SIM_mc_generation/MCGTrickOps/.gitignore new file mode 100644 index 00000000..2211df63 --- /dev/null +++ b/test/SIM_mc_generation/MCGTrickOps/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/test/SIM_mc_generation/MCGTrickOps/MCGWorkFlow.py b/test/SIM_mc_generation/MCGTrickOps/MCGWorkFlow.py new file mode 100755 index 00000000..c5845fe4 --- /dev/null +++ b/test/SIM_mc_generation/MCGTrickOps/MCGWorkFlow.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +import os, sys + +sys.path.append(os.path.abspath("../../../share/trick/trickops/")) + +from TrickWorkflow import * +class ExampleWorkflow(TrickWorkflow): + def __init__( self, quiet, trick_top_level=os.path.abspath("../../../")): + # Base Class initialize, this creates internal management structures + if not(os.path.isdir(os.path.abspath("./MCGTrickOpsLog"))): + logDirectory = "MCGTrickOpsLog" + logDirectoryParent = os.path.abspath(".") + logPath = os.path.join(logDirectoryParent, logDirectory) + os.mkdir(logPath) + TrickWorkflow.__init__(self, project_top_level=trick_top_level, + log_dir=os.path.join(trick_top_level,'test/SIM_mc_generation/MCGTrickOps/MCGTrickOpsLog'), + trick_dir=trick_top_level, + config_file="test/SIM_mc_generation/MCGTrickOps/MCGenerationTest.yml", + cpus=3, quiet=quiet) + def run( self): + build_jobs = self.get_jobs(kind='build') + run_jobs = self.get_jobs(kind='run') + builds_status = self.execute_jobs(build_jobs, max_concurrent=3, header='Executing all sim builds.') + runs_status = self.execute_jobs(run_jobs, max_concurrent=1, header='Executing all sim runs.') + c = self.compare() + self.report() # Print Verbose report + self.status_summary() # Print a Succinct summary + return (builds_status or runs_status or self.config_errors or c) +if __name__ == "__main__": + ExampleWorkflow(quiet=True).run() diff --git a/test/SIM_mc_generation/MCGTrickOps/MCGenerationTest.yml b/test/SIM_mc_generation/MCGTrickOps/MCGenerationTest.yml new file mode 100644 index 00000000..f6b6aabf --- /dev/null +++ b/test/SIM_mc_generation/MCGTrickOps/MCGenerationTest.yml @@ -0,0 +1,256 @@ +SIM_mc_generation: + path: test/SIM_mc_generation + runs: + RUN_nominal/input_a.py: + MONTE_RUN_nominal/RUN_000/monte_input_a.py: + MONTE_RUN_nominal/RUN_001/monte_input_a.py: + RUN_random_normal_truncate_abs/input.py: + MONTE_RUN_random_normal_truncate_abs/RUN_0/monte_input.py: + MONTE_RUN_random_normal_truncate_abs/RUN_1/monte_input.py: + MONTE_RUN_random_normal_truncate_abs/RUN_2/monte_input.py: + MONTE_RUN_random_normal_truncate_abs/RUN_3/monte_input.py: + MONTE_RUN_random_normal_truncate_abs/RUN_4/monte_input.py: + MONTE_RUN_random_normal_truncate_abs/RUN_5/monte_input.py: + MONTE_RUN_random_normal_truncate_abs/RUN_6/monte_input.py: + MONTE_RUN_random_normal_truncate_abs/RUN_7/monte_input.py: + MONTE_RUN_random_normal_truncate_abs/RUN_8/monte_input.py: + MONTE_RUN_random_normal_truncate_abs/RUN_9/monte_input.py: + RUN_random_normal_truncate_rel/input.py: + MONTE_RUN_random_normal_truncate_rel/RUN_0/monte_input.py: + MONTE_RUN_random_normal_truncate_rel/RUN_1/monte_input.py: + MONTE_RUN_random_normal_truncate_rel/RUN_2/monte_input.py: + MONTE_RUN_random_normal_truncate_rel/RUN_3/monte_input.py: + MONTE_RUN_random_normal_truncate_rel/RUN_4/monte_input.py: + MONTE_RUN_random_normal_truncate_rel/RUN_5/monte_input.py: + MONTE_RUN_random_normal_truncate_rel/RUN_6/monte_input.py: + MONTE_RUN_random_normal_truncate_rel/RUN_7/monte_input.py: + MONTE_RUN_random_normal_truncate_rel/RUN_8/monte_input.py: + MONTE_RUN_random_normal_truncate_rel/RUN_9/monte_input.py: + RUN_random_normal_truncate_sd/input.py: + MONTE_RUN_random_normal_truncate_sd/RUN_0/monte_input.py: + MONTE_RUN_random_normal_truncate_sd/RUN_1/monte_input.py: + MONTE_RUN_random_normal_truncate_sd/RUN_2/monte_input.py: + MONTE_RUN_random_normal_truncate_sd/RUN_3/monte_input.py: + MONTE_RUN_random_normal_truncate_sd/RUN_4/monte_input.py: + MONTE_RUN_random_normal_truncate_sd/RUN_5/monte_input.py: + MONTE_RUN_random_normal_truncate_sd/RUN_6/monte_input.py: + MONTE_RUN_random_normal_truncate_sd/RUN_7/monte_input.py: + MONTE_RUN_random_normal_truncate_sd/RUN_8/monte_input.py: + MONTE_RUN_random_normal_truncate_sd/RUN_9/monte_input.py: + RUN_random_normal__untruncate/input.py: + MONTE_RUN_random_normal__untruncate/RUN_0/monte_input.py: + MONTE_RUN_random_normal__untruncate/RUN_1/monte_input.py: + MONTE_RUN_random_normal__untruncate/RUN_2/monte_input.py: + MONTE_RUN_random_normal__untruncate/RUN_3/monte_input.py: + MONTE_RUN_random_normal__untruncate/RUN_4/monte_input.py: + MONTE_RUN_random_normal__untruncate/RUN_5/monte_input.py: + MONTE_RUN_random_normal__untruncate/RUN_6/monte_input.py: + MONTE_RUN_random_normal__untruncate/RUN_7/monte_input.py: + MONTE_RUN_random_normal__untruncate/RUN_8/monte_input.py: + MONTE_RUN_random_normal__untruncate/RUN_9/monte_input.py: + RUN_random_normal_untruncated/input.py: + MONTE_RUN_random_normal_untruncated/RUN_0/monte_input.py: + MONTE_RUN_random_normal_untruncated/RUN_1/monte_input.py: + MONTE_RUN_random_normal_untruncated/RUN_2/monte_input.py: + MONTE_RUN_random_normal_untruncated/RUN_3/monte_input.py: + MONTE_RUN_random_normal_untruncated/RUN_4/monte_input.py: + MONTE_RUN_random_normal_untruncated/RUN_5/monte_input.py: + MONTE_RUN_random_normal_untruncated/RUN_6/monte_input.py: + MONTE_RUN_random_normal_untruncated/RUN_7/monte_input.py: + MONTE_RUN_random_normal_untruncated/RUN_8/monte_input.py: + MONTE_RUN_random_normal_untruncated/RUN_9/monte_input.py: + RUN_random_uniform/input.py: + MONTE_RUN_random_uniform/RUN_0/monte_input.py: + MONTE_RUN_random_uniform/RUN_1/monte_input.py: + MONTE_RUN_random_uniform/RUN_2/monte_input.py: + MONTE_RUN_random_uniform/RUN_3/monte_input.py: + MONTE_RUN_random_uniform/RUN_4/monte_input.py: + MONTE_RUN_random_uniform/RUN_5/monte_input.py: + MONTE_RUN_random_uniform/RUN_6/monte_input.py: + MONTE_RUN_random_uniform/RUN_7/monte_input.py: + MONTE_RUN_random_uniform/RUN_8/monte_input.py: + MONTE_RUN_random_uniform/RUN_9/monte_input.py: + RUN_ERROR_file_inconsistent_skip/input.py: + MONTE_RUN_ERROR_file_inconsistent_skip/RUN_0/monte_input.py: + RUN_ERROR_invalid_call/input.py: + MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py: + RUN_ERROR_invalid_name/input.py: + MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py: + RUN_ERROR_invalid_sequence/input.py: + MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py: + RUN_ERROR_invalid_sequencing/input.py: + MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py: + RUN_ERROR_out_of_domain_error/input.py: + MONTE_RUN_ERROR_out_of_domain_error/RUN_0/monte_input.py: + RUN_ERROR_random_value_truncation/input.py: + MONTE_RUN_ERROR_random_value_truncation/RUN_0/monte_input.py: + MONTE_RUN_ERROR_random_value_truncation/RUN_1/monte_input.py: + RUN_generate_meta_data_early/input.py: + RUN_file_sequential/input.py: + MONTE_RUN_file_sequential/RUN_0/monte_input.py: + MONTE_RUN_file_sequential/RUN_1/monte_input.py: + MONTE_RUN_file_sequential/RUN_2/monte_input.py: + MONTE_RUN_file_sequential/RUN_3/monte_input.py: + MONTE_RUN_file_sequential/RUN_4/monte_input.py: + MONTE_RUN_file_sequential/RUN_5/monte_input.py: + MONTE_RUN_file_sequential/RUN_6/monte_input.py: + MONTE_RUN_file_sequential/RUN_7/monte_input.py: + MONTE_RUN_file_sequential/RUN_8/monte_input.py: + MONTE_RUN_file_sequential/RUN_9/monte_input.py: + RUN_file_skip/input.py: + MONTE_RUN_file_skip/RUN_0/monte_input.py: + MONTE_RUN_file_skip/RUN_1/monte_input.py: + MONTE_RUN_file_skip/RUN_2/monte_input.py: + MONTE_RUN_file_skip/RUN_3/monte_input.py: + MONTE_RUN_file_skip/RUN_4/monte_input.py: + MONTE_RUN_file_skip/RUN_5/monte_input.py: + MONTE_RUN_file_skip/RUN_6/monte_input.py: + MONTE_RUN_file_skip/RUN_7/monte_input.py: + MONTE_RUN_file_skip/RUN_8/monte_input.py: + MONTE_RUN_file_skip/RUN_9/monte_input.py: + RUN_file_skip2/input.py: + MONTE_RUN_file_skip2/RUN_0/monte_input.py: + MONTE_RUN_file_skip2/RUN_1/monte_input.py: + MONTE_RUN_file_skip2/RUN_2/monte_input.py: + MONTE_RUN_file_skip2/RUN_3/monte_input.py: + MONTE_RUN_file_skip2/RUN_4/monte_input.py: + RUN_remove_variable/input.py: + RUN_WARN_config_error/input.py: + MONTE_RUN_WARN_config_error/RUN_0/monte_input.py: + RUN_WARN_invalid_name/input.py: + MONTE_RUN_WARN_invalid_name/RUN_0/monte_input.py: + RUN_WARN_overconstrained_config/input.py: + MONTE_RUN_WARN_overconstrained_config/RUN_0/monte_input.py: + FAIL_config_error/input.py: + returns: 1 + FAIL_duplicate_variable/input.py: + returns: 1 + FAIL_illegal_config/input.py: + returns: 1 + FAIL_invalid_config/input.py: + returns: 1 + FAIL_invalid_data_file/input.py: + returns: 1 + FAIL_IO_error/input.py: + returns: 1 + FAIL_malformed_data_file/input.py: + returns: 1 + compare: + - test/SIM_mc_generation/verif_data/MonteCarlo_Meta_data_output vs. test/SIM_mc_generation/MonteCarlo_Meta_data_output + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/RUN_000/monte_input_a.py vs. test/SIM_mc_generation/MONTE_RUN_nominal/RUN_000/monte_input_a.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/RUN_000/log_test_data.csv vs. test/SIM_mc_generation/MONTE_RUN_nominal/RUN_000/log_test_data.csv + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/RUN_001/monte_input_a.py vs. test/SIM_mc_generation/MONTE_RUN_nominal/RUN_001/monte_input_a.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/RUN_001/log_test_data.csv vs. test/SIM_mc_generation/MONTE_RUN_nominal/RUN_001/log_test_data.csv + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_nominal/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/monte_variables vs. test/SIM_mc_generation/MONTE_RUN_nominal/monte_variables + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/MonteCarlo_Meta_data_output vs. test/SIM_mc_generation/MONTE_RUN_nominal/MonteCarlo_Meta_data_output + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/monte_variables vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/monte_variables + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_file_inconsistent_skip/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_file_inconsistent_skip/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_out_of_domain_error/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_out_of_domain_error/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_random_value_truncation/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_random_value_truncation/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_random_value_truncation/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_random_value_truncation/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_generate_meta_data_early/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_generate_meta_data_early/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_generate_meta_data_early/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_generate_meta_data_early/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_file_skip/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_both_variables/monte_variables vs. test/SIM_mc_generation/MONTE_RUN_remove_variable/RUN_both_variables/monte_variables + - test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_one_variable/monte_variables vs. test/SIM_mc_generation/MONTE_RUN_remove_variable/RUN_one_variable/monte_variables + - test/SIM_mc_generation/verif_data/MONTE_RUN_WARN_config_error/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_WARN_config_error/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_WARN_invalid_name/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_WARN_invalid_name/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_WARN_overconstrained_config/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_WARN_overconstrained_config/RUN_0/monte_input.py diff --git a/test/SIM_mc_generation/MCGTrickOps/MCGenerationTestNoGenerate.yml b/test/SIM_mc_generation/MCGTrickOps/MCGenerationTestNoGenerate.yml new file mode 100644 index 00000000..4de55deb --- /dev/null +++ b/test/SIM_mc_generation/MCGTrickOps/MCGenerationTestNoGenerate.yml @@ -0,0 +1,156 @@ +SIM_mc_generation: + path: test/SIM_mc_generation + runs: + RUN_nominal/input_a.py: + RUN_random_normal_truncate_abs/input.py: + RUN_random_normal_truncate_rel/input.py: + RUN_random_normal_truncate_sd/input.py: + RUN_random_normal__untruncate/input.py: + RUN_random_normal_untruncated/input.py: + RUN_random_uniform/input.py: + RUN_ERROR_file_inconsistent_skip/input.py: + RUN_ERROR_invalid_call/input.py: + RUN_ERROR_invalid_name/input.py: + RUN_ERROR_invalid_sequence/input.py: + RUN_ERROR_invalid_sequencing/input.py: + RUN_ERROR_out_of_domain_error/input.py: + RUN_ERROR_random_value_truncation/input.py: + RUN_generate_meta_data_early/input.py: + RUN_file_sequential/input.py: + RUN_file_skip/input.py: + RUN_file_skip2/input.py: + RUN_remove_variable/input.py: + RUN_WARN_config_error/input.py: + RUN_WARN_invalid_name/input.py: + RUN_WARN_overconstrained_config/input.py: + FAIL_config_error/input.py: + returns: 1 + FAIL_duplicate_variable/input.py: + returns: 1 + FAIL_illegal_config/input.py: + returns: 1 + FAIL_invalid_config/input.py: + returns: 1 + FAIL_invalid_data_file/input.py: + returns: 1 + FAIL_IO_error/input.py: + returns: 1 + FAIL_malformed_data_file/input.py: + returns: 1 + compare: + - test/SIM_mc_generation/verif_data/MonteCarlo_Meta_data_output vs. test/SIM_mc_generation/MonteCarlo_Meta_data_output + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/RUN_000/monte_input_a.py vs. test/SIM_mc_generation/MONTE_RUN_nominal/RUN_000/monte_input_a.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/RUN_001/monte_input_a.py vs. test/SIM_mc_generation/MONTE_RUN_nominal/RUN_001/monte_input_a.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_nominal/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/monte_variables vs. test/SIM_mc_generation/MONTE_RUN_nominal/monte_variables + - test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/MonteCarlo_Meta_data_output vs. test/SIM_mc_generation/MONTE_RUN_nominal/MonteCarlo_Meta_data_output + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_abs/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_abs/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_rel/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_rel/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_truncate_sd/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_truncate_sd/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/monte_variables vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/monte_variables + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal__untruncate/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal__untruncate/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_normal_untruncated/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_normal_untruncated/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_random_uniform/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_random_uniform/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_file_inconsistent_skip/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_file_inconsistent_skip/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_out_of_domain_error/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_out_of_domain_error/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_random_value_truncation/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_random_value_truncation/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_random_value_truncation/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_ERROR_random_value_truncation/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_generate_meta_data_early/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_generate_meta_data_early/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_generate_meta_data_early/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_generate_meta_data_early/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_file_skip/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_5/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_5/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_6/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_6/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_7/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_7/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_8/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_8/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip/RUN_9/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip/RUN_9/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/monte_values_all_runs vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/monte_values_all_runs + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/RUN_1/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/RUN_1/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/RUN_2/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/RUN_2/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/RUN_3/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/RUN_3/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_file_skip2/RUN_4/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_file_skip2/RUN_4/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_both_variables/monte_variables vs. test/SIM_mc_generation/MONTE_RUN_remove_variable/RUN_both_variables/monte_variables + - test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_one_variable/monte_variables vs. test/SIM_mc_generation/MONTE_RUN_remove_variable/RUN_one_variable/monte_variables + - test/SIM_mc_generation/verif_data/MONTE_RUN_WARN_config_error/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_WARN_config_error/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_WARN_invalid_name/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_WARN_invalid_name/RUN_0/monte_input.py + - test/SIM_mc_generation/verif_data/MONTE_RUN_WARN_overconstrained_config/RUN_0/monte_input.py vs. test/SIM_mc_generation/MONTE_RUN_WARN_overconstrained_config/RUN_0/monte_input.py diff --git a/test/SIM_mc_generation/MONTE_IO_FAIL/git_hold b/test/SIM_mc_generation/MONTE_IO_FAIL/git_hold new file mode 100644 index 00000000..eecc0d1b --- /dev/null +++ b/test/SIM_mc_generation/MONTE_IO_FAIL/git_hold @@ -0,0 +1,2 @@ +This file holds MONTE_FAIL_IO_error2 in the git repo so its permissions can be +modified to support the FAIL_IO_error2 test. diff --git a/test/SIM_mc_generation/MONTE_IO_RUN_ERROR1/git_hold b/test/SIM_mc_generation/MONTE_IO_RUN_ERROR1/git_hold new file mode 100644 index 00000000..85c67f6e --- /dev/null +++ b/test/SIM_mc_generation/MONTE_IO_RUN_ERROR1/git_hold @@ -0,0 +1,2 @@ +This file holds MONTE_ERROR_IO_error in the git repo so its permissions can be +modified to support the ERROR_IO_error test. diff --git a/test/SIM_mc_generation/MONTE_IO_RUN_ERROR2/MonteCarlo_Meta_data_output b/test/SIM_mc_generation/MONTE_IO_RUN_ERROR2/MonteCarlo_Meta_data_output new file mode 100644 index 00000000..0a466a73 --- /dev/null +++ b/test/SIM_mc_generation/MONTE_IO_RUN_ERROR2/MonteCarlo_Meta_data_output @@ -0,0 +1,21 @@ + + +*************************** SUMMARY ************************** +1 total assignments + - 0 constant values + - 0 calculated variables + - 1 prescribed (file-based) variables + - 0 random variables + - 0 files for execution + - 0 variables of undefined type + +********************* LIST OF VARIABLES, TYPES**************** +test.x_file_lookup[0], Prescribed +************************************************************** + + +***** LIST OF DATA FILES AND THE VARIABLES THEY POPULATE ***** +****** +Modified_data/datafile.txt +3 test.x_file_lookup[0] +************************************************************** diff --git a/test/SIM_mc_generation/MONTE_IO_RUN_ERROR2/RUN_0/monte_input.py b/test/SIM_mc_generation/MONTE_IO_RUN_ERROR2/RUN_0/monte_input.py new file mode 100644 index 00000000..cff2d504 --- /dev/null +++ b/test/SIM_mc_generation/MONTE_IO_RUN_ERROR2/RUN_0/monte_input.py @@ -0,0 +1,7 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('IO_RUN_ERROR2/input.py').read()) +monte_carlo.mc_master.monte_run_number = 0 + +test.x_file_lookup[0] = 2 diff --git a/test/SIM_mc_generation/MONTE_IO_RUN_ERROR2/monte_variables b/test/SIM_mc_generation/MONTE_IO_RUN_ERROR2/monte_variables new file mode 100644 index 00000000..cd043da2 --- /dev/null +++ b/test/SIM_mc_generation/MONTE_IO_RUN_ERROR2/monte_variables @@ -0,0 +1,2 @@ +run_number +test.x_file_lookup[0], diff --git a/test/SIM_mc_generation/Modified_data/datafile.txt b/test/SIM_mc_generation/Modified_data/datafile.txt new file mode 100644 index 00000000..405e062d --- /dev/null +++ b/test/SIM_mc_generation/Modified_data/datafile.txt @@ -0,0 +1,7 @@ +0 1 2 3 4 +# comment +10 11 12 13 14 + +20 21 22 23 24 + +30 31 32 33 34 diff --git a/test/SIM_mc_generation/Modified_data/datafile_1.txt b/test/SIM_mc_generation/Modified_data/datafile_1.txt new file mode 100644 index 00000000..405e062d --- /dev/null +++ b/test/SIM_mc_generation/Modified_data/datafile_1.txt @@ -0,0 +1,7 @@ +0 1 2 3 4 +# comment +10 11 12 13 14 + +20 21 22 23 24 + +30 31 32 33 34 diff --git a/test/SIM_mc_generation/Modified_data/empty_file.txt b/test/SIM_mc_generation/Modified_data/empty_file.txt new file mode 100644 index 00000000..e69de29b diff --git a/test/SIM_mc_generation/Modified_data/monte_variables.py b/test/SIM_mc_generation/Modified_data/monte_variables.py new file mode 100644 index 00000000..310950df --- /dev/null +++ b/test/SIM_mc_generation/Modified_data/monte_variables.py @@ -0,0 +1,104 @@ +mc_var = trick.MonteCarloVariableRandomUniform( "test.x_uniform", 0, 10, 20) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal", 2, 10, 2) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[0]", 2, 10, 2) +mc_var.truncate(0.5, trick.MonteCarloVariableRandomNormal.StandardDeviation) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[1]", 2, 10, 2) +mc_var.truncate(-0.5, 0.7, trick.MonteCarloVariableRandomNormal.Relative) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[2]", 2, 10, 2) +mc_var.truncate(9.9,11, trick.MonteCarloVariableRandomNormal.Absolute) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[3]", 2, 10, 2) +mc_var.truncate_low(9.9, trick.MonteCarloVariableRandomNormal.Absolute) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[4]", 2, 10, 2) +mc_var.truncate_high(4, trick.MonteCarloVariableRandomNormal.Absolute) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_length", 2, 10, 2) +mc_var.units = "ft" +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomUniformInt( "test.x_integer", 1, 0, 2) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomStringSet( "test.x_string", 3) +mc_var.add_string("\"ABC\"") +mc_var.add_string("\"DEF\"") +mc_var.add_string("'GHIJKL'") +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloPythonLineExec( "test.x_line_command", + "test.x_integer * test.x_uniform") +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloPythonLineExec( + "test.standalone_function( test.x_normal)") +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloPythonFileExec( "Modified_data/sample.py") +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomBool( "test.x_boolean", 4) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# call this one mc_var1 because I'm going to use it as the seed for the +# MonteCarloVariableSemiFixed later. +mc_var1 = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 3) +mc_var1.thisown = False +monte_carlo.mc_master.add_variable(mc_var1) + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[1]", + "Modified_data/datafile.txt", + 2) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[2]", + "Modified_data/datafile.txt", + 1) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + + +mc_var = trick.MonteCarloVariableFixed( "test.x_fixed_value_int", 7) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableFixed( "test.x_fixed_value_double", 7.0) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableFixed( "test.x_fixed_value_string", "\"7\"") +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableSemiFixed( "test.x_semi_fixed_value", mc_var1 ) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + diff --git a/test/SIM_mc_generation/Modified_data/sample.py b/test/SIM_mc_generation/Modified_data/sample.py new file mode 100644 index 00000000..fd5e7827 --- /dev/null +++ b/test/SIM_mc_generation/Modified_data/sample.py @@ -0,0 +1,3 @@ +test.x_file_command[0] = 1 +test.x_file_command[1] = monte_carlo.mc_master.monte_run_number +test.x_file_command[2] = test.x_file_command[0] + test.x_file_command[1] diff --git a/test/SIM_mc_generation/Modified_data/single_col_1.txt b/test/SIM_mc_generation/Modified_data/single_col_1.txt new file mode 100644 index 00000000..97b3d1a5 --- /dev/null +++ b/test/SIM_mc_generation/Modified_data/single_col_1.txt @@ -0,0 +1,15 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 diff --git a/test/SIM_mc_generation/Modified_data/single_col_2.txt b/test/SIM_mc_generation/Modified_data/single_col_2.txt new file mode 100644 index 00000000..5e565722 --- /dev/null +++ b/test/SIM_mc_generation/Modified_data/single_col_2.txt @@ -0,0 +1,15 @@ +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 diff --git a/test/SIM_mc_generation/Modified_data/single_col_3.txt b/test/SIM_mc_generation/Modified_data/single_col_3.txt new file mode 100644 index 00000000..a96c36d5 --- /dev/null +++ b/test/SIM_mc_generation/Modified_data/single_col_3.txt @@ -0,0 +1,15 @@ +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 diff --git a/test/SIM_mc_generation/README b/test/SIM_mc_generation/README new file mode 100644 index 00000000..fa76a9b6 --- /dev/null +++ b/test/SIM_mc_generation/README @@ -0,0 +1,44 @@ +Verification simulation of monte_carlo. + + +The following tests rely on setting the directories to be non-writable. Their +purpose is to detect situations in which the monte-carlo model cannot generate +certain files. These tests by their very nature are difficult to run within an +automated scripted testing system. + + IO_FAIL + IO_RUN_ERROR1 + IO_RUN_ERROR2 + + +The following cases are expecting either a warning or an error but +the simulation does not terminate. Instead, ZERO is returned. +The purpose of these cases is to emit the error or warning, not generate +viable datasets. Do not evaluate any of these cases for good dispersions. +There is no telling what state the data is in after the warning / error +message is emitted. + + RUN_ERROR_file_inconsistent_skip + RUN_ERROR_invalid_call + RUN_ERROR_invalid_name + RUN_ERROR_invalid_sequence + RUN_ERROR_invalid_sequencing + RUN_ERROR_IO_error + RUN_ERROR_IO_error2 + RUN_ERROR_out_of_domain_error + RUN_ERROR_random_value_truncation + RUN_WARN_config_error + RUN_WARN_invalid_name + RUN_WARN_overconstrained_config + + +The following cases emit a fatal error and the simulation halts in its tracks: + + FAIL_config_error + FAIL_duplicate_variable + FAIL_illegal_config + FAIL_invalid_config + FAIL_invalid_data_file + FAIL_IO_error + FAIL_IO_error2 + FAIL_malformed_data_file diff --git a/test/SIM_mc_generation/RUN_ERROR_file_inconsistent_skip/input.py b/test/SIM_mc_generation/RUN_ERROR_file_inconsistent_skip/input.py new file mode 100644 index 00000000..becba617 --- /dev/null +++ b/test/SIM_mc_generation/RUN_ERROR_file_inconsistent_skip/input.py @@ -0,0 +1,46 @@ +monte_carlo.mc_master.activate("RUN_ERROR_file_inconsistent_skip") +monte_carlo.mc_master.set_num_runs(1) + +print('*********************************************************************') +print('these messages are expected:') +print(' Error Invalid configuration') +print(' It is not permissible for two variables looking at the same file to') +print(' operate under different line-selection criteria.') +print(' test.x_file_lookup[1]') +print(' will be switched to the behavior of') +print(' test.x_file_lookup_rl[0],') +print(' which has a setting for the maximum number of lines to skip of 3') +print('') +print(' Error Invalid configuration') +print(' It is not permissible for two variables looking at the same file to') +print(' operate under different line-selection criteria.') +print(' test.x_file_lookup[2]') +print(' will be switched to the behavior of') +print(' test.x_file_lookup_rl[0],') +print(' which has a setting for the maximum number of lines to skip of 3') +print('*********************************************************************') + + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 3) +mc_var.thisown = False +mc_var.max_skip = 3 +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[1]", + "Modified_data/datafile.txt", + 2) +mc_var.thisown = False +mc_var.max_skip = 2 +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[2]", + "Modified_data/datafile.txt", + 1) +mc_var.thisown = False +mc_var.max_skip = 1 +monte_carlo.mc_master.add_variable(mc_var) + + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_ERROR_invalid_call/input.py b/test/SIM_mc_generation/RUN_ERROR_invalid_call/input.py new file mode 100644 index 00000000..120ec268 --- /dev/null +++ b/test/SIM_mc_generation/RUN_ERROR_invalid_call/input.py @@ -0,0 +1,21 @@ +monte_carlo.mc_master.activate("RUN_ERROR_invalid_call") +monte_carlo.mc_master.set_num_runs(1) + +print('*********************************************************************') +print('this message is expected:') +print(' Error Invalid call') +print(' Attempted to register a dependent identified with NULL pointer with') +print(' the MonteCarloVariableFile for variable test.x_file_lookup[0].') +print(' This is not a valid action.') +print(' Registration failed, exiting without action.') +print('*********************************************************************') + + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 3) +# the next command is the source of the error! +mc_var.register_dependent(None) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_ERROR_invalid_name/input.py b/test/SIM_mc_generation/RUN_ERROR_invalid_name/input.py new file mode 100644 index 00000000..e52baf8a --- /dev/null +++ b/test/SIM_mc_generation/RUN_ERROR_invalid_name/input.py @@ -0,0 +1,15 @@ +monte_carlo.mc_master.activate("RUN_ERROR_invalid_name") +monte_carlo.mc_master.set_num_runs(1) + +print('*********************************************************************') +print('this message is expected:\n'+ + ' Error Invalid name\n' + + ' Could not find MonteCarlo variable with name test.x_uniform.\n'+ + ' Returning a NULL pointer.') +print('*********************************************************************') + +# empty monte carlo master without any variables. +# lets ask it find us a variable. +monte_carlo.mc_master.find_variable("test.x_uniform") + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_ERROR_invalid_sequence/input.py b/test/SIM_mc_generation/RUN_ERROR_invalid_sequence/input.py new file mode 100644 index 00000000..4271cca8 --- /dev/null +++ b/test/SIM_mc_generation/RUN_ERROR_invalid_sequence/input.py @@ -0,0 +1,37 @@ +monte_carlo.mc_master.activate("RUN_ERROR_invalid_sequence") +monte_carlo.mc_master.set_num_runs(1) + +print('*****************************************************************************************************************************') +print('three (3) types of messages are expected:') +print(' Error Invalid sequence') +print(' Attempted to set the number of runs to 10, but the input files have') +print(' already been generated.') +print('') +print(' Error Invalid sequence') +print(' Attempted to add a new variable test.x_normal to run RUN_invalid_sequence, but the input files have already been generated.') +print(' Cannot modify input files to accommodate this new variable.') +print(' Addition of variable rejected.') +print('') +print(' Error Invalid sequence') +print(' Attempted to generate a set of input files, but this action has') +print(' already been completed. Keeping the original set of input files.') +print(' Ignoring the later instruction.') +print('*****************************************************************************************************************************') + + +mc_var = trick.MonteCarloVariableRandomUniform( "test.x_uniform", 0, 10, 20) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# Trigger the "Invalid sequence" errors in MonteCarloMaster +monte_carlo.mc_master.prepare_input_files() + +# Change run-number after prepping inputs +monte_carlo.mc_master.set_num_runs(10) + +# Add a new variable after prepping inputs +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal", 2, 10, 2) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_ERROR_invalid_sequencing/input.py b/test/SIM_mc_generation/RUN_ERROR_invalid_sequencing/input.py new file mode 100644 index 00000000..99de57ea --- /dev/null +++ b/test/SIM_mc_generation/RUN_ERROR_invalid_sequencing/input.py @@ -0,0 +1,18 @@ +monte_carlo.mc_master.activate("RUN_ERROR_invalid_sequencing") +monte_carlo.mc_master.set_num_runs(1) + +print('***********************************************************************************') +print('this message is expected:') +print(' Error Invalid sequencing') +print(' For variable test.x_semi_fixed_value, the necessary pre-dispersion to obtain the') +print(' random value for assignment has not been completed.') +print(' Cannot generate the assignment for this variable.') +print('***********************************************************************************') + + +mc_var = trick.MonteCarloVariableSemiFixed("test.x_semi_fixed_value", + test.mc_var_file) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_ERROR_out_of_domain_error/input.py b/test/SIM_mc_generation/RUN_ERROR_out_of_domain_error/input.py new file mode 100644 index 00000000..495e08cf --- /dev/null +++ b/test/SIM_mc_generation/RUN_ERROR_out_of_domain_error/input.py @@ -0,0 +1,18 @@ +monte_carlo.mc_master.activate("RUN_ERROR_out_of_domain_error") +monte_carlo.mc_master.set_num_runs(1) + +print('***********************************************************************') +print('these messages are expected:') +print(' Error Out-of-domain error') +print(' Negative double-sided truncation specified for variable test.x_normal') +print(' truncate() must receive either two limits or one positive limit!') +print(' Using absolute value of limit.') +print('***********************************************************************') + + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal" ) +# the next command is the source of the error! +mc_var.truncate(-1.0) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_ERROR_random_value_truncation/input.py b/test/SIM_mc_generation/RUN_ERROR_random_value_truncation/input.py new file mode 100644 index 00000000..4c97d2aa --- /dev/null +++ b/test/SIM_mc_generation/RUN_ERROR_random_value_truncation/input.py @@ -0,0 +1,57 @@ +monte_carlo.mc_master.activate("RUN_ERROR_random_value_truncation") +monte_carlo.mc_master.set_num_runs(2) + +print('******************************************************************************************') +print('multiple error messages are expected:') +print(' Error Random value truncation failure') +print(' Could not generate a value for test.x_normal_trunc[#] within the specified domain within') +print(' the specified maximum number of tries (1).') +print(' Assuming a value equal to:') +print(' - midpoint value for a distribution truncated at both ends') +print(' - truncation value for a distribution truncated at only one end.') +print('') +print('NOTE: three tests are included here to test different code path after error') +print(' message is emitted') +print('******************************************************************************************') + + +# give some crazy initial values to the class +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[0]", 9956453, 10, 3.5) +mc_var.thisown = False +# give very small truncate_low & truncate_high distrubution values +mc_var.truncate_low(0.1) +mc_var.truncate_high(0.2) +# lower max_num_tries to an unreasonable # to generate the desired error +mc_var.max_num_tries = 1 +monte_carlo.mc_master.add_variable(mc_var) + +# note: this test also covers mc_variable_random_normal.cc lines 84-86 +# in the same code section after the message + + +# give some crazy initial values to the class +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[1]", 99565644453, 10, 3.5) +mc_var.thisown = False +# give very small truncate_low distrubution value +mc_var.truncate_low(0.1) +# lower max_num_tries to an unreasonable # to generate the desired error +mc_var.max_num_tries = 1 +monte_carlo.mc_master.add_variable(mc_var) + +# note: this test also covers mc_variable_random_normal.cc lines 78-80 +# in the same code section after the message + + +# give some crazy initial values to the class +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[2]", 99565644453, 10, 3.5) +mc_var.thisown = False +# give negative truncate_high distrubution value +mc_var.truncate_high(-50.2) +# lower max_num_tries to an unreasonable # to generate the desired error +mc_var.max_num_tries = 1 +monte_carlo.mc_master.add_variable(mc_var) + +# note: this test also covers mc_variable_random_normal.cc lines 81-83 +# in the same code section after the message + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_WARN_config_error/input.py b/test/SIM_mc_generation/RUN_WARN_config_error/input.py new file mode 100644 index 00000000..95fe91f8 --- /dev/null +++ b/test/SIM_mc_generation/RUN_WARN_config_error/input.py @@ -0,0 +1,20 @@ +monte_carlo.mc_master.activate("RUN_WARN_config_error") +monte_carlo.mc_master.set_num_runs(1) + +print('***************************************************************************************') +print('this message is expected:') +print(' Warning Configuration error') +print(' Zero truncation specified for variable test.x_normal which will produce a fixed point') +print('***************************************************************************************') + + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal", 24858569, 10, 2) + +# this call generates the warning! +mc_var.truncate(0.0) + +# fix up the values before generating the assignment to avoid spurrious errors +mc_var.truncate(7.0, 13.0, trick.MonteCarloVariableRandomNormal.Absolute) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_WARN_invalid_name/input.py b/test/SIM_mc_generation/RUN_WARN_invalid_name/input.py new file mode 100644 index 00000000..bc0eb5e0 --- /dev/null +++ b/test/SIM_mc_generation/RUN_WARN_invalid_name/input.py @@ -0,0 +1,22 @@ +monte_carlo.mc_master.activate("RUN_WARN_invalid_name") +monte_carlo.mc_master.set_num_runs(1) + +print('*********************************************************************************') +print('this message is expected:') +print(' Warning Invalid name') +print(' Attempt to remove MonteCarlo variable with name monte_carlo.not_found_variable FAILED.') +print(' Did not find a variable with that name.') +print('*********************************************************************************') + + +# give the simulation something to do +# add a variable into memory so list is not empty when we go to +# remove a non-existant variable from memory. +mc_var = trick.MonteCarloVariableRandomUniform( "test.x_uniform", 0, 10, 20) +monte_carlo.mc_master.add_variable(mc_var) + +# Trigger the "Invalid Name" warning in MonteCarloMaster +# by calling remove_variable() to remove a non-existant variable name +monte_carlo.mc_master.remove_variable("monte_carlo.not_found_variable") + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_WARN_overconstrained_config/input.py b/test/SIM_mc_generation/RUN_WARN_overconstrained_config/input.py new file mode 100644 index 00000000..c7df0b1b --- /dev/null +++ b/test/SIM_mc_generation/RUN_WARN_overconstrained_config/input.py @@ -0,0 +1,19 @@ +monte_carlo.mc_master.activate("RUN_WARN_overconstrained_config") +monte_carlo.mc_master.set_num_runs(1) + +print('*******************************************************') +print('these messages are expected:') +print(' Warning Overconstrained configuration') +print(' For variable The distribution collapses to a point.') +print(' the specified minimum allowable value and') +print(' the specified maximum allowable value are equal (14).') +print('*******************************************************') + + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal", 2, 10, 2) +# the next two commands are neded to produce the warning! +mc_var.truncate_low(2.0) +mc_var.truncate_high(2.0) +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_file_sequential/input.py b/test/SIM_mc_generation/RUN_file_sequential/input.py new file mode 100644 index 00000000..735c3405 --- /dev/null +++ b/test/SIM_mc_generation/RUN_file_sequential/input.py @@ -0,0 +1,48 @@ +monte_carlo.mc_master.activate("RUN_file_sequential") +monte_carlo.mc_master.set_num_runs(10) + + +mc_var0 = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 3) +monte_carlo.mc_master.add_variable(mc_var0) + +mc_var1 = trick.MonteCarloVariableFile( "test.x_file_lookup[1]", + "Modified_data/datafile.txt", + 2) +monte_carlo.mc_master.add_variable(mc_var1) + +mc_var2 = trick.MonteCarloVariableFile( "test.x_file_lookup[2]", + "Modified_data/datafile.txt", + 1) +monte_carlo.mc_master.add_variable(mc_var2) + + +print("\nmc_var0.has_dependents() returns: " + str(mc_var0.has_dependents())) +print("mc_var0.get_column_number() returns: " + str(mc_var0.get_column_number())) +print("mc_var0.get_first_column_number() returns: " + str(mc_var0.get_first_column_number())) +print("mc_var0.get_filename() returns: '" + mc_var0.get_filename() + "'") +print("\nmc_var1.has_dependents() returns: " + str(mc_var1.has_dependents())) +print("mc_var1.get_column_number() returns: " + str(mc_var1.get_column_number())) +print("mc_var1.get_first_column_number() returns: " + str(mc_var1.get_first_column_number())) +print("mc_var1.get_filename() returns: '" + mc_var1.get_filename() + "'") +print("\nmc_var2.has_dependents() returns: " + str(mc_var2.has_dependents())) +print("mc_var2.get_column_number() returns: " + str(mc_var2.get_column_number())) +print("mc_var2.get_first_column_number() returns: " + str(mc_var2.get_first_column_number())) +print("mc_var2.get_filename() returns: '" + mc_var2.get_filename() + "'") + +# call parent class' "virtual int get_seed() const" method. should return ZERO. +# code coverage for: mc_variable.hh, line 70 +print("\ncode coverage for parent's get_seed() virtual method... should return ZERO.") +print("mc_var2.get_seed() returns: " + str(mc_var2.get_seed())) + +# Check the validity of looking up a variable by name. +print("\nmonte_carloing 'find_variable' and 'get_variable_name' for test.x_file_lookup[0]: "+ + "returns: " + + monte_carlo.mc_master.find_variable("test.x_file_lookup[0]").get_variable_name()) +print("monte_carloing 'find_variable' and 'get_variable_name' for test.x_file_lookup[1]: "+ + "returns: " + + monte_carlo.mc_master.find_variable("test.x_file_lookup[1]").get_variable_name() + "\n") + + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_file_skip/input.py b/test/SIM_mc_generation/RUN_file_skip/input.py new file mode 100644 index 00000000..1915b627 --- /dev/null +++ b/test/SIM_mc_generation/RUN_file_skip/input.py @@ -0,0 +1,26 @@ +monte_carlo.mc_master.activate("RUN_file_skip") +monte_carlo.mc_master.set_num_runs(10) + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/datafile.txt", + 3) +mc_var.thisown = False +mc_var.max_skip = 3 +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[1]", + "Modified_data/datafile.txt", + 2) +mc_var.thisown = False +mc_var.max_skip = 3 +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[2]", + "Modified_data/datafile.txt", + 1) +mc_var.thisown = False +mc_var.max_skip = 3 +monte_carlo.mc_master.add_variable(mc_var) + + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_file_skip2/input.py b/test/SIM_mc_generation/RUN_file_skip2/input.py new file mode 100644 index 00000000..b4cc1e4a --- /dev/null +++ b/test/SIM_mc_generation/RUN_file_skip2/input.py @@ -0,0 +1,32 @@ +monte_carlo.mc_master.activate("RUN_file_skip2") +# For regression monte_carloing, use 5 runs +# For verification, setting this value to 250 results in 2 duplications. +monte_carlo.mc_master.set_num_runs(5) +#monte_carlo.mc_master.set_num_runs(250) + + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[0]", + "Modified_data/single_col_1.txt", + 0, + 0) +mc_var.max_skip = 1 +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[1]", + "Modified_data/single_col_2.txt", + 0, + 0) +mc_var.max_skip = 2 +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableFile( "test.x_file_lookup[2]", + "Modified_data/single_col_3.txt", + 0, + 0) +mc_var.max_skip = 3 +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_generate_meta_data_early/input.py b/test/SIM_mc_generation/RUN_generate_meta_data_early/input.py new file mode 100644 index 00000000..7ad59c17 --- /dev/null +++ b/test/SIM_mc_generation/RUN_generate_meta_data_early/input.py @@ -0,0 +1,11 @@ +monte_carlo.mc_master.activate("RUN_generate_meta_data_early") +monte_carlo.mc_master.set_num_runs(1) +monte_carlo.mc_master.generate_meta_data = True +monte_carlo.mc_master.input_file_name = "input.py" + +exec(open("Modified_data/monte_variables.py").read()) + +# By running this early, the MonteCarlo_Meta_data_output file +# should end up in the sim directory instead of the MONTE_RUN.. +# directory +monte_carlo.mc_master.collate_meta_data() diff --git a/test/SIM_mc_generation/RUN_nominal/input.py b/test/SIM_mc_generation/RUN_nominal/input.py new file mode 100644 index 00000000..f6b14d38 --- /dev/null +++ b/test/SIM_mc_generation/RUN_nominal/input.py @@ -0,0 +1,48 @@ +# The Monte Carlo tool uses a double execution of the S-main: +# - pass #1 uses the scenario input.py file to process the variables identified +# for dispersion. A specified number, N, of values {v_1, ..., v_N} is +# generated for each variable v, with the values constrained by the specified +# distribution of v; N is specified in the input file. +# A set of N files, {RUN_1/monte_input.py, ... , RUN_N/monte_input.py} is +# created, with each file containing one of the set of values for each +# variable. Once these files are generated, the simulation is complete for +# pass #1 and it terminates. +# - pass #2 uses one of the generated files (monte_input.py) as the input file +# for a regular execution of the simulation. There will typically be many +# executions of the sim, one for each of the generated monte_input.py files. + +# This input file provides one example of how to test this two-pass process, +# although it is admittedly a bit convoluted and hard to read. TODO: Once +# TrickOps is capable of operating with this monte-carlo implementation, that +# framework can manage both the generation and local execution of generated +# monte_input.py files, removing the need for this type of "sim that launches a +# sim" test methodology -Jordan 10/2022 + +# For the purpose of expedient testing, we generate and run only 2 files. +# This is sufficient to demonstrate "multiple" without unnecessarily +# burning CPU time. + +import os +exename = "S_main_" + os.getenv("TRICK_HOST_CPU") + ".exe" + +# Pass #1 Generate the set of scenarios with unique dispersions +print("Processing Pass #1 for run RUN_nominal") +input_file = "RUN_nominal/input_a.py" +ret = os.system("./" + exename + " " + input_file) +if ret != 0: + trick.exec_terminate_with_return(1, "double_pass.py", 34, "Error running " + input_file) + +# Pass #2 Run the scenarios. Logged data will go into each scenario's folder +print("") +print("") +print("Processing Pass #2 for run RUN_nominal") +for ii in range(2): + input_file = "MONTE_RUN_nominal"+"/RUN_00%d/monte_input_a.py" %ii + print ("**************** %s" %input_file) + ret = os.system("./" + exename + " " + input_file) + if ret != 0: + trick.exec_terminate_with_return(1, "double_pass.py", 43, "Error running " + input_file) + +# To be compatible with our current unit-sim framework, this file has to be a +# simulation input file. Therefore it needs a stop time so it doesn't run forever. +trick.stop(0.0) diff --git a/test/SIM_mc_generation/RUN_nominal/input_a.py b/test/SIM_mc_generation/RUN_nominal/input_a.py new file mode 100644 index 00000000..d5ecb48d --- /dev/null +++ b/test/SIM_mc_generation/RUN_nominal/input_a.py @@ -0,0 +1,37 @@ +# Instruct sim to generate MC files for RUN_verif. +# This could be done in a top-level MC-launch script +monte_carlo.mc_master.activate("RUN_nominal") +monte_carlo.mc_master.set_num_runs(2) +monte_carlo.mc_master.generate_meta_data = True +monte_carlo.mc_master.input_file_name = "input_a.py" +monte_carlo.mc_master.minimum_padding = 3 + + + +# Standard if-tests for a regular multi-purpose input file, allowing for a +# MC-implementation of a general scenario. + +# NOTE: in this case, the first test is redundant because this input file is +# ALWAYS going to have mc-master be active. But this is likely to get +# copied and used as a template. + +# Quick breakdown: +# - if running with MC: +# (this test allows a general input file to have MC-specific content) +# +# - setup logging and any other MC-specific configurations +# +# - if generating dispersions, generate them. +# (This test separates out the execution of pass#1 (which generates the +# dispersions) from that of pass#2 (which executes with those +# dispersions). Without this test blocking the generation on pass#2, the +# dispersions would get regenerated for every actual run, which is +# completely unnecessary.) +if monte_carlo.mc_master.active: + # Logging + exec(open("Log_data/log_nominal.py").read()) + + if monte_carlo.mc_master.generate_dispersions: + exec(open("Modified_data/monte_variables.py").read()) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_random_normal__untruncate/input.py b/test/SIM_mc_generation/RUN_random_normal__untruncate/input.py new file mode 100644 index 00000000..0ab6ad12 --- /dev/null +++ b/test/SIM_mc_generation/RUN_random_normal__untruncate/input.py @@ -0,0 +1,22 @@ +monte_carlo.mc_master.activate("RUN_random_normal__untruncate") +monte_carlo.mc_master.set_num_runs(10) + +# generate a set of numbers +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[0]", 2, 10, 2) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[1]", 2, 10, 2) +# signal an absolute truncation +mc_var.truncate(8, 12, trick.MonteCarloVariableRandomNormal.Absolute) +# changed my mind. no longer wish to truncate. +# this method turns off 'truncated_low' and 'truncated_high' indicators, leaving +# the original variable values alone! +# NOTE: the two values in this sim should match! +# +# code coverage for untruncate() method, mc_variable_random_normal.cc, lines 204-205 +mc_var.untruncate() +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_random_normal_truncate_abs/input.py b/test/SIM_mc_generation/RUN_random_normal_truncate_abs/input.py new file mode 100644 index 00000000..7419212b --- /dev/null +++ b/test/SIM_mc_generation/RUN_random_normal_truncate_abs/input.py @@ -0,0 +1,34 @@ +monte_carlo.mc_master.activate("RUN_random_normal_truncate_abs") +# Use 10 runs for regression comparison; use more (10,000) for confirming +# statistical distribution. +monte_carlo.mc_master.set_num_runs(10) + +# should keep values between -10 and 10, exclusive +# this one calls 'truncate_low(-10)' and 'truncate_high(10)' to establish +# truncation bounds so we need to pivot around ZERO. Otherwise, if mean +# is not ZERO, ex. 25, the truncation bounds would be -25 and 25 which +# does not match the desired values of +/- 10 of mean. +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[0]", 11122, 0, 5) +mc_var.truncate(10, trick.MonteCarloVariableRandomNormal.Absolute) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# should keep values 72.5 thru 85.0, exclusive +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[1]", 77546, 75, 5) +mc_var.truncate(72.5, 85, trick.MonteCarloVariableRandomNormal.Absolute) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# should keep values greater than 90.0 +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[2]", 60540, 100, 5) +mc_var.truncate_low(90, trick.MonteCarloVariableRandomNormal.Absolute) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# should keep values less than 135.0 +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[3]", 77077, 125, 5) +mc_var.truncate_high(135, trick.MonteCarloVariableRandomNormal.Absolute) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_random_normal_truncate_rel/input.py b/test/SIM_mc_generation/RUN_random_normal_truncate_rel/input.py new file mode 100644 index 00000000..f26828ae --- /dev/null +++ b/test/SIM_mc_generation/RUN_random_normal_truncate_rel/input.py @@ -0,0 +1,33 @@ +monte_carlo.mc_master.activate("RUN_random_normal_truncate_rel") +# Use 10 runs for regression comparison; use more (10,000) for confirming +# statistical distribution. +monte_carlo.mc_master.set_num_runs(10) + +# should keep values between -10 and 10, exclusive +# this one computes mean+10, resulting in calls to +# 'truncate_low(-10)' and 'truncate_high(10)' to establish +# truncation bounds. +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[0]", 11122, 0, 5) +mc_var.truncate(10, trick.MonteCarloVariableRandomNormal.Relative) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# should keep values 72.5 thru 85.0, exclusive +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[1]", 77546, 75, 5) +mc_var.truncate(-2.5, 10, trick.MonteCarloVariableRandomNormal.Relative) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# should keep values greater than 90.0 +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[2]", 60540, 100, 5) +mc_var.truncate_low(-10, trick.MonteCarloVariableRandomNormal.Relative) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# should keep values less than 135.0 +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[3]", 77077, 125, 5) +mc_var.truncate_high(10, trick.MonteCarloVariableRandomNormal.Relative) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_random_normal_truncate_sd/input.py b/test/SIM_mc_generation/RUN_random_normal_truncate_sd/input.py new file mode 100644 index 00000000..861fc92b --- /dev/null +++ b/test/SIM_mc_generation/RUN_random_normal_truncate_sd/input.py @@ -0,0 +1,33 @@ +monte_carlo.mc_master.activate("RUN_random_normal_truncate_sd") +# Use 10 runs for regression comparison; use more (10,000) for confirming +# statistical distribution. +monte_carlo.mc_master.set_num_runs(10) + +# should keep values between -10 and 10, exclusive +# this one computes (2*std_dev)+mean, resulting in calls to +# 'truncate_low(-10)' and 'truncate_high(10)' to establish +# truncation bounds. +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[0]", 11122, 0, 5) +mc_var.truncate(2, trick.MonteCarloVariableRandomNormal.StandardDeviation) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# should keep values 72.5 thru 85.0, exclusive +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[1]", 77546, 75, 5) +mc_var.truncate(-0.5, 2, trick.MonteCarloVariableRandomNormal.StandardDeviation) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# should keep values greater than 90.0 +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[2]", 60540, 100, 5) +mc_var.truncate_low(-2, trick.MonteCarloVariableRandomNormal.StandardDeviation) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# should keep values less than 135.0 +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[3]", 77077, 125, 5) +mc_var.truncate_high(2, trick.MonteCarloVariableRandomNormal.StandardDeviation) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_random_normal_untruncated/input.py b/test/SIM_mc_generation/RUN_random_normal_untruncated/input.py new file mode 100644 index 00000000..2048263f --- /dev/null +++ b/test/SIM_mc_generation/RUN_random_normal_untruncated/input.py @@ -0,0 +1,26 @@ +monte_carlo.mc_master.activate("RUN_random_normal_untruncated") +# Use 10 runs for regression comparison; use more (10,000) for confirming +# statistical distribution. +monte_carlo.mc_master.set_num_runs(10) + +# normal distribution from approximately -17.5 to 18.1 +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[0]", 11122, 0, 5) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# normal distribution from approximately 53.1 to 94.5 +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[1]", 77546, 75, 5) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# normal distribution from approximately 81.4 to 119.75 +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[2]", 60540, 100, 5) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# normal distribution from approximately 106.3 to 144.7 +mc_var = trick.MonteCarloVariableRandomNormal( "test.x_normal_trunc[3]", 77077, 125, 5) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_random_uniform/input.py b/test/SIM_mc_generation/RUN_random_uniform/input.py new file mode 100644 index 00000000..b75d7df5 --- /dev/null +++ b/test/SIM_mc_generation/RUN_random_uniform/input.py @@ -0,0 +1,16 @@ +monte_carlo.mc_master.activate("RUN_random_uniform") +# Use 10 runs for regression comparison; use more (10,000) for confirming +# statistical distribution. +monte_carlo.mc_master.set_num_runs(10) + +# generate random uniformly distributed floating point values from 100.0 to 100,000.0 +mc_var = trick.MonteCarloVariableRandomUniform( "test.x_uniform", 77545, 100.0, 100000.0) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +# generate random uniformly distributed integer values from 100.0 to 100,000.0 +mc_var = trick.MonteCarloVariableRandomUniformInt( "test.x_integer", 77001, 100, 100000) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_remove_variable/input.py b/test/SIM_mc_generation/RUN_remove_variable/input.py new file mode 100644 index 00000000..7671c1ed --- /dev/null +++ b/test/SIM_mc_generation/RUN_remove_variable/input.py @@ -0,0 +1,33 @@ +# The purpose of this test is to: +# Execute the first input file to add two variables into MonteCarloMaster. +# Execute the second input file to add the same variables into +# MonteCarloMaster, but removes one of them before monte carlo RUN files +# are generated. +# Then it compares the variable lists from the monte carlo runs. If they differ, +# the test terminate with a non-zero return. + +import os +exename = "S_main_" + os.getenv("TRICK_HOST_CPU") + ".exe" + +print("Processing 1st input file for run RUN_remove_variable") +input_file = "RUN_remove_variable/input_a.py" +ret = os.system("./" + exename + " " + input_file) +if ret != 0: + trick.exec_terminate_with_return(1, "input.py", 16, "Error running " + input_file) + +print("Processing 2nd input file for run RUN_remove_variable") +input_file = "RUN_remove_variable/input_b.py" +ret = os.system("./" + exename + " " + input_file) +if ret != 0: + trick.exec_terminate_with_return(1, "input.py", 22, "Error running " + input_file) + +print('Checking if the variable was successfully removed') +ret = os.system("diff -q MONTE_RUN_remove_variable/RUN_0/monte_variables MONTE_RUN_remove_variable/RUN_1/monte_variables > /dev/null") +if ret != 0: + trick.exec_terminate_with_return(0, "input.py", 27, "variable successfully removed!") +else: + trick.exec_terminate_with_return(1, "input.py", 29, "variable 'test.x_fixed_value_int' was not removed!") + +# To be compatible with our current unit-sim framework, this file has to be a +# simulation input file. Therefore it needs a stop time so it doesn't run forever. +trick.stop(0.0) diff --git a/test/SIM_mc_generation/RUN_remove_variable/input_a.py b/test/SIM_mc_generation/RUN_remove_variable/input_a.py new file mode 100644 index 00000000..0f9eace5 --- /dev/null +++ b/test/SIM_mc_generation/RUN_remove_variable/input_a.py @@ -0,0 +1,7 @@ +monte_carlo.mc_master.activate("RUN_remove_variable/RUN_both_variables") +monte_carlo.mc_master.set_num_runs(1) +monte_carlo.mc_master.input_file_name = "input_a.py" + +exec(open("RUN_remove_variable/variable_list.py").read()) + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_remove_variable/input_b.py b/test/SIM_mc_generation/RUN_remove_variable/input_b.py new file mode 100644 index 00000000..9d78d6f3 --- /dev/null +++ b/test/SIM_mc_generation/RUN_remove_variable/input_b.py @@ -0,0 +1,11 @@ +monte_carlo.mc_master.activate("RUN_remove_variable/RUN_one_variable") +monte_carlo.mc_master.set_num_runs(1) +monte_carlo.mc_master.input_file_name = "input_b.py" + +exec(open("RUN_remove_variable/variable_list.py").read()) + +# execute remove_variable() (success path) +# code coverage for mc_master.cc, remove_variable(), lines 288-289 +monte_carlo.mc_master.remove_variable("test.x_fixed_value_int") + +trick.stop(1) diff --git a/test/SIM_mc_generation/RUN_remove_variable/variable_list.py b/test/SIM_mc_generation/RUN_remove_variable/variable_list.py new file mode 100644 index 00000000..ae36a273 --- /dev/null +++ b/test/SIM_mc_generation/RUN_remove_variable/variable_list.py @@ -0,0 +1,8 @@ + +mc_var = trick.MonteCarloVariableRandomUniform( "test.x_uniform", 0, 10, 20) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) + +mc_var = trick.MonteCarloVariableFixed( "test.x_fixed_value_int", 7) +mc_var.thisown = False +monte_carlo.mc_master.add_variable(mc_var) diff --git a/test/SIM_mc_generation/S_define b/test/SIM_mc_generation/S_define new file mode 100644 index 00000000..ba39de48 --- /dev/null +++ b/test/SIM_mc_generation/S_define @@ -0,0 +1,58 @@ +/***************************************************************************** +PURPOSE: SIM object used to Exercise the features and failure cases of the +MonteCarloGeneration model. +PROGRAMMERS: + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +*****************************************************************************/ +#include "sim_objects/default_trick_sys.sm" +#include "sim_objects/MonteCarloGenerate.sm" + +/***************************************************************************** +MC_Test sim object +****************************************************************************/ + +class MC_TestSimObject : public Trick::SimObject +{ + public: + double x_uniform; + double x_normal; + double x_normal_trunc[5]; + double x_normal_length; // (m) Dispersed in ft. + double x_line_command; + double x_file_command[3]; + double x_file_lookup[3]; + double x_fixed_value_double; + double x_semi_fixed_value; + int x_fixed_value_int; + int x_integer; + bool x_boolean; + std::string x_string; + std::string x_fixed_value_string; + int x_sdefine_routine_called; + + MonteCarloVariableFile mc_var_file; + + MC_TestSimObject() + : + mc_var_file("test.x_file_lookup[0]", "Modified_data/datafile.txt", 3) + { + ("initialization") monte_carlo.generate_dispersions(); + (1.0,"environment") output_strings(); + }; + + // standalone_function is used to test using an instance of + // MonteCarloPythonLineExec to execute a standalone function. + void standalone_function( double value) + { + std::cout << "\nStandalone_function received a value of " << value << "\n"; + x_sdefine_routine_called = 1; + } + private: + void output_strings() { + std::cout << "\nstrings : " << x_string << " : " << x_fixed_value_string << "\n"; + } + MC_TestSimObject( const MC_TestSimObject&); + MC_TestSimObject & operator= ( const MC_TestSimObject&); + +}; +MC_TestSimObject test; diff --git a/test/SIM_mc_generation/S_overrides.mk b/test/SIM_mc_generation/S_overrides.mk new file mode 100644 index 00000000..7e7c1958 --- /dev/null +++ b/test/SIM_mc_generation/S_overrides.mk @@ -0,0 +1,8 @@ +TRICK_CFLAGS += -g -Wall -Wextra +TRICK_CXXFLAGS += -g -std=c++11 -Wall -Wextra +# We can't yet make warnings to be errors on MacOS, because +# MACOS deprecates and warns about sprintf. But SWIG +# still generates code containing sprintf.. +ifneq ($(TRICK_HOST_TYPE), Darwin) +TRICK_CXXFLAGS += -Werror -Wno-stringop-truncation +endif diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py new file mode 100644 index 00000000..9e1bad17 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py @@ -0,0 +1,7 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_ERROR_invalid_call/input.py').read()) +monte_carlo.mc_master.monte_run_number = 0 + +test.x_file_lookup[0] = 2 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py new file mode 100644 index 00000000..f877043b --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py @@ -0,0 +1,6 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_ERROR_invalid_name/input.py').read()) +monte_carlo.mc_master.monte_run_number = 0 + diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py new file mode 100644 index 00000000..436d46e3 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py @@ -0,0 +1,7 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_ERROR_invalid_sequence/input.py').read()) +monte_carlo.mc_master.monte_run_number = 0 + +test.x_uniform = 15.92844616516683 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py new file mode 100644 index 00000000..3339b9a1 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py @@ -0,0 +1,6 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_ERROR_invalid_sequencing/input.py').read()) +monte_carlo.mc_master.monte_run_number = 0 + diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_0/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_0/monte_input.py new file mode 100644 index 00000000..a7dcf583 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_0/monte_input.py @@ -0,0 +1,9 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_file_sequential/input.py').read()) +monte_carlo.mc_master.monte_run_number = 0 + +test.x_file_lookup[0] = 2 +test.x_file_lookup[1] = 1 +test.x_file_lookup[2] = 0 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_1/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_1/monte_input.py new file mode 100644 index 00000000..d6789b49 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_1/monte_input.py @@ -0,0 +1,9 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_file_sequential/input.py').read()) +monte_carlo.mc_master.monte_run_number = 1 + +test.x_file_lookup[0] = 12 +test.x_file_lookup[1] = 11 +test.x_file_lookup[2] = 10 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_2/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_2/monte_input.py new file mode 100644 index 00000000..89683e01 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_2/monte_input.py @@ -0,0 +1,9 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_file_sequential/input.py').read()) +monte_carlo.mc_master.monte_run_number = 2 + +test.x_file_lookup[0] = 22 +test.x_file_lookup[1] = 21 +test.x_file_lookup[2] = 20 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_3/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_3/monte_input.py new file mode 100644 index 00000000..d7c22687 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_3/monte_input.py @@ -0,0 +1,9 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_file_sequential/input.py').read()) +monte_carlo.mc_master.monte_run_number = 3 + +test.x_file_lookup[0] = 32 +test.x_file_lookup[1] = 31 +test.x_file_lookup[2] = 30 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_4/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_4/monte_input.py new file mode 100644 index 00000000..ecc75837 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_4/monte_input.py @@ -0,0 +1,9 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_file_sequential/input.py').read()) +monte_carlo.mc_master.monte_run_number = 4 + +test.x_file_lookup[0] = 2 +test.x_file_lookup[1] = 1 +test.x_file_lookup[2] = 0 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_5/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_5/monte_input.py new file mode 100644 index 00000000..3e3cd4e7 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_5/monte_input.py @@ -0,0 +1,9 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_file_sequential/input.py').read()) +monte_carlo.mc_master.monte_run_number = 5 + +test.x_file_lookup[0] = 12 +test.x_file_lookup[1] = 11 +test.x_file_lookup[2] = 10 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_6/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_6/monte_input.py new file mode 100644 index 00000000..60074fed --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_6/monte_input.py @@ -0,0 +1,9 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_file_sequential/input.py').read()) +monte_carlo.mc_master.monte_run_number = 6 + +test.x_file_lookup[0] = 22 +test.x_file_lookup[1] = 21 +test.x_file_lookup[2] = 20 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_7/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_7/monte_input.py new file mode 100644 index 00000000..3d01585a --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_7/monte_input.py @@ -0,0 +1,9 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_file_sequential/input.py').read()) +monte_carlo.mc_master.monte_run_number = 7 + +test.x_file_lookup[0] = 32 +test.x_file_lookup[1] = 31 +test.x_file_lookup[2] = 30 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_8/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_8/monte_input.py new file mode 100644 index 00000000..38be7fe3 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_8/monte_input.py @@ -0,0 +1,9 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_file_sequential/input.py').read()) +monte_carlo.mc_master.monte_run_number = 8 + +test.x_file_lookup[0] = 2 +test.x_file_lookup[1] = 1 +test.x_file_lookup[2] = 0 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_9/monte_input.py b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_9/monte_input.py new file mode 100644 index 00000000..dda981fc --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_9/monte_input.py @@ -0,0 +1,9 @@ +monte_carlo.mc_master.active = True +monte_carlo.mc_master.generate_dispersions = False + +exec(open('RUN_file_sequential/input.py').read()) +monte_carlo.mc_master.monte_run_number = 9 + +test.x_file_lookup[0] = 12 +test.x_file_lookup[1] = 11 +test.x_file_lookup[2] = 10 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/monte_values_all_runs b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/monte_values_all_runs new file mode 100644 index 00000000..4eb16b27 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/monte_values_all_runs @@ -0,0 +1,10 @@ +0, 2, 1, 0 +1, 12, 11, 10 +2, 22, 21, 20 +3, 32, 31, 30 +4, 2, 1, 0 +5, 12, 11, 10 +6, 22, 21, 20 +7, 32, 31, 30 +8, 2, 1, 0 +9, 12, 11, 10 diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/MonteCarlo_Meta_data_output b/test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/MonteCarlo_Meta_data_output new file mode 100644 index 00000000..c4743ab8 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/MonteCarlo_Meta_data_output @@ -0,0 +1,74 @@ + + +*************************** SUMMARY ************************** +21 total assignments + - 4 constant values + - 1 calculated variables + - 3 prescribed (file-based) variables + - 11 random variables + - 2 files for execution + - 0 variables of undefined type + +********************* LIST OF VARIABLES, TYPES**************** +test.x_boolean, Random +test.x_file_lookup[0], Prescribed +test.x_file_lookup[1], Prescribed +test.x_file_lookup[2], Prescribed +test.x_fixed_value_double, Constant +test.x_fixed_value_int, Constant +test.x_fixed_value_string, Constant +test.x_integer, Random +test.x_line_command, Calculated +test.x_normal, Random +test.x_normal_length, Random +test.x_normal_trunc[0], Random +test.x_normal_trunc[1], Random +test.x_normal_trunc[2], Random +test.x_normal_trunc[3], Random +test.x_normal_trunc[4], Random +test.x_semi_fixed_value, Constant +test.x_string, Random +test.x_uniform, Random +************************************************************** + + +*********** LIST OF EXECUTABLE FILES AND FUNCTIONS *********** +test.standalone_function( test.x_normal) +*** +Modified_data/sample.py +*** +************************************************************** + + +***** LIST OF DATA FILES AND THE VARIABLES THEY POPULATE ***** +****** +Modified_data/datafile.txt +1 test.x_file_lookup[2] +2 test.x_file_lookup[1] +3 test.x_file_lookup[0] +************************************************************** + + +*****Duplicate seeds; check for intentional correlations***** +2 test.x_normal +2 test.x_normal_trunc[0] +2 test.x_normal_trunc[1] +2 test.x_normal_trunc[2] +2 test.x_normal_trunc[3] +2 test.x_normal_trunc[4] +2 test.x_normal_length +************************************************************** + + +************************ ALL SEEDS ************************* +0 test.x_uniform +1 test.x_integer +2 test.x_normal +2 test.x_normal_trunc[0] +2 test.x_normal_trunc[1] +2 test.x_normal_trunc[2] +2 test.x_normal_trunc[3] +2 test.x_normal_trunc[4] +2 test.x_normal_length +3 test.x_string +4 test.x_boolean diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/monte_variables b/test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/monte_variables new file mode 100644 index 00000000..2a836cdf --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/monte_variables @@ -0,0 +1,17 @@ +run_number +test.x_uniform, +test.x_normal, +test.x_normal_trunc[0], +test.x_normal_trunc[1], +test.x_normal_trunc[2], +test.x_normal_trunc[3], +test.x_normal_trunc[4], +test.x_normal_length, ft +test.x_integer, +test.x_boolean, +test.x_file_lookup[0], +test.x_file_lookup[1], +test.x_file_lookup[2], +test.x_fixed_value_int, +test.x_fixed_value_double, +test.x_semi_fixed_value, diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_both_variables/monte_variables b/test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_both_variables/monte_variables new file mode 100644 index 00000000..a3d0916b --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_both_variables/monte_variables @@ -0,0 +1,3 @@ +run_number +test.x_uniform, +test.x_fixed_value_int, diff --git a/test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_one_variable/monte_variables b/test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_one_variable/monte_variables new file mode 100644 index 00000000..0e31b6c5 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_one_variable/monte_variables @@ -0,0 +1,2 @@ +run_number +test.x_uniform, diff --git a/test/SIM_mc_generation/verif_data/MonteCarlo_Meta_data_output b/test/SIM_mc_generation/verif_data/MonteCarlo_Meta_data_output new file mode 100644 index 00000000..bbe69dd8 --- /dev/null +++ b/test/SIM_mc_generation/verif_data/MonteCarlo_Meta_data_output @@ -0,0 +1,78 @@ +Generating meta-data on the current configuration. +The input files have not yet been generated which means this +configuration has not been finalized and is subject to change. +Sending meta-data to top-level directory. + + +*************************** SUMMARY ************************** +21 total assignments + - 4 constant values + - 1 calculated variables + - 3 prescribed (file-based) variables + - 11 random variables + - 2 files for execution + - 0 variables of undefined type + +********************* LIST OF VARIABLES, TYPES**************** +test.x_boolean, Random +test.x_file_lookup[0], Prescribed +test.x_file_lookup[1], Prescribed +test.x_file_lookup[2], Prescribed +test.x_fixed_value_double, Constant +test.x_fixed_value_int, Constant +test.x_fixed_value_string, Constant +test.x_integer, Random +test.x_line_command, Calculated +test.x_normal, Random +test.x_normal_length, Random +test.x_normal_trunc[0], Random +test.x_normal_trunc[1], Random +test.x_normal_trunc[2], Random +test.x_normal_trunc[3], Random +test.x_normal_trunc[4], Random +test.x_semi_fixed_value, Constant +test.x_string, Random +test.x_uniform, Random +************************************************************** + + +*********** LIST OF EXECUTABLE FILES AND FUNCTIONS *********** +test.standalone_function( test.x_normal) +*** +Modified_data/sample.py +*** +************************************************************** + + +***** LIST OF DATA FILES AND THE VARIABLES THEY POPULATE ***** +****** +Modified_data/datafile.txt +3 test.x_file_lookup[0] +2 test.x_file_lookup[1] +1 test.x_file_lookup[2] +************************************************************** + + +*****Duplicate seeds; check for intentional correlations***** +2 test.x_normal +2 test.x_normal_trunc[0] +2 test.x_normal_trunc[1] +2 test.x_normal_trunc[2] +2 test.x_normal_trunc[3] +2 test.x_normal_trunc[4] +2 test.x_normal_length +************************************************************** + + +************************ ALL SEEDS ************************* +0 test.x_uniform +1 test.x_integer +2 test.x_normal +2 test.x_normal_trunc[0] +2 test.x_normal_trunc[1] +2 test.x_normal_trunc[2] +2 test.x_normal_trunc[3] +2 test.x_normal_trunc[4] +2 test.x_normal_length +3 test.x_string +4 test.x_boolean diff --git a/test_overrides.mk b/test_overrides.mk index c80a7618..9e793302 100644 --- a/test_overrides.mk +++ b/test_overrides.mk @@ -6,4 +6,4 @@ include ${TRICK_HOME}/share/trick/makefiles/Makefile.common unexport TRICK_PYTHON_PATH sim_test: - python3 trickops.py ${TRICK_HOME} + python3 trickops.py --quiet diff --git a/test_sims.yml b/test_sims.yml index 00de016a..777e60ba 100644 --- a/test_sims.yml +++ b/test_sims.yml @@ -5,9 +5,9 @@ SIM_alloc_test: path: test/SIM_alloc_test SIM_anon_enum: path: test/SIM_anon_enum -SIM_default_member_initializer: +SIM_default_member_initializer: path: test/SIM_default_member_initializer -SIM_delete_default_constructor: +SIM_delete_default_constructor: path: test/SIM_delete_default_constructor SIM_demo_inputfile: path: test/SIM_demo_inputfile @@ -19,7 +19,7 @@ SIM_measurement_units: path: test/SIM_measurement_units SIM_parse_s_define: path: test/SIM_parse_s_define -SIM_target_specific_variables: +SIM_target_specific_variables: path: test/SIM_target_specific_variables SIM_test_abstract: path: test/SIM_test_abstract @@ -35,163 +35,163 @@ SIM_satellite: path: trick_sims/SIM_satellite # Normal case compile and run sims -SIM_demo_sdefine: +SIM_demo_sdefine: path: test/SIM_demo_sdefine - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_exec_set_time_tic_value: path: test/SIM_exec_set_time_tic_value - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_python_namespace: path: test/SIM_python_namespace - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_rti: path: test/SIM_rti - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_test_dp: path: test/SIM_test_dp - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_test_icg_file_skipped: path: test/SIM_test_icg_file_skipped - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_test_io: path: test/SIM_test_io - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_test_ip: path: test/SIM_test_ip - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_test_sched: path: test/SIM_test_sched - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_test_templates: path: test/SIM_test_templates - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_threads: path: test/SIM_threads - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_trickified: path: test/SIM_trickified - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_ball_L1: path: trick_sims/Ball/SIM_ball_L1 - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_ball_L2: path: trick_sims/Ball/SIM_ball_L2 - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_ball_L3: path: trick_sims/Ball/SIM_ball_L3 - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_cannon_aero: path: trick_sims/Cannon/SIM_cannon_aero - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_cannon_analytic: path: trick_sims/Cannon/SIM_cannon_analytic - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_cannon_eulercromer: path: trick_sims/Cannon/SIM_cannon_eulercromer - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_cannon_numeric: path: trick_sims/Cannon/SIM_cannon_numeric - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_cannon_jet: path: trick_sims/Cannon/SIM_cannon_jet - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_Ball++_L1: path: trick_sims/SIM_Ball++_L1 - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_sun: path: trick_sims/SIM_sun - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: returns: 0 SIM_earlyterm: path: test/SIM_earlyterm - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/input.py: @@ -201,19 +201,20 @@ SIM_earlyterm: # setup.py dumps a checkpoint # unit_test.py loads that checkpoint and verifies the data -SIM_stls: +SIM_stls: path: test/SIM_stls - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/setup.py: + phase: -1 returns: 0 RUN_test/unit_test.py: returns: 0 SIM_test_dr: path: test/SIM_test_dr - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: @@ -225,42 +226,36 @@ SIM_test_dr: # All the dump.py runs dump a checkpoint # All the unit_test.py runs load that checkpoint and then compare against expected logs -SIM_checkpoint_data_recording: +SIM_checkpoint_data_recording: path: test/SIM_checkpoint_data_recording - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: - RUN_test1/dump.py: - returns: 0 - RUN_test2/dump.py: - returns: 0 - RUN_test3/dump.py: - returns: 0 - RUN_test4/dump.py: - returns: 0 - RUN_test5/dump.py: + RUN_test[1-5]/dump.py: + phase: -1 returns: 0 RUN_test6/dump.py: returns: 0 + # Note we could use the [1-5] notation here if RUN_test2 didn't stand out as not matching the pattern -Jordan 1/2023 RUN_test1/unit_test.py: returns: 0 - compare: + compare: - test/SIM_checkpoint_data_recording/RUN_test1/ref_log_foo.csv vs. test/SIM_checkpoint_data_recording/RUN_test1/log_foo.csv RUN_test2/unit_test.py: returns: 0 analyze: './test/SIM_checkpoint_data_recording/RUN_test2/check_log.sh' RUN_test3/unit_test.py: returns: 0 - compare: + compare: - test/SIM_checkpoint_data_recording/RUN_test3/ref_log_foo.csv vs. test/SIM_checkpoint_data_recording/RUN_test3/log_foo.csv RUN_test4/unit_test.py: returns: 0 - compare: + compare: - test/SIM_checkpoint_data_recording/RUN_test4/ref_log_foo.csv vs. test/SIM_checkpoint_data_recording/RUN_test4/log_foo.csv RUN_test5/unit_test.py: returns: 0 - compare: + compare: - test/SIM_checkpoint_data_recording/RUN_test5/ref_log_foo.csv vs. test/SIM_checkpoint_data_recording/RUN_test5/log_foo.csv RUN_test6/unit_test.py: returns: 0 @@ -269,7 +264,7 @@ SIM_checkpoint_data_recording: SIM_events: path: test/SIM_events - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" runs: RUN_test/unit_test.py: @@ -285,7 +280,7 @@ SIM_events: # The variable server client and SIM_amoeba sometimes fail to connect and need to be retried SIM_test_varserv: path: test/SIM_test_varserv - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" labels: - retries_allowed @@ -294,10 +289,136 @@ SIM_test_varserv: returns: 0 SIM_amoeba: path: trick_sims/Cannon/SIM_amoeba - build_command: "trick-CP -t" + build_args: "-t" binary: "T_main_{cpu}_test.exe" labels: - retries_allowed runs: RUN_test/unit_test.py: returns: 0 + +#TODO: all compares should be vs. , need to swap order! +SIM_mc_generation: + path: test/SIM_mc_generation + runs: + RUN_nominal/input_a.py: + phase: -1 + compare: + - test/SIM_mc_generation/MONTE_RUN_nominal/MonteCarlo_Meta_data_output vs. test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/MonteCarlo_Meta_data_output + - test/SIM_mc_generation/MONTE_RUN_nominal/monte_variables vs. test/SIM_mc_generation/verif_data/MONTE_RUN_nominal/monte_variables + MONTE_RUN_nominal/RUN_[000-001]/monte_input_a.py: + compare: + RUN_random_normal_truncate_abs/input.py: + phase: -1 + compare: + MONTE_RUN_random_normal_truncate_abs/RUN_[0-9]/monte_input.py: + compare: + RUN_random_normal_truncate_rel/input.py: + phase: -1 + compare: + MONTE_RUN_random_normal_truncate_rel/RUN_[0-9]/monte_input.py: + compare: + RUN_random_normal_truncate_sd/input.py: + compare: + phase: -1 + MONTE_RUN_random_normal_truncate_sd/RUN_[0-9]/monte_input.py: + compare: + RUN_random_normal__untruncate/input.py: + compare: + phase: -1 + MONTE_RUN_random_normal__untruncate/RUN_[0-9]/monte_input.py: + compare: + RUN_random_normal_untruncated/input.py: + compare: + phase: -1 + MONTE_RUN_random_normal_untruncated/RUN_[0-9]/monte_input.py: + compare: + RUN_random_uniform/input.py: + compare: + phase: -1 + MONTE_RUN_random_uniform/RUN_[0-9]/monte_input.py: + compare: + RUN_ERROR_file_inconsistent_skip/input.py: + phase: -1 + MONTE_RUN_ERROR_file_inconsistent_skip/RUN_0/monte_input.py: + RUN_ERROR_invalid_call/input.py: + compare: + - test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py vs. test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py + phase: -1 + MONTE_RUN_ERROR_invalid_call/RUN_0/monte_input.py: + RUN_ERROR_invalid_name/input.py: + compare: + - test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py vs. test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py + phase: -1 + MONTE_RUN_ERROR_invalid_name/RUN_0/monte_input.py: + RUN_ERROR_invalid_sequence/input.py: + compare: + - test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py vs. test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py + phase: -1 + MONTE_RUN_ERROR_invalid_sequence/RUN_0/monte_input.py: + RUN_ERROR_invalid_sequencing/input.py: + compare: + - test/SIM_mc_generation/MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py vs. test/SIM_mc_generation/verif_data/MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py + phase: -1 + MONTE_RUN_ERROR_invalid_sequencing/RUN_0/monte_input.py: + RUN_ERROR_out_of_domain_error/input.py: + compare: + phase: -1 + MONTE_RUN_ERROR_out_of_domain_error/RUN_0/monte_input.py: + RUN_ERROR_random_value_truncation/input.py: + phase: -1 + MONTE_RUN_ERROR_random_value_truncation/RUN_[0-1]/monte_input.py: + compare: + RUN_generate_meta_data_early/input.py: + compare: + - test/SIM_mc_generation/MonteCarlo_Meta_data_output vs. test/SIM_mc_generation/verif_data/MonteCarlo_Meta_data_output + phase: -1 + RUN_file_sequential/input.py: + compare: + - test/SIM_mc_generation/MONTE_RUN_file_sequential/monte_values_all_runs vs. test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/monte_values_all_runs + phase: -1 + MONTE_RUN_file_sequential/RUN_[0-9]/monte_input.py: + compare: + # Note that technically monte_input.py is an output of RUN_file_sequential, not these runs. The comparisons + # are only placed here due to limitations of TrickOps [min-max] range notation. See the comment block in + # TrickWorkflow.Run.multiply() for details. This workaround is also used in more runs for this sim. -Jordan 1/2023 + - test/SIM_mc_generation/MONTE_RUN_file_sequential/RUN_[0-9]/monte_input.py vs. test/SIM_mc_generation/verif_data/MONTE_RUN_file_sequential/RUN_[0-9]/monte_input.py + RUN_file_skip/input.py: + phase: -1 + MONTE_RUN_file_skip/RUN_[0-9]/monte_input.py: + compare: + RUN_file_skip2/input.py: + phase: -1 + MONTE_RUN_file_skip2/RUN_[0-4]/monte_input.py: + RUN_remove_variable/input_a.py: + compare: + - test/SIM_mc_generation/MONTE_RUN_remove_variable/RUN_both_variables/monte_variables vs. test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_both_variables/monte_variables + RUN_remove_variable/input_b.py: + compare: + - test/SIM_mc_generation/MONTE_RUN_remove_variable/RUN_one_variable/monte_variables vs. test/SIM_mc_generation/verif_data/MONTE_RUN_remove_variable/RUN_one_variable/monte_variables + RUN_WARN_config_error/input.py: + compare: + phase: -1 + MONTE_RUN_WARN_config_error/RUN_0/monte_input.py: + RUN_WARN_invalid_name/input.py: + compare: + phase: -1 + MONTE_RUN_WARN_invalid_name/RUN_0/monte_input.py: + RUN_WARN_overconstrained_config/input.py: + compare: + phase: -1 + MONTE_RUN_WARN_overconstrained_config/RUN_0/monte_input.py: + FAIL_config_error/input.py: + returns: 1 + FAIL_duplicate_variable/input.py: + returns: 1 + FAIL_illegal_config/input.py: + returns: 1 + FAIL_invalid_config/input.py: + returns: 1 + FAIL_invalid_data_file/input.py: + returns: 1 + FAIL_IO_error/input.py: + returns: 1 + FAIL_malformed_data_file/input.py: + returns: 1 diff --git a/trick_source/sim_services/MonteCarloGeneration/Makefile b/trick_source/sim_services/MonteCarloGeneration/Makefile new file mode 100644 index 00000000..915b9526 --- /dev/null +++ b/trick_source/sim_services/MonteCarloGeneration/Makefile @@ -0,0 +1,4 @@ +include $(dir $(lastword $(MAKEFILE_LIST)))../../../share/trick/makefiles/Makefile.common +include ${TRICK_HOME}/share/trick/makefiles/Makefile.tricklib +TRICK_CXXFLAGS += -std=c++11 +-include Makefile_deps diff --git a/trick_source/sim_services/MonteCarloGeneration/Makefile_deps b/trick_source/sim_services/MonteCarloGeneration/Makefile_deps new file mode 100644 index 00000000..7f215187 --- /dev/null +++ b/trick_source/sim_services/MonteCarloGeneration/Makefile_deps @@ -0,0 +1,24 @@ +object_${TRICK_HOST_CPU}/mc_master.o: mc_master.cc \ + ${TRICK_HOME}/include/trick/mc_master.hh \ + ${TRICK_HOME}/include/trick/message_type.h \ + ${TRICK_HOME}/include/trick/message_proto.h \ + ${TRICK_HOME}/include/trick/exec_proto.h +object_${TRICK_HOST_CPU}/mc_variable.o: mc_variable.cc \ + ${TRICK_HOME}/include/trick/mc_variable.hh \ + ${TRICK_HOME}/include/trick/message_type.h \ + ${TRICK_HOME}/include/trick/message_proto.h \ + ${TRICK_HOME}/include/trick/exec_proto.h +object_${TRICK_HOST_CPU}/mc_variable_file.o: mc_variable_file.cc \ + ${TRICK_HOME}/include/trick/mc_variable_file.hh \ + ${TRICK_HOME}/include/trick/message_type.h \ + ${TRICK_HOME}/include/trick/message_proto.h \ + ${TRICK_HOME}/include/trick/exec_proto.h +object_${TRICK_HOST_CPU}/mc_variable_random_normal.o: mc_variable_random_normal.cc \ + ${TRICK_HOME}/include/trick/mc_variable_random_normal.hh \ + ${TRICK_HOME}/include/trick/message_type.h \ + ${TRICK_HOME}/include/trick/message_proto.h \ + ${TRICK_HOME}/include/trick/exec_proto.h +object_${TRICK_HOST_CPU}/mc_variable_random_string.o: mc_variable_random_string.cc \ + ${TRICK_HOME}/include/trick/mc_variable_random_string.hh +object_${TRICK_HOST_CPU}/mc_variable_random_string.o: mc_variable_random_uniform.cc \ + ${TRICK_HOME}/include/trick/mc_variable_random_uniform.hh diff --git a/trick_source/sim_services/MonteCarloGeneration/README.md b/trick_source/sim_services/MonteCarloGeneration/README.md new file mode 120000 index 00000000..13cd31eb --- /dev/null +++ b/trick_source/sim_services/MonteCarloGeneration/README.md @@ -0,0 +1 @@ +../../../docs/documentation/miscellaneous_trick_tools/MonteCarloGeneration.md \ No newline at end of file diff --git a/trick_source/sim_services/MonteCarloGeneration/mc_master.cc b/trick_source/sim_services/MonteCarloGeneration/mc_master.cc new file mode 100644 index 00000000..7fd4ba45 --- /dev/null +++ b/trick_source/sim_services/MonteCarloGeneration/mc_master.cc @@ -0,0 +1,583 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: (Provides the front-end interface to the monte-carlo model) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#include "trick/mc_master.hh" + +#include // std::prev +#include // std::ofstream +#include // system +#include "trick/message_type.h" +#include "trick/message_proto.h" +#include "trick/exec_proto.h" + + +/***************************************************************************** +Constructor +*****************************************************************************/ +MonteCarloMaster::MonteCarloMaster( + std::string location_) + : + active(false), + generate_dispersions(true), + run_name(), + monte_dir(), + input_file_name("input.py"), + generate_meta_data(false), + generate_summary(true), + minimum_padding(0), + monte_run_number(0), + input_files_prepared(false), + location(location_), + variables(), + num_runs(0) +{} + +/***************************************************************************** +activate +Purpose:() +*****************************************************************************/ +void +MonteCarloMaster::activate( + std::string run_name_) +{ + run_name = run_name_; + monte_dir = "MONTE_"+run_name; + active = true; +} + +/***************************************************************************** +prepare_input_files +Purpose:(Creates the top-level MONTE_ directory, clearing out any + existing content. + Creates the RUN_ subdirectories + Creates the monte_ in each RUN_ subdirectory.) +*****************************************************************************/ +bool +MonteCarloMaster::prepare_input_files() +{ + if (input_files_prepared) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Invalid sequence\n" + "Attempted to " + "generate a set of input files, but this action has\nalready been " + "completed. Keeping the original set of input files.\nIgnoring " + "the later instruction.\n"; + message_publish(MSG_ERROR, message.c_str()); + return true; + } + + // Create the new MONTE_ dir name where runs will go + std::string command = "mkdir -p "+monte_dir; + system( command.c_str()); + command = "rm -rf "+monte_dir+"/RUN_*"; + // TODO should we check return code for failure here? -Jordan 2/2020 + system( command.c_str()); + + // Based on the number of runs, generate an appropriately-sized string to + // contain the run number. + // E.g. For 1-10 runs, need only 1 numeric character to supply: + // RUN_0, RUN_1, RUN_2, ..., RUN_9 + // For between 10001-100000 runs, need 5 numeric characters to supply: + // RUN_00000, RUN_00001, ..., RUN_99999 + // This string length can be set to a MINIMUM value with the variable + // minimum_padding. + int max_length = std::to_string(num_runs-1).size(); + if (max_length < minimum_padding) { + max_length = minimum_padding; + } + std::string run_num_base(max_length, '0'); + + + // create the master list of varaibles being recorded in the monte_values + // files. note that this is not all the variables, only those whose values + // are being recorded ina s eparate file for every run. For a full list of + // variables and their type, see the meta-data file. + if (generate_summary) { + // Create the summary file and variable list + std::string filename = monte_dir + "/monte_variables"; + std::ofstream variable_list(filename); + // Check for success of file creation + if (!variable_list.is_open()) { + std::string message = std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", I/O error\nUnable to open the " + + "variable summary files for writing.\nDispersion summary will not " + + "be generated.\n"; + message_publish(MSG_ERROR, message.c_str()); + generate_summary = false; + } + else { + // Write the variable list + variable_list << "run_number\n"; + for (auto var_it : variables) { + if (var_it->include_in_summary) { + variable_list << var_it->get_variable_name() + << ", " << var_it->units << "\n"; + } + } + variable_list.close(); + } + } + + // Process each input file one at a time, and write all variables into each + // file before moving on to the next file. This is better than trying to + // keep a large number of files open so each variable can be written into + // all files before moving on to the next variable. + for (unsigned int run_num = 0; run_num < num_runs; ++run_num) { + std::string run_num_str(run_num_base); + std::string run_num_str_partial = std::to_string(run_num); + int length = run_num_str_partial.size(); + + // I'm going to be replacing contents of the string of zeros with the + // run-number; make sure the run-number doesn't contain more characters + // than in the string of zeros. Because the zeros-string is as long as + // the largest number, this should always pass. + // Unreachable code in current implementation. run_num_str is sized to + // accommodate run_num_base, which ahs been given as many zeroes as the + // number of characters in the largest run number. + if (run_num_str_partial.size() > run_num_str.size()) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Sizing Error\nAttempted to create a " + + "filename with a run-number that exceeds the\npre-generated size " + + "(e.g. trying to fit the number 10000 into 4 characters.\nThis " + + "should never happen.\n"; + message_publish(MSG_ERROR, message.c_str()); + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + } + // else + run_num_str.replace( max_length - length, length, run_num_str_partial); + + // Create the directories. + command = "mkdir -p " +monte_dir+"/RUN_"+run_num_str; + system( command.c_str()); + + // Next write the input file into the created directory + std::string filename_root = monte_dir + "/RUN_" + run_num_str + "/monte_"; + std::string filename = filename_root + input_file_name; + std::ofstream input_file(filename); + // Check for success of file-open using ofstream's failbit. + if (input_file.fail()) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", I/O error\nUnable to open file " + + filename.c_str() + " for writing."; + message_publish(MSG_ERROR, message.c_str()); + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + } + + // print the default (common) content to the top of the file and add the + // run-number identification for any variables that may depend on this. + input_file << + location << ".active = True" + "\n" << location << ".generate_dispersions = False\n" + "\nexec(open('"<generate_assignment(); + input_file << var_it->get_command(); + } + input_file << "\n"; + input_file.close(); + + // Generate the run-level dispersions list + if (generate_summary) { + std::ofstream disp_list(filename_root + "values"); + if (disp_list.is_open()) { + disp_list << run_num_str; + // Print the run number at the beginning of the line + for (auto var_it : variables) { + if (var_it->include_in_summary) { + disp_list << ", " << var_it->get_assignment(); + } + } + disp_list << "\n"; + disp_list.close(); + } + // Unreachable code in current implementation. + else { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Output failure\nFailed to record " + + "summary data for run " + run_num_str.c_str() + ".\n"; + message_publish(MSG_ERROR, message.c_str()); + generate_summary = false; + } + } + } + + // Consolidate all individual run monte_values files. + if (generate_summary) { + std::string all_runs_filename = monte_dir + "/monte_values_all_runs"; + command = "rm -f " + all_runs_filename; + system( command.c_str()); + command = "cat " + monte_dir + "/RUN_*/monte_values >> " + + all_runs_filename; + system( command.c_str()); + command = "for r in " + monte_dir + "/RUN_*;" + + "do ln -s ../monte_variables $r/monte_variables; done"; + system( command.c_str()); + } + + + input_files_prepared = true; + return true; +} + +/***************************************************************************** +add_variable +Purpose:(Adds a pointer to an instantiated MonteCarloVariable to the + master-list. This master-list will be processed in generating the + random assignments that are recorded in the monte_input.py files.) +*****************************************************************************/ +void +MonteCarloMaster::add_variable( + MonteCarloVariable & variable) +{ + if (input_files_prepared) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Invalid sequence\nAttempted to add a " + + "new variable " + variable.get_variable_name().c_str() + " to run " + + run_name.c_str() + ", but the input files have already been " + + "generated.\nCannot modify input files to accommodate this new " + + "variable.\nAddition of variable rejected.\n"; + message_publish(MSG_ERROR, message.c_str()); + return; + } + + // check for uniqueness + for (auto var_it = variables.begin(); var_it != variables.end(); ++var_it) { + if ( (**var_it).get_variable_name() == variable.get_variable_name()) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Duplicated variable.\nAttempted " + + "to add two settings for variable " + + variable.get_variable_name().c_str() + + ".\nTerminating to allow resolution of which setting to use.\n"; + message_publish(MSG_ERROR, message.c_str()); + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + } + } + + // if the variable is of type MonteCarloVariableFile, check for + // other MonteCarloVariableFiles that use the same file. + MonteCarloVariableFile * file_variable = + dynamic_cast< MonteCarloVariableFile *> (&variable); + if (file_variable != NULL) { + std::string filename = file_variable->get_filename(); + bool found_file = false; + for (auto it = file_list.begin(); it != file_list.end(); ++it) { + if (filename == (*it).first) { + (*it).second->register_dependent( file_variable); + found_file = true; + break; + } + } + if (!found_file) { + std::pair< std::string, MonteCarloVariableFile *> + new_pair(filename, file_variable); + file_list.push_back( new_pair); + } + } + + // Finally, add this variable to the list + variables.push_back(&variable); +} + +/***************************************************************************** +find_variable +Purpose: + Get a pointer to a MonteCarloVariable instance based on its "name" -- i.e. + the variable for which it is generating a value. +Limitations: + - Only returns the base-class pointer, so this has limited + versatility. If the desire is to modify the distribution parameters + or other characteristic of a MCVariable after it has been + generated, the returned pointer may require an additional + dynamic-cast to make the characteristics of the distribution type + available. + - Return value must be checked for NULL +*****************************************************************************/ +MonteCarloVariable * +MonteCarloMaster::find_variable( std::string var_name) +{ + for (auto it : variables) { + if (var_name == it->get_variable_name()) { + return it; + } + } + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Invalid name\nCould not find MonteCarlo " + + "variable with name " + var_name.c_str() + + ".\nReturning a NULL pointer.\n"; + message_publish(MSG_ERROR, message.c_str()); + return NULL; +} + + +/***************************************************************************** +remove_variable +Purpose:(remove a variable from distribution after it has been added) +Limitation:(Must be run before "execute"; once the files have been generated, it is too late to remove a variable) +*****************************************************************************/ +void +MonteCarloMaster::remove_variable( std::string var_name) +{ + // NOTE - cannot use find_variable(...); that returns a pointer to the + // variable and this method needs the list iterator addressing the variable. + for (auto it = variables.begin(); it != variables.end(); ++it) { + if (var_name == (*it)->get_variable_name()) { + variables.erase(it); + return; + } + } + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Invalid name\nAttempt to remove " + + "MonteCarlo variable with name " + var_name.c_str() + " FAILED.\nDid " + + "not find a variable with that name.\n"; + message_publish(MSG_WARNING, message.c_str()); +} + + +/***************************************************************************** +set_num_runs +Purpose:(Sets the intended number of runs for this scenario) +*****************************************************************************/ +void +MonteCarloMaster::set_num_runs( + unsigned int num_runs_) +{ + if (input_files_prepared) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Invalid sequence\nAttempted to set the " + + "number of runs to " + std::to_string(num_runs_) + ", but the " + + "input files have\nalready been generated."; + message_publish(MSG_ERROR, message.c_str()); + } + else { + num_runs = num_runs_; + } +} + +/***************************************************************************** +execute +Purpose:(The main executive. This should be run as an initialization-class + job very early in the initialization cycle, after all the input + processor operations have completed.) +*****************************************************************************/ +void +MonteCarloMaster::execute() +{ + if (!active || !generate_dispersions) { + return; + } + + for (auto it = file_list.begin(); it != file_list.end(); ++it) { + (*it).second->initialize_file(); + } + + prepare_input_files(); + + if (generate_meta_data) { + collate_meta_data(); + } + + // clean up any lingering aspects of the MonteCarloVariable instances. + std::list::iterator var_it = variables.begin(); + for (; var_it != variables.end(); ++var_it) { + (**var_it).shutdown(); + } + + std::string message = + "\nMonte-Input files generated in "+ monte_dir + + "\nRuns can now be launched using the generated monte-input "+ + "files and\nmanaged with an external load-management system, "+ + "such as SLURM.\n"+ + "This simulation is complete. Exiting.\n"; + exec_terminate_with_return(0, __FILE__, __LINE__,message.c_str()); +} + +/***************************************************************************** +collate_meta_data +Purpose:(Generates an output of metadata describing the variables, their + types, and other identifying characteristics) +*****************************************************************************/ +void +MonteCarloMaster::collate_meta_data() +{ + std::string filename = "MonteCarlo_Meta_data_output"; + if (input_files_prepared) { // so directory exists + filename = monte_dir + "/" + filename; + } + + std::ofstream meta_data( filename); + // Check for success of file-open. + if (meta_data.fail()) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", I/O error\nUnable to open file " + + filename.c_str() + " for writing.\nAborting generation of meta-data.\n"; + message_publish(MSG_WARNING, message.c_str()); + return; + } + + if (!input_files_prepared) { + meta_data << "Generating meta-data on the current configuration.\n" << + "The input files have not yet been generated which means this\n" << + "configuration has not been finalized and is subject to change.\n" << + "Sending meta-data to top-level directory.\n"; + } + + // Capture and alphabetize all variable names with their respective variable + // type; count the number of each type. + std::list< std::string > variable_names; + std::list< std::string > exec_file_names; + std::list< std::pair < unsigned int, std::string> > random_variables; + unsigned int count_calc = 0; + unsigned int count_const = 0; + unsigned int count_exec = 0; + unsigned int count_presc = 0; + unsigned int count_rand = 0; + unsigned int count_undef = 0; + + for (auto const * var_it : variables) { + switch (var_it->get_type()) { + // Unreachable case in current implementation. + // All current variable classes have been given a "type" + default: + variable_names.push_back (var_it->get_variable_name() + ", Undefined_type"); + count_undef++; + break; + case MonteCarloVariable::Calculated: + variable_names.push_back (var_it->get_variable_name() + ", Calculated"); + count_calc++; + break; + case MonteCarloVariable::Constant: + variable_names.push_back (var_it->get_variable_name() + ", Constant"); + count_const++; + break; + case MonteCarloVariable::Execute: + exec_file_names.push_back (var_it->get_variable_name()); + count_exec++; + break; + case MonteCarloVariable::Prescribed: + variable_names.push_back (var_it->get_variable_name() + ", Prescribed"); + count_presc++; + break; + case MonteCarloVariable::Random: + variable_names.push_back (var_it->get_variable_name() + ", Random"); + count_rand++; + std::pair< unsigned int, std::string> var(var_it->get_seed(), + var_it->get_variable_name()); + random_variables.push_back(var); + break; + } + } + variable_names.sort(); + + meta_data << + "\n\n*************************** SUMMARY **************************\n" << + variables.size() << " total assignments\n - " << + count_const << " constant values\n - " << + count_calc << " calculated variables\n - " << + count_presc << " prescribed (file-based) variables\n - " << + count_rand << " random variables\n - " << + count_exec << " files for execution\n - " << + count_undef << " variables of undefined type" << + "\n\n********************* LIST OF VARIABLES, TYPES****************\n"; + + std::list< std::string >::iterator var_name_it = variable_names.begin(); + for (; var_name_it != variable_names.end(); ++var_name_it) { + meta_data << (*var_name_it) << "\n"; + } + meta_data << + "**************************************************************\n"; + + if (!exec_file_names.empty()) { + meta_data << + "\n\n*********** LIST OF EXECUTABLE FILES AND FUNCTIONS ***********\n"; + for (auto const & var_name_it_temp : exec_file_names) { + meta_data << var_name_it_temp << "\n***\n"; + } + meta_data << + "**************************************************************\n"; + } + + if (!file_list.empty()) { + meta_data << + "\n\n***** LIST OF DATA FILES AND THE VARIABLES THEY POPULATE *****"; + for (auto const & file_it : file_list) { + meta_data << "\n******\n" << file_it.first << "\n"; + const std::list & dependents = + file_it.second->get_dependents(); + for (auto const & itv : dependents) { + meta_data << itv->get_column_number() << " " << + itv->get_variable_name() << "\n"; + } + } + meta_data << + "**************************************************************\n"; + } + + // Need to check the seeds on the random variables for inadvertent + // correlation between variables. If there are no random variables, there is + // nothing to do. + if (!random_variables.empty()) { + random_variables.sort(seed_sort); + + meta_data << + "\n\n*****Duplicate seeds; check for intentional correlations*****\n"; + std::list< std::pair< unsigned int, std::string> >::iterator rand_it = + random_variables.begin(); + unsigned int prev_seed = (*rand_it).first; + // start checking at the second element ... obviously the first element + // doesn't match its previous value, and starting with the second element + // guarantees that there will be a previous element. + ++rand_it; + bool in_duplicate = false; + for (; rand_it != random_variables.end(); ++rand_it) { + // if this seed matches the value of the previous seed, need to + // record it and the variable associated with it. + // Also record the variable associated with the previous list entry (i.e. + // the one with the same seed), but need to be careful here in the case + // of having 3 (or more) variables with matching seeds to avoid + // multiple records being made of the middle duplicates. + if ((*rand_it).first == prev_seed) { + if (!in_duplicate) { + // if this is the first match for a given seed, record the previous + // entry as well. + meta_data << (*std::prev(rand_it)).first << " " << + (*std::prev(rand_it)).second << "\n"; + in_duplicate = true; + } + // Then record this entry for all matches. + meta_data << (*rand_it).first << " " << (*rand_it).second << "\n"; + } + else { + in_duplicate = false; + prev_seed = (*rand_it).first; + } + } + meta_data << + "**************************************************************\n"; + meta_data << + "\n\n************************ ALL SEEDS *************************\n"; + for (rand_it = random_variables.begin(); + rand_it != random_variables.end(); + ++rand_it) { + meta_data << (*rand_it).first << " " << (*rand_it).second << "\n"; + } + } + meta_data.close(); +} diff --git a/trick_source/sim_services/MonteCarloGeneration/mc_variable.cc b/trick_source/sim_services/MonteCarloGeneration/mc_variable.cc new file mode 100644 index 00000000..63af93e3 --- /dev/null +++ b/trick_source/sim_services/MonteCarloGeneration/mc_variable.cc @@ -0,0 +1,118 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Base class for the MonteCarloVariable type) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ + +#include "trick/mc_variable.hh" +#include "trick/message_proto.h" +#include "trick/message_type.h" +#include "trick/exec_proto.h" +#include // ostringstream + +/***************************************************************************** +Constructor +*****************************************************************************/ +MonteCarloVariable::MonteCarloVariable( + const std::string & var_name) + : + units(), + include_in_summary(true), + variable_name(var_name), + assignment(), + command(), + type(Undefined) +{} + +/***************************************************************************** +insert_units +Purpose:(Provides a unit-conversion insertion into an established command) +*****************************************************************************/ +void +MonteCarloVariable::insert_units() +{ + // If no units specified, nothing to do here. + if (units.empty()) { + return; + } + + // Unreachable code in current implementation. insert_units is only ever + // called immediately after generate_command, which cannot possibly generate + // an empty string. + if (command.empty()) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Sequencing error\nVariable " + + variable_name.c_str() + " has units specified (" + units.c_str() + + ") but no command generated.\nThe command must be generated before " + + "applying units.Will attempt to generate the command to " + "avoid terminal fault but this\nmay not be what was intended.\n"; + message_publish(MSG_ERROR, message.c_str()); + } + // parse the command + size_t pos_equ = command.find("="); + if (pos_equ == std::string::npos) { + // Unreachable code in current implementation. insert_units is only ever + // called immediately after generate_command, and even if all else fails, + // generate_command produces a command with an = symbol in it, So an = + // will always be found. + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + " Invalid command\nFor variable " + + variable_name.c_str() + ", the command is poorly formed.\nCannot " + + "apply units to this command.\n"; + message_publish(MSG_ERROR, message.c_str()); + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + return; + } + + // TODO: Pick a unit-conversion mechanism + // Right now, the only one available is Trick: + trick_units( pos_equ+1); +} + +/***************************************************************************** +trick_units +Purpose:(inserts the Trick unit conversion mechanism into an established + command string.) +*****************************************************************************/ +void +MonteCarloVariable::trick_units( + size_t insertion_pt) +{ + command.insert(insertion_pt, " trick.attach_units(\"" + units + "\","); + command.append(")"); +} + +/***************************************************************************** +generate_assignment_internal +Purpose:() +*****************************************************************************/ +void +MonteCarloVariable::assign_double( + double value) +{ + std::ostringstream ostring; + ostring.precision(16); + ostring << value; + assignment = ostring.str(); + generate_command(); + insert_units(); +} +/****************************************************************************/ +void +MonteCarloVariable::assign_int( + int value) +{ + assignment = std::to_string(value); + generate_command(); + insert_units(); +} +/****************************************************************************/ +void +MonteCarloVariable::generate_command() +{ + command = "\n" + variable_name + " = " + assignment; +} diff --git a/trick_source/sim_services/MonteCarloGeneration/mc_variable_file.cc b/trick_source/sim_services/MonteCarloGeneration/mc_variable_file.cc new file mode 100644 index 00000000..3122f649 --- /dev/null +++ b/trick_source/sim_services/MonteCarloGeneration/mc_variable_file.cc @@ -0,0 +1,273 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a file-lookup assignment + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ + +#include // all_of +#include // istringstream +#include "trick/exec_proto.h" +#include "trick/message_proto.h" +#include "trick/message_type.h" +#include "trick/mc_variable_file.hh" + +/***************************************************************************** +Constructor +*****************************************************************************/ +MonteCarloVariableFile::MonteCarloVariableFile( + const std::string & var_name, + const std::string & filename_, + size_t column_number_, + size_t first_column_number_) + : + MonteCarloVariable(var_name), + max_skip(0), + is_dependent(false), + rand_gen(0), + filename(filename_), + column_number(column_number_), + first_column_number(first_column_number_), + dependents(), + file() +{ + // make this a dependent of itself so that when it reads the data file, it + // populates its own "assignment" variable. + dependents.push_back(this); + type = MonteCarloVariable::Prescribed; +} + +/***************************************************************************** +initialize_file +Purpose:(Opens the file identified by filename as an ifstream) +*****************************************************************************/ +void +MonteCarloVariableFile::initialize_file() +{ + // At this time, the list of dependencies has been finalized. We can sort + // this list by column number of each of the dependencies, in increasing + // order. + if (dependents.size() > 1) { + dependents.sort(sort_by_col_num); + } + // Check that the specified first_column_number is no larger than the + // smallest column number: + MonteCarloVariableFile * first_var =dependents.front(); + if (first_column_number > first_var->get_column_number()) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Configuration Error\nIn configuring " + + "the file for variable " + first_var->get_variable_name().c_str() + + ", it was identified that\nit was specified to draw data from column " + + std::to_string(first_var->get_column_number()) + ", but that the " + + "first\ncolumn was identified as having index " + + std::to_string(first_column_number) + ".\n"; + message_publish(MSG_ERROR, message.c_str()); + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + } + + // Now we can get to reading the file. + file.open(filename); + + if (file.fail()) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", I/O error\nUnable to open file " + + filename.c_str() + " for reading.\nRequired for variable " + + variable_name.c_str() + ".\n"; + message_publish(MSG_ERROR, message.c_str()); + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + } + // Sanity check -- make sure the file has at least 1 line of data: + std::string line; + do { + // if reached the end of the file, not found anything good. Fail out. + if (file.eof()) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + " Invalid data file\nData file " + + filename.c_str() + " contains no recognized lines of data\n" + + "Required for variable " + variable_name.c_str() + ".\n"; + message_publish(MSG_ERROR, message.c_str()); + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + } + std::getline( file, line); + // keep looking if the line is empty, starts with a "#" character or + // "/" character or is completely whitespace. + } while (line.empty() || + line.front() == '#' || + line.front() == '/' || + std::all_of( line.begin(), line.end(), isspace)); + // Rewind the file + file.seekg(0, file.beg); + +} + +/***************************************************************************** +generate_assignment +Purpose:(generates the command line that is to be embedded in the monte-input + file currently being generated.) +*****************************************************************************/ +void +MonteCarloVariableFile::generate_assignment() +{ + // if this instance is not dependent on another, need to read the file. + if (!is_dependent) { + process_line(); // provides "assignment" + } + generate_command(); + insert_units(); +} + + +/***************************************************************************** +register_dependent +Purpose:(Registers another MonteCarloVariableFile instance with this one, +allowing this instance to read the data for the other.) +*****************************************************************************/ +void +MonteCarloVariableFile::register_dependent( + MonteCarloVariableFile * new_var) +{ + if (new_var == NULL) { + std::string message = + std::string("File: ") + __FILE__ + ", line " + + std::to_string(__LINE__) + ", Invalid call\nAttempted to register " + + "a dependent identified with NULL pointer with \nthe " + + "MonteCarloVariableFile for variable " + variable_name.c_str() + + ".\nThis is not a valid action.\nRegistration failed, exiting " + + "without action.\n"; + message_publish(MSG_ERROR, message.c_str()); + return; + } + if (new_var->has_dependents()) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Invalid configuration\nError in " + + "attempting to make " + new_var->get_variable_name().c_str() + + " be dependent on " + variable_name.c_str() + ".\n" + + new_var->get_variable_name().c_str() + " cannot be marked as " + + "dependent when it has dependencies of its own.\nThe dependency " + + "hierarchy can only be one level deep."; + message_publish(MSG_ERROR, message.c_str()); + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + } + if (new_var->max_skip != max_skip) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Invalid configuration\nIt is not " + + "permissible for two variables looking at the same file to\noperate " + + "under different line-selection criteria.\n" + + new_var->get_variable_name().c_str() + "\nwill be switched to the " + + "behavior of\n" + variable_name.c_str() + ",\nwhich " + + "as a setting for the maximum number of lines to skip of " + + std::to_string(max_skip) + "\n"; + message_publish(MSG_ERROR, message.c_str()); + } + dependents.push_back(new_var); + new_var->is_dependent = true; +} + + +/***************************************************************************** +process_line +Purpose:(extract and process a line of data from the file, breaking it + into words and extracting the approriate word number.) +*****************************************************************************/ +void +MonteCarloVariableFile::process_line() +{ + size_t skip_count = 0; + if (max_skip > 0) { + std::uniform_int_distribution skip_distrib(0,max_skip); + skip_count = skip_distrib( rand_gen); + } + + std::string line; + + for (size_t ii = 0; ii <= skip_count; ++ii) { + // keep reading the next line until a "good" line is found + do { + // read the next line + std::getline( file, line); + + // if reached the end of the file, clear the error flag and go back + // to the beginning. + if (file.eof()) { + file.clear(); + file.seekg(0, file.beg); + } + // keep looking if the line is empty, starts with a "#" character or + // "/" character or is completely whitespace. + } while ( line.empty() || + line.front() == '#' || + line.front() == '/' || + std::all_of( line.begin(), line.end(), isspace)); + + // A good line was found; return to the top of the for loop to see if + // we have skipped over enough good lines yet. + } + + // Have the line containing the data. Need to assign some subset of the + // words in this line to a set of variables, knowing the column number for + // each variable. + // Capture the first word, and associate it with the user-specified + // column number of the first word. + std::istringstream word(line); + std::string scratch_assignment; + word >> scratch_assignment; + size_t current_column_number = first_column_number; + + // Now for each dependent (including itself) find and assign the appropriate + // word + for (auto it = dependents.begin(); it != dependents.end(); ++it) { + // The dependents have already been sorted according to their column + // number, so the next column we need data from is the column number of + // this next variable. + size_t next_column_needed = (*it)->get_column_number(); + + // This next-needed column can be no earlier than + // the current column, but it could be the current column -- could be + // the first column, or we could have multiple variables collecting data + // from the same column. So check whether we need to advance the + // string-stream. + while (next_column_needed > current_column_number && word) { + // for as long as the next needed column is to the right of where we + // currently are, and there is another word to the right, extract + // the next word and advance the current column number. + word >> scratch_assignment; + current_column_number++; + } + // There are two ways to get past the while loop -- we ran out of + // words, or the current column number reached the target next-needed + // column number (including the case where it was already there). + // If we ran out of words before reaching a specified column, we have + // a problem + if (current_column_number < next_column_needed) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Malformed data file\nData file " + + "for variable " + variable_name.c_str() + " includes this line:\n" + + line.c_str() + "\nWhich has only " + + std::to_string(current_column_number-1) + " values.\nVariable " + + variable_name.c_str() + " uses the value from position " + + std::to_string(column_number) + + ", which does not exist in this line\n"; + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + } + // and if we found the desired column, send its value to the variable: + (*it)->assignment = scratch_assignment; + } +} + +/***************************************************************************** +sort_by_col_num +Purpose:(sorts the dependent list by column number +*****************************************************************************/ +bool MonteCarloVariableFile::sort_by_col_num( + MonteCarloVariableFile * left, + MonteCarloVariableFile * right) +{ + return left->get_column_number() < right->get_column_number(); +} diff --git a/trick_source/sim_services/MonteCarloGeneration/mc_variable_random_normal.cc b/trick_source/sim_services/MonteCarloGeneration/mc_variable_random_normal.cc new file mode 100644 index 00000000..ebf7ed5e --- /dev/null +++ b/trick_source/sim_services/MonteCarloGeneration/mc_variable_random_normal.cc @@ -0,0 +1,232 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a class to support generation and assignment + of a random value distributed normally.) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#include "trick/mc_variable_random_normal.hh" + +#include "trick/exec_proto.h" +#include "trick/message_proto.h" +#include "trick/message_type.h" +#include "trick/compareFloatingPoint.hh" +#include +#include +#include +#include + +/***************************************************************************** +Constructor +*****************************************************************************/ +MonteCarloVariableRandomNormal::MonteCarloVariableRandomNormal( + const std::string & var_name, + unsigned int seed, + double mean, + double stdev) + : + MonteCarloVariableRandom( var_name, seed), + max_num_tries(10000), + distribution(mean, stdev), + min_value(0.0), + max_value(0.0), + truncated_low(false), + truncated_high(false) +{} + +/***************************************************************************** +generate_assignment +Purpose:(generates the normally-distributed random number) +*****************************************************************************/ +void +MonteCarloVariableRandomNormal::generate_assignment() +{ + double assignment_d = distribution(random_generator); + + if (truncated_low && truncated_high) { + if (min_value > max_value) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Illegal configuration\nFor variable " + + variable_name.c_str() + " the specified minimum allowable value (" + + std::to_string(min_value) + ") >= the specified maximum allowable" + + + " value (" + std::to_string(max_value) + ").\nOne or both of the" + " limits must be changed to generate a random value.\n"; + message_publish(MSG_ERROR, message.c_str()); + exec_terminate_with_return(1, __FILE__, __LINE__, message.c_str()); + } + else if (Trick::dbl_is_near( min_value, max_value, 0.0)) { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Overconstrained configuration\nFor " + + "variable" + variable_name.c_str() + " the specified minimum " + + "allowable value and \nthe specified maximum allowable value are " + "equal (" + std::to_string(min_value) + ").\nThe distribution " + + " collapses to a point.\n"; + message_publish(MSG_WARNING, message.c_str()); + assignment_d = min_value; + } + } + + size_t num_tries = 0; + while ( (truncated_high && assignment_d > max_value) || + (truncated_low && assignment_d < min_value)) { + if ( num_tries < max_num_tries) { + assignment_d = distribution(random_generator); + num_tries++; + } + else { + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + " Random value truncation failure\n" + + "Could not generate a value for " + variable_name.c_str() + + " within the specified domain within\nthe specified maximum " + + "number of tries (" + + std::to_string(max_num_tries) + ").\nAssuming a value equal to:\n" + + " - midpoint value for a distribution truncated at both ends\n - " + + "truncation value for a distribution truncated at only one end.\n"; + message_publish(MSG_ERROR, message.c_str()); + // Note - at least one truncation must be defined in order to be in + // this part of the code. + if (!truncated_high) { // i.e. truncated-low only + assignment_d = min_value; + } + else if (!truncated_low) { // i.e. truncated-high only + assignment_d = max_value; + } + else { // i.e. truncated both sides + assignment_d = (max_value + min_value) / 2.0; + } + // Stop trying to generate an acceptable value at this point: + break; + } + } + assign_double(assignment_d); +} + +/***************************************************************************** +truncate +Purpose:(Truncates the normal distribution to be within +- limit.) +*****************************************************************************/ +void +MonteCarloVariableRandomNormal::truncate( + double limit, + TruncationType truncType) +{ + if (limit < 0) + { + std::string message = + std::string("File: ") + __FILE__ + ", Line " + + std::to_string(__LINE__) + ", Out-of-domain error\nNegative " + + "double-sided truncation specified for variable " + + std::to_string(max_num_tries) + "\ntruncate() must receive either " + + "two limits or one positive limit!\nUsing absolute value of limit." + + "\nUsing absolute value of limit.\n"; + message_publish(MSG_ERROR, message.c_str()); + limit = -limit; + } + + if (Trick::dbl_is_near(limit, 0.0, 0.0)) + { + std::string message = + std::string("File: ") + __FILE__ + ", Line " + + std::to_string(__LINE__) + ", Configuration error\nZero truncation " + + "for specified for variable " + variable_name.c_str() + " which " + + "will produce a fixed point\n"; + message_publish(MSG_WARNING, message.c_str()); + } + + // Assign the truncation on both sides: + truncate_low(-limit, truncType); + truncate_high(limit, truncType); +} + +/*************************************************************************** +truncate +Purpose:(Truncates the normal distribution to be within asymmetric limits.) +*****************************************************************************/ +void +MonteCarloVariableRandomNormal::truncate( + double min, + double max, + TruncationType truncType) +{ + truncate_low(min, truncType); + truncate_high(max, truncType); +} + +/***************************************************************************** +truncate_low +Purpose:(Specifies the lower-bound of the truncation) +Note - min is often -- but not necessarily -- a negative value. +*****************************************************************************/ +void +MonteCarloVariableRandomNormal::truncate_low( + double min, + TruncationType truncType) +{ + switch (truncType) { + case StandardDeviation: + min_value = distribution.mean() + distribution.stddev() * min; + break; + case Relative: + min_value = distribution.mean() + min; + break; + case Absolute: + min_value = min; + break; + // Unreachable code. All types are covered. + default: + std::string message = + std::string("File: ") + __FILE__ + ", Line " + + std::to_string(__LINE__) + ", Invalid TruncationType\nInvalid " + + "truncation type passed to truncate_low for variable " + + variable_name.c_str() + ".\nMinimum will not be applied.\n"; + message_publish(MSG_ERROR, message.c_str()); + return; + } + truncated_low = true; +} +/***************************************************************************** +truncate_high +Purpose:(Specifies the upper-bound of the truncation) +*****************************************************************************/ +void +MonteCarloVariableRandomNormal::truncate_high( double max, + TruncationType truncType) +{ + switch (truncType) { + case StandardDeviation: + max_value = distribution.mean() + distribution.stddev() * max; + break; + case Relative: + max_value = distribution.mean() + max; + break; + case Absolute: + max_value = max; + break; + // Unreachable code. All types are covered. + default: + std::string message = + std::string("File: ") + __FILE__ + ", Line: " + + std::to_string(__LINE__) + ", Invalid TruncationType\nInvalid " + + "truncation type passed to truncate_high for variable " + + variable_name.c_str() + ".\nMaximum will not be applied.\n"; + message_publish(MSG_ERROR, message.c_str()); + return; + } + truncated_high = true; +} + +/***************************************************************************** +untruncate +Purpose:(Remove truncation flags.) +*****************************************************************************/ +void +MonteCarloVariableRandomNormal::untruncate() +{ + truncated_low = false; + truncated_high = false; +} + diff --git a/trick_source/sim_services/MonteCarloGeneration/mc_variable_random_string.cc b/trick_source/sim_services/MonteCarloGeneration/mc_variable_random_string.cc new file mode 100644 index 00000000..6d544b57 --- /dev/null +++ b/trick_source/sim_services/MonteCarloGeneration/mc_variable_random_string.cc @@ -0,0 +1,66 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a class to randomly pick one of a set of + character strings. These strings could represent actual string + variables, or enumerated types, or commands, or any other concept + that can be expressed as an assignment.) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#include "trick/mc_variable_random_string.hh" + +/***************************************************************************** +Constructor +*****************************************************************************/ +MonteCarloVariableRandomStringSet::MonteCarloVariableRandomStringSet( + const std::string & var_name, + unsigned int seed) + : + MonteCarloVariableRandomUniform( var_name, seed, 0, 1), + values() +{ + include_in_summary = false; // The summary file is comma-delimited. String + // variables are excluded by default because they + // may contain commas. +} + +/***************************************************************************** +generate_assignment +Purpose:(pick one string at random) +*****************************************************************************/ +void +MonteCarloVariableRandomStringSet::generate_assignment() +{ + // generate a random number on the interval [0,1) + double pre_ix = distribution(random_generator); + // convert to an integer between 0 and max-index of the "values" vector. + size_t ix = static_cast (pre_ix * values.size()); + // send the string at that index to the command-generation. + assignment = values[ix]; + generate_command(); +} +/***************************************************************************** +add_string +Purpose:(Adds an option to the set of strings) + +NOTE - When adding a string realize that the content of the selected string + will be assigned as written. Consequently, if the value is intended for + assignment to an actual string or char variable, the contents of the + string being added here should include the quotes. + E.g. the string might look like: "'actual_string'" so that the assignment + would look like: + variable = 'actual_string' + or "\"actual_string\"" + variable = "actual_string" + This is to support the use of a string to represent a non-string variable + such as an enumeration or command: + E.g. adding a string here that contains the characters "x * 3 + 2" can + be used to achieve an assignment like: + variable = x * 3 + 2 +*****************************************************************************/ +void +MonteCarloVariableRandomStringSet::add_string( std::string new_string) +{ + values.push_back(new_string); +} diff --git a/trick_source/sim_services/MonteCarloGeneration/mc_variable_random_uniform.cc b/trick_source/sim_services/MonteCarloGeneration/mc_variable_random_uniform.cc new file mode 100644 index 00000000..1287ae3d --- /dev/null +++ b/trick_source/sim_services/MonteCarloGeneration/mc_variable_random_uniform.cc @@ -0,0 +1,61 @@ +/*******************************TRICK HEADER****************************** +PURPOSE: ( Implementation of a class to support generation and assignment + of a random value distributed uniformally. + Provides float and integer distributions) + +PROGRAMMERS: + (((Gary Turner) (OSR) (October 2019) (Antares) (Initial))) + (((Isaac Reaves) (NASA) (November 2022) (Integration into Trick Core))) +**********************************************************************/ +#include "trick/mc_variable_random_uniform.hh" + +/***************************************************************************** +Constructor +*****************************************************************************/ +MonteCarloVariableRandomUniform::MonteCarloVariableRandomUniform( + const std::string & var_name, + unsigned int seed, + double lower_bound, + double upper_bound) + : + MonteCarloVariableRandom( var_name, seed), + distribution(lower_bound, upper_bound) +{} + + +/***************************************************************************** +generate_assignment +Purpose:(generates the random number and the input-file assignment) +*****************************************************************************/ +void +MonteCarloVariableRandomUniform::generate_assignment() +{ + assign_double(distribution(random_generator)); +} + + + + +/***************************************************************************** +Constructor +*****************************************************************************/ +MonteCarloVariableRandomUniformInt::MonteCarloVariableRandomUniformInt( + const std::string & var_name, + unsigned int seed, + double lower_bound, + double upper_bound) + : + MonteCarloVariableRandom( var_name, seed), + distribution(lower_bound, upper_bound) +{} + + +/***************************************************************************** +generate_assignment +Purpose:(generates the random number and the input-file assignment) +*****************************************************************************/ +void +MonteCarloVariableRandomUniformInt::generate_assignment() +{ + assign_int(distribution(random_generator)); +} diff --git a/trick_source/trick_swig/Makefile b/trick_source/trick_swig/Makefile index eafe6dee..a93bd517 100644 --- a/trick_source/trick_swig/Makefile +++ b/trick_source/trick_swig/Makefile @@ -25,7 +25,7 @@ PY_FILES = \ ${TRICK_HOME}/share/trick/swig/swig_ref.py SWIG_SRC_FILES = $(addprefix swig_${TRICK_HOST_CPU}/, $(notdir $(subst .o,.cpp,$(SWIG_OBJECT_FILES)))) -TRICK_CXXFLAGS += $(PYTHON_INCLUDES) -Wno-redundant-decls -Wno-shadow -Wno-unused-parameter -Wno-missing-field-initializers +TRICK_CXXFLAGS += $(PYTHON_INCLUDES) -Wno-redundant-decls -Wno-shadow -Wno-unused-parameter -Wno-missing-field-initializers -std=c++11 ifeq ($(IS_CC_CLANG), 1) TRICK_CXXFLAGS += -Wno-self-assign -Wno-sometimes-uninitialized diff --git a/trick_source/trick_swig/sim_services.i b/trick_source/trick_swig/sim_services.i index 10292a0c..452f4874 100644 --- a/trick_source/trick_swig/sim_services.i +++ b/trick_source/trick_swig/sim_services.i @@ -3,6 +3,7 @@ %{ #include +#include %} #include "trick/swig/trick_swig.i" @@ -101,6 +102,17 @@ #include "trick/MSSocket.hh" #include "trick/MSSharedMem.hh" #include "trick/Master.hh" +#include "trick/mc_master.hh" +#include "trick/mc_python_code.hh" +#include "trick/mc_variable_file.hh" +#include "trick/mc_variable_fixed.hh" +#include "trick/mc_variable.hh" +#include "trick/mc_variable_random_bool.hh" +#include "trick/mc_variable_random.hh" +#include "trick/mc_variable_random_normal.hh" +#include "trick/mc_variable_random_string.hh" +#include "trick/mc_variable_random_uniform.hh" +#include "trick/mc_variable_semi_fixed.hh" #include "trick/Slave.hh" #include "trick/master_proto.h" #include "trick/MemoryManager.hh" diff --git a/trickops.py b/trickops.py index 2f715f21..14027706 100644 --- a/trickops.py +++ b/trickops.py @@ -1,7 +1,8 @@ import sys import os -sys.path.append(sys.argv[1] + "/share/trick/trickops") +thisdir = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) +sys.path.append(os.path.join(thisdir,"share/trick/trickops")) from TrickWorkflow import * from WorkflowCommon import Job @@ -9,51 +10,36 @@ from WorkflowCommon import Job max_retries = 5 class SimTestWorkflow(TrickWorkflow): - def __init__( self, quiet, trick_top_level ): + def __init__( self, quiet, trick_top_level, cpus): + self.cpus = cpus # Create the trick_test directory if it doesn't already exist if not os.path.exists(trick_top_level + "/trick_test"): os.makedirs(trick_top_level + "/trick_test") # Base Class initialize, this creates internal management structures - num_cpus = os.cpu_count() if os.cpu_count() is not None else 8 TrickWorkflow.__init__(self, project_top_level=(trick_top_level), log_dir=(trick_top_level +'/trickops_logs/'), - trick_dir=trick_top_level, config_file=(trick_top_level + "/test_sims.yml"), cpus=num_cpus, quiet=quiet) - + trick_dir=trick_top_level, config_file=(trick_top_level + "/test_sims.yml"), cpus=self.cpus, quiet=quiet) def run( self ): - build_jobs = self.get_jobs(kind='build') - run_jobs = self.get_jobs(kind='run') + # Two sims have runs that require ordering via phases: + # - SIM_stls dumps a checkpoint that is then read in and checked by a subsequent run + # - SIM_checkpoint_data_recording dumps checkpoints that are read by subsequent runs + first_run_jobs = self.get_jobs(kind='run', phase=-1) # Get all jobs with early phase -1 + remaining_run_jobs = self.get_jobs(kind='run', phase=0) # Get all jobs with default phase 0 analysis_jobs = self.get_jobs(kind='analyze') - # This job in SIM_stls dumps a checkpoint that is then read in and checked by RUN_test/unit_test.py in the same sim - # This is a workaround to ensure that this run goes first. - first_phase_jobs = [] - stl_sim = self.get_sim('SIM_stls') - if stl_sim is not None: - stl_dump_job = stl_sim.get_run(input='RUN_test/setup.py').get_run_job() - first_phase_jobs.append(stl_dump_job) - run_jobs.remove(stl_dump_job) - - # Same with SIM_checkpoint_data_recording - half the runs dump checkpoints, the others read and verify. - # Make sure that the dump checkpoint runs go first. - if self.get_sim('SIM_checkpoint_data_recording') is not None: - num_dump_runs = int(len(self.get_sim('SIM_checkpoint_data_recording').get_runs())/2) - for i in range(num_dump_runs): - job = self.get_sim('SIM_checkpoint_data_recording').get_run(input=('RUN_test' + str(i+1) + '/dump.py')).get_run_job() - first_phase_jobs.append(job) - run_jobs.remove(job) - # Some tests fail intermittently for reasons not related to the tests themselves, mostly network weirdness. # Allow retries so that we can still cover some network-adjacent code retry_allowed_sims = self.get_sims(labels='retries_allowed') retry_allowed_jobs = [run.get_run_job() for run in [item for sublist in [sim.get_runs() for sim in retry_allowed_sims] for item in sublist]] for job in retry_allowed_jobs: - run_jobs.remove(job) - + # Note there's an assumption/dependency here that 'retries_allowed' runs + # are only in the remaining_run_jobs list. - Jordan 2/2023 + remaining_run_jobs.remove(job) builds_status = self.execute_jobs(build_jobs, max_concurrent=self.cpus, header='Executing all sim builds.') - first_phase_run_status = self.execute_jobs(first_phase_jobs, max_concurrent=self.cpus, header="Executing required first phase runs.") - runs_status = self.execute_jobs(run_jobs, max_concurrent=self.cpus, header='Executing all sim runs.') + first_phase_run_status = self.execute_jobs(first_run_jobs, max_concurrent=self.cpus, header="Executing first phase runs.") + runs_status = self.execute_jobs(remaining_run_jobs, max_concurrent=self.cpus, header='Executing remaining runs.') # Run the retry_allowed jobs self.execute_jobs(retry_allowed_jobs, max_concurrent=self.cpus, header='Executing retry-allowed runs.') @@ -67,7 +53,6 @@ class SimTestWorkflow(TrickWorkflow): status, final_job = self.retry_job(sim, run, max_retries) final_retry_jobs += [final_job] all_retried_status = all_retried_status or status - comparison_result = self.compare() analysis_status = self.execute_jobs(analysis_jobs, max_concurrent=self.cpus, header='Executing all analysis.') @@ -75,7 +60,7 @@ class SimTestWorkflow(TrickWorkflow): self.status_summary() # Print a Succinct summary # Dump failing logs - jobs = build_jobs + first_phase_jobs + run_jobs + final_retry_jobs + jobs = build_jobs + first_run_jobs + remaining_run_jobs + final_retry_jobs for job in jobs: if job.get_status() == Job.Status.FAILED: print("Failing job: ", job.name) @@ -83,7 +68,7 @@ class SimTestWorkflow(TrickWorkflow): print(open(job.log_file, "r").read()) print ("*"*120, "\n") - return (builds_status or runs_status or first_phase_run_status or all_retried_status or self.config_errors or comparison_result or analysis_status) + return (builds_status or runs_status or first_phase_run_status or all_retried_status or len(self.config_errors) > 0 or comparison_result or analysis_status) # Retries a job up to max_retries times and adds runs to the sim # Returns tuple of (job_status, final retry job) @@ -94,7 +79,7 @@ class SimTestWorkflow(TrickWorkflow): retry_job = None while tries < max_retries and job_failing: tries += 1 - retry_run = TrickWorkflow.Run(sim_dir=run.sim_dir, input=run.input, binary=run.binary, returns=run.returns,log_dir=run.log_dir) + retry_run = TrickWorkflow.Run(sim_dir=run.sim_dir, input_file=run.input_file, binary=run.binary, returns=run.returns,log_dir=run.log_dir) retry_job = retry_run.get_run_job() retry_job.name = retry_job.name + "_retry_" + str(tries) job_failing = self.execute_jobs([retry_job], max_concurrent=1, header="Retrying failed job") @@ -102,7 +87,13 @@ class SimTestWorkflow(TrickWorkflow): return (job_failing, retry_job) - if __name__ == "__main__": - should_be_quiet = os.getenv('CI') is not None - sys.exit(SimTestWorkflow(quiet=should_be_quiet, trick_top_level=sys.argv[1]).run()) + parser = argparse.ArgumentParser(description='Build, run, and compare all test sims for Trick', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( "--trick_top_level", type=str, help="Path to TRICK_HOME", default=thisdir) + parser.add_argument( "--quiet", action="store_true", help="Suppress progress bars.") + parser.add_argument( "--cpus", type=int, default=(os.cpu_count() if os.cpu_count() is not None else 8), + help="Number of cpus to use for testing. For builds this number is used for MAKEFLAGS *and* number of " + "concurrent builds (cpus^2). For sim runs this controls the maximum number of simultaneous runs.") + myargs = parser.parse_args() + sys.exit(SimTestWorkflow(quiet=myargs.quiet, trick_top_level=myargs.trick_top_level, cpus=myargs.cpus).run())