Merge pull request TemoaProject#88 from jeff-ws/main

Monte Carlo Functionality

data_files/monte_carlo/run_settings_1.csv

@@ -0,0 +1,5 @@
+run,param,index,mod,value,notes
+1,MaxCapacity,utopia|2010|TXD,a,-1.0,reduce the max capacity of TXD in region Utopia in period 2010 by 1.0 units (absolute)
+2,Demand,utopia|*|RH,r,0.5,make Res Heat costlier by 50% in all 3 periods
+2,CostVariable,*|1990/2000|IMPOIL1|*,s,20.0,substitute cost of 20.0 for var cost of IMPOIL in periods 1990/2000 in all regions (just utopia exists) for all vintages
+3,CostVariable,china|1990|IMPOIL1|*,s,1000,bad input: unknown region (china) should fail and be logged

data_files/my_configs/config_sample.toml

@@ -15,7 +15,7 @@ scenario = "zulu"
 # Scenaio Mode (Mandatory)
 # See documentation for explanations.  A standard single run is "perfect_foresight"
 # mode must be one of (case-insensitive):
-#     [perfect_foresight, MGA, myopic, method_of_morris, build_only, check]
+#     [perfect_foresight, MGA, myopic, method_of_morris, build_only, check, monte_carlo]
 scenario_mode = "perfect_foresight"
 
 # Input database (Mandatory)
@@ -108,3 +108,7 @@ emission_labels = ['co2', 'nox']
 capacity_labels = ['TXD', 'TXG']
 activity_labels = []
 
+[monte_carlo]
+# a path from the PROJECT ROOT to the settings file that contains the run data.
+run_settings = 'data_files/monte_carlo/run_settings_1.csv'
+

data_files/my_configs/mga_utopia.toml

@@ -18,13 +18,13 @@ scenario = "sierra"
 scenario_mode = "mga"
 
 # Input database (Mandatory)
-input_database = "data_files/utopia.sqlite"
+input_database = "data_files/example_dbs/utopia.sqlite"
 
 # Output file (Mandatory)
 # The output file must be an existing .sqlite file
 # For Pefrect Foresight, the user may target the same input file or a separate /
 # copied sqlite file in a different location.  Myopic requires that input_database = output_database
-output_database = "data_files/utopia.sqlite"
+output_database = "data_files/example_dbs/utopia.sqlite"
 
 # ------------------------------------
 #        DATA / MODEL CHECKS
@@ -79,7 +79,7 @@ save_lp_file = false
 [MGA]
 # see notes on these in the extensions/modeling_to_generate_alternatives folder readme.txt
 cost_epsilon = 0.03    # propotional relaxation on optimal cost (ex: 0.05 = bound at 105% of original optimal cost)
-iteration_limit = 55   # max iterations to perform
+iteration_limit = 20   # max iterations to perform
 time_limit_hrs = 1     # max time
 axis = "tech_category_activity"   # use the tech activity Manager to control exploration based on categories in Tech
 weighting = "hull_expansion"  # use a convex hull expansion algorithm to weight exploration

data_files/my_configs/monte_carlo_utopia.toml

@@ -0,0 +1,114 @@
+# ----------------------------------------------------------
+#            Configuration file for a Temoa Run
+# Allows specification of run type and associated parameters
+# ----------------------------------------------------------
+#
+# For toml format info see: https://toml.io/en/
+#   - comments may be added with hash
+#   - do NOT comment out table names in brackets like:  [<table>]
+
+# Scenario Name (Mandatory)
+# This scenario name is used to label results within the output .sqlite file
+# (cannot contain "-" dash)
+scenario = "Orange Squirrel"
+
+# Scenaio Mode (Mandatory)
+# See documentation for explanations.  A standard single run is "perfect_foresight"
+# mode must be one of (case-insensitive):
+#     [perfect_foresight, MGA, myopic, method_of_morris, build_only, check, monte_carlo]
+scenario_mode = "monte_carlo"
+
+# Input database (Mandatory)
+input_database = "data_files/example_dbs/utopia.sqlite"
+
+# Output file (Mandatory)
+# The output file must be an existing .sqlite file
+# For Pefrect Foresight, the user may target the same input file or a separate /
+# copied sqlite file in a different location.  Myopic, MGA require that input_database = output_database
+output_database = "data_files/example_dbs/utopia.sqlite"
+
+# ------------------------------------
+#        DATA / MODEL CHECKS
+#  To check data / cost integrity
+# ------------------------------------
+
+# See the documentation section on Data Quality for notes on the features below
+
+# Check the pricing structure for common errors, which are reported in the log file
+# Strongly recommended
+price_check = true
+
+# Check the network connectivity for processes in the model.  Strongly
+# recommended to ensure proper performance.  Results are reported in log file
+# This requires that source commodities be marked with 's' in Commodity table
+# This is required for Myopic runs
+source_trace = true
+
+# Produce HTML files for Commodity Networks.  Requires source_trace above
+plot_commodity_network = true
+
+# ------------------------------------
+#             SOLVER
+#        Solver Selection
+# ------------------------------------
+
+# use the NEOS server to solve.  (Currently NOT supported)
+neos = false
+
+# solver (Mandatory)
+# Depending on what client machine has installed.
+#  [cbc, appsi_highs, gurobi, cplex, ...]
+solver_name = "cbc"
+
+# ------------------------------------
+#             OUTPUTS
+# select desired output products/files
+# ------------------------------------
+
+# generate an Excel file in the output_files folder
+save_excel = true
+
+# save the duals in the output Database (may slow execution slightly?)
+save_duals = true
+
+# save a copy of the pyomo-generated lp file(s) to the outputs folder (maybe a large file(s)!)
+save_lp_file = false
+
+# ---------------------------------------------------
+#                   MODE OPTIONS
+# options below are mode-specific and will be ignored
+# if the run is not executed in that mode.
+# ---------------------------------------------------
+[MGA]
+# see notes on these in the extensions/modeling_to_generate_alternatives folder readme.txt
+cost_epsilon = 0.03    # propotional relaxation on optimal cost (ex: 0.05 = bound at 105% of original optimal cost)
+iteration_limit = 55   # max iterations to perform
+time_limit_hrs = 1     # max time
+axis = "tech_category_activity"   # use the tech activity Manager to control exploration based on categories in Tech
+weighting = "hull_expansion"  # use a convex hull expansion algorithm to weight exploration
+
+[myopic]
+view_depth = 2   # number of periods seen/analyzed per iteration
+step_size = 1    # number of periods to step by (must be <= view depth)
+
+[morris]
+perturbation = 0.10     # amount to perturb marked parameters (ex:  0.10 -> +/- 10%)
+levels = 8              # number of levels in param grid (must be even number)
+trajectories = 10        # number of Morris trajectories to generate/explore
+seed = false               # random seed for use in generation/analysis for repeatable results.  false=system derived
+cores = 0               # number of CPU cores to use.  0 (default) = cpu count
+# Note:  Problem size (in general) is (Groups + 1) * trajectories see the SALib Dox
+#        Groups = number of unique labels used in MM analysis columns in DB
+
+[SVMGA]
+cost_epsilon = 0.05
+# labels from appropriate tables in database.  It is recommended to only use one of the lists below and leave
+# the others blank
+emission_labels = ['co2', 'nox']
+capacity_labels = ['TXD', 'TXG']
+activity_labels = []
+
+[monte_carlo]
+# a path from the PROJECT ROOT to the settings file that contains the run data.
+run_settings = 'data_files/monte_carlo/run_settings_1.csv'
+

temoa/extensions/modeling_to_generate_alternatives/mga_sequencer.py

@@ -75,13 +75,13 @@ def __init__(self, config: TemoaConfig):
                 'Recommend selecting source trace in config file.'
             )
         if config.save_lp_file:
-            logger.info('Saving LP file is disabled during MGA runs.')
+            logger.warning('Saving LP file is disabled during MGA runs.')
             config.save_lp_file = False
         if config.save_duals:
-            logger.info('Saving duals is disabled during MGA runs.')
+            logger.warning('Saving duals is disabled during MGA runs.')
             config.save_duals = False
         if config.save_excel:
-            logger.info('Saving excel is disabled during MGA runs.')
+            logger.warning('Saving excel is disabled during MGA runs.')
             config.save_excel = False
         self.config = config
 
@@ -210,16 +210,18 @@ def start(self):
         )
 
         # 5.  Set up the Workers
+        num_workers = self.num_workers
         work_queue = Queue(1)  # restrict the queue to hold just 1 models in it max
-        result_queue = Queue(2)
+        result_queue = Queue(
+            num_workers + 1
+        )  # must be able to hold a shutdown signal from all workers at once!
         log_queue = Queue(50)
         # make workers
         workers = []
         kwargs = {
             'solver_name': self.config.solver_name,
             'solver_options': self.worker_solver_options,
         }
-        num_workers = self.num_workers
         # construct path for the solver logs
         s_path = Path(get_OUTPUT_PATH(), 'solver_logs')
         if not s_path.exists():
@@ -285,6 +287,8 @@ def start(self):
         if self.verbose:
             print('shutting it down')
         for _ in workers:
+            if self.verbose:
+                print('shutdown sent')
             work_queue.put('ZEBRA')  # shutdown signal
 
         # 7b.  Keep pulling results from the queue to empty it out

temoa/extensions/monte_carlo/MC_solver_options.toml

@@ -0,0 +1,26 @@
+# A container for solver options
+# the top level solver name in brackets should align with the solver name in the config.toml
+
+num_workers = 6
+
+[gurobi]
+
+Method= 2  # Barrier ONLY
+Threads= 20  # per solver instance
+BarConvTol = 0.01  # Relative Barrier Tolerance primal-dual
+FeasibilityTol= 1e-2  # pretty loose
+Crossover= 0  # Disabled
+TimeLimit= 18000  # 5 hrs
+
+# regarding BarConvTol:  https://www.gurobi.com/documentation/current/refman/barrier_logging.html
+# note that ref above seems to imply that FeasibilyTol is NOT used when using barrier only...?
+
+# for records ...
+# 'LogFile': './my_gurobi_log.log',
+# 'LPWarmStart': 2,  # pass basis
+
+[cbc]
+# tbd
+
+[appsi_highs]
+# tbd