diff --git a/canflood/build/dialog.py b/canflood/build/dialog.py index e9087d8a..a4abfc04 100644 --- a/canflood/build/dialog.py +++ b/canflood/build/dialog.py @@ -514,8 +514,8 @@ def upd_lfield(): #updating the field box #=========================================================================== # HELPERS---------- #=========================================================================== - def set_setup(self, set_cf_fp=True, set_finv=True, #attach parameters from setup tab - logger=None,): + def set_setup(self, set_cf_fp=True, set_finv=True, logger=None,): + """attach parameters from setup tab""" if logger is None: logger=self.logger log = logger.getChild('set_setup') #======================================================================= @@ -625,8 +625,7 @@ def build_scenario(self): # Generate a CanFlood project from scratch #======================================================================= # prechecks #======================================================================= - if self.radioButton_SS_fpRel.isChecked(): - raise Error('Relative filepaths not implemented') + self.feedback.upd_prog(10) diff --git a/canflood/build/validator.py b/canflood/build/validator.py index b2bf8ee3..1107501c 100644 --- a/canflood/build/validator.py +++ b/canflood/build/validator.py @@ -90,12 +90,20 @@ def config_cf(self, #helper to initilaize the configParser return cpars - def cf_check(self, #check the control file against a passed model's expectation handles - modObj #model object to run check against + def cf_check(self, # + modObj ): + """check the control file against a passed model's expectation handles + + Params + ---------- + modObj: class object + an uninitalized child of the model.modcom.Model() class + must have a `validate` method + """ - wrkr = modObj(cf_fp = self.cf_fp, logger=self.logger) #initilize it + wrkr = modObj(cf_fp = self.cf_fp, logger=self.logger, absolute_fp=self.absolute_fp) #initilize it #======================================================================= # check against expectations #======================================================================= diff --git a/canflood/hlpr/basic.py b/canflood/hlpr/basic.py index 919a8929..af69496c 100644 --- a/canflood/hlpr/basic.py +++ b/canflood/hlpr/basic.py @@ -301,14 +301,19 @@ def set_cf_pars(self, #update the control file w/ the passed parameters return - def _get_from_cpar(self, #special parameter extraction recognizing object's t ype + def _get_from_cpar(self, # cpars, sectName, varName, logger = None): - """each parameter should exist on teh class instance. - we use this to set the type""" + """special parameter extraction recognizing object's type + + each parameter should exist on teh class instance. + we use this to set the type + + + """ if logger is None: logger=self.logger log = logger.getChild('_get_from_cpar') diff --git a/canflood/hlpr/logr.py b/canflood/hlpr/logr.py index 035d2903..d29a5685 100644 --- a/canflood/hlpr/logr.py +++ b/canflood/hlpr/logr.py @@ -57,11 +57,16 @@ def basic_logger(root_lvl = logging.DEBUG, #=========================================================================== logger = logging.getLogger() #get the root logger logging.config.fileConfig(logcfg_file) #load the configuration file - logger.info('root logger initiated and configured from file: %s'%(logcfg_file)) + logger.info('root logger initiated and configured from file:\n %s'%(logcfg_file)) #override default level in the config file logger.setLevel(root_lvl) + # Retrieve filename and path of the file handler + for handler in logger.handlers: + if isinstance(handler, logging.FileHandler): + print(f"log file location: {handler.baseFilename}") # Prints the file path + diff --git a/canflood/model/modcom.py b/canflood/model/modcom.py index 84213c09..757370f0 100644 --- a/canflood/model/modcom.py +++ b/canflood/model/modcom.py @@ -364,7 +364,7 @@ def init_model(self, #load and attach control file parameters if 'absolute_fp' in self.pars['parameters']: absolute_fp = self.pars['parameters'].getboolean('absolute_fp') if not self.absolute_fp==absolute_fp: - log.warning(f'overwriting \'aboslute_fp\' with value from control file ({absolute_fp})') + log.warning(f'overwriting \'absolute_fp\' with value from control file ({absolute_fp})') self.absolute_fp=absolute_fp @@ -710,13 +710,7 @@ def _get_cf_miss(self, #collect mismatch between expectations and control file p cpars): assert isinstance(cpars, configparser.ConfigParser) -#=============================================================================== -# errors = [] -# -# for chk_d, opt_f in ((self.exp_pars_md,False), (self.exp_pars_op,True)): -# _, l = self.cf_chk_pars(cpars, copy.copy(chk_d), optional=opt_f) -# errors = errors + l -#=============================================================================== + #======================================================================= # mandatory @@ -2082,7 +2076,9 @@ def _par_hndl_chk(self, """ if logger is None: logger=self.logger - if absolute_fp= None: absolute_fp=self.aboslute_fp + if not hasattr(self, 'absolute_fp'): + raise AttributeError(f'object {self.__name__} missing attribute \'absolute_fp\'') + if absolute_fp is None: absolute_fp=self.absolute_fp log = logger.getChild('par_hndl_chk') #======================================================================= @@ -2108,6 +2104,9 @@ def _par_hndl_chk(self, assert isinstance(hvals, tuple), '%s.%s got bad type on hvals: %s'%(sect, varnm, type(hvals)) assert pval in hvals, '%s.%s unexpected value: \'%s\''%(sect, varnm, pval) + #=================================================================== + # filepaths + #=================================================================== elif chk_hndl == 'ext': #basic checks @@ -2118,11 +2117,9 @@ def _par_hndl_chk(self, #handl relative filepaths if not absolute_fp: - pval = os.path.join(self.cf_dir, pval) - - + pval = os.path.join(self.cf_dir, pval) - assert os.path.exists(pval), '%s.%s passed invalid filepath: \'%s\''%(sect, varnm, pval) + assert os.path.exists(pval), f'%s.%s passed invalid filepath (absolute_fp={absolute_fp}):\n %s'%(sect, varnm, pval) ext = os.path.splitext(os.path.split(pval)[1])[1] @@ -2143,22 +2140,23 @@ def _par_hndl_chk(self, - def validate(self, #validate this model object - cpars, #initilzied config parser - #so a session can pass a control file... rather than usin gthe workers init - logger=None, - ): - #if logger is None: logger=self.logger + def validate(self, cpars, logger=None,): - """only 1 check for now""" - #======================================================================= - # check the control file expectations - #======================================================================= - errors = self._get_cf_miss(cpars) + """run all validation checks on this model + only 1 check for now. check the control file expectations + children could over-write this method with custom validations + Params + --------- + cpars: config parser + so a session can pass a control file... rather than usin gthe workers init + + """ + #if logger is None: logger=self.logger + - return errors + return self._get_cf_miss(cpars) #check the control file expectations def check_attrimat(self, #check the logic of the attrimat atr_dxcol=None, diff --git a/tests2/conftest.py b/tests2/conftest.py index 952eae89..ea85ea98 100644 --- a/tests2/conftest.py +++ b/tests2/conftest.py @@ -59,9 +59,7 @@ def __init__(self, crs=None,logger=None,**kwargs): super().__init__(crsid = crs.authid(), logger=logger, #feedbac=MyFeedBackQ(logger=logger), - **kwargs) - - + **kwargs) self.logger.info('finished Session_pytest.__init__') @@ -101,6 +99,11 @@ def __exit__(self, exc_type, exc_value, traceback): #sys.exit() #wrap print('exiting DialTester') + +#=============================================================================== +# FIXTURES.FUNCTIONS------- +#=============================================================================== + @pytest.fixture(scope='function') def dialogClass(request): #always passing this as an indirect return request.param @@ -128,6 +131,7 @@ def session(tmp_path, """TODO: fix logger""" np.random.seed(100) + print(f'tmp_path\n {tmp_path}') #configure output out_dir=tmp_path @@ -157,19 +161,9 @@ def session(tmp_path, yield ses -@pytest.fixture(scope='session') -def write(): - - write=False - - - if write: - print('WARNING!!! runnig in write mode') - return write -#=============================================================================== -# function.fixtures------- -#=============================================================================== + + @pytest.fixture(scope='function') def out_dir(tmp_path): return tmp_path @@ -177,9 +171,40 @@ def out_dir(tmp_path): @pytest.fixture(scope='function') def test_name(request): return request.node.name.replace('[','_').replace(']', '_') + + +@pytest.fixture(scope='function') +def true_dir(write, tmp_path, test_dir): + true_dir = os.path.join(test_dir, os.path.basename(tmp_path)) + if write: + if os.path.exists(true_dir): + try: + shutil.rmtree(true_dir) + os.makedirs(true_dir) #add back an empty folder + #os.makedirs(os.path.join(true_dir, 'working')) #and the working folder + except Exception as e: + print('failed to cleanup the true_dir: %s w/ \n %s'%(true_dir, e)) + + """no... this is controlled with the out_dir on the session + #not found.. create a fresh one + if not os.path.exists(true_dir): + os.makedirs(true_dir)""" + + #assert os.path.exists(true_dir) + return true_dir + #=============================================================================== -# session.fixtures---------- +# FIXTURES.SESSIOn---------- #=============================================================================== +@pytest.fixture(scope='session') +def write(): + + write=False + + + if write: + print('WARNING!!! runnig in write mode') + return write #=============================================================================== # logger @@ -232,9 +257,7 @@ def logger(): """simple backend loggers""" return mod_logger -#=============================================================================== -# directories---------- -#=============================================================================== + @pytest.fixture(scope='session') def base_dir(): from definitions import base_dir @@ -248,25 +271,7 @@ def test_dir(base_dir): -@pytest.fixture -def true_dir(write, tmp_path, test_dir): - true_dir = os.path.join(test_dir, os.path.basename(tmp_path)) - if write: - if os.path.exists(true_dir): - try: - shutil.rmtree(true_dir) - os.makedirs(true_dir) #add back an empty folder - #os.makedirs(os.path.join(true_dir, 'working')) #and the working folder - except Exception as e: - print('failed to cleanup the true_dir: %s w/ \n %s'%(true_dir, e)) - - """no... this is controlled with the out_dir on the session - #not found.. create a fresh one - if not os.path.exists(true_dir): - os.makedirs(true_dir)""" - #assert os.path.exists(true_dir) - return true_dir #=============================================================================== @@ -280,7 +285,7 @@ def _build_dialog_validate_handler(dial): but in a test environment, we want to raise these """ QTest.mouseClick(dial.pushButton_Validate, Qt.LeftButton) -# Loop through errors generated by Model.cf_chk_pars() and raise them + # Loop through errors generated by Model.cf_chk_pars() and raise them for vtag, error_l in dial.validation_result_d.items(): for error in error_l: if isinstance(error, Exception): diff --git a/tests2/data/test_02_build_inv_tests2__data0/CanFlood_test_01.txt b/tests2/data/test_02_build_inv_tests2__data0/CanFlood_test_01.txt index eefc1648..625377d2 100644 --- a/tests2/data/test_02_build_inv_tests2__data0/CanFlood_test_01.txt +++ b/tests2/data/test_02_build_inv_tests2__data0/CanFlood_test_01.txt @@ -16,10 +16,10 @@ apply_miti = False #whether to apply mitigation algorthihims [dmg_fps] curves = #damage curve library filepath -finv = finv_test_02_32_tut2.csv +finv = tests2\data\test_02_build_inv_tests2__data0\finv_test_02_32_tut2.csv expos = #exposure data filepath gels = #ground elevation data filepath -#'finv' file path set from prepr.py at 2022-06-26 14.30.12 + [risk_fps] dmgs = #damage data results filepath diff --git a/tests2/data/test_05_build_evals_tests2__da0/CanFlood_test_01.txt b/tests2/data/test_05_build_evals_tests2__da0/CanFlood_test_01.txt index 885b50a2..8974ffe2 100644 --- a/tests2/data/test_05_build_evals_tests2__da0/CanFlood_test_01.txt +++ b/tests2/data/test_05_build_evals_tests2__da0/CanFlood_test_01.txt @@ -23,8 +23,8 @@ gels = #ground elevation data filepath [risk_fps] dmgs = #damage data results filepath exlikes = #secondary exposure likelihood data filepath -evals = C:\LS\09_REPOS\03_TOOLS\CanFlood\_git\tests2\data\test_05_build_evals_tests2__da0\evals_4_test_05.csv -#evals file path set from build.dialog.py at 2022-06-26 18.06.20 +evals = tests2\data\test_05_build_evals_tests2__da0\evals_4_test_05.csv + [validation] risk1 = False diff --git a/tests2/data/test_06_build_dtm_tutorials__20/CanFlood_test_01.txt b/tests2/data/test_06_build_dtm_tutorials__20/CanFlood_test_01.txt index 15c6af6c..7bc9a676 100644 --- a/tests2/data/test_06_build_dtm_tutorials__20/CanFlood_test_01.txt +++ b/tests2/data/test_06_build_dtm_tutorials__20/CanFlood_test_01.txt @@ -19,18 +19,18 @@ curves = tests2\data\test_03_build_inv_curves_tests0\cLib_test_03_2022-06-26_180 finv = tests2\data\test_02_build_inv_tests2__data0\finv_test_02_32_tut2.csv expos = tests2\data\test_04_build_hsamp_tutorials_0\expos_test_04_4_32.csv gels = tests2\data\test_06_build_dtm_tutorials__20\gels_test_06_1_32.csv -#'gels' file path set from rsamp.py at 2022-06-26 18.06.21 [risk_fps] dmgs = #damage data results filepath exlikes = #secondary exposure likelihood data filepath -evals = C:\LS\09_REPOS\03_TOOLS\CanFlood\_git\tests2\data\test_05_build_evals_tests2__da0\evals_4_test_05.csv +evals = tests2\data\test_05_build_evals_tests2__da0\evals_4_test_05.csv [validation] risk1 = False dmg2 = False risk2 = False risk3 = False +# 'dmg2' validated by validator.py at 2025-01-24 16.59.58 [results_fps] attrimat02 = #lvl2 attribution matrix fp (post dmg model) diff --git a/tests2/test_build.py b/tests2/test_build.py index 28358b54..2fb852d0 100644 --- a/tests2/test_build.py +++ b/tests2/test_build.py @@ -14,7 +14,7 @@ ''' -import pytest, os, shutil +import pytest, os, shutil, configparser import pandas as pd @@ -115,7 +115,7 @@ def test_02_build_inv(session, base_dir, finv_vlay, cf_fp): #retrieve the filepath from the control file fp = dial.get_cf_par(cf_fp, sectName='dmg_fps', varName='finv') assert not fp is None, f'no finv specified' - assert os.path.exists(fp), fp + assert os.path.exists(fp), f'failed to write control file\n {fp}' df = pd.read_csv(fp) assert len(df)==finv_vlay.dataProvider().featureCount() @@ -298,6 +298,7 @@ def test_05_build_evals(session, base_dir, cf_fp, true_dir): def test_06_build_dtm(session, base_dir, cf_fp, true_dir, finv_vlay, dtm_fp): dial = session.Dialog + #=========================================================================== # setup #=========================================================================== @@ -354,7 +355,8 @@ def test_06_build_dtm(session, base_dir, cf_fp, true_dir, finv_vlay, dtm_fp): @pytest.mark.parametrize('cf_fp',[r'tests2\data\test_06_build_dtm_tutorials__20\CanFlood_test_01.txt']) #from test_06 def test_07_build_valid(session, base_dir, cf_fp): dial = session.Dialog - + """NO... this is set with the radio button""" + #assert not dial.absolute_fp #=========================================================================== # setup #=========================================================================== @@ -388,6 +390,9 @@ def _build_setup(base_dir, cf_fp, dial, out_dir, testName='testName'): #set the relative filepath flag dial.radioButton_SS_fpRel.setChecked(True) + #=========================================================================== + # copy over files + #=========================================================================== #copy over the control file """need to copy everything @@ -396,22 +401,54 @@ def _build_setup(base_dir, cf_fp, dial, out_dir, testName='testName'): then build the cf_fp = os.path.join(dir, cf_+fp) """ assert os.path.exists(os.path.join(base_dir, cf_fp)) - par_dir = os.path.join(base_dir, cf_fp) - directory_path = os.path.dirname(par_dir) - cf_fp = shutil.copy2(os.path.join(base_dir, cf_fp), os.path.join(out_dir, os.path.basename(cf_fp))) - - for item in os.listdir(directory_path): - source_item = os.path.join(directory_path, item) - destination_item = os.path.join(out_dir, item) - if os.path.isfile(source_item): - shutil.copy2(source_item, destination_item) - #set the working directory + cf_fp = os.path.join(base_dir, cf_fp) + cf_fn = os.path.basename(cf_fp) + #directory_path = os.path.dirname(par_dir) + + #copy control file + cf_fp_new =os.path.join(out_dir, cf_fn) + _ = shutil.copy2(cf_fp, cf_fp_new) + + + #extract datafilepaths from parameter file + params_lib = _extract_fps_parameters(cf_fp) + + new_fp_d = dict() + for _, d in params_lib.items(): + for k, fp1 in d.items(): + try: + ofp = os.path.join(out_dir, fp1) + os.makedirs(os.path.dirname(ofp), exist_ok=True) + new_fp_d[k] = shutil.copy2(os.path.join(base_dir, fp1), ofp) + except Exception as e: + raise IOError(f'failed to copy datafile \n {fp1}\n {cf_fp}\n {e}') + + + print(f'copied over {len(new_fp_d)} data files to \n {out_dir}') + #=========================================================================== + # #set the working directory + #=========================================================================== dial.lineEdit_wdir.setText(str(out_dir)) dial.linEdit_ScenTag.setText(testName) #set the control file - dial.lineEdit_cf_fp.setText(cf_fp) + dial.lineEdit_cf_fp.setText(cf_fp_new) + + return cf_fp_new + + +# Function to load the parameter file and extract _fps sections +def _extract_fps_parameters(file_path): + config = configparser.ConfigParser(allow_no_value=True) # `allow_no_value=True` allows empty values + config.read(file_path) + + fps_data = {} + + # Iterate over all sections + for section in config.sections(): + if section.endswith("_fps"): + fps_data[section] = {key: value for key, value in config.items(section) if not value.startswith('#')} - return cf_fp + return fps_data