From cba0f466ec4de929c71d09a8e3a5a7f09ef57376 Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Mon, 8 Apr 2024 05:43:24 -0700 Subject: [PATCH 01/22] Dump current state of development effort The code is currently a bit messy and by no means final. That's what feature branches are for, I guess. Storing it here on this branch to eliminate the need of keeping it in a stash and risk accidentally purging it. --- pelicun/db.py | 306 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 304 insertions(+), 2 deletions(-) diff --git a/pelicun/db.py b/pelicun/db.py index 86a18bf52..99fa59b8a 100644 --- a/pelicun/db.py +++ b/pelicun/db.py @@ -55,6 +55,9 @@ """ +# todo +# fmt: off + import re import json from pathlib import Path @@ -63,8 +66,8 @@ from scipy.stats import norm import pandas as pd -from . import base -from .uq import fit_distribution_to_percentiles +from pelicun import base +from pelicun.uq import fit_distribution_to_percentiles idx = base.idx @@ -2913,3 +2916,302 @@ def create_Hazus_EQ_bldg_injury_db( print("Successfully parsed and saved the injury consequence data from Hazus " "EQ") + +# todo +# fmt: on + + +def create_Hazus_HU_fragility_db( + source_file, + target_meta_file='damage_DB_SimCenter_Hazus_HU_bldg.csv', +): + """ + Create a database metadata file for the HAZUS Hurricane fragilities. + + This method was developed to add a json file with metadata + accompanying `damage_DB_SimCenter_Hazus_HU_bldg.csv`. That file + contains fragility curves fitted to Hazus Hurricane data relaetd + to the Hazus Hurricane Technical Manual v4.2. + + Parameters + ---------- + source_file: string + Path to the Hazus Hurricane fragility data. + target_meta_file: string + Path where the fragility metadata should be saved. A json file is + expected. + + """ + + source_file = 'pelicun/resources/SimCenterDBDL/damage_DB_SimCenter_Hazus_HU_bldg.csv' # todo + + # general_information: { + # "ShortName": "Hazus Hurricane Methodology", + # "Description": "The models in this dataset are based on version 4.2 of the Hazus Hurricane Model Technical Manual", + # "Version": "1.0", + # # "ComponentGroups": { + # # "GF - Geotechnical Failure": [ + # # "GF.H - Horizontal Spreading", + # # "GF.V - Vertical Settlement", + # # ], + # # "LF - Lifeline Facilities": [ + # # "LF.W1 - Wood, Light Frame", + # # "LF.W2 - Wood, Commercial & Industrial", + # # "LF.S1 - Steel Moment Frame", + # # "LF.S2 - Steel Braced Frame", + # # "LF.S3 - Steel Light Frame", + # # "LF.S4 - Steel Frame with Cast-in-Place Concrete Shear Walls", + # # "LF.S5 - Steel Frame with Unreinforced Masonry Infill Walls", + # # "LF.C1 - Concrete Moment Frame", + # # "LF.C2 - Concrete Shear Walls", + # # "LF.C3 - Concrete Frame with Unreinforced Masonry Infill Walls", + # # "LF.PC1 - Precast Concrete Tilt-Up Walls", + # # "LF.PC2 - Precast Concrete Frames with Concrete Shear Walls", + # # "LF.RM1 - Reinforced Masonry Bearing Walls with Wood or Metal Deck Diaphragms", + # # "LF.RM2 - Reinforced Masonry Bearing Walls with Precast Concrete Diaphragms", + # # "LF.URM - Unreinforced Masonry Bearing Walls", + # # "LF.MH - Mobile Homes", + # # ], + # # "NSA - Non-Structural Acceleration-Sensitive": [], + # # "NSD - Non-Structural Drift-Sensitive": [], + # # "STR - Structural": [ + # # "STR.W1 - Wood, Light Frame", + # # "STR.W2 - Wood, Commercial & Industrial", + # # "STR.S1 - Steel Moment Frame", + # # "STR.S2 - Steel Braced Frame", + # # "STR.S3 - Steel Light Frame", + # # "STR.S4 - Steel Frame with Cast-in-Place Concrete Shear Walls", + # # "STR.S5 - Steel Frame with Unreinforced Masonry Infill Walls", + # # "STR.C1 - Concrete Moment Frame", + # # "STR.C2 - Concrete Shear Walls", + # # "STR.C3 - Concrete Frame with Unreinforced Masonry Infill Walls", + # # "STR.PC1 - Precast Concrete Tilt-Up Walls", + # # "STR.PC2 - Precast Concrete Frames with Concrete Shear Walls", + # # "STR.RM1 - Reinforced Masonry Bearing Walls with Wood or Metal Deck Diaphragms", + # # "STR.RM2 - Reinforced Masonry Bearing Walls with Precast Concrete Diaphragms", + # # "STR.URM - Unreinforced Masonry Bearing Walls", + # # "STR.MH - Mobile Homes", + # # ], + # # }, + # } + # todo: update component groups + + # ds_descriptions = { + # 'DS1': ( + # 'Minor Damage. Maximum of one broken window, door or garage door. ' + # 'Moderate roof cover loss that can be covered to prevent additional ' + # 'water entering the building. Marks or dents on walls requiring ' + # 'painting or patching for repair.' + # ), + # 'DS2': ( + # 'Moderate Damage. Major roof cover damage, moderate window breakage.' + # 'Minor roof sheathing failure. Some resulting damage to interior of' + # 'building from water.' + # ), + # 'DS3': ( + # 'Severe Damage. Major window damage or roof sheathing loss. Major roof' + # 'cover loss. Extensive damage to interior from water.' + # ), + # 'DS4': ( + # 'Destruction. Complete roof failure and/or, failure of wall frame. Loss' + # 'of more than 50% of roof sheathing.' + # ), + # } + + fragility_data = pd.read_csv(source_file) + ids = fragility_data['ID'].str.split('.') + code_strings = set([x for y in ids.to_list() for x in y]) + + roof_shape = { + 'flt': 'Flat roof', + 'gab': 'Gable roof', + 'hip': 'Hip roof.', + } + + secondary_water_resistance = { + '1': 'Secondary water resistance.', + '0': 'No secondary water resistance.', + } + + roof_deck_attachment = { + '6d': '6d roof deck nails', + '6s': '6s roof deck nails', + '8d': '8d roof deck nails', + '8s': '8s roof deck nails', + 'st': 'Standard roof deck attachment', + 'su': 'Superior roof deck attachment', + } + + roof_wall_connection = { + 'tnail': 'Roof-to-wall toe nails.', + 'strap': 'Roof-to-wall straps.', + } + + garage_presence = { + 'no': 'No garage.', + 'wkd': 'Weak garage door.', + 'std': 'Standard garage door.', + 'sup': 'Strong garage door.', + } + + shutters = {'1': 'Has Shutters.', '0': 'No shutters.'} + + roof_cover = { + 'bur': 'Built-up roof cover.', + 'spm': 'Single-ply membrane roof cover.', + 'smtl': 'Sheet metal roof cover.', + 'cshl': 'Shingle roof cover.', + } + + roof_quality = { + 'god': 'Good roof quality.', + 'por': 'Poor roof quality.', + } + + masonry_reinforcing = { + '1': 'Has masonry reinforcing.', + '0': 'No masonry reinforcing.', + } + + roof_frame_type = { + 'trs': 'Wood truss roof frame.', + 'ows': 'OWSJ roof frame.', + } + + wind_debris_environment = { + 'A': 'Residentiao/commercial wind debris environment.', + 'B': 'Wind debris environment varies by direction.', + 'C': 'Residential wind debris environment.', + 'D': 'No wind debris environment.', + } + + roof_deck_age = { + 'god': 'New or average roof age.', + 'por': 'Old roof age.', + } + + roof_metal_deck_attachment_quality = { + 'std': 'Standard metal deck roof attachment.', + 'sup': 'Superior metal deck roof attachment.', + } + + number_of_units = { + 'sgl': 'Single unit.', + 'mlt': 'Multi-unit.', + } + + joist_spacing = { + '4': '4 ft joist spacing', + '6': '6 ft foot joist spacing', + } + + window_area = { + 'low': 'Low window area.', + 'med': 'Medium window area.', + 'hig': 'High window area.', + } + + +class_types = { + 'W.SF.1': 'Wood, Single-family, One-story', + 'W.SF.2': 'Wood, Single-family, Two or More Stories', + 'W.MUH.1': 'Wood, Multi-Unit Housing, One-story', + 'W.MUH.2': 'Wood, Multi-Unit Housing, Two Stories', + 'W.MUH.3': 'Wood, Multi-Unit Housing, Three or More Stories', + 'M.SF.1': 'Masonry, Single-family, One-story', + 'M.SF.2': 'Masonry, Single-family, Two or More Stories', + 'M.MUH.1': 'Masonry, Multi-Unit Housing, One-story', + 'M.MUH.2': 'Masonry, Multi-Unit Housing, Two Stories', + 'M.MUH.3': 'Masonry, Multi-Unit Housing, Three or More Stories', + 'M.LRM.1': 'Masonry, Low-Rise Strip Mall, Up to 15 Feet', + 'M.LRM.2': 'Masonry, Low-Rise Strip Mall, More than 15 Feet', + 'M.LRI': 'Masonry, Low-Rise Industrial/Warehouse/Factory Buildings', + 'M.ERB.L': 'Masonry, Engineered Residential Building, Low-Rise (1-2 Stories)', + 'M.ERB.M': 'Masonry, Engineered Residential Building, Mid-Rise (3-5 Stories)', + 'M.ERB.H': 'Masonry, Engineered Residential Building, High-Rise (6+ Stories)', + 'M.ECB.L': 'Masonry, Engineered Commercial Building, Low-Rise (1-2 Stories)', + 'M.ECB.M': 'Masonry, Engineered Commercial Building, Mid-Rise (3-5 Stories)', + 'M.ECB.H': 'Masonry, Engineered Commercial Building, High-Rise (6+ Stories)', + 'C.ERB.L': 'Concrete, Engineered Residential Building, Low-Rise (1-2 Stories)', + 'C.ERB.M': 'Concrete, Engineered Residential Building, Mid-Rise (3-5 Stories)', + 'C.ERB.H': 'Concrete, Engineered Residential Building, High-Rise (6+ Stories)', + 'C.ECB.L': 'Concrete, Engineered Commercial Building, Low-Rise (1-2 Stories)', + 'C.ECB.M': 'Concrete, Engineered Commercial Building, Mid-Rise (3-5 Stories)', + 'C.ECB.H': 'Concrete, Engineered Commercial Building, High-Rise (6+ Stories)', + 'S.PMB.S': 'Steel, Pre-Engineered Metal Building, Small', + 'S.PMB.M': 'Steel, Pre-Engineered Metal Building, Medium', + 'S.PMB.L': 'Steel, Pre-Engineered Metal Building, Large', + 'S.ERB.L': 'Steel, Engineered Residential Building, Low-Rise (1-2 Stories)', + 'S.ERB.M': 'Steel, Engineered Residential Building, Mid-Rise (3-5 Stories)', + 'S.ERB.H': 'Steel, Engineered Residential Building, High-Rise (6+ Stories)', + 'S.ECB.L': 'Steel, Engineered Commercial Building, Low-Rise (1-2 Stories)', + 'S.ECB.M': 'Steel, Engineered Commercial Building, Mid-Rise (3-5 Stories)', + 'S.ECB.H': 'Steel, Engineered Commercial Building, High-Rise (6+ Stories)', + 'MH.PHUD': 'Manufactured Home, Pre-Housing and Urban Development (HUD)', + 'MH.76HUD': 'Manufactured Home, 1976 HUD', + 'MH.94HUDI': 'Manufactured Home, 1994 HUD - Wind Zone I', + 'MH.94HUDII': 'Manufactured Home, 1994 HUD - Wind Zone II', + 'MH.94HUDIII': 'Manufactured Home, 1994 HUD - Wind Zone III', + 'HUEF.FS': 'Fire Station', + 'HUEF.H.S': 'Small Hospital, Hospital with fewer than 50 Beds', + 'HUEF.H.M': 'Medium Hospital, Hospital with beds between 50 & 150', + 'HUEF.H.L': 'Large Hospital, Hospital with more than 150 Beds', + 'HUEF.S.L': '???', + 'HUEF.S.S': '???', + 'HUEF.S.M': '???', + 'HUEF.EO': 'Emergency Operation Centers', + 'HUEF.PS': 'Police Station', +} + +my_dictionaries = { + 'roof_shape': roof_shape, + 'secondary_water_resistance': secondary_water_resistance, + 'roof_deck_attachment': roof_deck_attachment, + 'roof_wall_connection': roof_wall_connection, + 'garage_presence': garage_presence, + 'shutters': shutters, + 'roof_cover': roof_cover, + 'roof_quality': roof_quality, + 'masonry_reinforcing': masonry_reinforcing, + 'roof_frame_type': roof_frame_type, + 'wind_debris_environment': wind_debris_environment, + 'roof_deck_age': roof_deck_age, + 'roof_metal_deck_attachment_quality': roof_metal_deck_attachment_quality, + 'number_of_units': number_of_units, + 'joist_spacing': joist_spacing, + 'window_area': window_area, +} + + +def find_class_type(entry): + entry_elements = entry.split('.') + for nper in range(1, len(entry_elements)): + first_parts = '.'.join(entry_elements[:nper]) + if first_parts in class_types: + return first_parts + return None + + +all_entry_attr_dcts = {} +for entry in fragility_data['ID']: + + class_type = find_class_type(entry) + assert class_type is not None + + attrs = entry.replace(f'{class_type}.', '').split('.') + all_attr_dcts = {} + for attr in attrs: + attr_dcts = [] + for dct_name, dct in my_dictionaries.items(): + if attr in dct: + attr_dcts.append(dct_name) + all_attr_dcts[attr] = attr_dcts + all_entry_attr_dcts[entry] = all_attr_dcts + +ddf = pd.DataFrame(all_entry_attr_dcts.values(), index=all_entry_attr_dcts.keys()) + +for col in ddf.columns: + print(col) + print(ddf[col].value_counts()) + print() + print() From e4cf02f954682f71647def9210922f920e69f1b8 Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Sat, 13 Apr 2024 14:27:13 -0700 Subject: [PATCH 02/22] Complete definition of `create_Hazus_HU_fragility_db` - Finalizes create_Hazus_HU_fragility_db. - Introduces a .json file with predefined metadata - Added code is linted and type-hinted. --- pelicun/db.py | 676 +++++++++++++----- ...e_DB_SimCenter_Hazus_HU_bldg_template.json | 198 +++++ 2 files changed, 681 insertions(+), 193 deletions(-) create mode 100644 pelicun/resources/SimCenterDBDL/damage_DB_SimCenter_Hazus_HU_bldg_template.json diff --git a/pelicun/db.py b/pelicun/db.py index 99fa59b8a..aec483420 100644 --- a/pelicun/db.py +++ b/pelicun/db.py @@ -58,6 +58,7 @@ # todo # fmt: off +from __future__ import annotations import re import json from pathlib import Path @@ -2922,9 +2923,15 @@ def create_Hazus_EQ_bldg_injury_db( def create_Hazus_HU_fragility_db( - source_file, - target_meta_file='damage_DB_SimCenter_Hazus_HU_bldg.csv', -): + source_file: str = ( + 'pelicun/resources/SimCenterDBDL/' 'damage_DB_SimCenter_Hazus_HU_bldg.csv' + ), + meta_file: str = ( + 'pelicun/resources/SimCenterDBDL/' + 'damage_DB_SimCenter_Hazus_HU_bldg_template.json' + ), + target_meta_file: str = 'damage_DB_SimCenter_Hazus_HU_bldg.json', +) -> None: """ Create a database metadata file for the HAZUS Hurricane fragilities. @@ -2937,114 +2944,70 @@ def create_Hazus_HU_fragility_db( ---------- source_file: string Path to the Hazus Hurricane fragility data. + meta_file: string + Path to a predefined fragility metadata file. target_meta_file: string Path where the fragility metadata should be saved. A json file is expected. """ - source_file = 'pelicun/resources/SimCenterDBDL/damage_DB_SimCenter_Hazus_HU_bldg.csv' # todo - - # general_information: { - # "ShortName": "Hazus Hurricane Methodology", - # "Description": "The models in this dataset are based on version 4.2 of the Hazus Hurricane Model Technical Manual", - # "Version": "1.0", - # # "ComponentGroups": { - # # "GF - Geotechnical Failure": [ - # # "GF.H - Horizontal Spreading", - # # "GF.V - Vertical Settlement", - # # ], - # # "LF - Lifeline Facilities": [ - # # "LF.W1 - Wood, Light Frame", - # # "LF.W2 - Wood, Commercial & Industrial", - # # "LF.S1 - Steel Moment Frame", - # # "LF.S2 - Steel Braced Frame", - # # "LF.S3 - Steel Light Frame", - # # "LF.S4 - Steel Frame with Cast-in-Place Concrete Shear Walls", - # # "LF.S5 - Steel Frame with Unreinforced Masonry Infill Walls", - # # "LF.C1 - Concrete Moment Frame", - # # "LF.C2 - Concrete Shear Walls", - # # "LF.C3 - Concrete Frame with Unreinforced Masonry Infill Walls", - # # "LF.PC1 - Precast Concrete Tilt-Up Walls", - # # "LF.PC2 - Precast Concrete Frames with Concrete Shear Walls", - # # "LF.RM1 - Reinforced Masonry Bearing Walls with Wood or Metal Deck Diaphragms", - # # "LF.RM2 - Reinforced Masonry Bearing Walls with Precast Concrete Diaphragms", - # # "LF.URM - Unreinforced Masonry Bearing Walls", - # # "LF.MH - Mobile Homes", - # # ], - # # "NSA - Non-Structural Acceleration-Sensitive": [], - # # "NSD - Non-Structural Drift-Sensitive": [], - # # "STR - Structural": [ - # # "STR.W1 - Wood, Light Frame", - # # "STR.W2 - Wood, Commercial & Industrial", - # # "STR.S1 - Steel Moment Frame", - # # "STR.S2 - Steel Braced Frame", - # # "STR.S3 - Steel Light Frame", - # # "STR.S4 - Steel Frame with Cast-in-Place Concrete Shear Walls", - # # "STR.S5 - Steel Frame with Unreinforced Masonry Infill Walls", - # # "STR.C1 - Concrete Moment Frame", - # # "STR.C2 - Concrete Shear Walls", - # # "STR.C3 - Concrete Frame with Unreinforced Masonry Infill Walls", - # # "STR.PC1 - Precast Concrete Tilt-Up Walls", - # # "STR.PC2 - Precast Concrete Frames with Concrete Shear Walls", - # # "STR.RM1 - Reinforced Masonry Bearing Walls with Wood or Metal Deck Diaphragms", - # # "STR.RM2 - Reinforced Masonry Bearing Walls with Precast Concrete Diaphragms", - # # "STR.URM - Unreinforced Masonry Bearing Walls", - # # "STR.MH - Mobile Homes", - # # ], - # # }, - # } - # todo: update component groups - - # ds_descriptions = { - # 'DS1': ( - # 'Minor Damage. Maximum of one broken window, door or garage door. ' - # 'Moderate roof cover loss that can be covered to prevent additional ' - # 'water entering the building. Marks or dents on walls requiring ' - # 'painting or patching for repair.' - # ), - # 'DS2': ( - # 'Moderate Damage. Major roof cover damage, moderate window breakage.' - # 'Minor roof sheathing failure. Some resulting damage to interior of' - # 'building from water.' - # ), - # 'DS3': ( - # 'Severe Damage. Major window damage or roof sheathing loss. Major roof' - # 'cover loss. Extensive damage to interior from water.' - # ), - # 'DS4': ( - # 'Destruction. Complete roof failure and/or, failure of wall frame. Loss' - # 'of more than 50% of roof sheathing.' - # ), - # } - fragility_data = pd.read_csv(source_file) - ids = fragility_data['ID'].str.split('.') - code_strings = set([x for y in ids.to_list() for x in y]) + + with open(meta_file, 'r', encoding='utf-8') as f: + meta_dict = json.load(f) + + # retrieve damage state descriptions and remove that part from + # `hazus_hu_metadata` + damage_state_classes = meta_dict.pop('DamageStateClasses') + damage_state_descriptions = meta_dict.pop('DamageStateDescriptions') + + # Procedure Overview: + # (1) We define several dictionaries mapping chunks of the + # composite asset ID (the parts between periods) to human-readable + # (`-h` for short) representations. + # (2) We define -h asset type descriptions and map them to the + # first-most relevant ID chunks (`primary chunks`) + # (3) We map asset class codes with general asset classes + # (4) We define the required dictionaries from (1) that decode the + # ID chunks after the `primary chunks` for each general asset + # class + # (5) We decode: + # ID -> asset class -> general asset class -> dictionaries + # -> ID turns to -h text by combining the description of the asset class + # from the `primary chunks` and the decoded description of the + # following chunks using the dictionaries. + + # + # (1) Dictionaries + # roof_shape = { - 'flt': 'Flat roof', - 'gab': 'Gable roof', + 'flt': 'Flat roof.', + 'gab': 'Gable roof.', 'hip': 'Hip roof.', } secondary_water_resistance = { '1': 'Secondary water resistance.', '0': 'No secondary water resistance.', + 'null': 'No information on secondary water resistance.', } roof_deck_attachment = { - '6d': '6d roof deck nails', - '6s': '6s roof deck nails', - '8d': '8d roof deck nails', - '8s': '8s roof deck nails', - 'st': 'Standard roof deck attachment', - 'su': 'Superior roof deck attachment', + '6d': '6d roof deck nails.', + '6s': '6s roof deck nails.', + '8d': '8d roof deck nails.', + '8s': '8s roof deck nails.', + 'st': 'Standard roof deck attachment.', + 'su': 'Superior roof deck attachment.', + 'null': 'Missing roof deck attachment information.', } roof_wall_connection = { 'tnail': 'Roof-to-wall toe nails.', 'strap': 'Roof-to-wall straps.', + 'null': 'Missing roof-to-wall connection information.', } garage_presence = { @@ -3052,6 +3015,7 @@ def create_Hazus_HU_fragility_db( 'wkd': 'Weak garage door.', 'std': 'Standard garage door.', 'sup': 'Strong garage door.', + 'null': 'No information on garage.', } shutters = {'1': 'Has Shutters.', '0': 'No shutters.'} @@ -3061,16 +3025,19 @@ def create_Hazus_HU_fragility_db( 'spm': 'Single-ply membrane roof cover.', 'smtl': 'Sheet metal roof cover.', 'cshl': 'Shingle roof cover.', + 'null': 'No information on roof cover.', } roof_quality = { 'god': 'Good roof quality.', 'por': 'Poor roof quality.', + 'null': 'No information on roof quality.', } masonry_reinforcing = { '1': 'Has masonry reinforcing.', '0': 'No masonry reinforcing.', + 'null': 'Unknown information on masonry reinfocing.', } roof_frame_type = { @@ -3088,21 +3055,25 @@ def create_Hazus_HU_fragility_db( roof_deck_age = { 'god': 'New or average roof age.', 'por': 'Old roof age.', + 'null': 'Missing roof age information.', } roof_metal_deck_attachment_quality = { 'std': 'Standard metal deck roof attachment.', 'sup': 'Superior metal deck roof attachment.', + 'null': 'Missing roof attachment quality information.', } number_of_units = { 'sgl': 'Single unit.', 'mlt': 'Multi-unit.', + 'null': 'Unknown number of units.', } joist_spacing = { - '4': '4 ft joist spacing', - '6': '6 ft foot joist spacing', + '4': '4 ft joist spacing.', + '6': '6 ft foot joist spacing.', + 'null': 'Unknown joist spacing.', } window_area = { @@ -3111,107 +3082,426 @@ def create_Hazus_HU_fragility_db( 'hig': 'High window area.', } + tie_downs = {'1': 'Tie downs.', '0': 'No tie downs.'} + + terrain_surface_roughness = { + '3': 'Terrain surface roughness: 0.03 m.', + '15': 'Terrain surface roughness: 0.15 m.', + '35': 'Terrain surface roughness: 0.35 m.', + '70': 'Terrain surface roughness: 0.7 m.', + '100': 'Terrain surface roughness: 1 m.', + } + + # + # (2) Asset type descriptions + # + + # maps class type code to -h description + class_types = { + # ------------------------ + 'W.SF.1': 'Wood, Single-family, One-story.', + 'W.SF.2': 'Wood, Single-family, Two or More Stories.', + # ------------------------ + 'W.MUH.1': 'Wood, Multi-Unit Housing, One-story.', + 'W.MUH.2': 'Wood, Multi-Unit Housing, Two Stories.', + 'W.MUH.3': 'Wood, Multi-Unit Housing, Three or More Stories.', + # ------------------------ + 'M.SF.1': 'Masonry, Single-family, One-story.', + 'M.SF.2': 'Masonry, Single-family, Two or More Stories.', + # ------------------------ + 'M.MUH.1': 'Masonry, Multi-Unit Housing, One-story.', + 'M.MUH.2': 'Masonry, Multi-Unit Housing, Two Stories.', + 'M.MUH.3': 'Masonry, Multi-Unit Housing, Three or More Stories.', + # ------------------------ + 'M.LRM.1': 'Masonry, Low-Rise Strip Mall, Up to 15 Feet.', + 'M.LRM.2': 'Masonry, Low-Rise Strip Mall, More than 15 Feet.', + # ------------------------ + 'M.LRI': 'Masonry, Low-Rise Industrial/Warehouse/Factory Buildings.', + # ------------------------ + 'M.ERB.L': ( + 'Masonry, Engineered Residential Building, Low-Rise (1-2 Stories).' + ), + 'M.ERB.M': ( + 'Masonry, Engineered Residential Building, Mid-Rise (3-5 Stories).' + ), + 'M.ERB.H': ( + 'Masonry, Engineered Residential Building, High-Rise (6+ Stories).' + ), + # ------------------------ + 'M.ECB.L': ( + 'Masonry, Engineered Commercial Building, Low-Rise (1-2 Stories).' + ), + 'M.ECB.M': ( + 'Masonry, Engineered Commercial Building, Mid-Rise (3-5 Stories).' + ), + 'M.ECB.H': ( + 'Masonry, Engineered Commercial Building, High-Rise (6+ Stories).' + ), + # ------------------------ + 'C.ERB.L': ( + 'Concrete, Engineered Residential Building, Low-Rise (1-2 Stories).' + ), + 'C.ERB.M': ( + 'Concrete, Engineered Residential Building, Mid-Rise (3-5 Stories).' + ), + 'C.ERB.H': ( + 'Concrete, Engineered Residential Building, High-Rise (6+ Stories).' + ), + # ------------------------ + 'C.ECB.L': ( + 'Concrete, Engineered Commercial Building, Low-Rise (1-2 Stories).' + ), + 'C.ECB.M': ( + 'Concrete, Engineered Commercial Building, Mid-Rise (3-5 Stories).' + ), + 'C.ECB.H': ( + 'Concrete, Engineered Commercial Building, High-Rise (6+ Stories).' + ), + # ------------------------ + 'S.PMB.S': 'Steel, Pre-Engineered Metal Building, Small.', + 'S.PMB.M': 'Steel, Pre-Engineered Metal Building, Medium.', + 'S.PMB.L': 'Steel, Pre-Engineered Metal Building, Large.', + # ------------------------ + 'S.ERB.L': 'Steel, Engineered Residential Building, Low-Rise (1-2 Stories).', + 'S.ERB.M': 'Steel, Engineered Residential Building, Mid-Rise (3-5 Stories).', + 'S.ERB.H': 'Steel, Engineered Residential Building, High-Rise (6+ Stories).', + # ------------------------ + 'S.ECB.L': 'Steel, Engineered Commercial Building, Low-Rise (1-2 Stories).', + 'S.ECB.M': 'Steel, Engineered Commercial Building, Mid-Rise (3-5 Stories).', + 'S.ECB.H': 'Steel, Engineered Commercial Building, High-Rise (6+ Stories).', + # ------------------------ + 'MH.PHUD': 'Manufactured Home, Pre-Housing and Urban Development (HUD).', + 'MH.76HUD': 'Manufactured Home, 1976 HUD.', + 'MH.94HUDI': 'Manufactured Home, 1994 HUD - Wind Zone I.', + 'MH.94HUDII': 'Manufactured Home, 1994 HUD - Wind Zone II.', + 'MH.94HUDIII': 'Manufactured Home, 1994 HUD - Wind Zone III.', + # ------------------------ + 'HUEF.H.S': 'Small Hospital, Hospital with fewer than 50 Beds.', + 'HUEF.H.M': 'Medium Hospital, Hospital with beds between 50 & 150.', + 'HUEF.H.L': 'Large Hospital, Hospital with more than 150 Beds.', + # ------------------------ + 'HUEF.S.S': 'Elementary School.', + 'HUEF.S.M': 'High school, two-story.', + 'HUEF.S.L': 'Large high school, three-story.', + # ------------------------ + 'HUEF.EO': 'Emergency Operation Centers.', + 'HUEF.FS': 'Fire Station.', + 'HUEF.PS': 'Police Station.', + # ------------------------ + } + + def find_class_type(entry: str) -> str | None: + """ + Find the class type code from an entry string based on + predefined patterns. + + Parameters + ---------- + entry : str + A string representing the entry, consisting of delimited + segments that correspond to various attributes of an + asset. + + Returns + ------- + str or None + The class type code if a matching pattern is found; + otherwise, None if no pattern matches the input string. + + """ + entry_elements = entry.split('.') + for nper in range(1, len(entry_elements)): + first_parts = '.'.join(entry_elements[:nper]) + if first_parts in class_types: + return first_parts + return None + + # + # (3) General asset class + # + + # maps class code type to general class code + general_classes = { + # ------------------------ + 'W.SF.1': 'WSF', + 'W.SF.2': 'WSF', + # ------------------------ + 'W.MUH.1': 'WMUH', + 'W.MUH.2': 'WMUH', + 'W.MUH.3': 'WMUH', + # ------------------------ + 'M.SF.1': 'MSF', + 'M.SF.2': 'MSF', + # ------------------------ + 'M.MUH.1': 'MMUH', + 'M.MUH.2': 'MMUH', + 'M.MUH.3': 'MMUH', + # ------------------------ + 'M.LRM.1': 'MLRM1', + 'M.LRM.2': 'MLRM2', + # ------------------------ + 'M.LRI': 'MLRI', + # ------------------------ + 'M.ERB.L': 'MERB', + 'M.ERB.M': 'MERB', + 'M.ERB.H': 'MERB', + # ------------------------ + 'M.ECB.L': 'MECB', + 'M.ECB.M': 'MECB', + 'M.ECB.H': 'MECB', + # ------------------------ + 'C.ERB.L': 'CERB', + 'C.ERB.M': 'CERB', + 'C.ERB.H': 'CERB', + # ------------------------ + 'C.ECB.L': 'CECB', + 'C.ECB.M': 'CECB', + 'C.ECB.H': 'CECB', + # ------------------------ + 'S.PMB.S': 'SPMB', + 'S.PMB.M': 'SPMB', + 'S.PMB.L': 'SPMB', + # ------------------------ + 'S.ERB.L': 'SERB', + 'S.ERB.M': 'SERB', + 'S.ERB.H': 'SERB', + # ------------------------ + 'S.ECB.L': 'SECB', + 'S.ECB.M': 'SECB', + 'S.ECB.H': 'SECB', + # ------------------------ + 'MH.PHUD': 'MH', + 'MH.76HUD': 'MH', + 'MH.94HUDI': 'MH', + 'MH.94HUDII': 'MH', + 'MH.94HUDIII': 'MH', + # ------------------------ + 'HUEF.H.S': 'HUEFH', + 'HUEF.H.M': 'HUEFH', + 'HUEF.H.L': 'HUEFH', + # ------------------------ + 'HUEF.S.S': 'HUEFS', + 'HUEF.S.M': 'HUEFS', + 'HUEF.S.L': 'HUEFS', + # ------------------------ + 'HUEF.EO': 'HUEFEO', + 'HUEF.FS': 'HUEFFS', + 'HUEF.PS': 'HUEFPS', + # ------------------------ + } -class_types = { - 'W.SF.1': 'Wood, Single-family, One-story', - 'W.SF.2': 'Wood, Single-family, Two or More Stories', - 'W.MUH.1': 'Wood, Multi-Unit Housing, One-story', - 'W.MUH.2': 'Wood, Multi-Unit Housing, Two Stories', - 'W.MUH.3': 'Wood, Multi-Unit Housing, Three or More Stories', - 'M.SF.1': 'Masonry, Single-family, One-story', - 'M.SF.2': 'Masonry, Single-family, Two or More Stories', - 'M.MUH.1': 'Masonry, Multi-Unit Housing, One-story', - 'M.MUH.2': 'Masonry, Multi-Unit Housing, Two Stories', - 'M.MUH.3': 'Masonry, Multi-Unit Housing, Three or More Stories', - 'M.LRM.1': 'Masonry, Low-Rise Strip Mall, Up to 15 Feet', - 'M.LRM.2': 'Masonry, Low-Rise Strip Mall, More than 15 Feet', - 'M.LRI': 'Masonry, Low-Rise Industrial/Warehouse/Factory Buildings', - 'M.ERB.L': 'Masonry, Engineered Residential Building, Low-Rise (1-2 Stories)', - 'M.ERB.M': 'Masonry, Engineered Residential Building, Mid-Rise (3-5 Stories)', - 'M.ERB.H': 'Masonry, Engineered Residential Building, High-Rise (6+ Stories)', - 'M.ECB.L': 'Masonry, Engineered Commercial Building, Low-Rise (1-2 Stories)', - 'M.ECB.M': 'Masonry, Engineered Commercial Building, Mid-Rise (3-5 Stories)', - 'M.ECB.H': 'Masonry, Engineered Commercial Building, High-Rise (6+ Stories)', - 'C.ERB.L': 'Concrete, Engineered Residential Building, Low-Rise (1-2 Stories)', - 'C.ERB.M': 'Concrete, Engineered Residential Building, Mid-Rise (3-5 Stories)', - 'C.ERB.H': 'Concrete, Engineered Residential Building, High-Rise (6+ Stories)', - 'C.ECB.L': 'Concrete, Engineered Commercial Building, Low-Rise (1-2 Stories)', - 'C.ECB.M': 'Concrete, Engineered Commercial Building, Mid-Rise (3-5 Stories)', - 'C.ECB.H': 'Concrete, Engineered Commercial Building, High-Rise (6+ Stories)', - 'S.PMB.S': 'Steel, Pre-Engineered Metal Building, Small', - 'S.PMB.M': 'Steel, Pre-Engineered Metal Building, Medium', - 'S.PMB.L': 'Steel, Pre-Engineered Metal Building, Large', - 'S.ERB.L': 'Steel, Engineered Residential Building, Low-Rise (1-2 Stories)', - 'S.ERB.M': 'Steel, Engineered Residential Building, Mid-Rise (3-5 Stories)', - 'S.ERB.H': 'Steel, Engineered Residential Building, High-Rise (6+ Stories)', - 'S.ECB.L': 'Steel, Engineered Commercial Building, Low-Rise (1-2 Stories)', - 'S.ECB.M': 'Steel, Engineered Commercial Building, Mid-Rise (3-5 Stories)', - 'S.ECB.H': 'Steel, Engineered Commercial Building, High-Rise (6+ Stories)', - 'MH.PHUD': 'Manufactured Home, Pre-Housing and Urban Development (HUD)', - 'MH.76HUD': 'Manufactured Home, 1976 HUD', - 'MH.94HUDI': 'Manufactured Home, 1994 HUD - Wind Zone I', - 'MH.94HUDII': 'Manufactured Home, 1994 HUD - Wind Zone II', - 'MH.94HUDIII': 'Manufactured Home, 1994 HUD - Wind Zone III', - 'HUEF.FS': 'Fire Station', - 'HUEF.H.S': 'Small Hospital, Hospital with fewer than 50 Beds', - 'HUEF.H.M': 'Medium Hospital, Hospital with beds between 50 & 150', - 'HUEF.H.L': 'Large Hospital, Hospital with more than 150 Beds', - 'HUEF.S.L': '???', - 'HUEF.S.S': '???', - 'HUEF.S.M': '???', - 'HUEF.EO': 'Emergency Operation Centers', - 'HUEF.PS': 'Police Station', -} - -my_dictionaries = { - 'roof_shape': roof_shape, - 'secondary_water_resistance': secondary_water_resistance, - 'roof_deck_attachment': roof_deck_attachment, - 'roof_wall_connection': roof_wall_connection, - 'garage_presence': garage_presence, - 'shutters': shutters, - 'roof_cover': roof_cover, - 'roof_quality': roof_quality, - 'masonry_reinforcing': masonry_reinforcing, - 'roof_frame_type': roof_frame_type, - 'wind_debris_environment': wind_debris_environment, - 'roof_deck_age': roof_deck_age, - 'roof_metal_deck_attachment_quality': roof_metal_deck_attachment_quality, - 'number_of_units': number_of_units, - 'joist_spacing': joist_spacing, - 'window_area': window_area, -} - - -def find_class_type(entry): - entry_elements = entry.split('.') - for nper in range(1, len(entry_elements)): - first_parts = '.'.join(entry_elements[:nper]) - if first_parts in class_types: - return first_parts - return None - - -all_entry_attr_dcts = {} -for entry in fragility_data['ID']: - - class_type = find_class_type(entry) - assert class_type is not None - - attrs = entry.replace(f'{class_type}.', '').split('.') - all_attr_dcts = {} - for attr in attrs: - attr_dcts = [] - for dct_name, dct in my_dictionaries.items(): - if attr in dct: - attr_dcts.append(dct_name) - all_attr_dcts[attr] = attr_dcts - all_entry_attr_dcts[entry] = all_attr_dcts - -ddf = pd.DataFrame(all_entry_attr_dcts.values(), index=all_entry_attr_dcts.keys()) - -for col in ddf.columns: - print(col) - print(ddf[col].value_counts()) - print() - print() + # + # (4) Relevant dictionaries + # + + # maps general class code to list of dicts where the -h attribute + # descriptions will be pulled from + dictionaries_of_interest = { + 'WSF': [ + roof_shape, + secondary_water_resistance, + roof_deck_attachment, + roof_wall_connection, + garage_presence, + shutters, + terrain_surface_roughness, + ], + 'WMUH': [ + roof_shape, + roof_cover, + roof_quality, + secondary_water_resistance, + roof_deck_attachment, + roof_wall_connection, + shutters, + terrain_surface_roughness, + ], + 'MSF': [ + roof_shape, + roof_wall_connection, + roof_frame_type, + roof_deck_attachment, + shutters, + secondary_water_resistance, + garage_presence, + masonry_reinforcing, + roof_cover, + terrain_surface_roughness, + ], + 'MMUH': [ + roof_shape, + secondary_water_resistance, + roof_cover, + roof_quality, + roof_deck_attachment, + roof_wall_connection, + shutters, + masonry_reinforcing, + terrain_surface_roughness, + ], + 'MLRM1': [ + roof_cover, + shutters, + masonry_reinforcing, + wind_debris_environment, + roof_frame_type, + roof_deck_attachment, + roof_wall_connection, + roof_deck_age, + roof_metal_deck_attachment_quality, + terrain_surface_roughness, + ], + 'MLRM2': [ + roof_cover, + shutters, + masonry_reinforcing, + wind_debris_environment, + roof_frame_type, + roof_deck_attachment, + roof_wall_connection, + roof_deck_age, + roof_metal_deck_attachment_quality, + number_of_units, + joist_spacing, + terrain_surface_roughness, + ], + 'MLRI': [ + shutters, + masonry_reinforcing, + roof_deck_age, + roof_metal_deck_attachment_quality, + terrain_surface_roughness, + ], + 'MERB': [ + roof_cover, + shutters, + wind_debris_environment, + roof_metal_deck_attachment_quality, + window_area, + terrain_surface_roughness, + ], + 'MECB': [ + roof_cover, + shutters, + wind_debris_environment, + roof_metal_deck_attachment_quality, + window_area, + terrain_surface_roughness, + ], + 'CERB': [ + roof_cover, + shutters, + wind_debris_environment, + window_area, + terrain_surface_roughness, + ], + 'CECB': [ + roof_cover, + shutters, + wind_debris_environment, + window_area, + terrain_surface_roughness, + ], + 'SPMB': [ + shutters, + roof_deck_age, + roof_metal_deck_attachment_quality, + terrain_surface_roughness, + ], + 'SERB': [ + roof_cover, + shutters, + wind_debris_environment, + roof_metal_deck_attachment_quality, + window_area, + terrain_surface_roughness, + ], + 'SECB': [ + roof_cover, + shutters, + wind_debris_environment, + roof_metal_deck_attachment_quality, + window_area, + terrain_surface_roughness, + ], + 'MH': [shutters, tie_downs, terrain_surface_roughness], + 'HUEFH': [ + roof_cover, + wind_debris_environment, + roof_metal_deck_attachment_quality, + shutters, + terrain_surface_roughness, + ], + 'HUEFS': [ + roof_cover, + shutters, + wind_debris_environment, + roof_deck_age, + roof_metal_deck_attachment_quality, + terrain_surface_roughness, + ], + 'HUEFEO': [ + roof_cover, + shutters, + wind_debris_environment, + roof_metal_deck_attachment_quality, + window_area, + terrain_surface_roughness, + ], + 'HUEFFS': [ + roof_cover, + shutters, + wind_debris_environment, + roof_deck_age, + roof_metal_deck_attachment_quality, + terrain_surface_roughness, + ], + 'HUEFPS': [ + roof_cover, + shutters, + wind_debris_environment, + roof_metal_deck_attachment_quality, + window_area, + terrain_surface_roughness, + ], + } + + # + # (5) Decode IDs and extend metadata with the individual records + # + + for fragility_id in fragility_data['ID'].to_list(): + + class_type = find_class_type(fragility_id) + + class_type_human_readable = class_types[class_type] + + general_class = general_classes[class_type] + dictionaries = dictionaries_of_interest[general_class] + remaining_chunks = fragility_id.replace(f'{class_type}.', '').split('.') + assert len(remaining_chunks) == len(dictionaries) + human_description = [class_type_human_readable] + for chunk, dictionary in zip(remaining_chunks, dictionaries): + human_description.append(dictionary[chunk]) + human_description_str = ' '.join(human_description) + + damage_state_class = damage_state_classes[class_type] + damage_state_description = damage_state_descriptions[damage_state_class] + + limit_states = {} + for damage_state, description in damage_state_description.items(): + limit_state = damage_state.replace('DS', 'LS') + limit_states[limit_state] = {damage_state: description} + + record = { + 'Description': human_description_str, + 'SuggestedComponentBlockSize': '1 EA', + 'RoundUpToIntegerQuantity': 'True', + 'LimitStates': limit_states, + } + + meta_dict[fragility_id] = record + + # save the metadata + with open(target_meta_file, 'w+', encoding='utf-8') as f: + json.dump(meta_dict, f, indent=2) diff --git a/pelicun/resources/SimCenterDBDL/damage_DB_SimCenter_Hazus_HU_bldg_template.json b/pelicun/resources/SimCenterDBDL/damage_DB_SimCenter_Hazus_HU_bldg_template.json new file mode 100644 index 000000000..34c871bd0 --- /dev/null +++ b/pelicun/resources/SimCenterDBDL/damage_DB_SimCenter_Hazus_HU_bldg_template.json @@ -0,0 +1,198 @@ +{ + "_GeneralInformation": { + "ShortName": "Hazus Hurricane Methodology - Buildings", + "Description": "The models in this dataset are based on version 4.2 of the Hazus Hurricane Model Technical Manual", + "Version": "1.0", + "ComponentGroups": { + "W - Wood": { + "W.SF - Wood, Single-family": [ + "W.SF.1 - Wood, Single-family, One-story.", + "W.SF.2 - Wood, Single-family, Two or More Stories." + ], + "W.MUH - Wood, Multi-Unit Housing": [ + "W.MUH.1 - Wood, Multi-Unit Housing, One-story.", + "W.MUH.2 - Wood, Multi-Unit Housing, Two Stories.", + "W.MUH.3 - Wood, Multi-Unit Housing, Three or More Stories." + ] + }, + "M - Masonry": { + "M.SF - Masonry, Single-family": [ + "M.SF.1 - Masonry, Single-family, One-story.", + "M.SF.2 - Masonry, Single-family, Two or More Stories." + ], + "M.MUH - Masonry, Multi-Unit Housing": [ + "M.MUH.1 - Masonry, Multi-Unit Housing, One-story.", + "M.MUH.2 - Masonry, Multi-Unit Housing, Two Stories.", + "M.MUH.3 - Masonry, Multi-Unit Housing, Three or More Stories." + ], + "M.LRM - Masonry, Low-Rise Strip Mall": [ + "M.LRM.1 - Masonry, Low-Rise Strip Mall, Up to 15 Feet.", + "M.LRM.2 - Masonry, Low-Rise Strip Mall, More than 15 Feet." + ], + "M.LRI - Masonry, Low-Rise Industrial/Warehouse/Factory Buildings": [ + "M.LRI - Masonry, Low-Rise Industrial/Warehouse/Factory Buildings." + ], + "M.ERB - Masonry, Engineered Residential Building": [ + "M.ERB.L - Masonry, Engineered Residential Building, Low-Rise (1-2 Stories).", + "M.ERB.M - Masonry, Engineered Residential Building, Mid-Rise (3-5 Stories).", + "M.ERB.H - Masonry, Engineered Residential Building, High-Rise (6+ Stories)." + ], + "M.ECB - Masonry, Engineered Commercial Building": [ + "M.ECB.L - Masonry, Engineered Commercial Building, Low-Rise (1-2 Stories).", + "M.ECB.M - Masonry, Engineered Commercial Building, Mid-Rise (3-5 Stories).", + "M.ECB.H - Masonry, Engineered Commercial Building, High-Rise (6+ Stories)." + ] + }, + "C - Concrete": { + "C.ERB - Concrete, Engineered Residential Building": [ + "C.ERB.L - Concrete, Engineered Residential Building, Low-Rise (1-2 Stories).", + "C.ERB.M - Concrete, Engineered Residential Building, Mid-Rise (3-5 Stories).", + "C.ERB.H - Concrete, Engineered Residential Building, High-Rise (6+ Stories)." + ], + "C.ECB - Concrete, Engineered Commercial Building": [ + "C.ECB.L - Concrete, Engineered Commercial Building, Low-Rise (1-2 Stories).", + "C.ECB.M - Concrete, Engineered Commercial Building, Mid-Rise (3-5 Stories).", + "C.ECB.H - Concrete, Engineered Commercial Building, High-Rise (6+ Stories)." + ] + }, + "S - Steel": { + "S.PMB - Steel, Pre-Engineered Metal Building": [ + "S.PMB.S - Steel, Pre-Engineered Metal Building, Small.", + "S.PMB.M - Steel, Pre-Engineered Metal Building, Medium.", + "S.PMB.L - Steel, Pre-Engineered Metal Building, Large." + ], + "S.ERB - Steel, Engineered Residential Building": [ + "S.ERB.L - Steel, Engineered Residential Building, Low-Rise (1-2 Stories).", + "S.ERB.M - Steel, Engineered Residential Building, Mid-Rise (3-5 Stories).", + "S.ERB.H - Steel, Engineered Residential Building, High-Rise (6+ Stories)." + ], + "S.ECB - Steel, Engineered Commercial Building": [ + "S.ECB.L - Steel, Engineered Commercial Building, Low-Rise (1-2 Stories).", + "S.ECB.M - Steel, Engineered Commercial Building, Mid-Rise (3-5 Stories).", + "S.ECB.H - Steel, Engineered Commercial Building, High-Rise (6+ Stories)." + ] + }, + "MH - Manufactured Home": [ + "MH.PHUD - Manufactured Home, Pre-Housing and Urban Development (HUD).", + "MH.76HUD - Manufactured Home, 1976 HUD.", + "MH.94HUDI - Manufactured Home, 1994 HUD - Wind Zone I.", + "MH.94HUDII - Manufactured Home, 1994 HUD - Wind Zone II.", + "MH.94HUDIII - Manufactured Home, 1994 HUD - Wind Zone III." + ], + "HUEF - Health, Utilities, Education, and Fire": { + "HUEF.H - Hospital": [ + "HUEF.H.S - Small Hospital, Hospital with fewer than 50 Beds.", + "HUEF.H.M - Medium Hospital, Hospital with beds between 50 & 150.", + "HUEF.H.L - Large Hospital, Hospital with more than 150 Beds." + ], + "HUEF.S - School": [ + "HUEF.S.S - Elementary School.", + "HUEF.S.M - High school, two-story.", + "HUEF.S.L - Large high school, three-story." + ], + "HUEF.EO - Emergency Operation Centers": [ + "HUEF.EO - Emergency Operation Centers." + ], + "HUEF.FS - Fire Station": [ + "HUEF.FS - Fire Station." + ], + "HUEF.PS - Police Station": [ + "HUEF.PS - Police Station." + ] + } + } + }, + "DamageStateClasses": { + "W.SF.1": "5-175", + "W.SF.2": "5-175", + "W.MUH.1": "5-175", + "W.MUH.2": "5-175", + "W.MUH.3": "5-175", + "M.SF.1": "5-175", + "M.SF.2": "5-175", + "M.MUH.1": "5-175", + "M.MUH.2": "5-175", + "M.MUH.3": "5-175", + "M.LRM.1": "5-208", + "M.LRM.2": "5-208", + "M.LRI": "5-239", + "M.ERB.L": "5-230", + "M.ERB.M": "5-230", + "M.ERB.H": "5-230", + "M.ECB.L": "5-230", + "M.ECB.M": "5-230", + "M.ECB.H": "5-230", + "C.ERB.L": "5-230", + "C.ERB.M": "5-230", + "C.ERB.H": "5-230", + "C.ECB.L": "5-230", + "C.ECB.M": "5-230", + "C.ECB.H": "5-230", + "S.PMB.S": "5-227", + "S.PMB.M": "5-227", + "S.PMB.L": "5-227", + "S.ERB.L": "5-230", + "S.ERB.M": "5-230", + "S.ERB.H": "5-230", + "S.ECB.L": "5-230", + "S.ECB.M": "5-230", + "S.ECB.H": "5-230", + "MH.PHUD": "5-195", + "MH.76HUD": "5-195", + "MH.94HUDI": "5-195", + "MH.94HUDII": "5-195", + "MH.94HUDIII": "5-195", + "HUEF.H.S": "5-208", + "HUEF.H.M": "5-208", + "HUEF.H.L": "5-208", + "HUEF.S.S": "5-208", + "HUEF.S.M": "5-208", + "HUEF.S.L": "5-208", + "HUEF.EO": "5-208", + "HUEF.FS": "5-208", + "HUEF.PS": "5-208" + }, + "DamageStateDescriptions": { + "5-175": { + "DS1": "Minor Damage: Maximum of one broken window, door or garage door. Moderate roof cover loss that can be covered to prevent additional water entering the building. Marks or dents on walls requiring painting or patching for repair.", + "DS2": "Moderate Damage: Major roof cover damage, moderate window breakage. Minor roof sheathing failure. Some resulting damage to interior of building from water", + "DS3": "Severe Damage: Major window damage or roof sheathing loss. Major roof cover loss. Extensive damage to interior from water.", + "DS4": "Destruction: Complete roof failure and/or, failure of wall frame. Loss of more than 50% of roof sheathing." + }, + "5-195": { + "DS0": "No Damage or Very Minor Damage - Little or no visible damage from the outside Slight shifting on the blocks that would suggest re-leveling, but not off the blocks Some cracked windows, but no resulting water damage.", + "DS1": "Minor Damage - Shifting off the blocks or so that blocks press up to the floor; re- leveling required. Walls, doors, etc., buckled slightly, but able to be corrected by re-leveling. Minor eave and upper wall damage, with slight water damage but roof not pulled all the way back. Minor pulling away of siding with slight water damage Minor missile and/or tree damage. Slight window breakage and attendant water damage.", + "DS2": "Moderate Damage (Still Livable) - Severe shifting off the blocks with some attendant floor and superstructure damage (punching, racking, etc). Roof removed over a portion or all of the home but the joists remain intact, walls not collapsed Missile and/or tree damage to a section of the wall or roof, including deep dents or punctures. Serious water damage from holes in roof, walls, windows, doors or floors.", + "DS3": "Severe Damage (Not Livable, but Repairable) - Unit rolled onto side but frame intact. Extreme shifting causing severe racking and separations in the superstructure. Roof off, joists damaged or removed, walls damaged from lack of lateral support at top. Severe tree damage, including crushing of one wall or roof section. Superstructure partially separated from under frame.", + "DS4": "Destruction (Not Livable) - Unit rolled onto top or rolled several times. Unit tossed or vaulted through the air. Superstructure separated from the underframe or collapsed to side of the underframe. Roof off, joists removed, and walls collapsed. Destruction of a major section by a falling tree." + }, + "5-208": { + "DS0": "No Damage or Very Minor Damage Little or no visible damage from the outside. No broken windows, or failed roof deck. Minimal loss of roof cover, with no or very limited water penetration.", + "DS1": "Minor Damage Maximum of one broken window or door. Moderate roof cover loss that can be covered to prevent additional water entering the building. Marks or dents on walls requiring painting or patching for repair.", + "DS2": "Moderate Damage Major roof cover damage, moderate window breakage. Minor roof sheathing failure. Some resulting damage to interior of building from water.", + "DS3": "Severe Damage Major window damage or roof sheathing loss. Major roof cover loss. Extensive damage to interior from water. Limited, local joist failures. Failure of one wall.", + "DS4": "Destruction Complete roof failure on 1/3 or more of the units and/or failure of more than one wall. Loss of more than 25% of roof sheathing." + }, + "5-227": { + "DS0": "No Damage or Very Minor Damage Little or no visible damage from the outside. No broken windows, or failed roof deck. None or very limited water penetration.", + "DS1": "Minor Damage Maximum of one broken window or, door or wall panel. Marks or dents on walls requiring painting or patching for repair.", + "DS2": "Moderate Damage Moderate fenestration failures. Minor roof panel failures, or wall panel failures. Some resulting damage to interior of building from water.", + "DS3": "Severe Damage Major window damage or roof sheathing loss. Extensive damage to the interior from water. Some frame damage likely.", + "DS4": "Destruction Significant failures of fenestrations, significant roof and wall panel failures. Significant frame damage likely." + }, + "5-230": { + "DS0": "No Damage or Very Minor Damage Little or no visible damage from the outside. No broken windows, or failed roof deck. Minimal loss of roof cover, with no or very limited water penetration.", + "DS1": "Minor Damage Maximum of one broken window or door. Moderate roof cover loss that can be covered to prevent additional water entering the building. Marks or dents on walls requiring painting or patching for repair.", + "DS2": "Moderate Damage Major roof cover damage, moderate window breakage. Minor roof deck failure. Some resulting damage to interior of building from water.", + "DS3": "Severe Damage Major window damage or roof sheathing loss. Major roof cover loss. Extensive damage to interior from water. Limited, local joist failures.", + "DS4": "Destruction Essentially complete roof failure and/or of more than 25% of roof sheathing. Significant amount of the wall envelope opened through windows failure. Extensive damage to interior." + }, + "5-239": { + "DS0": "No Damage or Very Minor Damage Little or no visible damage from the outside. No failed doors or roof deck. Minimal loss of roof cover, with no or very limited water penetration.", + "DS1": "Minor Damage Maximum of one failed door. Moderate roof cover loss that can be covered to prevent additional water entering the building. Marks or dents on walls requiring painting or patching for repair.", + "DS2": "Moderate Damage Major roof cover damage, moderate window breakage. Minor roof sheathing failure. Some resulting damage to interior of building from water.", + "DS3": "Severe Damage Major window damage or roof sheathing loss. Major roof cover loss. Extensive damage to interior from water. Limited, local joist failures. Failure of one wall.", + "DS4": "Destruction Complete roof failure on 1/3 or more of the units and/or failure of more than one wall. Loss of more than 25% of roof sheathing." + } + } +} From c785cb38e22ecdf508b39129e31654dc0838a2ef Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Sat, 13 Apr 2024 14:31:48 -0700 Subject: [PATCH 03/22] Remove forgotten temporary lines --- pelicun/db.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pelicun/db.py b/pelicun/db.py index aec483420..f9ec3cdf0 100644 --- a/pelicun/db.py +++ b/pelicun/db.py @@ -55,9 +55,6 @@ """ -# todo -# fmt: off - from __future__ import annotations import re import json @@ -2915,11 +2912,9 @@ def create_Hazus_EQ_bldg_injury_db( # with open(target_meta_file, 'w+') as f: # json.dump(meta_dict, f, indent=2) - print("Successfully parsed and saved the injury consequence data from Hazus " - "EQ") - -# todo -# fmt: on + print( + "Successfully parsed and saved the injury consequence data from Hazus " "EQ" + ) def create_Hazus_HU_fragility_db( From a31f32eca8a0a42f865006597cdbcb3a8073180d Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Sat, 13 Apr 2024 17:49:32 -0700 Subject: [PATCH 04/22] Preliminary refactoring - remove commented out code --- pelicun/model/loss_model.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index b6e5e94b9..db9b45cb4 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -348,14 +348,6 @@ def __init__(self, assessment): self.loss_type = 'Repair' - # def load_model(self, data_paths, mapping_path): - - # super().load_model(data_paths, mapping_path) - - # def calculate(self): - - # super().calculate() - def _create_DV_RVs(self, case_list): """ Prepare the random variables used for repair cost and time simulation. From a503900bd7fa0f10c64f66354504e42d84487dc7 Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Sat, 13 Apr 2024 19:09:29 -0700 Subject: [PATCH 05/22] Preliminary refactoring - remove _sample property It's simpler to just use the `sample` attribute directly here. --- pelicun/model/loss_model.py | 15 ++++----------- pelicun/tests/test_model.py | 10 +++++----- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index db9b45cb4..4c2240d31 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -78,18 +78,11 @@ class LossModel(PelicunModel): def __init__(self, assessment): super().__init__(assessment) - self._sample = None + self.sample = None self.loss_map = None self.loss_params = None self.loss_type = 'Generic' - @property - def sample(self): - """ - sample property - """ - return self._sample - def save_sample(self, filepath=None, save_units=False): """ Save loss sample to a csv file @@ -139,7 +132,7 @@ def load_sample(self, filepath): self.log_div() self.log_msg('Loading loss sample...') - self._sample = file_io.load_data( + self.sample = file_io.load_data( filepath, self._asmnt.unit_conversion_factors, log=self._asmnt.log ) @@ -867,7 +860,7 @@ def _generate_DV_sample(self, dmg_quantities, sample_size): # If everything is undamaged there are no losses if set(dmg_quantities.columns.get_level_values('ds')) == {'0'}: - self._sample = None + self.sample = None self.log_msg( "There is no damage---DV sample is set to None.", prepend_timestamp=False, @@ -1097,7 +1090,7 @@ def _generate_DV_sample(self, dmg_quantities, sample_size): DV_sample.loc[id_replacement, idx[:, :, :, :, locs]] = 0.0 - self._sample = DV_sample + self.sample = DV_sample self.log_msg("Successfully obtained DV sample.", prepend_timestamp=False) diff --git a/pelicun/tests/test_model.py b/pelicun/tests/test_model.py index 872632d85..8164c18f0 100644 --- a/pelicun/tests/test_model.py +++ b/pelicun/tests/test_model.py @@ -1714,7 +1714,7 @@ def test_init(self, loss_model): assert loss_model.log_msg assert loss_model.log_div - assert loss_model._sample is None + assert loss_model.sample is None assert loss_model.loss_type == 'Generic' def test_load_sample_save_sample(self, loss_model): @@ -1774,7 +1774,7 @@ def test_load_sample_save_sample(self, loss_model): pd.testing.assert_frame_equal( sample, - loss_model._sample, + loss_model.sample, check_index_type=False, check_column_type=False, ) @@ -1882,7 +1882,7 @@ def test_init(self, repair_model): assert repair_model.log_msg assert repair_model.log_div - assert repair_model._sample is None + assert repair_model.sample is None assert repair_model.loss_type == 'Repair' def test__create_DV_RVs(self, repair_model, loss_params_A): @@ -1950,7 +1950,7 @@ def test__calc_median_consequence(self, repair_model, loss_params_A): assert medians['Time'].to_dict() == {(0, '1'): {0: 22.68, 1: 20.16}} def test_aggregate_losses(self, repair_model, loss_params_A): - repair_model._sample = pd.DataFrame( + repair_model.sample = pd.DataFrame( ((100.00, 1.00),), columns=pd.MultiIndex.from_tuples( ( @@ -2138,7 +2138,7 @@ def test__generate_DV_sample(self, repair_model): repair_model._generate_DV_sample(dmg_quantities, 4) - assert repair_model._sample.to_dict() == expected_sample[(ecods, ecofl)] + assert repair_model.sample.to_dict() == expected_sample[(ecods, ecofl)] # _____ _ _ From 381a77efdec306dccf5eb67c7c4cae6e907581a2 Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Sun, 14 Apr 2024 02:57:40 -0700 Subject: [PATCH 06/22] Reorder methods --- pelicun/tests/test_model.py | 72 ++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/pelicun/tests/test_model.py b/pelicun/tests/test_model.py index 8164c18f0..8c6430664 100644 --- a/pelicun/tests/test_model.py +++ b/pelicun/tests/test_model.py @@ -1949,42 +1949,6 @@ def test__calc_median_consequence(self, repair_model, loss_params_A): assert medians['Cost'].to_dict() == {(0, '1'): {0: 25704.0, 1: 22848.0}} assert medians['Time'].to_dict() == {(0, '1'): {0: 22.68, 1: 20.16}} - def test_aggregate_losses(self, repair_model, loss_params_A): - repair_model.sample = pd.DataFrame( - ((100.00, 1.00),), - columns=pd.MultiIndex.from_tuples( - ( - ( - "Cost", - "some.test.component", - "some.test.component", - "1", - "1", - "1", - ), - ( - "Time", - "some.test.component", - "some.test.component", - "1", - "1", - "1", - ), - ), - names=("dv", "loss", "dmg", "ds", "loc", "dir"), - ), - ) - - repair_model.loss_params = loss_params_A - - df_agg = repair_model.aggregate_losses() - - assert df_agg.to_dict() == { - ('repair_cost', ''): {0: 100.0}, - ('repair_time', 'parallel'): {0: 1.0}, - ('repair_time', 'sequential'): {0: 1.0}, - } - def test__generate_DV_sample(self, repair_model): expected_sample = { (True, True): { @@ -2140,6 +2104,42 @@ def test__generate_DV_sample(self, repair_model): assert repair_model.sample.to_dict() == expected_sample[(ecods, ecofl)] + def test_aggregate_losses(self, repair_model, loss_params_A): + repair_model.sample = pd.DataFrame( + ((100.00, 1.00),), + columns=pd.MultiIndex.from_tuples( + ( + ( + "Cost", + "some.test.component", + "some.test.component", + "1", + "1", + "1", + ), + ( + "Time", + "some.test.component", + "some.test.component", + "1", + "1", + "1", + ), + ), + names=("dv", "loss", "dmg", "ds", "loc", "dir"), + ), + ) + + repair_model.loss_params = loss_params_A + + df_agg = repair_model.aggregate_losses() + + assert df_agg.to_dict() == { + ('repair_cost', ''): {0: 100.0}, + ('repair_time', 'parallel'): {0: 1.0}, + ('repair_time', 'sequential'): {0: 1.0}, + } + # _____ _ _ # | ___| _ _ __ ___| |_(_) ___ _ __ ___ From c286fcd1106a66e556a8aba5440d1895190cc494 Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Sun, 14 Apr 2024 06:14:50 -0700 Subject: [PATCH 07/22] Preliminary refactoring - sample_size & changes in DL_calculation.py In an effort to modularize the loss model, sample_size is turned into an argument that has to be passed in `calculate` instead of having to infer it from other assessment models. This small change evolved into a larger refactoring effort to make sampling_size more explicit across methods. - The config schema was modified to include sampling information as part of "Options". - `sampling_size` is included in all `calcualte` methods as an explicit argument and no longer has to be infered with `self._samnt.` - Test suite updated DL_calculation is modified to assign sample_size when calling `.calculate` methods. Also: - The `config` dictionary is used explictly instead of assigning sub-dictionaries into variables. This helps understand the full path to the retrieved item at any point in the code, instead of having to remember what key corresponds to what variable. - Improved some parts where there was strange syntax like `if out_config.get('Demand', None) is not None` -> `if 'Demand' is in out_config` --- pelicun/base.py | 2 +- pelicun/model/damage_model.py | 18 ++- pelicun/model/loss_model.py | 13 +- pelicun/settings/default_config.json | 66 +++++----- pelicun/tests/test_base.py | 5 + pelicun/tests/test_model.py | 8 +- pelicun/tools/DL_calculation.py | 178 ++++++++++++++------------- 7 files changed, 161 insertions(+), 129 deletions(-) diff --git a/pelicun/base.py b/pelicun/base.py index da87daf32..1a3fa4b6a 100644 --- a/pelicun/base.py +++ b/pelicun/base.py @@ -187,7 +187,7 @@ def __init__(self, user_config_options, assessment=None): user_config_options) self._seed = merged_config_options['Seed'] - self.sampling_method = merged_config_options['SamplingMethod'] + self.sampling_method = merged_config_options['Sampling']['SamplingMethod'] self.list_all_ds = merged_config_options['ListAllDamageStates'] self.units_file = merged_config_options['UnitsFile'] diff --git a/pelicun/model/damage_model.py b/pelicun/model/damage_model.py index e5986573d..1d4703e4e 100644 --- a/pelicun/model/damage_model.py +++ b/pelicun/model/damage_model.py @@ -1469,7 +1469,21 @@ def _complete_ds_cols(self, dmg_sample): return res def calculate( - self, dmg_process=None, block_batch_size=1000, scaling_specification=None + self, sample_size=None, dmg_process=None, block_batch_size=1000, scaling_specification=None + ): + """ + Wrapper around the new calculate method that requires sample size. + Exists for backwards compatibility + """ + if not sample_size: + # todo: Deprecation warning + sample_size = self._asmnt.demand.sample.shape[0] + self.calculate_internal(sample_size, dmg_process, block_batch_size, scaling_specification) + + + def calculate_internal( + self, sample_size, dmg_process=None, block_batch_size=1000, scaling_specification=None + ): """ Calculate the damage state of each component block in the asset. @@ -1479,8 +1493,6 @@ def calculate( self.log_div() self.log_msg('Calculating damages...') - sample_size = self._asmnt.demand.sample.shape[0] - # Break up damage calculation and perform it by performance group. # Compared to the simultaneous calculation of all PGs, this approach # reduces demands on memory and increases the load on CPU. This leads diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index 4c2240d31..4ee473ad2 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -296,7 +296,18 @@ def _generate_DV_sample(self, dmg_quantities, sample_size): """ raise NotImplementedError - def calculate(self): + def calculate(self, sample_size=None): + """ + Wrapper method around new calculate that requires sample size. + Exists for backwards compatibility. + """ + if not sample_size: + # todo: deprecation warning + sample_size = self._asmnt.demand.sample.shape[0] + self.calculate_internal(sample_size) + + + def calculate_internal(self, sample_size): """ Calculate the consequences of each component block damage in the asset. diff --git a/pelicun/settings/default_config.json b/pelicun/settings/default_config.json index 6ae92499b..47fe005aa 100644 --- a/pelicun/settings/default_config.json +++ b/pelicun/settings/default_config.json @@ -1,38 +1,38 @@ { "Options": { - "Verbose": false, - "Seed": null, - "LogShowMS": false, - "LogFile": null, - "UnitsFile": null, - "PrintLog": false, - "ShowWarnings": false, - "DemandOffset": { - "PFA": -1, - "PFV": -1 - }, - "SamplingMethod": "LHS", - "ListAllDamageStates": false, - "NonDirectionalMultipliers": { - "ALL": 1.2 - }, - "EconomiesOfScale": { - "AcrossFloors": true, - "AcrossDamageStates": false - }, - "RepairCostAndTimeCorrelation": 0.0 + "Verbose": false, + "Seed": null, + "LogShowMS": false, + "LogFile": null, + "UnitsFile": null, + "PrintLog": false, + "ShowWarnings": false, + "DemandOffset": { + "PFA": -1, + "PFV": -1 }, - "DemandAssessment": { - "Calibration": { - "Marginals": { - "ALL": { - "DistributionFamily": "lognormal" - } - } - }, - "Sampling": { - "SampleSize": 1000, - "PreserveRawOrder": false - } + "ListAllDamageStates": false, + "NonDirectionalMultipliers": { + "ALL": 1.2 + }, + "EconomiesOfScale": { + "AcrossFloors": true, + "AcrossDamageStates": false + }, + "Sampling": { + "SamplingMethod": "LHS", + "SampleSize": 1000, + "PreserveRawOrder": false + }, + "RepairCostAndTimeCorrelation": 0.0 + }, + "DemandAssessment": { + "Calibration": { + "Marginals": { + "ALL": { + "DistributionFamily": "lognormal" + } + } + } } } diff --git a/pelicun/tests/test_base.py b/pelicun/tests/test_base.py index 3ae89ce31..6312a92d7 100644 --- a/pelicun/tests/test_base.py +++ b/pelicun/tests/test_base.py @@ -66,6 +66,11 @@ def test_options_init(): "LogFile": 'test_log_file', "PrintLog": False, "DemandOffset": {"PFA": -1, "PFV": -1}, + "Sampling": { + "SamplingMethod": "MonteCarlo", + "SampleSize": 1000, + "PreserveRawOrder": False, + }, "SamplingMethod": "MonteCarlo", "NonDirectionalMultipliers": {"ALL": 1.2}, "EconomiesOfScale": {"AcrossFloors": True, "AcrossDamageStates": True}, diff --git a/pelicun/tests/test_model.py b/pelicun/tests/test_model.py index 8c6430664..11e1e8cfe 100644 --- a/pelicun/tests/test_model.py +++ b/pelicun/tests/test_model.py @@ -1033,7 +1033,7 @@ def damage_model_with_sample(self, assessment_instance): ) ), ) - assessment_instance.damage.calculate(dmg_process=dmg_process) + assessment_instance.damage.calculate(sample_size=4, dmg_process=dmg_process) assessment_instance.asset.cmp_units = pd.Series( ['ea'] * len(assessment_instance.damage.sample.columns), index=assessment_instance.damage.sample.columns, @@ -1666,12 +1666,12 @@ def test_calculate_multilinear_CDF(self, damage_model): # A damage calculation test utilizing a multilinear CDF RV for # the capcity. - num_realizations = 1000 + sample_size = 1000 # define the demand conversion_factor = assessment_instance.unit_conversion_factors['inps2'] demand_model.sample = pd.DataFrame( - np.full(num_realizations, 0.50 * conversion_factor), + np.full(sample_size, 0.50 * conversion_factor), columns=(('PGV', '0', '1'),), ) @@ -1699,7 +1699,7 @@ def test_calculate_multilinear_CDF(self, damage_model): ) # calculate damage - damage_model.calculate() + damage_model.calculate(sample_size) res = damage_model.sample.value_counts() assert res.to_dict() == {(1.0, 0.0): 750, (0.0, 1.0): 250} diff --git a/pelicun/tools/DL_calculation.py b/pelicun/tools/DL_calculation.py index ef4aafe41..4a66f453c 100644 --- a/pelicun/tools/DL_calculation.py +++ b/pelicun/tools/DL_calculation.py @@ -60,6 +60,7 @@ # this is exceptional code +# (so let's run pylint everywhere /except/ here.) # pylint: disable=consider-using-namedtuple-or-dataclass # pylint: disable=too-many-locals # pylint: disable=too-many-statements @@ -369,8 +370,7 @@ def run_pelicun( # f"{config['commonFileDir']}/CustomDLModels/" custom_dl_file_path = custom_model_dir - DL_config = config.get('DL', None) - if not DL_config: + if 'DL' not in config: log_msg("Damage and Loss configuration missing from config file. ") if auto_script_path is not None: @@ -390,7 +390,7 @@ def run_pelicun( # add the demand information config_ap['DL']['Demands'].update( - {'DemandFilePath': f'{demand_file}', 'SampleSize': f'{realizations}'} + {'DemandFilePath': f'{demand_file}'} ) if coupled_EDP is True: @@ -428,32 +428,42 @@ def run_pelicun( with open(config_ap_path, 'w') as f: json.dump(config_ap, f, indent=2) - DL_config = config_ap.get('DL', None) + config['DL'] = config_ap.get('DL', None) else: log_msg("Terminating analysis.") return -1 - GI_config = config.get('GeneralInformation', None) - - asset_config = DL_config.get('Asset', None) - demand_config = DL_config.get('Demands', None) - damage_config = DL_config.get('Damage', None) - loss_config = DL_config.get('Losses', None) - out_config = DL_config.get('Outputs', None) + # + # sample size: backwards compatibility + # + sample_size_str = ( + # expected location + config.get('Options', {}).get('Sampling', {}).get('SampleSize', None) + ) + if not sample_size_str: + # try previous location + sample_size_str = ( + config.get('DL', {}).get('Demands', {}).get('SampleSize', None) + ) + if not sample_size_str: + # give up + print('Sampling size not provided in config file.') + return -1 + sample_size = int(sample_size_str) # provide all outputs if the files are not specified - if out_config is None: - out_config = full_out_config + if 'Outputs' not in config: + config['DL']['Outputs'] = full_out_config # provide outputs in CSV by default - if ('Format' in out_config.keys()) is False: - out_config.update({'Format': {'CSV': True, 'JSON': False}}) + if ('Format' in config['DL']['Outputs'].keys()) is False: + config['DL']['Outputs'].update({'Format': {'CSV': True, 'JSON': False}}) # override file format specification if the output_format is provided if output_format is not None: - out_config.update( + config['DL']['Outputs'].update( { 'Format': { 'CSV': 'csv' in output_format, @@ -463,20 +473,20 @@ def run_pelicun( ) # add empty Settings to output config to simplify code below - if ('Settings' in out_config.keys()) is False: - out_config.update({'Settings': pbe_settings}) + if ('Settings' in config['DL']['Outputs'].keys()) is False: + config['DL']['Outputs'].update({'Settings': pbe_settings}) - if asset_config is None: + if 'Asset' not in config['DL']: log_msg("Asset configuration missing. Terminating analysis.") return -1 - if demand_config is None: + if 'Demands' not in config['DL']: log_msg("Demand configuration missing. Terminating analysis.") return -1 # get the length unit from the config file try: - length_unit = GI_config['units']['length'] + length_unit = config['GeneralInformation']['units']['length'] except KeyError: log_msg( "No default length unit provided in the input file. " @@ -485,12 +495,8 @@ def run_pelicun( return -1 - # if out_config is None: - # log_msg("Output configuration missing. Terminating analysis.") - # return -1 - # initialize the Pelicun Assessement - options = DL_config.get("Options", {}) + options = config['DL'].get("Options", {}) options.update({"LogFile": "pelicun_log.txt", "Verbose": True}) # If the user did not prescribe anything for ListAllDamageStates, @@ -504,8 +510,8 @@ def run_pelicun( # Demand Assessment ----------------------------------------------------------- # check if there is a demand file location specified in the config file - if demand_config.get('DemandFilePath', False): - demand_path = Path(demand_config['DemandFilePath']).resolve() + if config['DL']['Demands'].get('DemandFilePath', False): + demand_path = Path(config['DL']['Demands']['DemandFilePath']).resolve() else: # otherwise assume that there is a response.csv file next to the config file @@ -515,7 +521,7 @@ def run_pelicun( raw_demands = pd.read_csv(demand_path, index_col=0) # remove excessive demands that are considered collapses, if needed - if demand_config.get('CollapseLimits', False): + if config['DL']['Demands'].get('CollapseLimits', False): raw_demands = convert_to_MultiIndex(raw_demands, axis=1) if 'Units' in raw_demands.index: @@ -527,7 +533,7 @@ def run_pelicun( DEM_to_drop = np.full(raw_demands.shape[0], False) - for DEM_type, limit in demand_config['CollapseLimits'].items(): + for DEM_type, limit in config['DL']['Demands']['CollapseLimits'].items(): if raw_demands.columns.nlevels == 4: DEM_to_drop += raw_demands.loc[:, idx[:, DEM_type, :, :]].max( axis=1 @@ -560,9 +566,9 @@ def run_pelicun( PAL.demand.load_sample(demands) # get the calibration information - if demand_config.get('Calibration', False): + if config['DL']['Demands'].get('Calibration', False): # then use it to calibrate the demand model - PAL.demand.calibrate_model(demand_config['Calibration']) + PAL.demand.calibrate_model(config['DL']['Demands']['Calibration']) else: # if no calibration is requested, @@ -570,13 +576,11 @@ def run_pelicun( PAL.demand.calibrate_model({"ALL": {"DistributionFamily": "empirical"}}) # and generate a new demand sample - sample_size = int(demand_config['SampleSize']) - PAL.demand.generate_sample( { "SampleSize": sample_size, - 'PreserveRawOrder': demand_config.get('CoupledDemands', False), - 'DemandCloning': demand_config.get('DemandCloning', False) + 'PreserveRawOrder': config['DL']['Demands'].get('CoupledDemands', False), + 'DemandCloning': config['DL']['Demands'].get('DemandCloning', False) } ) @@ -586,8 +590,8 @@ def run_pelicun( demand_sample = pd.concat([demand_sample, demand_units.to_frame().T]) # get residual drift estimates, if needed - if demand_config.get('InferResidualDrift', False): - RID_config = demand_config['InferResidualDrift'] + if config['DL']['Demands'].get('InferResidualDrift', False): + RID_config = config['DL']['Demands']['InferResidualDrift'] if RID_config['method'] == 'FEMA P-58': RID_list = [] @@ -626,8 +630,8 @@ def run_pelicun( PAL.demand.load_sample(convert_to_SimpleIndex(demand_sample, axis=1)) # save results - if out_config.get('Demand', None) is not None: - out_reqs = [out if val else "" for out, val in out_config['Demand'].items()] + if 'Demand' in config['DL']['Outputs']: + out_reqs = [out if val else "" for out, val in config['DL']['Outputs']['Demand'].items()] if np.any(np.isin(['Sample', 'Statistics'], out_reqs)): demand_sample, demand_units = PAL.demand.save_sample(save_units=True) @@ -685,13 +689,13 @@ def run_pelicun( # Asset Definition ------------------------------------------------------------ # set the number of stories - if asset_config.get('NumberOfStories', False): - PAL.stories = int(asset_config['NumberOfStories']) + if config['DL']['Asset'].get('NumberOfStories', False): + PAL.stories = int(config['DL']['Asset']['NumberOfStories']) # load a component model and generate a sample - if asset_config.get('ComponentAssignmentFile', False): + if config['DL']['Asset'].get('ComponentAssignmentFile', False): cmp_marginals = pd.read_csv( - asset_config['ComponentAssignmentFile'], + config['DL']['Asset']['ComponentAssignmentFile'], index_col=0, encoding_errors='replace', ) @@ -699,8 +703,8 @@ def run_pelicun( DEM_types = demand_sample.columns.unique(level=0) # add component(s) to support collapse calculation - if 'CollapseFragility' in damage_config.keys(): - coll_DEM = damage_config['CollapseFragility']["DemandType"] + if 'CollapseFragility' in config['DL']['Damage'].keys(): + coll_DEM = config['DL']['Damage']['CollapseFragility']["DemandType"] if coll_DEM.startswith('SA'): # we have a global demand and evaluate collapse directly pass @@ -737,7 +741,7 @@ def run_pelicun( cmp_marginals.loc['collapse', 'Theta_0'] = 1.0 # add components to support irreparable damage calculation - if 'IrreparableDamage' in damage_config.keys(): + if 'IrreparableDamage' in config['DL']['Damage'].keys(): if 'RID' in DEM_types: # excessive RID is added on every floor to detect large RIDs cmp_marginals.loc['excessiveRID', 'Units'] = 'ea' @@ -771,16 +775,16 @@ def run_pelicun( PAL.asset.generate_cmp_sample() # if requested, load the quantity sample from a file - elif asset_config.get('ComponentSampleFile', False): - PAL.asset.load_cmp_sample(asset_config['ComponentSampleFile']) + elif config['DL']['Asset'].get('ComponentSampleFile', False): + PAL.asset.load_cmp_sample(config['DL']['Asset']['ComponentSampleFile']) # if requested, save results - if out_config.get('Asset', None) is not None: + if 'Asset' in config['DL']['Outputs']: cmp_sample, cmp_units = PAL.asset.save_cmp_sample(save_units=True) cmp_units = cmp_units.to_frame().T if ( - out_config['Settings'].get('AggregateColocatedComponentResults', False) + config['DL']['Outputs']['Settings'].get('AggregateColocatedComponentResults', False) is True ): cmp_units = cmp_units.groupby(level=[0, 1, 2], axis=1).first() @@ -791,7 +795,7 @@ def run_pelicun( cmp_groupby_uid.count() == 0, np.nan ) - out_reqs = [out if val else "" for out, val in out_config['Asset'].items()] + out_reqs = [out if val else "" for out, val in config['DL']['Outputs']['Asset'].items()] if np.any(np.isin(['Sample', 'Statistics'], out_reqs)): if 'Sample' in out_reqs: @@ -851,18 +855,18 @@ def run_pelicun( # Damage Assessment ----------------------------------------------------------- # if a damage assessment is requested - if damage_config is not None: + if 'Damage' in config['DL']: # load the fragility information - if asset_config['ComponentDatabase'] in default_DBs['fragility'].keys(): + if config['DL']['Asset']['ComponentDatabase'] in default_DBs['fragility'].keys(): component_db = [ 'PelicunDefault/' - + default_DBs['fragility'][asset_config['ComponentDatabase']], + + default_DBs['fragility'][config['DL']['Asset']['ComponentDatabase']], ] else: component_db = [] - if asset_config.get('ComponentDatabasePath', False) is not False: - extra_comps = asset_config['ComponentDatabasePath'] + if config['DL']['Asset'].get('ComponentDatabasePath', False) is not False: + extra_comps = config['DL']['Asset']['ComponentDatabasePath'] extra_comps = extra_comps.replace( 'CustomDLDataFolder', custom_dl_file_path @@ -880,8 +884,8 @@ def run_pelicun( adf = pd.DataFrame(columns=P58_data.columns) - if 'CollapseFragility' in damage_config.keys(): - coll_config = damage_config['CollapseFragility'] + if 'CollapseFragility' in config['DL']['Damage'].keys(): + coll_config = config['DL']['Damage']['CollapseFragility'] if 'excessive.coll.DEM' in cmp_marginals.index: # if there is story-specific evaluation @@ -953,7 +957,7 @@ def run_pelicun( adf.loc['collapse', ('LS1', 'Theta_0')] = 1e10 adf.loc['collapse', 'Incomplete'] = 0 - elif "Water" not in asset_config['ComponentDatabase']: + elif "Water" not in config['DL']['Asset']['ComponentDatabase']: # add a placeholder collapse fragility that will never trigger # collapse, but allow damage processes to work with collapse @@ -964,8 +968,8 @@ def run_pelicun( adf.loc['collapse', ('LS1', 'Theta_0')] = 1e10 adf.loc['collapse', 'Incomplete'] = 0 - if 'IrreparableDamage' in damage_config.keys(): - irrep_config = damage_config['IrreparableDamage'] + if 'IrreparableDamage' in config['DL']['Damage'].keys(): + irrep_config = config['DL']['Damage']['IrreparableDamage'] # add excessive RID fragility according to settings provided in the # input file @@ -999,7 +1003,7 @@ def run_pelicun( # TODO: we can improve this by creating a water # network-specific assessment class - if "Water" in asset_config['ComponentDatabase']: + if "Water" in config['DL']['Asset']['ComponentDatabase']: # add a placeholder aggregate fragility that will never trigger # damage, but allow damage processes to aggregate the @@ -1015,8 +1019,8 @@ def run_pelicun( # load the damage process if needed dmg_process = None - if damage_config.get('DamageProcess', False) is not False: - dp_approach = damage_config['DamageProcess'] + if config['DL']['Damage'].get('DamageProcess', False) is not False: + dp_approach = config['DL']['Damage']['DamageProcess'] if dp_approach in damage_processes: dmg_process = damage_processes[dp_approach] @@ -1082,7 +1086,7 @@ def run_pelicun( elif dp_approach == "User Defined": # load the damage process from a file with open( - damage_config['DamageProcessFilePath'], 'r', encoding='utf-8' + config['DL']['Damage']['DamageProcessFilePath'], 'r', encoding='utf-8' ) as f: dmg_process = json.load(f) @@ -1096,15 +1100,15 @@ def run_pelicun( ) # calculate damages - PAL.damage.calculate(dmg_process=dmg_process) + PAL.damage.calculate(sample_size, dmg_process=dmg_process) # if requested, save results - if out_config.get('Damage', None) is not None: + if 'Damage' in config['DL']['Outputs']: damage_sample, damage_units = PAL.damage.save_sample(save_units=True) damage_units = damage_units.to_frame().T if ( - out_config['Settings'].get( + config['DL']['Outputs']['Settings'].get( 'AggregateColocatedComponentResults', False ) is True @@ -1122,7 +1126,7 @@ def run_pelicun( ) out_reqs = [ - out if val else "" for out, val in out_config['Damage'].items() + out if val else "" for out, val in config['DL']['Outputs']['Damage'].items() ] if np.any( @@ -1157,7 +1161,7 @@ def run_pelicun( if np.any(np.isin(['GroupedSample', 'GroupedStatistics'], out_reqs)): if ( - out_config['Settings'].get( + config['DL']['Outputs']['Settings'].get( 'AggregateColocatedComponentResults', False ) is True @@ -1184,7 +1188,7 @@ def run_pelicun( ) # if requested, condense DS output - if out_config['Settings'].get('CondenseDS', False) is True: + if config['DL']['Outputs']['Settings'].get('CondenseDS', False) is True: # replace non-zero values with 1 grp_damage = grp_damage.mask( grp_damage.astype(np.float64).values > 0, 1 @@ -1276,21 +1280,21 @@ def run_pelicun( # Loss Assessment ----------------------------------------------------------- # if a loss assessment is requested - if loss_config is not None: - out_config_loss = out_config.get('Loss', {}) + if 'Losses' in config['DL']: + out_config_loss = config['DL']['Outputs'].get('Loss', {}) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # backwards-compatibility for v3.2 and earlier | remove after v4.0 - if loss_config.get('BldgRepair', False): - loss_config['Repair'] = loss_config['BldgRepair'] + if config['DL']['Losses'].get('BldgRepair', False): + config['DL']['Losses']['Repair'] = config['DL']['Losses']['BldgRepair'] if out_config_loss.get('BldgRepair', False): out_config_loss['Repair'] = out_config_loss['BldgRepair'] # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # if requested, calculate repair consequences - if loss_config.get('Repair', False): - repair_config = loss_config['Repair'] + if config['DL']['Losses'].get('Repair', False): + repair_config = config['DL']['Losses']['Repair'] # load the fragility information if ( @@ -1356,7 +1360,7 @@ def run_pelicun( ) # DL_method = repair_config['ConsequenceDatabase'] - DL_method = damage_config.get('DamageProcess', 'User Defined') + DL_method = config['DL']['Damage'].get('DamageProcess', 'User Defined') rc = ('replacement', 'Cost') if 'ReplacementCost' in repair_config.keys(): @@ -1426,7 +1430,7 @@ def run_pelicun( adf.loc[rt, ('DV', 'Unit')] = 'day' # load the replacement time that corresponds to total loss - occ_type = asset_config['OccupancyType'] + occ_type = config['DL']['Asset']['OccupancyType'] adf.loc[rt, ('DS1', 'Theta_0')] = conseq_df.loc[ (f"STR.{occ_type}", 'Time'), ('DS5', 'Theta_0') ] @@ -1527,7 +1531,7 @@ def run_pelicun( ]: # with Hazus Earthquake we assume that consequence # archetypes are only differentiated by occupancy type - occ_type = asset_config.get('OccupancyType', None) + occ_type = config['DL']['Asset'].get('OccupancyType', None) for dmg_cmp in dmg_cmps: if dmg_cmp == 'collapse': @@ -1587,7 +1591,7 @@ def run_pelicun( decision_variables=DV_list, ) - PAL.repair.calculate() + PAL.repair.calculate(sample_size) agg_repair = PAL.repair.aggregate_losses() @@ -1599,7 +1603,7 @@ def run_pelicun( repair_units = repair_units.to_frame().T if ( - out_config['Settings'].get( + config['DL']['Outputs']['Settings'].get( 'AggregateColocatedComponentResults', False ) is True @@ -1747,7 +1751,7 @@ def run_pelicun( else: damage_sample_s['irreparable'] = np.zeros(damage_sample_s.shape[0]) - if loss_config is not None: + if 'Losses' in config['DL']: if 'agg_repair' not in locals(): agg_repair = PAL.repair.aggregate_losses() @@ -1773,11 +1777,11 @@ def run_pelicun( output_files.append('DL_summary_stats.csv') # create json outputs if needed - if out_config['Format']['JSON'] is True: + if config['DL']['Outputs']['Format']['JSON'] is True: for filename in output_files: filename_json = filename[:-3] + 'json' - if out_config['Settings'].get('SimpleIndexInJSON', False) is True: + if config['DL']['Outputs']['Settings'].get('SimpleIndexInJSON', False) is True: df = pd.read_csv(output_path / filename, index_col=0) else: df = convert_to_MultiIndex( @@ -1809,7 +1813,7 @@ def run_pelicun( json.dump(out_dict, f, indent=2) # remove csv outputs if they were not requested - if out_config['Format']['CSV'] is False: + if config['DL']['Outputs']['Format']['CSV'] is False: for filename in output_files: # keep the DL_summary and DL_summary_stats files if 'DL_summary' in filename: From f793a334c4370c48e72d70d0f693aa5d1c9e0726 Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Sun, 14 Apr 2024 06:27:31 -0700 Subject: [PATCH 08/22] Preliminary refactoring - reorder methods --- pelicun/model/loss_model.py | 208 ++++++++++++++++++------------------ 1 file changed, 104 insertions(+), 104 deletions(-) diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index 4ee473ad2..07e86cee3 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -742,110 +742,6 @@ def _calc_median_consequence(self, eco_qnt): return medians - def aggregate_losses(self): - """ - Aggregates repair consequences across components. - - Repair costs are simply summed up for each realization while repair - times are aggregated to provide lower and upper limits of the total - repair time using the assumption of parallel and sequential repair of - floors, respectively. Repairs within each floor are assumed to occur - sequentially. - """ - - self.log_div() - self.log_msg("Aggregating repair consequences...") - - DV = self.sample - - if DV is None: - return - - # group results by DV type and location - DVG = DV.groupby(level=[0, 4], axis=1).sum() - - # create the summary DF - df_agg = pd.DataFrame( - index=DV.index, - columns=[ - 'repair_cost', - 'repair_time-parallel', - 'repair_time-sequential', - 'repair_carbon', - 'repair_energy', - ], - ) - - if 'Cost' in DVG.columns: - df_agg['repair_cost'] = DVG['Cost'].sum(axis=1) - else: - df_agg = df_agg.drop('repair_cost', axis=1) - - if 'Time' in DVG.columns: - df_agg['repair_time-sequential'] = DVG['Time'].sum(axis=1) - - df_agg['repair_time-parallel'] = DVG['Time'].max(axis=1) - else: - df_agg = df_agg.drop( - ['repair_time-parallel', 'repair_time-sequential'], axis=1 - ) - - if 'Carbon' in DVG.columns: - df_agg['repair_carbon'] = DVG['Carbon'].sum(axis=1) - else: - df_agg = df_agg.drop('repair_carbon', axis=1) - - if 'Energy' in DVG.columns: - df_agg['repair_energy'] = DVG['Energy'].sum(axis=1) - else: - df_agg = df_agg.drop('repair_energy', axis=1) - - # convert units - - cmp_units = ( - self.loss_params[('DV', 'Unit')] - .groupby( - level=[ - 1, - ] - ) - .agg(lambda x: x.value_counts().index[0]) - ) - - dv_units = pd.Series(index=df_agg.columns, name='Units', dtype='object') - - if 'Cost' in DVG.columns: - dv_units['repair_cost'] = cmp_units['Cost'] - - if 'Time' in DVG.columns: - dv_units['repair_time-parallel'] = cmp_units['Time'] - dv_units['repair_time-sequential'] = cmp_units['Time'] - - if 'Carbon' in DVG.columns: - dv_units['repair_carbon'] = cmp_units['Carbon'] - - if 'Energy' in DVG.columns: - dv_units['repair_energy'] = cmp_units['Energy'] - - df_agg = file_io.save_to_csv( - df_agg, - None, - units=dv_units, - unit_conversion_factors=self._asmnt.unit_conversion_factors, - use_simpleindex=False, - log=self._asmnt.log, - ) - - df_agg.drop("Units", inplace=True) - - # convert header - - df_agg = base.convert_to_MultiIndex(df_agg, axis=1) - - self.log_msg("Repair consequences successfully aggregated.") - - return df_agg.astype(float) - def _generate_DV_sample(self, dmg_quantities, sample_size): """ Generate a sample of repair costs and times. @@ -1105,6 +1001,110 @@ def _generate_DV_sample(self, dmg_quantities, sample_size): self.log_msg("Successfully obtained DV sample.", prepend_timestamp=False) + def aggregate_losses(self): + """ + Aggregates repair consequences across components. + + Repair costs are simply summed up for each realization while repair + times are aggregated to provide lower and upper limits of the total + repair time using the assumption of parallel and sequential repair of + floors, respectively. Repairs within each floor are assumed to occur + sequentially. + """ + + self.log_div() + self.log_msg("Aggregating repair consequences...") + + DV = self.sample + + if DV is None: + return + + # group results by DV type and location + DVG = DV.groupby(level=[0, 4], axis=1).sum() + + # create the summary DF + df_agg = pd.DataFrame( + index=DV.index, + columns=[ + 'repair_cost', + 'repair_time-parallel', + 'repair_time-sequential', + 'repair_carbon', + 'repair_energy', + ], + ) + + if 'Cost' in DVG.columns: + df_agg['repair_cost'] = DVG['Cost'].sum(axis=1) + else: + df_agg = df_agg.drop('repair_cost', axis=1) + + if 'Time' in DVG.columns: + df_agg['repair_time-sequential'] = DVG['Time'].sum(axis=1) + + df_agg['repair_time-parallel'] = DVG['Time'].max(axis=1) + else: + df_agg = df_agg.drop( + ['repair_time-parallel', 'repair_time-sequential'], axis=1 + ) + + if 'Carbon' in DVG.columns: + df_agg['repair_carbon'] = DVG['Carbon'].sum(axis=1) + else: + df_agg = df_agg.drop('repair_carbon', axis=1) + + if 'Energy' in DVG.columns: + df_agg['repair_energy'] = DVG['Energy'].sum(axis=1) + else: + df_agg = df_agg.drop('repair_energy', axis=1) + + # convert units + + cmp_units = ( + self.loss_params[('DV', 'Unit')] + .groupby( + level=[ + 1, + ] + ) + .agg(lambda x: x.value_counts().index[0]) + ) + + dv_units = pd.Series(index=df_agg.columns, name='Units', dtype='object') + + if 'Cost' in DVG.columns: + dv_units['repair_cost'] = cmp_units['Cost'] + + if 'Time' in DVG.columns: + dv_units['repair_time-parallel'] = cmp_units['Time'] + dv_units['repair_time-sequential'] = cmp_units['Time'] + + if 'Carbon' in DVG.columns: + dv_units['repair_carbon'] = cmp_units['Carbon'] + + if 'Energy' in DVG.columns: + dv_units['repair_energy'] = cmp_units['Energy'] + + df_agg = file_io.save_to_csv( + df_agg, + None, + units=dv_units, + unit_conversion_factors=self._asmnt.unit_conversion_factors, + use_simpleindex=False, + log=self._asmnt.log, + ) + + df_agg.drop("Units", inplace=True) + + # convert header + + df_agg = base.convert_to_MultiIndex(df_agg, axis=1) + + self.log_msg("Repair consequences successfully aggregated.") + + return df_agg.astype(float) + def prep_constant_median_DV(median): """ From 50ac41d3c443c2084690d2d567dda30f731e9e43 Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Sun, 14 Apr 2024 06:40:55 -0700 Subject: [PATCH 09/22] Preliminary refactoring - format code with black Only formatting changes here. --- pelicun/__init__.py | 8 +- pelicun/assessment.py | 15 +- pelicun/base.py | 155 ++++++++------- pelicun/db.py | 180 +++++++++--------- pelicun/file_io.py | 99 ++++++---- pelicun/model/damage_model.py | 20 +- pelicun/model/loss_model.py | 1 - pelicun/model/pelicun_model.py | 12 +- .../resources/auto/Hazus_Earthquake_Story.py | 2 + pelicun/tests/test_auto.py | 4 +- pelicun/tools/DL_calculation.py | 91 +++++---- pelicun/tools/HDF_to_CSV.py | 2 +- pelicun/tools/export_DB.py | 6 +- 13 files changed, 321 insertions(+), 274 deletions(-) diff --git a/pelicun/__init__.py b/pelicun/__init__.py index 1742922e1..07ed2014a 100644 --- a/pelicun/__init__.py +++ b/pelicun/__init__.py @@ -43,8 +43,10 @@ __version__ = '3.3' -__copyright__ = ("Copyright (c) 2018 Leland Stanford " - "Junior University and The Regents " - "of the University of California") +__copyright__ = ( + "Copyright (c) 2018 Leland Stanford " + "Junior University and The Regents " + "of the University of California" +) __license__ = "BSD 3-Clause License" diff --git a/pelicun/assessment.py b/pelicun/assessment.py index 3c30201f3..4a303b3fd 100644 --- a/pelicun/assessment.py +++ b/pelicun/assessment.py @@ -90,13 +90,15 @@ def __init__(self, config_options=None): self.options = base.Options(config_options, self) - self.unit_conversion_factors = base.parse_units( - self.options.units_file) + self.unit_conversion_factors = base.parse_units(self.options.units_file) self.log = self.options.log - self.log.msg(f'pelicun {pelicun_version} | \n', - prepend_timestamp=False, prepend_blank_space=False) + self.log.msg( + f'pelicun {pelicun_version} | \n', + prepend_timestamp=False, + prepend_blank_space=False, + ) self.log.print_system_info() @@ -232,8 +234,9 @@ def calc_unit_scale_factor(self, unit): scale_factor = unit_count * self.unit_conversion_factors[unit_name] except KeyError as exc: - raise KeyError(f"Specified unit not recognized: " - f"{unit_count} {unit_name}") from exc + raise KeyError( + f"Specified unit not recognized: {unit_count} {unit_name}" + ) from exc return scale_factor diff --git a/pelicun/base.py b/pelicun/base.py index 1a3fa4b6a..094a73633 100644 --- a/pelicun/base.py +++ b/pelicun/base.py @@ -91,7 +91,6 @@ class Options: - """ Options objects store analysis options and the logging configuration. @@ -183,8 +182,7 @@ def __init__(self, user_config_options, assessment=None): self._seed = None self._rng = np.random.default_rng() - merged_config_options = merge_default_config( - user_config_options) + merged_config_options = merge_default_config(user_config_options) self._seed = merged_config_options['Seed'] self.sampling_method = merged_config_options['Sampling']['SamplingMethod'] @@ -203,7 +201,8 @@ def __init__(self, user_config_options, assessment=None): merged_config_options['ShowWarnings'], merged_config_options['LogShowMS'], merged_config_options['LogFile'], - merged_config_options['PrintLog']) + merged_config_options['PrintLog'], + ) def nondir_multi(self, EDP_type): """ @@ -237,7 +236,8 @@ def nondir_multi(self, EDP_type): f"calculation of {EDP_type} not specified.\n" f"Please add {EDP_type} in the configuration dictionary " f"under ['Options']['NonDirectionalMultipliers']" - " = {{'edp_type': value, ...}}") + " = {{'edp_type': value, ...}}" + ) @property def seed(self): @@ -277,7 +277,6 @@ def units_file(self, value): class Logger: - """ Logger objects are used to generate log files documenting execution events and related messages. @@ -314,6 +313,7 @@ class Logger: (see settings/default_config.json in the pelicun source code). """ + # TODO: finalize docstring def __init__(self, verbose, show_warnings, log_show_ms, log_file, print_log): @@ -427,10 +427,12 @@ def log_file(self, value): f.write('') except BaseException as err: - print(f"WARNING: The filepath provided for the log file does " - f"not point to a valid location: {value}. \nPelicun " - f"cannot print the log to a file.\n" - f"The error was: '{err}'") + print( + f"WARNING: The filepath provided for the log file does " + f"not point to a valid location: {value}. \nPelicun " + f"cannot print the log to a file.\n" + f"The error was: '{err}'" + ) raise @property @@ -485,9 +487,10 @@ def msg(self, msg='', prepend_timestamp=True, prepend_blank_space=True): for msg_i, msg_line in enumerate(msg_lines): - if (prepend_timestamp and (msg_i == 0)): + if prepend_timestamp and (msg_i == 0): formatted_msg = '{} {}'.format( - datetime.now().strftime(self.log_time_format), msg_line) + datetime.now().strftime(self.log_time_format), msg_line + ) elif prepend_timestamp: formatted_msg = self.log_pref + msg_line elif prepend_blank_space: @@ -519,15 +522,16 @@ def print_system_info(self): """ self.msg( - 'System Information:', - prepend_timestamp=False, prepend_blank_space=False) + 'System Information:', prepend_timestamp=False, prepend_blank_space=False + ) self.msg( f'local time zone: {datetime.utcnow().astimezone().tzinfo}\n' f'start time: {datetime.now().strftime("%Y-%m-%dT%H:%M:%S")}\n' f'python: {sys.version}\n' f'numpy: {np.__version__}\n' f'pandas: {pd.__version__}\n', - prepend_timestamp=False) + prepend_timestamp=False, + ) # get the absolute path of the pelicun directory @@ -535,7 +539,6 @@ def print_system_info(self): def control_warnings(show): - """ Convenience function to turn warnings on/off @@ -552,14 +555,11 @@ def control_warnings(show): action = 'ignore' if not sys.warnoptions: - warnings.filterwarnings( - category=FutureWarning, action=action) + warnings.filterwarnings(category=FutureWarning, action=action) - warnings.filterwarnings( - category=DeprecationWarning, action=action) + warnings.filterwarnings(category=DeprecationWarning, action=action) - warnings.filterwarnings( - category=pd.errors.PerformanceWarning, action=action) + warnings.filterwarnings(category=pd.errors.PerformanceWarning, action=action) def load_default_options(): @@ -567,18 +567,16 @@ def load_default_options(): Load the default_config.json file to set options to default values """ - with open(pelicun_path / "settings/default_config.json", - 'r', encoding='utf-8') as f: + with open( + pelicun_path / "settings/default_config.json", 'r', encoding='utf-8' + ) as f: default_config = json.load(f) default_options = default_config['Options'] return default_options -def update_vals( - update, primary, - update_path, primary_path -): +def update_vals(update, primary, update_path, primary_path): """ Updates the values of the `update` nested dictionary with those provided in the `primary` nested dictionary. If a key @@ -635,8 +633,11 @@ def update_vals( ) # With both being dictionaries, we recurse. update_vals( - update[key], primary[key], - f'{update_path}["{key}"]', f'{primary_path}["{key}"]') + update[key], + primary[key], + f'{update_path}["{key}"]', + f'{primary_path}["{key}"]', + ) # if `primary[key]` is NOT a dictionary: else: # if `key` does not exist in `update`, we add it, with @@ -686,9 +687,7 @@ def merge_default_config(user_config): # We fill out the user's config with the values available in the # default config that were not set. # We use a recursive function to handle nesting. - update_vals( - config, default_config, - 'user_settings', 'default_settings') + update_vals(config, default_config, 'user_settings', 'default_settings') return config @@ -907,8 +906,9 @@ def _warning(message, category, filename, lineno, file=None, line=None): warnings.showwarning = _warning -def describe(df, percentiles=(0.001, 0.023, 0.10, 0.159, 0.5, 0.841, 0.90, - 0.977, 0.999)): +def describe( + df, percentiles=(0.001, 0.023, 0.10, 0.159, 0.5, 0.841, 0.90, 0.977, 0.999) +): """ Provide descriptive statistics. """ @@ -1005,7 +1005,9 @@ def process_loc(string, stories): """ try: res = int(string) - return [res, ] + return [ + res, + ] except ValueError: if "-" in string: s_low, s_high = string.split('-') @@ -1015,9 +1017,13 @@ def process_loc(string, stories): if string == "all": return list(range(1, stories + 1)) if string == "top": - return [stories, ] + return [ + stories, + ] if string == "roof": - return [stories, ] + return [ + stories, + ] return None @@ -1031,8 +1037,7 @@ def dedupe_index(dataframe, dtype=str): inames = dataframe.index.names dataframe.reset_index(inplace=True) - dataframe['uid'] = ( - dataframe.groupby([*inames]).cumcount()).astype(dtype) + dataframe['uid'] = (dataframe.groupby([*inames]).cumcount()).astype(dtype) dataframe.set_index([*inames] + ['uid'], inplace=True) dataframe.sort_index(inplace=True) @@ -1041,45 +1046,39 @@ def dedupe_index(dataframe, dtype=str): EDP_to_demand_type = { # Drifts - 'Story Drift Ratio': 'PID', - 'Peak Interstory Drift Ratio': 'PID', - 'Roof Drift Ratio': 'PRD', - 'Peak Roof Drift Ratio': 'PRD', - 'Damageable Wall Drift': 'DWD', - 'Racking Drift Ratio': 'RDR', - 'Mega Drift Ratio': 'PMD', - 'Residual Drift Ratio': 'RID', + 'Story Drift Ratio': 'PID', + 'Peak Interstory Drift Ratio': 'PID', + 'Roof Drift Ratio': 'PRD', + 'Peak Roof Drift Ratio': 'PRD', + 'Damageable Wall Drift': 'DWD', + 'Racking Drift Ratio': 'RDR', + 'Mega Drift Ratio': 'PMD', + 'Residual Drift Ratio': 'RID', 'Residual Interstory Drift Ratio': 'RID', - 'Peak Effective Drift Ratio': 'EDR', - + 'Peak Effective Drift Ratio': 'EDR', # Floor response - 'Peak Floor Acceleration': 'PFA', - 'Peak Floor Velocity': 'PFV', - 'Peak Floor Displacement': 'PFD', - + 'Peak Floor Acceleration': 'PFA', + 'Peak Floor Velocity': 'PFV', + 'Peak Floor Displacement': 'PFD', # Component response - 'Peak Link Rotation Angle': 'LR', - 'Peak Link Beam Chord Rotation': 'LBR', - + 'Peak Link Rotation Angle': 'LR', + 'Peak Link Beam Chord Rotation': 'LBR', # Wind Intensity - 'Peak Gust Wind Speed': 'PWS', - + 'Peak Gust Wind Speed': 'PWS', # Inundation Intensity - 'Peak Inundation Height': 'PIH', - + 'Peak Inundation Height': 'PIH', # Shaking Intensity - 'Peak Ground Acceleration': 'PGA', - 'Peak Ground Velocity': 'PGV', - 'Spectral Acceleration': 'SA', - 'Spectral Velocity': 'SV', - 'Spectral Displacement': 'SD', - 'Peak Spectral Acceleration': 'SA', - 'Peak Spectral Velocity': 'SV', - 'Peak Spectral Displacement': 'SD', - 'Permanent Ground Deformation': 'PGD', - + 'Peak Ground Acceleration': 'PGA', + 'Peak Ground Velocity': 'PGV', + 'Spectral Acceleration': 'SA', + 'Spectral Velocity': 'SV', + 'Spectral Displacement': 'SD', + 'Peak Spectral Acceleration': 'SA', + 'Peak Spectral Velocity': 'SV', + 'Peak Spectral Displacement': 'SD', + 'Permanent Ground Deformation': 'PGD', # Placeholder for advanced calculations - 'One': 'ONE' + 'One': 'ONE', } @@ -1168,11 +1167,11 @@ def get_contents(file_path, preserve_categories=False): def convert_units( - values: (float | list[float] | np.ndarray), + values: float | list[float] | np.ndarray, unit: str, to_unit: str, - category: (str | None) = None -) -> (float | list[float] | np.ndarray): + category: str | None = None, +) -> float | list[float] | np.ndarray: """ Converts numeric values between different units. @@ -1227,11 +1226,9 @@ def convert_units( units = all_units[category] for unt in unit, to_unit: if unt not in units: - raise ValueError( - f'Unknown unit: `{unt}`' - ) + raise ValueError(f'Unknown unit: `{unt}`') else: - unit_category: (str | None) = None + unit_category: str | None = None for key in all_units: units = all_units[key] if unit in units: diff --git a/pelicun/db.py b/pelicun/db.py index dd5e7db7c..b156c9f85 100644 --- a/pelicun/db.py +++ b/pelicun/db.py @@ -321,9 +321,7 @@ def create_FEMA_P58_fragility_db( # the additional fields are added to the description if they exist if cmp_meta['Construction_Quality'] != 'Not Specified': - comments += ( - f'\nConstruction Quality: {cmp_meta["Construction_Quality"]}' - ) + comments += f'\nConstruction Quality: {cmp_meta["Construction_Quality"]}' if cmp_meta['Seismic_Installation_Conditions'] not in [ 'Not Specified', @@ -429,9 +427,11 @@ def create_FEMA_P58_fragility_db( sim_weights.append( np.product( [ - weights[ds_i] - if ds_map[-ds_i - 1] == '1' - else 1.0 - weights[ds_i] + ( + weights[ds_i] + if ds_map[-ds_i - 1] == '1' + else 1.0 - weights[ds_i] + ) for ds_i in range(sim_ds_count) ] ) @@ -897,9 +897,11 @@ def create_FEMA_P58_repair_db( cost_vals = np.sum( [ - cost_est[f'DS{ds_i + 1}'] - if ds_map[-ds_i - 1] == '1' - else np.zeros(5) + ( + cost_est[f'DS{ds_i + 1}'] + if ds_map[-ds_i - 1] == '1' + else np.zeros(5) + ) for ds_i in range(sim_ds_count) ], axis=0, @@ -907,9 +909,11 @@ def create_FEMA_P58_repair_db( time_vals = np.sum( [ - time_est[f'DS{ds_i + 1}'] - if ds_map[-ds_i - 1] == '1' - else np.zeros(6) + ( + time_est[f'DS{ds_i + 1}'] + if ds_map[-ds_i - 1] == '1' + else np.zeros(6) + ) for ds_i in range(sim_ds_count) ], axis=0, @@ -917,9 +921,11 @@ def create_FEMA_P58_repair_db( carbon_vals = np.sum( [ - carbon_est[f'DS{ds_i + 1}'] - if ds_map[-ds_i - 1] == '1' - else np.zeros(3) + ( + carbon_est[f'DS{ds_i + 1}'] + if ds_map[-ds_i - 1] == '1' + else np.zeros(3) + ) for ds_i in range(sim_ds_count) ], axis=0, @@ -927,9 +933,11 @@ def create_FEMA_P58_repair_db( energy_vals = np.sum( [ - energy_est[f'DS{ds_i + 1}'] - if ds_map[-ds_i - 1] == '1' - else np.zeros(3) + ( + energy_est[f'DS{ds_i + 1}'] + if ds_map[-ds_i - 1] == '1' + else np.zeros(3) + ) for ds_i in range(sim_ds_count) ], axis=0, @@ -989,9 +997,9 @@ def create_FEMA_P58_repair_db( f"{cost_qnt_low:g},{cost_qnt_up:g}" ) - df_db.loc[ - (cmp.Index, 'Cost'), f'DS{DS_i}-Theta_1' - ] = f"{cost_theta[1]:g}" + df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Theta_1'] = ( + f"{cost_theta[1]:g}" + ) df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = family_hat @@ -1000,37 +1008,37 @@ def create_FEMA_P58_repair_db( f"{time_qnt_low:g},{time_qnt_up:g}" ) - df_db.loc[ - (cmp.Index, 'Time'), f'DS{DS_i}-Theta_1' - ] = f"{time_theta[1]:g}" + df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Theta_1'] = ( + f"{time_theta[1]:g}" + ) df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-LongLeadTime'] = int( time_vals[5] > 0 ) - df_db.loc[ - (cmp.Index, 'Carbon'), f'DS{DS_i}-Family' - ] = family_hat_carbon + df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Family'] = ( + family_hat_carbon + ) - df_db.loc[ - (cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_0' - ] = f"{carbon_theta[0]:g}" + df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_0'] = ( + f"{carbon_theta[0]:g}" + ) - df_db.loc[ - (cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_1' - ] = f"{carbon_theta[1]:g}" + df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_1'] = ( + f"{carbon_theta[1]:g}" + ) - df_db.loc[ - (cmp.Index, 'Energy'), f'DS{DS_i}-Family' - ] = family_hat_energy + df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Family'] = ( + family_hat_energy + ) - df_db.loc[ - (cmp.Index, 'Energy'), f'DS{DS_i}-Theta_0' - ] = f"{energy_theta[0]:g}" + df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Theta_0'] = ( + f"{energy_theta[0]:g}" + ) - df_db.loc[ - (cmp.Index, 'Energy'), f'DS{DS_i}-Theta_1' - ] = f"{energy_theta[1]:g}" + df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Theta_1'] = ( + f"{energy_theta[1]:g}" + ) if ds_map.count('1') == 1: ds_pure_id = ds_map[::-1].find('1') + 1 @@ -1071,9 +1079,9 @@ def create_FEMA_P58_repair_db( for DS_i in range(1, 6): # cost if not pd.isna(getattr(cmp, f'Best_Fit_DS{DS_i}')): - df_db.loc[ - (cmp.Index, 'Cost'), f'DS{DS_i}-Family' - ] = convert_family[getattr(cmp, f'Best_Fit_DS{DS_i}')] + df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Family'] = ( + convert_family[getattr(cmp, f'Best_Fit_DS{DS_i}')] + ) if not pd.isna(getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}')): theta_0_low = getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}') @@ -1082,9 +1090,9 @@ def create_FEMA_P58_repair_db( qnt_up = getattr(cmp, f'Upper_Qty_Cutoff_DS{DS_i}') if theta_0_low == 0.0 and theta_0_up == 0.0: - df_db.loc[ - (cmp.Index, 'Cost'), f'DS{DS_i}-Family' - ] = np.nan + df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Family'] = ( + np.nan + ) else: df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Theta_0'] = ( @@ -1092,9 +1100,9 @@ def create_FEMA_P58_repair_db( f"{qnt_low:g},{qnt_up:g}" ) - df_db.loc[ - (cmp.Index, 'Cost'), f'DS{DS_i}-Theta_1' - ] = f"{getattr(cmp, f'CV__Dispersion_DS{DS_i}'):g}" + df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Theta_1'] = ( + f"{getattr(cmp, f'CV__Dispersion_DS{DS_i}'):g}" + ) else: incomplete_cost = True @@ -1114,9 +1122,9 @@ def create_FEMA_P58_repair_db( # time if not pd.isna(getattr(cmp, f'Best_Fit_DS{DS_i}_1')): - df_db.loc[ - (cmp.Index, 'Time'), f'DS{DS_i}-Family' - ] = convert_family[getattr(cmp, f'Best_Fit_DS{DS_i}_1')] + df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = ( + convert_family[getattr(cmp, f'Best_Fit_DS{DS_i}_1')] + ) if not pd.isna(getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}_1')): theta_0_low = getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}_1') @@ -1125,9 +1133,9 @@ def create_FEMA_P58_repair_db( qnt_up = getattr(cmp, f'Upper_Qty_Cutoff_DS{DS_i}_1') if theta_0_low == 0.0 and theta_0_up == 0.0: - df_db.loc[ - (cmp.Index, 'Time'), f'DS{DS_i}-Family' - ] = np.nan + df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = ( + np.nan + ) else: df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Theta_0'] = ( @@ -1135,22 +1143,22 @@ def create_FEMA_P58_repair_db( f"{qnt_low:g},{qnt_up:g}" ) - df_db.loc[ - (cmp.Index, 'Time'), f'DS{DS_i}-Theta_1' - ] = f"{getattr(cmp, f'CV__Dispersion_DS{DS_i}_2'):g}" + df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Theta_1'] = ( + f"{getattr(cmp, f'CV__Dispersion_DS{DS_i}_2'):g}" + ) - df_db.loc[ - (cmp.Index, 'Time'), f'DS{DS_i}-LongLeadTime' - ] = int(getattr(cmp, f'DS_{DS_i}_Long_Lead_Time') == 'YES') + df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-LongLeadTime'] = ( + int(getattr(cmp, f'DS_{DS_i}_Long_Lead_Time') == 'YES') + ) else: incomplete_time = True # Carbon if not pd.isna(getattr(cmp, f'DS{DS_i}_Best_Fit')): - df_db.loc[ - (cmp.Index, 'Carbon'), f'DS{DS_i}-Family' - ] = convert_family[getattr(cmp, f'DS{DS_i}_Best_Fit')] + df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Family'] = ( + convert_family[getattr(cmp, f'DS{DS_i}_Best_Fit')] + ) df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_0'] = getattr( cmp, f'DS{DS_i}_Embodied_Carbon_kg_CO2eq' @@ -1162,9 +1170,9 @@ def create_FEMA_P58_repair_db( # Energy if not pd.isna(getattr(cmp, f'DS{DS_i}_Best_Fit_1')): - df_db.loc[ - (cmp.Index, 'Energy'), f'DS{DS_i}-Family' - ] = convert_family[getattr(cmp, f'DS{DS_i}_Best_Fit_1')] + df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Family'] = ( + convert_family[getattr(cmp, f'DS{DS_i}_Best_Fit_1')] + ) df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Theta_0'] = getattr( cmp, f'DS{DS_i}_Embodied_Energy_MJ' @@ -1221,9 +1229,7 @@ def create_FEMA_P58_repair_db( with open(target_meta_file, 'w+', encoding='utf-8') as f: json.dump(meta_dict, f, indent=2) - print( - "Successfully parsed and saved the repair consequence data from FEMA P58" - ) + print("Successfully parsed and saved the repair consequence data from FEMA P58") def create_FEMA_P58_bldg_redtag_db( @@ -1484,9 +1490,7 @@ def create_FEMA_P58_bldg_redtag_db( with open(target_meta_file, 'w+', encoding='utf-8') as f: json.dump(meta_dict, f, indent=2) - print( - "Successfully parsed and saved the red tag consequence data from FEMA P58" - ) + print("Successfully parsed and saved the red tag consequence data from FEMA P58") def create_Hazus_EQ_fragility_db( @@ -1712,9 +1716,9 @@ def create_Hazus_EQ_fragility_db( if LS_i == 4: p_coll = S_data['P_collapse'][bt] - df_db.loc[ - counter, f'LS{LS_i}-DamageStateWeights' - ] = f'{1.0 - p_coll} | {p_coll}' + df_db.loc[counter, f'LS{LS_i}-DamageStateWeights'] = ( + f'{1.0 - p_coll} | {p_coll}' + ) cmp_meta["LimitStates"].update( { @@ -1940,9 +1944,9 @@ def create_Hazus_EQ_fragility_db( if LS_i == 4: p_coll = LF_data['P_collapse'][bt] - df_db.loc[ - counter, f'LS{LS_i}-DamageStateWeights' - ] = f'{1.0 - p_coll} | {p_coll}' + df_db.loc[counter, f'LS{LS_i}-DamageStateWeights'] = ( + f'{1.0 - p_coll} | {p_coll}' + ) cmp_meta["LimitStates"].update( { @@ -2006,9 +2010,9 @@ def create_Hazus_EQ_fragility_db( f_depth ] p_complete = GF_data['P_Complete'] - df_db.loc[ - counter, 'LS1-DamageStateWeights' - ] = f'{1.0 - p_complete} | {p_complete}' + df_db.loc[counter, 'LS1-DamageStateWeights'] = ( + f'{1.0 - p_complete} | {p_complete}' + ) cmp_meta["LimitStates"].update( { @@ -2340,9 +2344,7 @@ def create_Hazus_EQ_repair_db( with open(target_meta_file, 'w+', encoding='utf-8') as f: json.dump(meta_dict, f, indent=2) - print( - "Successfully parsed and saved the repair consequence data from Hazus EQ" - ) + print("Successfully parsed and saved the repair consequence data from Hazus EQ") def create_Hazus_EQ_bldg_injury_db( @@ -2475,9 +2477,7 @@ def create_Hazus_EQ_bldg_injury_db( # with open(target_meta_file, 'w+') as f: # json.dump(meta_dict, f, indent=2) - print( - "Successfully parsed and saved the injury consequence data from Hazus " "EQ" - ) + print("Successfully parsed and saved the injury consequence data from Hazus EQ") def create_Hazus_HU_fragility_db( diff --git a/pelicun/file_io.py b/pelicun/file_io.py index be3063ae8..7fcf79d34 100644 --- a/pelicun/file_io.py +++ b/pelicun/file_io.py @@ -71,28 +71,35 @@ } dependency_to_acronym = { - 'btw. Fragility Groups': 'FG', + 'btw. Fragility Groups': 'FG', 'btw. Performance Groups': 'PG', - 'btw. Floors': 'LOC', - 'btw. Directions': 'DIR', - 'btw. Component Groups': 'CSG', - 'btw. Damage States': 'DS', - 'Independent': 'IND', - 'per ATC recommendation': 'ATC', + 'btw. Floors': 'LOC', + 'btw. Directions': 'DIR', + 'btw. Component Groups': 'CSG', + 'btw. Damage States': 'DS', + 'Independent': 'IND', + 'per ATC recommendation': 'ATC', } HAZUS_occ_converter = { - 'RES': 'Residential', - 'COM': 'Commercial', - 'REL': 'Commercial', - 'EDU': 'Educational', - 'IND': 'Industrial', - 'AGR': 'Industrial' + 'RES': 'Residential', + 'COM': 'Commercial', + 'REL': 'Commercial', + 'EDU': 'Educational', + 'IND': 'Industrial', + 'AGR': 'Industrial', } -def save_to_csv(data, filepath, units=None, unit_conversion_factors=None, - orientation=0, use_simpleindex=True, log=None): +def save_to_csv( + data, + filepath, + units=None, + unit_conversion_factors=None, + orientation=0, + use_simpleindex=True, + log=None, +): """ Saves data to a CSV file following standard SimCenter schema. @@ -143,9 +150,11 @@ def save_to_csv(data, filepath, units=None, unit_conversion_factors=None, """ if filepath is None: - if log: log.msg('Preparing data ...', prepend_timestamp=False) + if log: + log.msg('Preparing data ...', prepend_timestamp=False) - elif log: log.msg(f'Saving data to {filepath}...', prepend_timestamp=False) + elif log: + log.msg(f'Saving data to {filepath}...', prepend_timestamp=False) if data is not None: @@ -158,9 +167,11 @@ def save_to_csv(data, filepath, units=None, unit_conversion_factors=None, if unit_conversion_factors is None: raise ValueError( 'When units is not None, ' - 'unit_conversion_factors must be provided') + 'unit_conversion_factors must be provided' + ) - if log: log.msg('Converting units...', prepend_timestamp=False) + if log: + log.msg('Converting units...', prepend_timestamp=False) # if the orientation is 1, we might not need to scale all columns if orientation == 1: @@ -173,7 +184,7 @@ def save_to_csv(data, filepath, units=None, unit_conversion_factors=None, labels = units.loc[units == unit_name].index.values - unit_factor = 1. / unit_conversion_factors[unit_name] + unit_factor = 1.0 / unit_conversion_factors[unit_name] active_labels = [] @@ -204,7 +215,8 @@ def save_to_csv(data, filepath, units=None, unit_conversion_factors=None, data = pd.concat([units, data], axis=1) data.sort_index(inplace=True) - if log: log.msg('Unit conversion successful.', prepend_timestamp=False) + if log: + log.msg('Unit conversion successful.', prepend_timestamp=False) if use_simpleindex: # convert MultiIndex to regular index with '-' separators @@ -223,13 +235,16 @@ def save_to_csv(data, filepath, units=None, unit_conversion_factors=None, # save the contents of the DataFrame into a csv data.to_csv(filepath) - if log: log.msg('Data successfully saved to file.', - prepend_timestamp=False) + if log: + log.msg( + 'Data successfully saved to file.', prepend_timestamp=False + ) else: raise ValueError( f'ERROR: Unexpected file type received when trying ' - f'to save to csv: {filepath}') + f'to save to csv: {filepath}' + ) return None @@ -237,8 +252,8 @@ def save_to_csv(data, filepath, units=None, unit_conversion_factors=None, return data # at this line, data is None - if log: log.msg('WARNING: Data was empty, no file saved.', - prepend_timestamp=False) + if log: + log.msg('WARNING: Data was empty, no file saved.', prepend_timestamp=False) return None @@ -382,9 +397,11 @@ def load_data( log.msg('Converting units...', prepend_timestamp=False) conversion_factors = units.map( - lambda unit: 1.00 - if pd.isna(unit) - else unit_conversion_factors.get(unit, 1.00) + lambda unit: ( + 1.00 + if pd.isna(unit) + else unit_conversion_factors.get(unit, 1.00) + ) ) if orientation == 1: @@ -458,7 +475,8 @@ def load_from_file(filepath, log=None): If the file is not a CSV. """ - if log: log.msg(f'Loading data from {filepath}...') + if log: + log.msg(f'Loading data from {filepath}...') # check if the filepath is valid filepath = Path(filepath).resolve() @@ -466,19 +484,28 @@ def load_from_file(filepath, log=None): if not filepath.is_file(): raise FileNotFoundError( f"The filepath provided does not point to an existing " - f"file: {filepath}") + f"file: {filepath}" + ) if filepath.suffix == '.csv': # load the contents of the csv into a DataFrame - data = pd.read_csv(filepath, header=0, index_col=0, low_memory=False, - encoding_errors='replace') + data = pd.read_csv( + filepath, + header=0, + index_col=0, + low_memory=False, + encoding_errors='replace', + ) - if log: log.msg('File successfully opened.', prepend_timestamp=False) + if log: + log.msg('File successfully opened.', prepend_timestamp=False) else: - raise ValueError(f'ERROR: Unexpected file type received when trying ' - f'to load from csv: {filepath}') + raise ValueError( + f'ERROR: Unexpected file type received when trying ' + f'to load from csv: {filepath}' + ) return data diff --git a/pelicun/model/damage_model.py b/pelicun/model/damage_model.py index 1d4703e4e..f29866a4e 100644 --- a/pelicun/model/damage_model.py +++ b/pelicun/model/damage_model.py @@ -894,7 +894,7 @@ def _evaluate_damage_state( # initialize the DataFrames that store the damage states and # quantities ds_sample = pd.DataFrame( - 0, # fill value + 0, # fill value columns=capacity_sample.columns.droplevel('ls').unique(), index=capacity_sample.index, dtype='int32', @@ -1469,7 +1469,11 @@ def _complete_ds_cols(self, dmg_sample): return res def calculate( - self, sample_size=None, dmg_process=None, block_batch_size=1000, scaling_specification=None + self, + sample_size=None, + dmg_process=None, + block_batch_size=1000, + scaling_specification=None, ): """ Wrapper around the new calculate method that requires sample size. @@ -1478,12 +1482,16 @@ def calculate( if not sample_size: # todo: Deprecation warning sample_size = self._asmnt.demand.sample.shape[0] - self.calculate_internal(sample_size, dmg_process, block_batch_size, scaling_specification) - + self.calculate_internal( + sample_size, dmg_process, block_batch_size, scaling_specification + ) def calculate_internal( - self, sample_size, dmg_process=None, block_batch_size=1000, scaling_specification=None - + self, + sample_size, + dmg_process=None, + block_batch_size=1000, + scaling_specification=None, ): """ Calculate the damage state of each component block in the asset. diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index 07e86cee3..30ccb4b1a 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -305,7 +305,6 @@ def calculate(self, sample_size=None): # todo: deprecation warning sample_size = self._asmnt.demand.sample.shape[0] self.calculate_internal(sample_size) - def calculate_internal(self, sample_size): """ diff --git a/pelicun/model/pelicun_model.py b/pelicun/model/pelicun_model.py index bcf9a2e8d..3428c9e9e 100644 --- a/pelicun/model/pelicun_model.py +++ b/pelicun/model/pelicun_model.py @@ -214,13 +214,13 @@ def convert_marginal_params(self, marginal_params, units, arg_units=None): ) # and update the values in the DF - marginal_params.loc[ - row_id, ['Theta_0', 'Theta_1', 'Theta_2'] - ] = theta + marginal_params.loc[row_id, ['Theta_0', 'Theta_1', 'Theta_2']] = ( + theta + ) - marginal_params.loc[ - row_id, ['TruncateLower', 'TruncateUpper'] - ] = tr_limits + marginal_params.loc[row_id, ['TruncateLower', 'TruncateUpper']] = ( + tr_limits + ) # remove the added columns marginal_params = marginal_params[original_cols] diff --git a/pelicun/resources/auto/Hazus_Earthquake_Story.py b/pelicun/resources/auto/Hazus_Earthquake_Story.py index 0b2bd34eb..75ed5c47c 100644 --- a/pelicun/resources/auto/Hazus_Earthquake_Story.py +++ b/pelicun/resources/auto/Hazus_Earthquake_Story.py @@ -224,6 +224,7 @@ def auto_populate(AIM): if ground_failure: foundation_type = 'S' + # fmt: off FG_GF_H = f'GF.H.{foundation_type}' # noqa FG_GF_V = f'GF.V.{foundation_type}' # noqa CMP_GF = pd.DataFrame( # noqa @@ -231,6 +232,7 @@ def auto_populate(AIM): f'{FG_GF_V}':[ 'ea', 1, 3, 1, 'N/A']}, # noqa index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa ).T # noqa + # fmt: on CMP = pd.concat([CMP, CMP_GF], axis=0) diff --git a/pelicun/tests/test_auto.py b/pelicun/tests/test_auto.py index b1b50fa58..36458cb91 100644 --- a/pelicun/tests/test_auto.py +++ b/pelicun/tests/test_auto.py @@ -106,9 +106,7 @@ def test_pelicun_default_path_replacement( modified_path = setup_auto_script_path.replace( 'PelicunDefault/', setup_expected_base_path ) - assert modified_path.startswith( - setup_expected_base_path - ) + assert modified_path.startswith(setup_expected_base_path) def test_auto_population_script_execution( diff --git a/pelicun/tools/DL_calculation.py b/pelicun/tools/DL_calculation.py index 4a66f453c..fd421476f 100644 --- a/pelicun/tools/DL_calculation.py +++ b/pelicun/tools/DL_calculation.py @@ -69,6 +69,7 @@ # pd.set_option('display.max_rows', None) + def log_msg(msg): formatted_msg = f'{strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())} {msg}' @@ -389,9 +390,7 @@ def run_pelicun( return 0 # add the demand information - config_ap['DL']['Demands'].update( - {'DemandFilePath': f'{demand_file}'} - ) + config_ap['DL']['Demands'].update({'DemandFilePath': f'{demand_file}'}) if coupled_EDP is True: config_ap['DL']['Demands'].update({"CoupledDemands": True}) @@ -440,7 +439,9 @@ def run_pelicun( # sample_size_str = ( # expected location - config.get('Options', {}).get('Sampling', {}).get('SampleSize', None) + config.get('Options', {}) + .get('Sampling', {}) + .get('SampleSize', None) ) if not sample_size_str: # try previous location @@ -500,7 +501,7 @@ def run_pelicun( options.update({"LogFile": "pelicun_log.txt", "Verbose": True}) # If the user did not prescribe anything for ListAllDamageStates, - # then use True as default for DL_calculations regardless of what + # then use True as default for DL_calculations regardless of what # the Pelicun default is. if "ListAllDamageStates" not in options.keys(): options.update({"ListAllDamageStates": True}) @@ -580,7 +581,7 @@ def run_pelicun( { "SampleSize": sample_size, 'PreserveRawOrder': config['DL']['Demands'].get('CoupledDemands', False), - 'DemandCloning': config['DL']['Demands'].get('DemandCloning', False) + 'DemandCloning': config['DL']['Demands'].get('DemandCloning', False), } ) @@ -631,7 +632,10 @@ def run_pelicun( # save results if 'Demand' in config['DL']['Outputs']: - out_reqs = [out if val else "" for out, val in config['DL']['Outputs']['Demand'].items()] + out_reqs = [ + out if val else "" + for out, val in config['DL']['Outputs']['Demand'].items() + ] if np.any(np.isin(['Sample', 'Statistics'], out_reqs)): demand_sample, demand_units = PAL.demand.save_sample(save_units=True) @@ -784,7 +788,9 @@ def run_pelicun( cmp_units = cmp_units.to_frame().T if ( - config['DL']['Outputs']['Settings'].get('AggregateColocatedComponentResults', False) + config['DL']['Outputs']['Settings'].get( + 'AggregateColocatedComponentResults', False + ) is True ): cmp_units = cmp_units.groupby(level=[0, 1, 2], axis=1).first() @@ -795,7 +801,10 @@ def run_pelicun( cmp_groupby_uid.count() == 0, np.nan ) - out_reqs = [out if val else "" for out, val in config['DL']['Outputs']['Asset'].items()] + out_reqs = [ + out if val else "" + for out, val in config['DL']['Outputs']['Asset'].items() + ] if np.any(np.isin(['Sample', 'Statistics'], out_reqs)): if 'Sample' in out_reqs: @@ -857,10 +866,15 @@ def run_pelicun( # if a damage assessment is requested if 'Damage' in config['DL']: # load the fragility information - if config['DL']['Asset']['ComponentDatabase'] in default_DBs['fragility'].keys(): + if ( + config['DL']['Asset']['ComponentDatabase'] + in default_DBs['fragility'].keys() + ): component_db = [ 'PelicunDefault/' - + default_DBs['fragility'][config['DL']['Asset']['ComponentDatabase']], + + default_DBs['fragility'][ + config['DL']['Asset']['ComponentDatabase'] + ], ] else: component_db = [] @@ -917,9 +931,9 @@ def run_pelicun( adf.loc[coll_CMP_name, ('Demand', 'Type')] = coll_DEM_name else: - adf.loc[ - coll_CMP_name, ('Demand', 'Type') - ] = f'{coll_DEM_name}|{coll_DEM_spec}' + adf.loc[coll_CMP_name, ('Demand', 'Type')] = ( + f'{coll_DEM_name}|{coll_DEM_spec}' + ) coll_DEM_unit = add_units( pd.DataFrame( @@ -975,9 +989,9 @@ def run_pelicun( # input file adf.loc['excessiveRID', ('Demand', 'Directional')] = 1 adf.loc['excessiveRID', ('Demand', 'Offset')] = 0 - adf.loc[ - 'excessiveRID', ('Demand', 'Type') - ] = 'Residual Interstory Drift Ratio' + adf.loc['excessiveRID', ('Demand', 'Type')] = ( + 'Residual Interstory Drift Ratio' + ) adf.loc['excessiveRID', ('Demand', 'Unit')] = 'unitless' adf.loc['excessiveRID', ('LS1', 'Theta_0')] = irrep_config[ @@ -1086,7 +1100,9 @@ def run_pelicun( elif dp_approach == "User Defined": # load the damage process from a file with open( - config['DL']['Damage']['DamageProcessFilePath'], 'r', encoding='utf-8' + config['DL']['Damage']['DamageProcessFilePath'], + 'r', + encoding='utf-8', ) as f: dmg_process = json.load(f) @@ -1126,7 +1142,8 @@ def run_pelicun( ) out_reqs = [ - out if val else "" for out, val in config['DL']['Outputs']['Damage'].items() + out if val else "" + for out, val in config['DL']['Outputs']['Damage'].items() ] if np.any( @@ -1188,7 +1205,10 @@ def run_pelicun( ) # if requested, condense DS output - if config['DL']['Outputs']['Settings'].get('CondenseDS', False) is True: + if ( + config['DL']['Outputs']['Settings'].get('CondenseDS', False) + is True + ): # replace non-zero values with 1 grp_damage = grp_damage.mask( grp_damage.astype(np.float64).values > 0, 1 @@ -1297,21 +1317,14 @@ def run_pelicun( repair_config = config['DL']['Losses']['Repair'] # load the fragility information - if ( - repair_config['ConsequenceDatabase'] - in default_DBs['repair'].keys() - ): + if repair_config['ConsequenceDatabase'] in default_DBs['repair'].keys(): consequence_db = [ 'PelicunDefault/' - + default_DBs['repair'][ - repair_config['ConsequenceDatabase'] - ], + + default_DBs['repair'][repair_config['ConsequenceDatabase']], ] conseq_df = PAL.get_default_data( - default_DBs['repair'][repair_config['ConsequenceDatabase']][ - :-4 - ] + default_DBs['repair'][repair_config['ConsequenceDatabase']][:-4] ) else: consequence_db = [] @@ -1557,7 +1570,8 @@ def run_pelicun( loss_map_path = repair_config['MapFilePath'] loss_map_path = loss_map_path.replace( - 'CustomDLDataFolder', custom_dl_file_path) + 'CustomDLDataFolder', custom_dl_file_path + ) else: print("User defined loss map path missing. Terminating analysis") @@ -1573,9 +1587,7 @@ def run_pelicun( # assemble the list of requested decision variables DV_list = [] if repair_config.get('DecisionVariables', False) is not False: - for DV_i, DV_status in repair_config[ - 'DecisionVariables' - ].items(): + for DV_i, DV_status in repair_config['DecisionVariables'].items(): if DV_status is True: DV_list.append(DV_i) @@ -1594,12 +1606,10 @@ def run_pelicun( PAL.repair.calculate(sample_size) agg_repair = PAL.repair.aggregate_losses() - + # if requested, save results if out_config_loss.get('Repair', False): - repair_sample, repair_units = PAL.repair.save_sample( - save_units=True - ) + repair_sample, repair_units = PAL.repair.save_sample(save_units=True) repair_units = repair_units.to_frame().T if ( @@ -1781,7 +1791,10 @@ def run_pelicun( for filename in output_files: filename_json = filename[:-3] + 'json' - if config['DL']['Outputs']['Settings'].get('SimpleIndexInJSON', False) is True: + if ( + config['DL']['Outputs']['Settings'].get('SimpleIndexInJSON', False) + is True + ): df = pd.read_csv(output_path / filename, index_col=0) else: df = convert_to_MultiIndex( diff --git a/pelicun/tools/HDF_to_CSV.py b/pelicun/tools/HDF_to_CSV.py index 1e21e206d..a72b66eb0 100644 --- a/pelicun/tools/HDF_to_CSV.py +++ b/pelicun/tools/HDF_to_CSV.py @@ -46,7 +46,7 @@ def convert_HDF(HDF_path): HDF_ext = HDF_path.split('.')[-1] - CSV_base = HDF_path[:-len(HDF_ext) - 1] + CSV_base = HDF_path[: -len(HDF_ext) - 1] HDF_path = Path(HDF_path).resolve() diff --git a/pelicun/tools/export_DB.py b/pelicun/tools/export_DB.py index d1f87f4d1..b4b8b0343 100644 --- a/pelicun/tools/export_DB.py +++ b/pelicun/tools/export_DB.py @@ -62,8 +62,7 @@ def export_DB(data_path, target_dir): row_dict = convert_Series_to_dict(row) - with open(target_dir_data / f'{row_id}.json', 'w', - encoding='utf-8') as f: + with open(target_dir_data / f'{row_id}.json', 'w', encoding='utf-8') as f: json.dump(row_dict, f, indent=2) # add population if it exists @@ -78,8 +77,7 @@ def export_DB(data_path, target_dir): pop_dict.update({row_id: convert_Series_to_dict(row)}) - with open(target_dir / 'population.json', 'w', - encoding='utf-8') as f: + with open(target_dir / 'population.json', 'w', encoding='utf-8') as f: json.dump(pop_dict, f, indent=2) except (ValueError, NotImplementedError, FileNotFoundError): From 5e9bd5100c3f2d8913fa6b0965882cfbc27b872d Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Thu, 18 Apr 2024 12:02:37 -0700 Subject: [PATCH 10/22] Update `calculate` - Don't need a second method for the deprecation warning. I'm not sure why I thought we do. - Update .gitignore --- .gitignore | 3 ++- pelicun/model/loss_model.py | 26 +++++++++----------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index a42e24a97..3e390d12f 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,5 @@ __pycache__ /.emacs.desktop.lock /pelicun/coverage.xml .cov2emacs.log -flycheck*.py \ No newline at end of file +flycheck*.py +/notes/ diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index 30ccb4b1a..901ff9c8a 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -53,6 +53,7 @@ """ +import warnings import numpy as np import pandas as pd from .pelicun_model import PelicunModel @@ -297,34 +298,25 @@ def _generate_DV_sample(self, dmg_quantities, sample_size): raise NotImplementedError def calculate(self, sample_size=None): - """ - Wrapper method around new calculate that requires sample size. - Exists for backwards compatibility. - """ - if not sample_size: - # todo: deprecation warning - sample_size = self._asmnt.demand.sample.shape[0] - self.calculate_internal(sample_size) - - def calculate_internal(self, sample_size): """ Calculate the consequences of each component block damage in the asset. """ + if not sample_size: + sample_size = self._asmnt.demand.sample.shape[0] + warnings.warn( + 'Using default sample size is deprecated and will ' + 'be removed in future versions. ' + 'Please provide the `sample_size` explicitly.', + DeprecationWarning, + ) self.log_div() self.log_msg("Calculating losses...") drivers = [d for d, _ in self.loss_map['Driver']] - if 'DMG' in drivers: - sample_size = self._asmnt.damage.sample.shape[0] - elif 'DEM' in drivers: - sample_size = self._asmnt.demand.sample.shape[0] - else: - raise ValueError('Invalid loss drivers. Check the specified loss map.') - # First, get the damaged quantities in each damage state for # each component of interest. dmg_q = self._asmnt.damage.sample.copy() From 312b9084d6deb1f1e6d35278798d37619288433b Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Thu, 18 Apr 2024 12:13:15 -0700 Subject: [PATCH 11/22] Remove unused variable --- pelicun/model/loss_model.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index 901ff9c8a..36fa03cd9 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -315,8 +315,6 @@ def calculate(self, sample_size=None): self.log_div() self.log_msg("Calculating losses...") - drivers = [d for d, _ in self.loss_map['Driver']] - # First, get the damaged quantities in each damage state for # each component of interest. dmg_q = self._asmnt.damage.sample.copy() From 6fd424e3bfacc101fde56c1960fa0abb3a6a7cfa Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Thu, 18 Apr 2024 12:13:22 -0700 Subject: [PATCH 12/22] Remove unused methods We can always place them back when we need to use them. --- pelicun/db.py | 396 -------------------------------------------------- 1 file changed, 396 deletions(-) diff --git a/pelicun/db.py b/pelicun/db.py index b156c9f85..30a8ab8f2 100644 --- a/pelicun/db.py +++ b/pelicun/db.py @@ -46,12 +46,10 @@ create_FEMA_P58_fragility_db create_FEMA_P58_repair_db - create_FEMA_P58_bldg_injury_db create_FEMA_P58_bldg_redtag_db create_Hazus_EQ_fragility_db create_Hazus_EQ_repair_db - create_Hazus_EQ_bldg_injury_db """ @@ -1232,267 +1230,6 @@ def create_FEMA_P58_repair_db( print("Successfully parsed and saved the repair consequence data from FEMA P58") -def create_FEMA_P58_bldg_redtag_db( - source_file, - target_data_file='bldg_redtag_DB_FEMA_P58_2nd.csv', - target_meta_file='bldg_redtag_DB_FEMA_P58_2nd.json', -): - """ - Create an red tag consequence parameter database based on the FEMA P58 data - - The method was developed to process v3.1.2 of the FragilityDatabase xls - that is provided with FEMA P58 2nd edition. - - Parameters - ---------- - source_file: string - Path to the fragility database file. - target_data_file: string - Path where the consequence data file should be saved. A csv file is - expected. - target_meta_file: string - Path where the consequence metadata should be saved. A json file is - expected. - - """ - - # parse the source file - df = pd.read_excel( - source_file, - sheet_name='Summary', - header=2, - index_col=1, - true_values=["YES", "Yes", "yes"], - false_values=["NO", "No", "no"], - ) - - # take another pass with booleans because the first does not always work - for true_str in ("YES", "Yes", "yes"): - df.replace(true_str, True, inplace=True) - - for false_str in ("NO", "No", "no"): - df.replace(false_str, False, inplace=True) - - # remove empty rows and columns - df.dropna(axis=0, how='all', inplace=True) - df.dropna(axis=1, how='all', inplace=True) - - # filter the columns we need for the injury database - cols_to_db = [ - 'DS Hierarchy', - ] - for DS_i in range(1, 6): - cols_to_db += [ - f'DS {DS_i}, Unsafe Placard Trigger Flag', - f'DS {DS_i}, Unsafe Placard Damage Median', - f'DS {DS_i}, Unsafe Placard Damage Dispersion', - ] - - # filter the columns that we need for the metadata - cols_to_meta = [ - "Component Name", - "Component Description", - "Construction Quality:", - "Seismic Installation Conditions:", - "Comments / Notes", - "Author", - "Fragility Unit of Measure", - "Round to Integer Unit?", - "DS 1, Description", - "DS 2, Description", - "DS 3, Description", - "DS 4, Description", - "DS 5, Description", - ] - - # remove special characters to make it easier to work with column names - str_map = { - ord(' '): "_", - ord('.'): "_", - ord('-'): "_", - ord(':'): None, - ord('('): None, - ord(')'): None, - ord('?'): None, - ord('/'): None, - ord(','): None, - } - - df_db_source = df.loc[:, cols_to_db] - df_db_source.columns = [s.translate(str_map) for s in cols_to_db] - df_db_source.sort_index(inplace=True) - - df_meta = df.loc[:, cols_to_meta] - df_meta.columns = [s.translate(str_map) for s in cols_to_meta] - - df_db_source.replace('BY USER', np.nan, inplace=True) - df_db_source.replace('By User', np.nan, inplace=True) - - # initialize the output loss table - # define the columns - out_cols = [ - "Index", - "Incomplete", - ] - for DS_i in range(1, 6): - out_cols += [f"DS{DS_i}-Family", f"DS{DS_i}-Theta_0", f"DS{DS_i}-Theta_1"] - - # create the database index - comps = df_db_source.index.values - - df_db = pd.DataFrame(columns=out_cols, index=comps, dtype=float) - - # initialize the dictionary that stores the loss metadata - meta_dict = {} - - # for each component... - # (this approach is not efficient, but easy to follow which was considered - # more important than efficiency.) - for cmp in df_db_source.itertuples(): - ID = cmp.Index.split('.') - cmpID = f'{ID[0][0]}.{ID[0][1:3]}.{ID[0][3:5]}.{ID[1]}' - - # store the new index - df_db.loc[cmp.Index, 'Index'] = cmpID - - # assume the component information is complete - incomplete = False - - # get the raw metadata for the component - cmp_meta = df_meta.loc[cmp.Index, :] - - # store the global (i.e., not DS-specific) metadata - - # start with a comp. description - if not pd.isna(cmp_meta['Component_Description']): - comments = cmp_meta['Component_Description'] - else: - comments = '' - - # the additional fields are added to the description if they exist - if cmp_meta['Construction_Quality'] != 'Not Specified': - comments += ( - f'\nConstruction Quality: ' f'{cmp_meta["Construction_Quality"]}' - ) - - if cmp_meta['Seismic_Installation_Conditions'] not in [ - 'Not Specified', - 'Not applicable', - 'Unknown', - 'Any', - ]: - comments += ( - f'\nSeismic Installation Conditions: ' - f'{cmp_meta["Seismic_Installation_Conditions"]}' - ) - - if cmp_meta['Comments__Notes'] != 'None': - comments += f'\nNotes: {cmp_meta["Comments__Notes"]}' - - if cmp_meta['Author'] not in ['Not Given', 'By User']: - comments += f'\nAuthor: {cmp_meta["Author"]}' - - # get the suggested block size and replace the misleading values with ea - block_size = cmp_meta['Fragility_Unit_of_Measure'].split(' ')[::-1] - - meta_data = { - "Description": cmp_meta['Component_Name'], - "Comments": comments, - "SuggestedComponentBlockSize": ' '.join(block_size), - "RoundUpToIntegerQuantity": cmp_meta['Round_to_Integer_Unit'], - "ControllingDemand": "Damage Quantity", - "DamageStates": {}, - } - - # Handle components with simultaneous damage states separately - if 'Simul' in cmp.DS_Hierarchy: - pass - # Note that we are assuming that components with simultaneous - # damage states do not have damage that would trigger a red tag. - # This assumption holds for the second edition of FEMA P58, but it - # might need to be revisited in future editions. - - # for every other component... - else: - # now look at each Damage State - for DS_i in range(1, 6): - redtag_flag = getattr(cmp, f'DS_{DS_i}_Unsafe_Placard_Trigger_Flag') - - if redtag_flag is True: - theta_0 = getattr( - cmp, f'DS_{DS_i}_Unsafe_Placard_Damage_' f'Median' - ) - theta_1 = getattr( - cmp, f'DS_{DS_i}_Unsafe_Placard_Damage_' f'Dispersion' - ) - - if theta_0 != 0.0: - df_db.loc[cmp.Index, f'DS{DS_i}-Family'] = 'lognormal' - - df_db.loc[cmp.Index, f'DS{DS_i}-Theta_0'] = theta_0 - - df_db.loc[cmp.Index, f'DS{DS_i}-Theta_1'] = theta_1 - - if pd.isna(theta_0) or pd.isna(theta_1): - incomplete = True - - if ~np.isnan(redtag_flag): - meta_data['DamageStates'].update( - { - f"DS{DS_i}": { - "Description": cmp_meta[f"DS_{DS_i}_Description"] - } - } - ) - - df_db.loc[cmp.Index, 'Incomplete'] = int(incomplete) - - # store the metadata for this component - meta_dict.update({cmpID: meta_data}) - - # assign the Index column as the new ID - df_db.set_index('Index', inplace=True) - - # review the database and drop rows with no information - cmp_to_drop = [] - for cmp in df_db.index: - empty = True - - for DS_i in range(1, 6): - if not pd.isna(df_db.loc[cmp, f'DS{DS_i}-Family']): - empty = False - break - - if empty: - cmp_to_drop.append(cmp) - - df_db.drop(cmp_to_drop, axis=0, inplace=True) - cmp_kept = df_db.index.get_level_values(0).unique() - - cmp_to_drop = [] - for cmp in meta_dict: - if cmp not in cmp_kept: - cmp_to_drop.append(cmp) - - for cmp in cmp_to_drop: - del meta_dict[cmp] - - # convert to optimal datatypes to reduce file size - df_db = df_db.convert_dtypes() - - # rename the index - df_db.index.name = "ID" - - # save the consequence data - df_db.to_csv(target_data_file) - - # save the metadata - with open(target_meta_file, 'w+', encoding='utf-8') as f: - json.dump(meta_dict, f, indent=2) - - print("Successfully parsed and saved the red tag consequence data from FEMA P58") - - def create_Hazus_EQ_fragility_db( source_file, meta_file='', @@ -2347,139 +2084,6 @@ def create_Hazus_EQ_repair_db( print("Successfully parsed and saved the repair consequence data from Hazus EQ") -def create_Hazus_EQ_bldg_injury_db( - source_file, - target_data_file='bldg_injury_DB_Hazus_EQ.csv', - target_meta_file='bldg_injury_DB_Hazus_EQ.json', -): - """ - Create a database file based on the HAZUS EQ Technical Manual - - This method was developed to process a json file with tabulated data from - v4.2.3 of the Hazus Earthquake Technical Manual. The json file is included - under data_sources in the SimCenter DB_DamageAndLoss repo on GitHub. - - Parameters - ---------- - source_file: string - Path to the Hazus database file. - target_data_file: string - Path where the injury DB file should be saved. A csv file is - expected. - target_meta_file: string - Path where the injury DB metadata should be saved. A json file is - expected. - - """ - - # parse the source file - with open(source_file, 'r', encoding='utf-8') as f: - raw_data = json.load(f) - - # # parse the extra metadata file - # if Path(meta_file).is_file(): - # with open(meta_file, 'r') as f: - # frag_meta = json.load(f) - # else: - # frag_meta = {} - - # prepare lists of labels for various building features - building_types = list( - raw_data['Structural_Fragility_Groups']['P_collapse'].keys() - ) - - # initialize the output loss table - # define the columns - out_cols = [ - "Incomplete", - "Quantity-Unit", - "DV-Unit", - ] - for DS_i in range(1, 6): - out_cols += [ - f"DS{DS_i}-Theta_0", - ] - - # create the MultiIndex - cmp_types = ['STR', 'LF'] - comps = [f'{cmp_type}.{bt}' for cmp_type in cmp_types for bt in building_types] - DVs = ['S1', 'S2', 'S3', 'S4'] - df_MI = pd.MultiIndex.from_product([comps, DVs], names=['ID', 'DV']) - - df_db = pd.DataFrame(columns=out_cols, index=df_MI, dtype=float) - - # First, prepare the structural damage consequences - S_data = raw_data['Structural_Fragility_Groups'] - - for bt in building_types: - # create the component id - cmp_id = f'STR.{bt}' - - # store the consequence values for each Damage State - for DS_i in range(1, 6): - # DS5 is stored under 'collapse' - if DS_i == 5: - ds_i = 'Collapse' - else: - ds_i = f'DS{DS_i}' - - for S_i in range(1, 5): - s_label = f'S{S_i}' - df_db.loc[(cmp_id, s_label), f'DS{DS_i}-Theta_0'] = S_data[ - 'Injury_rates' - ][ds_i][bt][S_i - 1] - - # Second, the lifeline facilities - - for bt in building_types: - # create the component id - cmp_id = f'STR.{bt}' - - # store the consequence values for each Damage State - for DS_i in range(1, 6): - # DS5 is stored under 'collapse' - if DS_i == 5: - ds_i = 'Collapse' - else: - ds_i = f'DS{DS_i}' - - for S_i in range(1, 5): - s_label = f'S{S_i}' - df_db.loc[(cmp_id, s_label), f'DS{DS_i}-Theta_0'] = S_data[ - 'Injury_rates' - ][ds_i][bt][S_i - 1] - - # remove empty rows - df_db.dropna(how='all', inplace=True) - - # All Hazus components have complete fragility info, - df_db.loc[:, 'Incomplete'] = 0 - - # The damage quantity unit is the same for all consequence values - df_db.loc[:, 'Quantity-Unit'] = "1 EA" - - # The output units are also identical among all components - df_db.loc[:, 'DV-Unit'] = "injury_rate" - - # convert to simple index - df_db = base.convert_to_SimpleIndex(df_db, 0) - - # rename the index - df_db.index.name = "ID" - - # convert to optimal datatypes to reduce file size - df_db = df_db.convert_dtypes() - - # save the consequence data - df_db.to_csv(target_data_file) - - # save the metadata - later - # with open(target_meta_file, 'w+') as f: - # json.dump(meta_dict, f, indent=2) - - print("Successfully parsed and saved the injury consequence data from Hazus EQ") - - def create_Hazus_HU_fragility_db( source_file: str = ( 'pelicun/resources/SimCenterDBDL/' 'damage_DB_SimCenter_Hazus_HU_bldg.csv' From a6c06fd5a94735696db6cbb861374780d6bf2d5e Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Thu, 18 Apr 2024 12:16:58 -0700 Subject: [PATCH 13/22] Replace unused variable name with generic `_` `_` is commonly used for representing unused variables and makes our linters happy. --- pelicun/model/damage_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pelicun/model/damage_model.py b/pelicun/model/damage_model.py index f29866a4e..215cc4de2 100644 --- a/pelicun/model/damage_model.py +++ b/pelicun/model/damage_model.py @@ -988,7 +988,7 @@ def get_num_blocks(key): else: # otherwise assume 1 block regardless of # ('cmp', 'loc', 'dir', 'uid) key - def get_num_blocks(key): + def get_num_blocks(_): return 1.00 # ('cmp', 'loc', 'dir', 'uid', 'block') -> damage state series From b6273bbd877e95b0769669b272cae5864ab4f0d9 Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Thu, 18 Apr 2024 12:18:03 -0700 Subject: [PATCH 14/22] Remove unnecessary `else` after `return` and deindent --- pelicun/model/damage_model.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/pelicun/model/damage_model.py b/pelicun/model/damage_model.py index 215cc4de2..17fe405d3 100644 --- a/pelicun/model/damage_model.py +++ b/pelicun/model/damage_model.py @@ -1003,23 +1003,20 @@ def get_num_blocks(_): continue if dropzero and ds == 0: continue - else: + dmg_qnt_vals = np.where( + damage_state_series.values == ds, + component_quantities[component, location, direction, uid].values + / get_num_blocks((component, location, direction, uid)), + 0.00, + ) + if -1 in damage_state_set: dmg_qnt_vals = np.where( - damage_state_series.values == ds, - component_quantities[ - component, location, direction, uid - ].values - / get_num_blocks((component, location, direction, uid)), - 0.00, + damage_state_series.values != -1, dmg_qnt_vals, np.nan ) - if -1 in damage_state_set: - dmg_qnt_vals = np.where( - damage_state_series.values != -1, dmg_qnt_vals, np.nan - ) - dmg_qnt_series = pd.Series(dmg_qnt_vals) - dmg_qnt_series_collection[ - (component, location, direction, uid, block, str(ds)) - ] = dmg_qnt_series + dmg_qnt_series = pd.Series(dmg_qnt_vals) + dmg_qnt_series_collection[ + (component, location, direction, uid, block, str(ds)) + ] = dmg_qnt_series damage_quantities = pd.concat( dmg_qnt_series_collection.values(), From fa8cb6ad3d18048a30a380cbdb7db42f6930b56e Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Thu, 18 Apr 2024 12:24:51 -0700 Subject: [PATCH 15/22] Remove unused argument (PGB) Remove unused argument (PGB) from `DamageModel._prepare_dmg_quantities` --- pelicun/model/damage_model.py | 7 ++----- pelicun/tests/test_model.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pelicun/model/damage_model.py b/pelicun/model/damage_model.py index 17fe405d3..9f49f7b41 100644 --- a/pelicun/model/damage_model.py +++ b/pelicun/model/damage_model.py @@ -935,7 +935,7 @@ def _evaluate_damage_state( return ds_sample - def _prepare_dmg_quantities(self, PGB, damage_state_sample, dropzero=True): + def _prepare_dmg_quantities(self, damage_state_sample, dropzero=True): """ Combine component quantity and damage state information in one DataFrame. @@ -946,9 +946,6 @@ def _prepare_dmg_quantities(self, PGB, damage_state_sample, dropzero=True): Parameters ---------- - PGB: DataFrame - A DataFrame that contains the number of blocks for each - component. damage_state_sample: DataFrame A DataFrame that assigns a damage state to each component block in the asset model. @@ -1577,7 +1574,7 @@ def calculate_internal( ) qnt_sample = self._prepare_dmg_quantities( - pg_batch.reset_index('Batch', drop=True), ds_sample, dropzero=False + ds_sample, dropzero=False ) # If requested, extend the quantity table with all possible DSs diff --git a/pelicun/tests/test_model.py b/pelicun/tests/test_model.py index 11e1e8cfe..23554ebc3 100644 --- a/pelicun/tests/test_model.py +++ b/pelicun/tests/test_model.py @@ -1356,7 +1356,7 @@ def test__evaluate_damage_state_and_prepare_dmg_quantities( ) qnt_sample = damage_model._prepare_dmg_quantities( - PGB, ds_sample, dropzero=False + ds_sample, dropzero=False ) # note: the realized number of damage states is random, limiting From a95a85662a57d17e74c1d659eac8f216e08bce6c Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Thu, 18 Apr 2024 12:46:02 -0700 Subject: [PATCH 16/22] Remove inconsistent `return` If there is no sample `aggregate_losses` returns None. Aggregating losses without a repair sample is unusual and we need to rethink what should be done in that case instad of returning None, which is a bit unintuitive and turns the output type of the method to optional, making things more complicated. --- pelicun/model/loss_model.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index 36fa03cd9..ee5aae8fb 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -1006,9 +1006,6 @@ def aggregate_losses(self): DV = self.sample - if DV is None: - return - # group results by DV type and location DVG = DV.groupby(level=[0, 4], axis=1).sum() From 252e2b1f65eaaf9059d52d59d3ad6263c1ed9381 Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Sat, 20 Apr 2024 09:19:14 -0700 Subject: [PATCH 17/22] Update docstrings Addresses most pylint warnings after having enabled `pylint.extensions.docparams`. Left a few relating to empty property methods that are about to be removed. --- pelicun/assessment.py | 29 +++- pelicun/auto.py | 46 +++-- pelicun/base.py | 307 ++++++++++++++++++++++++++++++--- pelicun/db.py | 10 ++ pelicun/file_io.py | 100 ++++++----- pelicun/model/asset_model.py | 286 +++++++++++++++++++++++++++--- pelicun/model/damage_model.py | 153 ++++++++++++++-- pelicun/model/demand_model.py | 153 ++++++++++++++-- pelicun/model/loss_model.py | 124 +++++++++++-- pelicun/model/pelicun_model.py | 6 +- 10 files changed, 1068 insertions(+), 146 deletions(-) diff --git a/pelicun/assessment.py b/pelicun/assessment.py index 4a303b3fd..faca36e8c 100644 --- a/pelicun/assessment.py +++ b/pelicun/assessment.py @@ -163,15 +163,24 @@ def repair(self): def get_default_data(self, data_name): """ - Load a default data file and pass it to the user. + Loads a default data file by name and returns it. This method + is specifically designed to access predefined CSV files from a + structured directory path related to the SimCenter fragility + library. Parameters ---------- - data_name: string - Name of the csv file to be loaded + data_name : str + The name of the CSV file to be loaded, without the '.csv' + extension. This name is used to construct the full path to + the file. + Returns + ------- + pd.DataFrame + The DataFrame containing the data loaded from the + specified CSV file. """ - data_path = f'{base.pelicun_path}/resources/SimCenterDBDL/{data_name}.csv' return file_io.load_data( @@ -187,6 +196,11 @@ def get_default_metadata(self, data_name): data_name: string Name of the json file to be loaded + Returns + ------- + dict + Default metadata + """ data_path = f'{base.pelicun_path}/resources/SimCenterDBDL/{data_name}.json' @@ -210,7 +224,7 @@ def calc_unit_scale_factor(self, unit): Returns ------- - scale_factor: float + float Scale factor that convert values from unit to base unit Raises @@ -251,6 +265,11 @@ def scale_factor(self, unit): unit: str A unit name. + Returns + ------- + float + Scale factor + Raises ------ ValueError diff --git a/pelicun/auto.py b/pelicun/auto.py index e7a1ea864..b440f695f 100644 --- a/pelicun/auto.py +++ b/pelicun/auto.py @@ -52,28 +52,50 @@ import importlib from pathlib import Path -from . import base +from pelicun import base def auto_populate( config, auto_script_path, **kwargs # pylint: disable=unused-argument ): """ - Automatically populates the DL configuration for a Pelicun - calculation. + Automatically populates the Damage and Loss (DL) configuration for + a Pelicun calculation using predefined rules. + + This function modifies the provided configuration dictionary based + on an external Python script that defines auto-population + rules. It supports using built-in scripts or custom scripts + specified by the user. Parameters ---------- - config: dict - Configuration dictionary with a GeneralInformation key that - holds another dictionary with attributes of the asset of - interest. - auto_script_path: string - Path pointing to a python script with the auto-population - rules. Built-in scripts can be referenced using the - PelicunDefault/XY format where XY is the name of the script. + config : dict + A configuration dictionary with a 'GeneralInformation' key + that holds another dictionary with attributes of the asset of + interest. This dictionary is modified in-place with + auto-populated values. + auto_script_path : str + The path pointing to a Python script with the auto-population + rules. Built-in scripts can be referenced using the + 'PelicunDefault/XY' format where 'XY' is the name of the + script. + + Returns + ------- + tuple + A tuple containing two items: + 1. Updated configuration dictionary including new 'DL' key + with damage and loss information. + 2. A dictionary of component quantities (CMP) derived from the + auto-population process. + + Raises + ------ + ValueError + If the configuration dictionary does not contain necessary + asset information under 'GeneralInformation'. """ - + # try to get the AIM attributes AIM = config.get('GeneralInformation', None) if AIM is None: diff --git a/pelicun/base.py b/pelicun/base.py index 094a73633..dbb718eae 100644 --- a/pelicun/base.py +++ b/pelicun/base.py @@ -215,6 +215,11 @@ def nondir_multi(self, EDP_type): EDP_type: str EDP type (e.g. "PFA", "PFV", ..., "ALL") + Returns + ------- + float + Nondirectional component multiplicative factor. + Raises ------ ValueError @@ -242,7 +247,12 @@ def nondir_multi(self, EDP_type): @property def seed(self): """ - seed property + Seed property + + Returns + ------- + float + Seed value """ return self._seed @@ -258,6 +268,11 @@ def seed(self, value): def rng(self): """ rng property + + Returns + ------- + Generator + Random generator """ return self._rng @@ -265,6 +280,11 @@ def rng(self): def units_file(self): """ units file property + + Returns + ------- + str + Units file """ return self._units_file @@ -336,6 +356,11 @@ def __init__(self, verbose, show_warnings, log_show_ms, log_file, print_log): def verbose(self): """ verbose property + + Returns + ------- + bool + Verbose property value """ return self._verbose @@ -350,6 +375,11 @@ def verbose(self, value): def show_warnings(self): """ show_warnings property + + Returns + ------- + bool + show_warnings value """ return self._show_warnings @@ -366,6 +396,10 @@ def show_warnings(self, value): def log_show_ms(self): """ log_show_ms property + + Returns + bool + log_show_ms value """ return self._log_show_ms @@ -382,6 +416,11 @@ def log_show_ms(self, value): def log_pref(self): """ log_pref property + + Returns + ------- + str + log_pref value """ return self._log_pref @@ -389,6 +428,11 @@ def log_pref(self): def log_div(self): """ log_div property + + Returns + ------- + str + log_div value """ return self._log_div @@ -565,6 +609,11 @@ def control_warnings(show): def load_default_options(): """ Load the default_config.json file to set options to default values + + Returns + ------- + dict + Default options """ with open( @@ -667,14 +716,14 @@ def merge_default_config(user_config): not include some option available in the default options, then the default option is used in the merged config. - Parameters. + Parameters ---------- user_config: dict User-specified configuration dictionary Returns ------- - user_config: dict + dict Merged configuration dictionary """ @@ -713,7 +762,7 @@ def convert_to_SimpleIndex(data, axis=0, inplace=False): Returns ------- - data: DataFrame + DataFrame The modified DataFrame Raises @@ -786,7 +835,7 @@ def convert_to_MultiIndex(data, axis=0, inplace=False): Returns ------- - data: DataFrame + DataFrame The modified DataFrame. Raises @@ -873,7 +922,14 @@ def convert_dtypes(dataframe): def show_matrix(data, use_describe=False): """ - Print a matrix in a nice way using a DataFrame + Print a matrix in a nice way using a DataFrame. + Parameters + ---------- + data : array-like + The matrix data to display. Can be any array-like structure that pandas can convert to a DataFrame. + use_describe : bool, default: False + If True, provides a descriptive statistical summary of the matrix including specified percentiles. + If False, simply prints the matrix as is. """ if use_describe: pp.pprint( @@ -885,7 +941,29 @@ def show_matrix(data, use_describe=False): def _warning(message, category, filename, lineno, file=None, line=None): """ - Monkeypatch warnings to get prettier messages + Custom warning function to format and print warnings more + attractively. This function modifies how warning messages are + displayed, emphasizing the file path and line number from where + the warning originated. + + Parameters + ---------- + message : str + The warning message to be displayed. + category : Warning + The category of the warning (unused, but required for + compatibility with standard warning signature). + filename : str + The path of the file from which the warning is issued. The + function simplifies the path for display. + lineno : int + The line number in the file at which the warning is issued. + file : file-like object, optional + The target file object to write the warning to (unused, but + required for compatibility with standard warning signature). + line : str, optional + Line of code causing the warning (unused, but required for + compatibility with standard warning signature). """ # pylint:disable = unused-argument if '\\' in filename: @@ -910,7 +988,30 @@ def describe( df, percentiles=(0.001, 0.023, 0.10, 0.159, 0.5, 0.841, 0.90, 0.977, 0.999) ): """ - Provide descriptive statistics. + Provides extended descriptive statistics for given data, including + percentiles and log standard deviation for applicable columns. + + This function accepts both pandas Series and DataFrame objects + directly, or any array-like structure which can be converted to + them. It calculates common descriptive statistics and optionally + adds log standard deviation for columns where all values are + positive. + + Parameters + ---------- + df : pd.Series, pd.DataFrame, or array-like + The data to describe. If array-like, it is converted to a + DataFrame or Series before analysis. + percentiles : tuple of float, optional + Specific percentiles to include in the output. Default + includes an extensive range tailored to provide a detailed + summary. + + Returns + ------- + pd.DataFrame + A DataFrame containing the descriptive statistics of the input + data, transposed so that each descriptive statistic is a row. """ if not isinstance(df, (pd.Series, pd.DataFrame)): vals = df @@ -940,7 +1041,32 @@ def describe( def str2bool(v): """ - Converts various bool-like forms of string to actual booleans + Converts a string representation of truth to boolean True or + False. + + This function is designed to convert string inputs that represent + boolean values into actual Python boolean types. It handles + typical representations of truthiness and falsiness, and is case + insensitive. + + Parameters + ---------- + v : str or bool + The value to convert into a boolean. This can be a boolean + itself (in which case it is simply returned) or a string that + is expected to represent a boolean value. + + Returns + ------- + bool + The boolean value corresponding to the input. + + Raises + ------ + argparse.ArgumentTypeError + If `v` is a string that does not correspond to a boolean + value, an error is raised indicating that a boolean value was + expected. """ # courtesy of Maxim @ stackoverflow @@ -965,7 +1091,7 @@ def float_or_None(string): Returns ------- - res: float, optional + float or None A float, if the given string can be converted to a float. Otherwise, it returns None """ @@ -988,7 +1114,7 @@ def int_or_None(string): Returns ------- - res: int, optional + int or None An int, if the given string can be converted to an int. Otherwise, it returns None """ @@ -1001,7 +1127,35 @@ def int_or_None(string): def process_loc(string, stories): """ - Parses the location parameter. + Parses the 'location' parameter from input to determine the + specific locations to be processed. This function interprets + various string formats to output a list of integers representing + locations. + + Parameters + ---------- + string : str + A string that describes the location or range of locations of + the asset. It can be a single number, a range (e.g., '3-7'), + 'all', 'top', 'roof', or 'last'. + stories : int + The total number of locations in the asset, used to interpret + relative terms like 'top' or 'roof', or to generate a range + for 'all'. + + Returns + ------- + list of int or None + A list of integers representing each floor specified by the + string. Returns None if the string does not conform to + expected formats. + + Raises + ------ + ValueError + Raises an exception if the string contains a range that is not + interpretable (e.g., non-integer values or logical + inconsistencies in the range). """ try: res = int(string) @@ -1029,12 +1183,27 @@ def process_loc(string, stories): def dedupe_index(dataframe, dtype=str): """ - Adds an extra level to the index of a dataframe so that all - resulting index elements are unique. Assumes that the original - index is a MultiIndex with specified names. + Modifies the index of a DataFrame to ensure all index elements are + unique by adding an extra level. Assumes that the DataFrame's + original index is a MultiIndex with specified names. A unique + identifier ('uid') is added as an additional index level based on + the cumulative count of occurrences of the original index + combinations. - """ + Parameters + ---------- + dataframe : pd.DataFrame + The DataFrame whose index is to be modified. It must have a + MultiIndex. + dtype : type, optional + The data type for the new index level 'uid'. Defaults to str. + Notes + ----- + This function changes the DataFrame in place, hence it does not + return the DataFrame but modifies the original one provided. + + """ inames = dataframe.index.names dataframe.reset_index(inplace=True) dataframe['uid'] = (dataframe.groupby([*inames]).cumcount()).astype(dtype) @@ -1084,13 +1253,44 @@ def dedupe_index(dataframe, dtype=str): def dict_raise_on_duplicates(ordered_pairs): """ - Reject duplicate keys. + Constructs a dictionary from a list of key-value pairs, raising an + exception if duplicate keys are found. - https://stackoverflow.com/questions/14902299/ - json-loads-allows-duplicate-keys- - in-a-dictionary-overwriting-the-first-value + This function ensures that no two pairs have the same key. It is + particularly useful when parsing JSON-like data where unique keys + are expected but not enforced by standard parsing methods. + Parameters + ---------- + ordered_pairs : list of tuples + A list of tuples, each containing a key and a value. Keys are + expected to be unique across the list. + + Returns + ------- + dict + A dictionary constructed from the ordered_pairs without any + duplicates. + + Raises + ------ + ValueError + If a duplicate key is found in the input list, a ValueError is + raised with a message indicating the duplicate key. + + Examples + -------- + >>> dict_raise_on_duplicates( + ... [("key1", "value1"), ("key2", "value2"), ("key1", "value3")] + ... ) + ValueError: duplicate key: key1 + + Notes + ----- + This implementation is useful for contexts in which data integrity + is crucial and key uniqueness must be ensured. """ + d = {} for k, v in ordered_pairs: if k in d: @@ -1109,6 +1309,16 @@ def parse_units(custom_file=None, preserve_categories=False): If a custom file is provided, only the units specified in the custom file are used. + Returns + ------- + dict + A dictionary where keys are unit names and values are + their corresponding conversion factors. If + `preserve_categories` is True, the dictionary may maintain + its original nested structure based on the JSON file. If + `preserve_categories` is False, the dictionary is flattened + to have globally unique unit names. + Raises ------ KeyError @@ -1122,6 +1332,57 @@ def parse_units(custom_file=None, preserve_categories=False): """ def get_contents(file_path, preserve_categories=False): + """ + Parses a unit conversion factors JSON file and returns a + dictionary mapping unit names to conversion factors. + + This function allows the use of a custom JSON file for + defining unit conversion factors or defaults to a predefined + file. It ensures that each unit name is unique and that all + conversion factors are float values. Additionally, it supports + the option to preserve the original data types of category + values from the JSON. + + Parameters + ---------- + file_path : str + The file path to a JSON file containing unit conversion + factors. If not provided, a default file is used. + preserve_categories : bool, optional + If True, maintains the original data types of category + values from the JSON file. If False, converts all values + to floats and flattens the dictionary structure, ensuring + that each unit name is globally unique across categories. + + Returns + ------- + dict + A dictionary where keys are unit names and values are + their corresponding conversion factors. If + `preserve_categories` is True, the dictionary may maintain + its original nested structure based on the JSON file. + + Raises + ------ + FileNotFoundError + If the specified file does not exist. + ValueError + If a unit name is duplicated, a conversion factor is not a + float, or other JSON structure issues are present. + json.decoder.JSONDecodeError + If the file is not a valid JSON file. + TypeError + If any value that needs to be converted to float cannot be + converted. + + Examples + -------- + >>> parse_units('custom_units.json') + { 'm': 1.0, 'cm': 0.01, 'mm': 0.001 } + + >>> parse_units('custom_units.json', preserve_categories=True) + { 'Length': {'m': 1.0, 'cm': 0.01, 'mm': 0.001} } + """ try: with open(file_path, 'r', encoding='utf-8') as f: dictionary = json.load(f, object_pairs_hook=dict_raise_on_duplicates) @@ -1194,16 +1455,16 @@ def convert_units( Returns ------- - (float | list[float] | np.ndarray): + float or list[float] or np.ndarray The converted value(s) in the target unit, in the same data type as the input values. Raises ------ - TypeError: + TypeError If the input `values` are not of type float, list, or np.ndarray. - ValueError: + ValueError If the `unit`, `to_unit`, or `category` is unknown or if `unit` and `to_unit` are not in the same category. diff --git a/pelicun/db.py b/pelicun/db.py index 30a8ab8f2..3ca2548f2 100644 --- a/pelicun/db.py +++ b/pelicun/db.py @@ -75,6 +75,16 @@ def parse_DS_Hierarchy(DSH): """ Parses the FEMA P58 DS hierarchy into a set of arrays. + + Parameters + ---------- + DSH: str + Damage state hierarchy + + Returns + ------- + list + Damage state setup """ if DSH[:3] == 'Seq': DSH = DSH[4:-1] diff --git a/pelicun/file_io.py b/pelicun/file_io.py index 7fcf79d34..0f099259f 100644 --- a/pelicun/file_io.py +++ b/pelicun/file_io.py @@ -101,54 +101,63 @@ def save_to_csv( log=None, ): """ - Saves data to a CSV file following standard SimCenter schema. + Saves data to a CSV file following the standard SimCenter schema. - The produced CSV files have a single header line and an index column. The - second line may start with 'Units' in the index or the first column may be - 'Units' to provide the units for the data in the file. + The produced CSV files have a single header line and an index + column. The second line may start with 'Units' in the index or the + first column may be 'Units' to provide the units for the data in + the file. - The following data types in pelicun can be saved with this function: + The following data types in pelicun can be saved with this + function: - Demand Data: Each column in a table corresponds to a demand type; each - row corresponds to a simulation/sample. The header identifies each demand - type. The user guide section of the documentation provides more - information about the header format. Target need to be specified in the - second row of the DataFrame. + Demand Data: Each column in a table corresponds to a demand type; + each row corresponds to a simulation/sample. The header identifies + each demand type. The user guide section of the documentation + provides more information about the header format. Target need to + be specified in the second row of the DataFrame. Parameters ---------- - data: DataFrame - The data to save - filepath: string - The location of the destination file. If None, the data is not saved, - but returned in the end. - units: Series, optional + data : DataFrame + The data to save. + filepath : str + The location of the destination file. If None, the data is not + saved, but returned in the end. + units : Series, optional Provides a Series with variables and corresponding units. - unit_conversion_factors: dict + unit_conversion_factors : dict, optional Dictionary containing key-value pairs of unit names and their - corresponding factors. Conversion factors are defined as the number of - times a base unit fits in the alternative unit. - orientation: int, {0, 1}, default: 0 - If 0, variables are organized along columns; otherwise they are along - the rows. This is important when converting values to follow the - prescribed units. - use_simpleindex: bool, default: True - If True, MultiIndex columns and indexes are converted to SimpleIndex - before saving - log: Logger - Logger object to be used. If no object is specified, no logging - is performed. + corresponding factors. Conversion factors are defined as the + number of times a base unit fits in the alternative unit. + orientation : int, {0, 1}, default 0 + If 0, variables are organized along columns; otherwise, they + are along the rows. This is important when converting values + to follow the prescribed units. + use_simpleindex : bool, default True + If True, MultiIndex columns and indexes are converted to + SimpleIndex before saving. + log : Logger, optional + Logger object to be used. If no object is specified, no + logging is performed. Raises ------ ValueError - If units is not None but unit_conversion_factors is None + If units is not None but unit_conversion_factors is None. ValueError If writing to a file fails. ValueError If the provided file name does not have the `.csv` suffix. - """ + Returns + ------- + DataFrame or None + If `filepath` is None, returns the DataFrame with potential + unit conversions and reformatting applied. Otherwise, returns + None after saving the data to a CSV file. + """ + if filepath is None: if log: log.msg('Preparing data ...', prepend_timestamp=False) @@ -357,11 +366,19 @@ def load_data( Returns ------- - data: DataFrame - Parsed data. - units: Series - Labels from the data and corresponding units specified in the - data. Units are only returned if return_units is set to True. + tuple + data: DataFrame + Parsed data. + units: Series + Labels from the data and corresponding units specified in the + data. Units are only returned if return_units is set to True. + + Raises + ------ + TypeError + If `data_source` is neither a string nor a DataFrame, a TypeError is raised. + ValueError + If `unit_conversion_factors` contains keys that do not correspond to any units in the data, a ValueError may be raised during processing. """ if isinstance(data_source, pd.DataFrame): @@ -461,11 +478,12 @@ def load_from_file(filepath, log=None): Returns ------- - data: DataFrame - Data loaded from the file. - log: Logger - Logger object to be used. If no object is specified, no logging - is performed. + tuple + data: DataFrame + Data loaded from the file. + log: Logger + Logger object to be used. If no object is specified, no logging + is performed. Raises ------ diff --git a/pelicun/model/asset_model.py b/pelicun/model/asset_model.py index 3907ca692..9cf1f3747 100644 --- a/pelicun/model/asset_model.py +++ b/pelicun/model/asset_model.py @@ -82,10 +82,25 @@ def __init__(self, assessment): @property def cmp_sample(self): """ - Assigns the _cmp_sample attribute if it is None and returns - the component sample. - """ + A property that gets or creates a DataFrame representing the + component sample for the current assessment. + + If the component sample has not been previously set or + generated, this property will generate it by retrieving + samples from the component random variables (_cmp_RVs), + sorting the indexes, and converting the DataFrame to use a + MultiIndex. The component sample is structured to include + information on component ('cmp'), location ('loc'), direction + ('dir'), and unique identifier ('uid'). + + Returns + ------- + DataFrame + A DataFrame containing the component samples, indexed and + sorted appropriately. The columns are multi-indexed to + represent various dimensions of the component data. + """ if self._cmp_sample is None: cmp_sample = pd.DataFrame(self._cmp_RVs.RV_sample) cmp_sample.sort_index(axis=0, inplace=True) @@ -104,10 +119,50 @@ def cmp_sample(self): def save_cmp_sample(self, filepath=None, save_units=False): """ - Save component quantity sample to a csv file + Saves the component quantity sample to a CSV file or returns + it as a DataFrame with optional units. - """ + This method handles the storage of a sample of component + quantities, which can either be saved directly to a file or + returned as a DataFrame for further manipulation. When saving + to a file, additional information such as unit conversion + factors and column units can be included. If the data is not + being saved to a file, the method can return the DataFrame + with or without units as specified. + Parameters + ---------- + filepath : str, optional + The path to the file where the component quantity sample + should be saved. If not provided, the sample is not saved + to disk but returned. + save_units : bool, default: False + Indicates whether to include a row with unit information + in the returned DataFrame. This parameter is ignored if a + file path is provided. + + Returns + ------- + None or tuple + If `filepath` is provided, the function returns None after + saving the data. + If no `filepath` is specified, returns: + - DataFrame containing the component quantity sample. + - Optionally, a Series containing the units for each + column if `save_units` is True. + + Raises + ------ + IOError + Raises an IOError if there is an issue saving the file to + the specified `filepath`. + + Notes + ----- + The function utilizes internal logging to notify the start and + completion of the saving process. It adjusts index types and + handles unit conversions based on assessment configurations. + """ self.log_div() if filepath is not None: self.log_msg('Saving asset components sample...') @@ -146,10 +201,39 @@ def save_cmp_sample(self, filepath=None, save_units=False): def load_cmp_sample(self, filepath): """ - Load component quantity sample from a csv file + Loads a component quantity sample from a specified CSV file + into the system. - """ + This method reads a CSV file that contains component quantity + samples, setting up the necessary DataFrame structures within + the application. It also handles unit conversions using + predefined conversion factors and captures the units of each + component quantity from the CSV file. + Parameters + ---------- + filepath : str + The path to the CSV file from which to load the component + quantity sample. + + Notes + ----- + Upon successful loading, the method sets the component sample + and units as internal attributes of the class, making them + readily accessible for further operations. It also sets + appropriate column names for the DataFrame to match expected + indexing levels such as component ('cmp'), location ('loc'), + direction ('dir'), and unique identifier ('uid'). + + Examples + -------- + Assuming the filepath to the component sample CSV is known and + accessible: + + >>> model.load_cmp_sample('path/to/component_sample.csv') + # This will load the component quantity sample into the model + # from the specified file. + """ self.log_div() self.log_msg('Loading asset components sample...') @@ -172,27 +256,112 @@ def load_cmp_sample(self, filepath): def load_cmp_model(self, data_source): """ - Load the model that describes component quantities in the asset. + Loads the model describing component quantities in an asset + from specified data sources. + + This function is responsible for loading data related to the + component model of an asset. It supports loading from multiple + types of data sources. If the data source is a string, it is + treated as a prefix to filenames that contain the necessary + data. If it is a dictionary, it directly contains the data as + DataFrames. Parameters ---------- - data_source: string or dict - If string, the data_source is a file prefix ( in the - following description) that identifies the following files: - _marginals.csv, _empirical.csv, - _correlation.csv. If dict, the data source is a dictionary - with the following optional keys: 'marginals', 'empirical', and - 'correlation'. The value under each key shall be a DataFrame. - """ + data_source : str or dict + The source from where to load the component model data. If + it's a string, it should be the prefix for three files: + one for marginal distributions (`_marginals.csv`), + one for empirical data (`_empirical.csv`), and one + for correlation data (`_correlation.csv`). If it's + a dictionary, it should have keys 'marginals', + 'empirical', and 'correlation', with each key associated + with a DataFrame containing the corresponding data. + + Notes + ----- + The function utilizes helper functions to handle complex + location strings that can describe single locations, ranges, + lists, or special keywords like 'all', 'top', and + 'roof'. These are used to specify which parts of the asset the + data pertains to, especially useful in buildings with multiple + stories or sections. + + Examples + -------- + To load data using file prefixes: + + >>> model.load_cmp_model('path/to/data_prefix') + + To load data using a dictionary of DataFrames: + + >>> data_dict = { + 'marginals': df_marginals, + 'empirical': df_empirical, + 'correlation': df_correlation + } + >>> model.load_cmp_model(data_dict) + """ def get_locations(loc_str): + """ + Parses a location string to determine specific sections of + an asset to be processed. + + This function interprets various string formats to output + a list of strings representing sections or parts of the + asset. It can handle single numbers, ranges (e.g., + '3--7'), lists separated by commas (e.g., '1,2,5'), and + special keywords like 'all', 'top', or 'roof'. + + Parameters + ---------- + loc_str : str + A string that describes the location or range of + sections in the asset. It can be a single number, a + range, a comma-separated list, 'all', 'top', or + 'roof'. + + Returns + ------- + numpy.ndarray + An array of strings, each representing a section + number. These sections are processed based on the + input string, which can denote specific sections, + ranges of sections, or special keywords. + + Raises + ------ + ValueError + If the location string cannot be parsed into any + recognized format, a ValueError is raised with a + message indicating the problematic string. + + Examples + -------- + Given an asset with multiple sections: + + >>> get_locations('5') + array(['5']) + + >>> get_locations('3--7') + array(['3', '4', '5', '6', '7']) + + >>> get_locations('1,2,5') + array(['1', '2', '5']) + + >>> get_locations('all') + array(['1', '2', '3', ..., '10']) + + >>> get_locations('top') + array(['10']) + + >>> get_locations('roof') + array(['11']) + """ try: res = str(int(loc_str)) - return np.array( - [ - res, - ] - ) + return np.array([res]) except ValueError as exc: stories = self._asmnt.stories @@ -228,6 +397,56 @@ def get_locations(loc_str): ) from exc def get_directions(dir_str): + """ + Parses a direction string to determine specific + orientations or directions applicable within an asset. + + This function processes direction descriptions to output + an array of strings, each representing a specific + direction. It can handle single numbers, ranges (e.g., + '1--3'), lists separated by commas (e.g., '1,2,5'), and + null values that default to '1'. + + Parameters + ---------- + dir_str : str or None + A string that describes the direction or range of + directions in the asset. It can be a single number, a + range, a comma-separated list, or it can be null, + which defaults to representing a single default + direction ('1'). + + Returns + ------- + numpy.ndarray + An array of strings, each representing a + direction. These directions are processed based on the + input string, which can denote specific directions, + ranges of directions, or a list. + + Raises + ------ + ValueError + If the direction string cannot be parsed into any + recognized format, a ValueError is raised with a + message indicating the problematic string. + + Examples + -------- + Given an asset with multiple potential orientations: + + >>> get_directions(None) + array(['1']) + + >>> get_directions('2') + array(['2']) + + >>> get_directions('1--3') + array(['1', '2', '3']) + + >>> get_directions('1,2,5') + array(['1', '2', '5']) + """ if pd.isnull(dir_str): return np.ones(1).astype(str) @@ -256,6 +475,8 @@ def get_directions(dir_str): ) from exc def get_attribute(attribute_str, dtype=float, default=np.nan): + # pylint: disable=missing-return-doc + # pylint: disable=missing-return-type-doc if pd.isnull(attribute_str): return default return dtype(attribute_str) @@ -419,11 +640,26 @@ def _create_cmp_RVs(self): def generate_cmp_sample(self, sample_size=None): """ - Generates component quantity realizations. If a sample_size - is not specified, the sample size found in the demand model is - used. - """ + Generates a sample of component quantity realizations based on + predefined model parameters and optionally specified sample + size. If no sample size is provided, the function attempts to + use the sample size from an associated demand model. + Parameters + ---------- + sample_size: int, optional + The number of realizations to generate. If not specified, + the sample size is taken from the demand model associated + with the assessment. + + Raises + ------ + ValueError + If the model parameters are not loaded before sample + generation, or if neither sample size is specified nor can + be determined from the demand model. + """ + if self.cmp_marginal_params is None: raise ValueError( 'Model parameters have not been specified. Load' diff --git a/pelicun/model/damage_model.py b/pelicun/model/damage_model.py index 9f49f7b41..aed601650 100644 --- a/pelicun/model/damage_model.py +++ b/pelicun/model/damage_model.py @@ -93,8 +93,35 @@ def __init__(self, assessment): def save_sample(self, filepath=None, save_units=False): """ - Save damage sample to a csv file + Saves the damage sample data to a CSV file or returns it + directly with an option to include units. + This function handles saving the sample data of damage + assessments to a specified file path or, if no path is + provided, returns the data as a DataFrame. The function can + optionally include a row for unit information when returning + data. + + Parameters + ---------- + filepath : str, optional + The path to the file where the damage sample should be + saved. If not provided, the sample is not saved to disk + but returned. + save_units : bool, default: False + Indicates whether to include a row with unit information + in the returned DataFrame. This parameter is ignored if a + file path is provided. + + Returns + ------- + None or tuple + If `filepath` is provided, the function returns None after + saving the data. + If no `filepath` is specified, returns: + - DataFrame containing the damage sample. + - Optionally, a Series containing the units for each + column if `save_units` is True. """ self.log_div() self.log_msg('Saving damage sample...') @@ -248,9 +275,14 @@ def _handle_operation(self, initial_value, operation, other_value): Returns ------- - result: float + float The result of the operation + Raises + ------ + ValueError + If the operation is invalid. + """ if operation == '+': return initial_value + other_value @@ -293,14 +325,64 @@ def _create_dmg_RVs(self, PGB, scaling_specification=None): a float. The operation can be '+' for addition, '-' for subtraction, '*' for multiplication, and '/' for division. + Returns + ------- + tuple + A tuple containing two RandomVariableRegistry instances: + one for the capacity random variables and one for the LSDS + assignments. + + Raises + ------ + ValueError + Raises an error if the scaling specification is invalid or + if the input DataFrame does not meet the expected format. + Also, raises errors if there are any issues with the types + or formats of the data in the input DataFrame. """ def assign_lsds(ds_weights, ds_id, lsds_RV_reg, lsds_rv_tag): """ - Prepare random variables to handle mutually exclusive damage states. - + Assigns limit states to damage states using random + variables, updating the provided random variable registry. + This function either creates a deterministic random + variable for a single damage state or a multinomial random + variable for multiple damage states. + + Parameters + ---------- + ds_weights : str or None + A string representing the weights of different damage + states associated with a limit state, separated by + '|'. If None, indicates that there is only one damage + state associated with the limit state. + ds_id : int + The starting index for damage state IDs. This ID helps + in mapping damage states to limit states. + lsds_RV_reg : RandomVariableRegistry + The registry where the newly created random variables + (for mapping limit states to damage states) will be + added. + lsds_rv_tag : str + A unique identifier for the random variable being + created, typically including identifiers for + component, location, direction, and limit state. + + Returns + ------- + int + The updated damage state ID, incremented based on the + number of damage states handled in this call. + + Notes + ----- + This function supports detailed control over the mapping + from limit states to damage states within a stochastic + framework, enhancing the flexibility and accuracy of + probabilistic damage assessments. It dynamically adjusts + to the number of damage states specified and applies a + mapping function to correctly assign state IDs. """ - # If the limit state has a single damage state assigned # to it, we don't need random sampling if pd.isnull(ds_weights): @@ -321,6 +403,29 @@ def assign_lsds(ds_weights, ds_id, lsds_RV_reg, lsds_rv_tag): ) def map_ds(values, offset=int(ds_id + 1)): + """ + Maps an array of damage state indices to their + corresponding actual state IDs by applying an + offset. + + Parameters + ---------- + values : array-like + An array of indices representing damage + states. These indices are typically sequential + integers starting from zero. + offset : int + The value to be added to each element in + `values` to obtain the actual damage state + IDs. + + Returns + ------- + array + An array where each original index from + `values` has been incremented by `offset` to + reflect its actual damage state ID. + """ return values + offset lsds_RV_reg.add_RV( @@ -648,7 +753,7 @@ def _get_required_demand_type(self, PGB): Returns ------- - EDP_req: dict + dict A dictionary of EDP requirements, where each key is the EDP string (e.g., "Peak Ground Acceleration-0-0"), and the corresponding value is a list of tuples (component_id, @@ -839,7 +944,7 @@ def _evaluate_damage_state( Returns ------- - dmg_sample: DataFrame + DataFrame Assigns a Damage State to each component block in the asset model. """ @@ -955,12 +1060,13 @@ def _prepare_dmg_quantities(self, damage_state_sample, dropzero=True): Returns ------- - res_df: DataFrame + DataFrame A DataFrame that combines the component quantity and damage state information. """ + # pylint: disable=missing-return-doc if self._asmnt.log.verbose: self.log_msg('Calculating damage quantities...', prepend_timestamp=True) @@ -980,12 +1086,14 @@ def _prepare_dmg_quantities(self, damage_state_sample, dropzero=True): num_blocks = component_marginal_parameters['Blocks'].to_dict() def get_num_blocks(key): + # pylint: disable=missing-return-type-doc return float(num_blocks[key]) else: # otherwise assume 1 block regardless of # ('cmp', 'loc', 'dir', 'uid) key def get_num_blocks(_): + # pylint: disable=missing-return-type-doc return 1.00 # ('cmp', 'loc', 'dir', 'uid', 'block') -> damage state series @@ -1067,6 +1175,11 @@ def _perform_dmg_task(self, task, ds_sample): damage states of the components after the task has been performed. + Raises + ------ + ValueError + Raises an error if the source or target event descriptions + do not follow expected formats. """ if self._asmnt.log.verbose: @@ -1229,7 +1342,8 @@ def _perform_dmg_event_loc( def _get_pg_batches(self, block_batch_size): """ - Group performance groups into batches for efficient damage assessment. + Group performance groups into batches for efficient damage + assessment. The method takes as input the block_batch_size, which specifies the maximum number of blocks per batch. The method @@ -1253,6 +1367,23 @@ def _get_pg_batches(self, block_batch_size): direction, and returns a dataframe that shows the number of blocks for each batch. + Returns + ------- + DataFrame + A DataFrame indexed by batch number, component identifier, + location, direction, and unique ID, with a column + indicating the number of blocks assigned to each + batch. This dataframe facilitates the management and + execution of damage assessment tasks by grouping + components into manageable batches based on the specified + block batch size. + + Raises + ------ + Warning + Logs a warning if any performance groups do not have + corresponding damage model information and are therefore + excluded from the analysis. """ # Get the marginal parameters for the components from the @@ -1573,9 +1704,7 @@ def calculate_internal( "Damage processes successfully applied.", prepend_timestamp=False ) - qnt_sample = self._prepare_dmg_quantities( - ds_sample, dropzero=False - ) + qnt_sample = self._prepare_dmg_quantities(ds_sample, dropzero=False) # If requested, extend the quantity table with all possible DSs if self._asmnt.options.list_all_ds: diff --git a/pelicun/model/demand_model.py b/pelicun/model/demand_model.py index 474124c25..df3276a8a 100644 --- a/pelicun/model/demand_model.py +++ b/pelicun/model/demand_model.py @@ -109,6 +109,21 @@ def save_sample(self, filepath=None, save_units=False): """ Save demand sample to a csv file or return it in a DataFrame + Returns + ------- + None or tuple + If `filepath` is specified, the function saves the demand + sample to a CSV file and returns None. + If `filepath` is not specified, it returns the DataFrame + containing the demand sample. + If `save_units` is True, it returns a tuple of the + DataFrame and a Series containing the units. + + Raises + ------ + IOError + Raises an IOError if there is an issue saving the file to + the specified `filepath`. """ self.log_div() @@ -156,6 +171,35 @@ def load_sample(self, filepath): """ def parse_header(raw_header): + """ + Parses and cleans the header of a demand DataFrame from + raw multi-level index to a standardized format. + + This function adjusts the raw header of a DataFrame, + removing optional event IDs and whitespace, and + standardizing the format to facilitate further + processing. It is designed to handle headers with either + three or four levels, where the first level (event_ID) is + optional and not used in further analysis. Note: This + will soon change, and that first level will be enforced + instead of removed. + + Parameters + ---------- + raw_header : pd.MultiIndex + The original multi-level index (header) of the + DataFrame, which may contain an optional event_ID and + might have excess whitespace in the labels. + + Returns + ------- + pd.MultiIndex + A new MultiIndex for the DataFrame's columns that is + cleaned of any unwanted characters or levels. This + index has three levels: 'type', 'loc', and 'dir', + representing the type of demand, location, and + direction, respectively. + """ old_MI = raw_header # The first number (event_ID) in the demand labels is optional and @@ -240,18 +284,54 @@ def parse_header(raw_header): def estimate_RID(self, demands, params, method='FEMA P58'): """ - Estimate residual drift realizations based on other demands + Estimates residual inter-story drift (RID) realizations based + on peak inter-story drift (PID) and other demand parameters + using specified methods. + + This method calculates RID based on the peak inter-story drift + provided in the demands DataFrame and parameters such as yield + drift specified in the params dictionary. The calculation + adheres to the FEMA P-58 methodology, which includes + conditions for different ranges of drift. Parameters ---------- - demands: DataFrame - Sample of demands required for the method to estimate the RID values - params: dict - Parameters required for the method to estimate the RID values - method: {'FEMA P58'}, default: 'FEMA P58' - Method to use for the estimation - currently, only one is available. - """ + demands : DataFrame + A DataFrame containing samples of demands, specifically + peak inter-story drift (PID) values for various + location-direction pairs required for the estimation + method. + params : dict + A dictionary containing parameters required for the + estimation method, such as 'yield_drift', which is the + drift at which yielding is expected to occur. + method : str, optional + The method used to estimate the RID values. Currently, + only 'FEMA P58' is implemented. Defaults to 'FEMA P58'. + + Returns + ------- + DataFrame + A DataFrame containing the estimated residual inter-story + drift (RID) realizations, indexed and structured similarly + to the input demands DataFrame. + Raises + ------ + ValueError + Raises a ValueError if an unrecognized method is provided + or required parameters are missing in the `params` + dictionary. + + Notes + ----- + The FEMA P-58 estimation approach divides the drift into three + domains, with different transformation rules for + each. Additional stochastic variation is introduced to nonzero + RID values to model the inherent uncertainty. The method + ensures that the RID values do not exceed the corresponding + PID values. + """ if method == 'FEMA P58': # method is described in FEMA P-58 Volume 1 Section 5.4 & Appendix C @@ -324,6 +404,8 @@ def calibrate_model(self, config): def parse_settings(settings, demand_type): def parse_str_to_float(in_str, context_string): + # pylint: disable = missing-return-type-doc + # pylint: disable = missing-return-doc try: out_float = float(in_str) @@ -391,6 +473,8 @@ def parse_str_to_float(in_str, context_string): cal_df.loc[idx[cols, :, :], 'SigIncrease'] = sig_increase def get_filter_mask(lower_lims, upper_lims): + # pylint: disable=missing-return-doc + # pylint: disable=missing-return-type-doc demands_of_interest = demand_sample.iloc[:, pd.notna(upper_lims)] limits_of_interest = upper_lims[pd.notna(upper_lims)] upper_mask = np.all(demands_of_interest < limits_of_interest, axis=1) @@ -814,6 +898,8 @@ def clone_demands(self, demand_cloning): # turn the config entries to tuples def turn_to_tuples(demand_cloning): + # pylint: disable=missing-return-doc + # pylint: disable=missing-return-type-doc demand_cloning_tuples = {} for key, values in demand_cloning.items(): demand_cloning_tuples[tuple(key.split('-'))] = [ @@ -865,9 +951,56 @@ def turn_to_tuples(demand_cloning): def generate_sample(self, config): """ - Generates an RV sample with the specified configuration. - """ + Generates a sample of random variables (RVs) based on the + specified configuration for demand modeling. + + This method utilizes the current settings for marginal + distribution parameters to generate a sample of demand + variables. The configuration can specify details such as the + sample size, whether to preserve the order of raw data, and + whether to apply demand cloning. The generated sample is + stored internally and can be used for subsequent analysis. + + Parameters + ---------- + config : dict + A dictionary containing configuration options for the + sample generation. Key options include: + - 'SampleSize': The number of samples to generate. + - 'PreserveRawOrder': Boolean indicating whether to + preserve the order of the raw data. Defaults to False. + - 'DemandCloning': Specifies if and how demand cloning + should be applied. Can be a boolean or a detailed + configuration. + Raises + ------ + ValueError + If model parameters are not loaded or specified before + attempting to generate a sample. + + Notes + ----- + The function is responsible for the creation of random + variables based on the distribution parameters specified in + `marginal_params`. It ensures that the sample is properly + indexed and sorted according to type, location, and + direction. It also handles the configuration of demand cloning + if specified, which is a method to artificially augment the + variability in the sample based on existing realizations. + + Examples + -------- + >>> config = { + 'SampleSize': 1000, + 'PreserveRawOrder': True, + 'DemandCloning': False + } + >>> model.generate_sample(config) + # This will generate 1000 realizations of demand variables + # with the specified configuration. + """ + if self.marginal_params is None: raise ValueError( 'Model parameters have not been specified. Either' diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index ee5aae8fb..471f5e210 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -86,8 +86,43 @@ def __init__(self, assessment): def save_sample(self, filepath=None, save_units=False): """ - Save loss sample to a csv file + Saves the loss sample to a CSV file or returns it as a + DataFrame with optional units. + This method handles the storage of a sample of loss estimates, + which can either be saved directly to a file or returned as a + DataFrame for further manipulation. When saving to a file, + additional information such as unit conversion factors and + column units can be included. If the data is not being saved + to a file, the method can return the DataFrame with or without + units as specified. + + Parameters + ---------- + filepath : str, optional + The path to the file where the loss sample should be + saved. If not provided, the sample is not saved to disk + but returned. + save_units : bool, default: False + Indicates whether to include a row with unit information + in the returned DataFrame. This parameter is ignored if a + file path is provided. + + Returns + ------- + None or tuple + If `filepath` is provided, the function returns None after + saving the data. + If no `filepath` is specified, returns: + - DataFrame containing the loss sample. + - Optionally, a Series containing the units for each + column if `save_units` is True. + + Raises + ------ + IOError + Raises an IOError if there is an issue saving the file to + the specified `filepath`. """ self.log_div() if filepath is not None: @@ -343,18 +378,29 @@ def __init__(self, assessment): def _create_DV_RVs(self, case_list): """ - Prepare the random variables used for repair cost and time simulation. + Prepare the random variables associated with decision + variables, such as repair cost and time. Parameters ---------- case_list: MultiIndex - Index with cmp-loc-dir-ds descriptions that identify the RVs - we need for the simulation. + Index with cmp-loc-dir-ds descriptions that identify the + RVs we need for the simulation. + + Returns + ------- + RandomVariableRegistry or None + A RandomVariableRegistry containing all the generated + random variables necessary for the simulation. If no + random variables are generated (due to missing parameters + or conditions), returns None. Raises ------ ValueError - When any Loss Driver is not recognized. + If an unrecognized loss driver type is encountered, + indicating a configuration or data input error. + """ RV_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) @@ -596,10 +642,43 @@ def _create_DV_RVs(self, case_list): def _calc_median_consequence(self, eco_qnt): """ - Calculate the median repair consequence for each loss component. + Calculates the median repair consequences for each loss + component based on their quantities and the associated loss + parameters. - """ + This function evaluates the median consequences for different + types of decision variables (DV), such as repair costs or + repair time, based on the provided loss parameters. It + utilizes the eco_qnt DataFrame, which contains economic + quantity realizations for various damage states and + components, to compute the consequences. + Parameters + ---------- + eco_qnt : DataFrame + A DataFrame containing economic quantity realizations for + various components and damage states, indexed or + structured to align with the loss parameters. + + Returns + ------- + dict + A dictionary where keys are the types of decision variables + (DV) like 'COST' or 'TIME', and values are DataFrames + containing the median consequences for each component and + damage state. These DataFrames are structured with + MultiIndex columns that may include 'cmp' (component), + 'ds' (damage state), and potentially 'loc' (location), + depending on assessment options. + + Raises + ------ + ValueError + If any loss driver types or distribution types are not + recognized, or if the parameters are incomplete or + unsupported. + """ + medians = {} DV_types = self.loss_params.index.unique(level=1) @@ -994,13 +1073,24 @@ def aggregate_losses(self): """ Aggregates repair consequences across components. - Repair costs are simply summed up for each realization while repair - times are aggregated to provide lower and upper limits of the total - repair time using the assumption of parallel and sequential repair of - floors, respectively. Repairs within each floor are assumed to occur - sequentially. + Returns + ------- + DataFrame + A DataFrame containing aggregated repair + consequences. Columns include: + - 'repair_cost': Total repair costs across all components. + - 'repair_time-parallel': Minimum possible repair time + assuming repairs are conducted in parallel. + - 'repair_time-sequential': Maximum possible repair time + assuming sequential repairs. + - 'repair_carbon': Total carbon emissions associated with + repairs. + - 'repair_energy': Total energy usage associated with + repairs. + Each of these columns is summed or calculated based on the + repair data available. """ - + self.log_div() self.log_msg("Aggregating repair consequences...") @@ -1103,13 +1193,15 @@ def prep_constant_median_DV(median): Returns ------- - f: callable + callable A function that returns the constant median DV for all component quantities. """ def f(*args): # pylint: disable=unused-argument + # pylint: disable=missing-return-doc + # pylint: disable=missing-return-type-doc return median return f @@ -1135,12 +1227,14 @@ def prep_bounded_multilinear_median_DV(medians, quantities): Returns ------- - f: callable + callable A function that returns the median DV given the quantity of damaged components. """ def f(quantity): + # pylint: disable=missing-return-doc + # pylint: disable=missing-return-type-doc if quantity is None: raise ValueError( 'A bounded linear median Decision Variable function called ' diff --git a/pelicun/model/pelicun_model.py b/pelicun/model/pelicun_model.py index 3428c9e9e..8a6a5d5ed 100644 --- a/pelicun/model/pelicun_model.py +++ b/pelicun/model/pelicun_model.py @@ -51,8 +51,8 @@ import numpy as np import pandas as pd -from .. import base -from .. import uq +from pelicun import base +from pelicun import uq idx = base.idx @@ -100,7 +100,7 @@ def convert_marginal_params(self, marginal_params, units, arg_units=None): Returns ------- - marginal_params: DataFrame + DataFrame Same structure as the input DataFrame but with values scaled to represent internal Standard International units. From f0c8407d2819bd43bbb64a9756694d1d55a48050 Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Sat, 20 Apr 2024 09:20:08 -0700 Subject: [PATCH 18/22] Raise error when the string is invalid --- pelicun/base.py | 10 +++------- pelicun/tests/test_base.py | 3 ++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pelicun/base.py b/pelicun/base.py index dbb718eae..cd93fc4ac 100644 --- a/pelicun/base.py +++ b/pelicun/base.py @@ -1162,7 +1162,7 @@ def process_loc(string, stories): return [ res, ] - except ValueError: + except ValueError as exc: if "-" in string: s_low, s_high = string.split('-') s_low = process_loc(s_low, stories) @@ -1170,15 +1170,11 @@ def process_loc(string, stories): return list(range(s_low[0], s_high[0] + 1)) if string == "all": return list(range(1, stories + 1)) - if string == "top": + if string in {"top", "roof", "last"}: return [ stories, ] - if string == "roof": - return [ - stories, - ] - return None + raise ValueError(f'Invalid string: {string}') from exc def dedupe_index(dataframe, dtype=str): diff --git a/pelicun/tests/test_base.py b/pelicun/tests/test_base.py index 6312a92d7..f234188f7 100644 --- a/pelicun/tests/test_base.py +++ b/pelicun/tests/test_base.py @@ -608,7 +608,8 @@ def test_process_loc(): ] # Test when string cannot be converted to an int or recognized - assert base.process_loc('abc', 10) is None + with pytest.raises(ValueError): + base.process_loc('abc', 10) def test_run_input_specs(): From 078ef0146eff7a687888a744ff32007218062c1f Mon Sep 17 00:00:00 2001 From: John Vouvakis Manousakis Date: Tue, 7 May 2024 12:23:20 -0700 Subject: [PATCH 19/22] FIX - Aggregating pipe damage states. --- pelicun/tools/DL_calculation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pelicun/tools/DL_calculation.py b/pelicun/tools/DL_calculation.py index 3d033dfb8..d9f4488ba 100644 --- a/pelicun/tools/DL_calculation.py +++ b/pelicun/tools/DL_calculation.py @@ -1028,6 +1028,7 @@ def run_pelicun( adf.loc['aggregate', ('Demand', 'Type')] = 'Peak Ground Velocity' adf.loc['aggregate', ('Demand', 'Unit')] = 'mps' adf.loc['aggregate', ('LS1', 'Theta_0')] = 1e10 + adf.loc['aggregate', ('LS2', 'Theta_0')] = 1e10 adf.loc['aggregate', 'Incomplete'] = 0 PAL.damage.load_damage_model(component_db + [adf]) From 1f1cdfab8eb25d9d3975564d1a356dac37a8be72 Mon Sep 17 00:00:00 2001 From: Adam Zsarnoczay <33822153+zsarnoczay@users.noreply.github.com> Date: Tue, 7 May 2024 14:01:35 -0700 Subject: [PATCH 20/22] azs - apply black -S --- pelicun/assessment.py | 3 +- pelicun/auto.py | 2 +- pelicun/base.py | 15 +- pelicun/db.py | 202 +++++++----------- pelicun/file_io.py | 32 +-- pelicun/model/asset_model.py | 14 +- pelicun/model/damage_model.py | 36 +--- pelicun/model/demand_model.py | 16 +- pelicun/model/loss_model.py | 54 ++--- pelicun/model/pelicun_model.py | 14 +- pelicun/resources/auto/Hazus_Earthquake_IM.py | 26 +-- .../resources/auto/Hazus_Earthquake_Story.py | 6 +- pelicun/tests/reset_tests.py | 3 +- pelicun/tests/test_auto.py | 4 +- pelicun/tests/test_base.py | 12 +- pelicun/tests/test_file_io.py | 4 +- pelicun/tests/test_model.py | 52 ++--- pelicun/tests/test_uq.py | 20 +- pelicun/uq.py | 12 +- 19 files changed, 173 insertions(+), 354 deletions(-) diff --git a/pelicun/assessment.py b/pelicun/assessment.py index faca36e8c..2165ced17 100644 --- a/pelicun/assessment.py +++ b/pelicun/assessment.py @@ -269,7 +269,7 @@ def scale_factor(self, unit): ------- float Scale factor - + Raises ------ ValueError @@ -278,7 +278,6 @@ def scale_factor(self, unit): """ if unit is not None: - if unit in self.unit_conversion_factors: scale_factor = self.unit_conversion_factors[unit] diff --git a/pelicun/auto.py b/pelicun/auto.py index b440f695f..4fe4622f6 100644 --- a/pelicun/auto.py +++ b/pelicun/auto.py @@ -95,7 +95,7 @@ def auto_populate( If the configuration dictionary does not contain necessary asset information under 'GeneralInformation'. """ - + # try to get the AIM attributes AIM = config.get('GeneralInformation', None) if AIM is None: diff --git a/pelicun/base.py b/pelicun/base.py index cd93fc4ac..39555a826 100644 --- a/pelicun/base.py +++ b/pelicun/base.py @@ -460,9 +460,7 @@ def log_file(self, value): self._log_file = None else: - try: - filepath = Path(value).resolve() self._log_file = str(filepath) @@ -530,7 +528,6 @@ def msg(self, msg='', prepend_timestamp=True, prepend_blank_space=True): msg_lines = msg.split('\n') for msg_i, msg_line in enumerate(msg_lines): - if prepend_timestamp and (msg_i == 0): formatted_msg = '{} {}'.format( datetime.now().strftime(self.log_time_format), msg_line @@ -772,17 +769,14 @@ def convert_to_SimpleIndex(data, axis=0, inplace=False): """ if axis in {0, 1}: - if inplace: data_mod = data else: data_mod = data.copy() if axis == 0: - # only perform this if there are multiple levels if data.index.nlevels > 1: - simple_name = '-'.join( [n if n is not None else "" for n in data.index.names] ) @@ -794,10 +788,8 @@ def convert_to_SimpleIndex(data, axis=0, inplace=False): data_mod.index.name = simple_name elif axis == 1: - # only perform this if there are multiple levels if data.columns.nlevels > 1: - simple_name = '-'.join( [n if n is not None else "" for n in data.columns.names] ) @@ -848,7 +840,6 @@ def convert_to_MultiIndex(data, axis=0, inplace=False): if ((axis == 0) and (isinstance(data.index, pd.MultiIndex))) or ( (axis == 1) and (isinstance(data.columns, pd.MultiIndex)) ): - # if yes, return the data unchanged return data @@ -864,7 +855,6 @@ def convert_to_MultiIndex(data, axis=0, inplace=False): max_lbl_len = np.max([len(labels) for labels in index_labels]) for l_i, labels in enumerate(index_labels): - if len(labels) != max_lbl_len: labels += [ '', @@ -874,7 +864,6 @@ def convert_to_MultiIndex(data, axis=0, inplace=False): index_labels = np.array(index_labels) if index_labels.shape[1] > 1: - if inplace: data_mod = data else: @@ -932,9 +921,7 @@ def show_matrix(data, use_describe=False): If False, simply prints the matrix as is. """ if use_describe: - pp.pprint( - pd.DataFrame(data).describe(percentiles=[0.01, 0.1, 0.5, 0.9, 0.99]) - ) + pp.pprint(pd.DataFrame(data).describe(percentiles=[0.01, 0.1, 0.5, 0.9, 0.99])) else: pp.pprint(pd.DataFrame(data)) diff --git a/pelicun/db.py b/pelicun/db.py index 3ca2548f2..30ec97fcb 100644 --- a/pelicun/db.py +++ b/pelicun/db.py @@ -413,9 +413,7 @@ def create_FEMA_P58_fragility_db( ls_meta.update( { f"DS{ds_id}": { - "Description": cmp_meta[ - f"DS_{ds_id}_Description" - ], + "Description": cmp_meta[f"DS_{ds_id}_Description"], "RepairAction": repair_action, } } @@ -1005,9 +1003,9 @@ def create_FEMA_P58_repair_db( f"{cost_qnt_low:g},{cost_qnt_up:g}" ) - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Theta_1'] = ( - f"{cost_theta[1]:g}" - ) + df_db.loc[ + (cmp.Index, 'Cost'), f'DS{DS_i}-Theta_1' + ] = f"{cost_theta[1]:g}" df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = family_hat @@ -1016,37 +1014,33 @@ def create_FEMA_P58_repair_db( f"{time_qnt_low:g},{time_qnt_up:g}" ) - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Theta_1'] = ( - f"{time_theta[1]:g}" - ) + df_db.loc[ + (cmp.Index, 'Time'), f'DS{DS_i}-Theta_1' + ] = f"{time_theta[1]:g}" df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-LongLeadTime'] = int( time_vals[5] > 0 ) - df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Family'] = ( - family_hat_carbon - ) + df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Family'] = family_hat_carbon - df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_0'] = ( - f"{carbon_theta[0]:g}" - ) + df_db.loc[ + (cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_0' + ] = f"{carbon_theta[0]:g}" - df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_1'] = ( - f"{carbon_theta[1]:g}" - ) + df_db.loc[ + (cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_1' + ] = f"{carbon_theta[1]:g}" - df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Family'] = ( - family_hat_energy - ) + df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Family'] = family_hat_energy - df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Theta_0'] = ( - f"{energy_theta[0]:g}" - ) + df_db.loc[ + (cmp.Index, 'Energy'), f'DS{DS_i}-Theta_0' + ] = f"{energy_theta[0]:g}" - df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Theta_1'] = ( - f"{energy_theta[1]:g}" - ) + df_db.loc[ + (cmp.Index, 'Energy'), f'DS{DS_i}-Theta_1' + ] = f"{energy_theta[1]:g}" if ds_map.count('1') == 1: ds_pure_id = ds_map[::-1].find('1') + 1 @@ -1073,8 +1067,7 @@ def create_FEMA_P58_repair_db( meta_data['DamageStates'].update( { f"DS{DS_i}": { - "Description": 'Combination of ' - + ' & '.join(ds_combo), + "Description": 'Combination of ' + ' & '.join(ds_combo), "RepairAction": 'Combination of pure DS repair ' 'actions.', } @@ -1087,9 +1080,9 @@ def create_FEMA_P58_repair_db( for DS_i in range(1, 6): # cost if not pd.isna(getattr(cmp, f'Best_Fit_DS{DS_i}')): - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Family'] = ( - convert_family[getattr(cmp, f'Best_Fit_DS{DS_i}')] - ) + df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Family'] = convert_family[ + getattr(cmp, f'Best_Fit_DS{DS_i}') + ] if not pd.isna(getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}')): theta_0_low = getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}') @@ -1098,9 +1091,7 @@ def create_FEMA_P58_repair_db( qnt_up = getattr(cmp, f'Upper_Qty_Cutoff_DS{DS_i}') if theta_0_low == 0.0 and theta_0_up == 0.0: - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Family'] = ( - np.nan - ) + df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Family'] = np.nan else: df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Theta_0'] = ( @@ -1108,9 +1099,9 @@ def create_FEMA_P58_repair_db( f"{qnt_low:g},{qnt_up:g}" ) - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Theta_1'] = ( - f"{getattr(cmp, f'CV__Dispersion_DS{DS_i}'):g}" - ) + df_db.loc[ + (cmp.Index, 'Cost'), f'DS{DS_i}-Theta_1' + ] = f"{getattr(cmp, f'CV__Dispersion_DS{DS_i}'):g}" else: incomplete_cost = True @@ -1130,9 +1121,9 @@ def create_FEMA_P58_repair_db( # time if not pd.isna(getattr(cmp, f'Best_Fit_DS{DS_i}_1')): - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = ( - convert_family[getattr(cmp, f'Best_Fit_DS{DS_i}_1')] - ) + df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = convert_family[ + getattr(cmp, f'Best_Fit_DS{DS_i}_1') + ] if not pd.isna(getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}_1')): theta_0_low = getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}_1') @@ -1141,9 +1132,7 @@ def create_FEMA_P58_repair_db( qnt_up = getattr(cmp, f'Upper_Qty_Cutoff_DS{DS_i}_1') if theta_0_low == 0.0 and theta_0_up == 0.0: - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = ( - np.nan - ) + df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = np.nan else: df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Theta_0'] = ( @@ -1151,12 +1140,12 @@ def create_FEMA_P58_repair_db( f"{qnt_low:g},{qnt_up:g}" ) - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Theta_1'] = ( - f"{getattr(cmp, f'CV__Dispersion_DS{DS_i}_2'):g}" - ) + df_db.loc[ + (cmp.Index, 'Time'), f'DS{DS_i}-Theta_1' + ] = f"{getattr(cmp, f'CV__Dispersion_DS{DS_i}_2'):g}" - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-LongLeadTime'] = ( - int(getattr(cmp, f'DS_{DS_i}_Long_Lead_Time') == 'YES') + df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-LongLeadTime'] = int( + getattr(cmp, f'DS_{DS_i}_Long_Lead_Time') == 'YES' ) else: @@ -1164,9 +1153,9 @@ def create_FEMA_P58_repair_db( # Carbon if not pd.isna(getattr(cmp, f'DS{DS_i}_Best_Fit')): - df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Family'] = ( - convert_family[getattr(cmp, f'DS{DS_i}_Best_Fit')] - ) + df_db.loc[ + (cmp.Index, 'Carbon'), f'DS{DS_i}-Family' + ] = convert_family[getattr(cmp, f'DS{DS_i}_Best_Fit')] df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_0'] = getattr( cmp, f'DS{DS_i}_Embodied_Carbon_kg_CO2eq' @@ -1178,9 +1167,9 @@ def create_FEMA_P58_repair_db( # Energy if not pd.isna(getattr(cmp, f'DS{DS_i}_Best_Fit_1')): - df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Family'] = ( - convert_family[getattr(cmp, f'DS{DS_i}_Best_Fit_1')] - ) + df_db.loc[ + (cmp.Index, 'Energy'), f'DS{DS_i}-Family' + ] = convert_family[getattr(cmp, f'DS{DS_i}_Best_Fit_1')] df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Theta_0'] = getattr( cmp, f'DS{DS_i}_Embodied_Energy_MJ' @@ -1290,13 +1279,9 @@ def create_Hazus_EQ_fragility_db( frag_meta = {} # prepare lists of labels for various building features - design_levels = list( - raw_data['Structural_Fragility_Groups']['EDP_limits'].keys() - ) + design_levels = list(raw_data['Structural_Fragility_Groups']['EDP_limits'].keys()) - building_types = list( - raw_data['Structural_Fragility_Groups']['P_collapse'].keys() - ) + building_types = list(raw_data['Structural_Fragility_Groups']['P_collapse'].keys()) convert_design_level = { 'High_code': 'HC', @@ -1398,9 +1383,7 @@ def create_Hazus_EQ_fragility_db( "Description": ( frag_meta['Meta']['Collections']['STR']['Description'] + ", " - + frag_meta['Meta']['StructuralSystems'][st][ - 'Description' - ] + + frag_meta['Meta']['StructuralSystems'][st]['Description'] + ", " + frag_meta['Meta']['HeightClasses'][hc]['Description'] + ", " @@ -1428,9 +1411,7 @@ def create_Hazus_EQ_fragility_db( "Description": ( frag_meta['Meta']['Collections']['STR']['Description'] + ", " - + frag_meta['Meta']['StructuralSystems'][st][ - 'Description' - ] + + frag_meta['Meta']['StructuralSystems'][st]['Description'] + ", " + frag_meta['Meta']['DesignLevels'][ convert_design_level[dl] @@ -1454,18 +1435,18 @@ def create_Hazus_EQ_fragility_db( ds_meta = frag_meta['Meta']['StructuralSystems'][st]['DamageStates'] for LS_i in range(1, 5): df_db.loc[counter, f'LS{LS_i}-Family'] = 'lognormal' - df_db.loc[counter, f'LS{LS_i}-Theta_0'] = S_data['EDP_limits'][ + df_db.loc[counter, f'LS{LS_i}-Theta_0'] = S_data['EDP_limits'][dl][ + bt + ][LS_i - 1] + df_db.loc[counter, f'LS{LS_i}-Theta_1'] = S_data['Fragility_beta'][ dl - ][bt][LS_i - 1] - df_db.loc[counter, f'LS{LS_i}-Theta_1'] = S_data[ - 'Fragility_beta' - ][dl] + ] if LS_i == 4: p_coll = S_data['P_collapse'][bt] - df_db.loc[counter, f'LS{LS_i}-DamageStateWeights'] = ( - f'{1.0 - p_coll} | {p_coll}' - ) + df_db.loc[ + counter, f'LS{LS_i}-DamageStateWeights' + ] = f'{1.0 - p_coll} | {p_coll}' cmp_meta["LimitStates"].update( { @@ -1480,9 +1461,7 @@ def create_Hazus_EQ_fragility_db( cmp_meta["LimitStates"].update( { f"LS{LS_i}": { - f"DS{LS_i}": { - "Description": ds_meta[f"DS{LS_i}"] - } + f"DS{LS_i}": {"Description": ds_meta[f"DS{LS_i}"]} } } ) @@ -1558,9 +1537,7 @@ def create_Hazus_EQ_fragility_db( "Comments": ( frag_meta['Meta']['Collections']['NSA']['Comment'] + "\n" - + frag_meta['Meta']['DesignLevels'][convert_design_level[dl]][ - 'Comment' - ] + + frag_meta['Meta']['DesignLevels'][convert_design_level[dl]]['Comment'] ), "SuggestedComponentBlockSize": "1 EA", "RoundUpToIntegerQuantity": "True", @@ -1622,9 +1599,7 @@ def create_Hazus_EQ_fragility_db( 'Description' ] + ", " - + frag_meta['Meta']['HeightClasses'][hc][ - 'Description' - ] + + frag_meta['Meta']['HeightClasses'][hc]['Description'] + ", " + frag_meta['Meta']['DesignLevels'][ convert_design_level[dl] @@ -1633,9 +1608,7 @@ def create_Hazus_EQ_fragility_db( "Comments": ( frag_meta['Meta']['Collections']['LF']['Comment'] + "\n" - + frag_meta['Meta']['StructuralSystems'][st][ - 'Comment' - ] + + frag_meta['Meta']['StructuralSystems'][st]['Comment'] + "\n" + frag_meta['Meta']['HeightClasses'][hc]['Comment'] + "\n" @@ -1663,9 +1636,7 @@ def create_Hazus_EQ_fragility_db( "Comments": ( frag_meta['Meta']['Collections']['LF']['Comment'] + "\n" - + frag_meta['Meta']['StructuralSystems'][st][ - 'Comment' - ] + + frag_meta['Meta']['StructuralSystems'][st]['Comment'] + "\n" + frag_meta['Meta']['DesignLevels'][ convert_design_level[dl] @@ -1677,23 +1648,21 @@ def create_Hazus_EQ_fragility_db( } # store the Limit State parameters - ds_meta = frag_meta['Meta']['StructuralSystems'][st][ - 'DamageStates' - ] + ds_meta = frag_meta['Meta']['StructuralSystems'][st]['DamageStates'] for LS_i in range(1, 5): df_db.loc[counter, f'LS{LS_i}-Family'] = 'lognormal' - df_db.loc[counter, f'LS{LS_i}-Theta_0'] = LF_data[ - 'EDP_limits' - ][dl][bt][LS_i - 1] + df_db.loc[counter, f'LS{LS_i}-Theta_0'] = LF_data['EDP_limits'][ + dl + ][bt][LS_i - 1] df_db.loc[counter, f'LS{LS_i}-Theta_1'] = LF_data[ 'Fragility_beta' ][dl] if LS_i == 4: p_coll = LF_data['P_collapse'][bt] - df_db.loc[counter, f'LS{LS_i}-DamageStateWeights'] = ( - f'{1.0 - p_coll} | {p_coll}' - ) + df_db.loc[ + counter, f'LS{LS_i}-DamageStateWeights' + ] = f'{1.0 - p_coll} | {p_coll}' cmp_meta["LimitStates"].update( { @@ -1757,9 +1726,9 @@ def create_Hazus_EQ_fragility_db( f_depth ] p_complete = GF_data['P_Complete'] - df_db.loc[counter, 'LS1-DamageStateWeights'] = ( - f'{1.0 - p_complete} | {p_complete}' - ) + df_db.loc[ + counter, 'LS1-DamageStateWeights' + ] = f'{1.0 - p_complete} | {p_complete}' cmp_meta["LimitStates"].update( { @@ -1867,9 +1836,7 @@ def create_Hazus_EQ_repair_db( # create the MultiIndex cmp_types = ['STR', 'NSD', 'NSA', 'LF'] comps = [ - f'{cmp_type}.{occ_type}' - for cmp_type in cmp_types - for occ_type in occupancies + f'{cmp_type}.{occ_type}' for cmp_type in cmp_types for occ_type in occupancies ] DVs = ['Cost', 'Time'] df_MI = pd.MultiIndex.from_product([comps, DVs], names=['ID', 'DV']) @@ -1969,9 +1936,9 @@ def create_Hazus_EQ_repair_db( {f"DS{DS_i}": {"Description": ds_meta[f"DS{DS_i}"]}} ) - df_db.loc[(cmp_id, 'Cost'), f'DS{DS_i}-Theta_0'] = NSD_data[ - 'Repair_cost' - ][occ_type][DS_i - 1] + df_db.loc[(cmp_id, 'Cost'), f'DS{DS_i}-Theta_0'] = NSD_data['Repair_cost'][ + occ_type + ][DS_i - 1] # store metadata meta_dict.update({cmp_id: cmp_meta}) @@ -2006,9 +1973,9 @@ def create_Hazus_EQ_repair_db( {f"DS{DS_i}": {"Description": ds_meta[f"DS{DS_i}"]}} ) - df_db.loc[(cmp_id, 'Cost'), f'DS{DS_i}-Theta_0'] = NSA_data[ - 'Repair_cost' - ][occ_type][DS_i - 1] + df_db.loc[(cmp_id, 'Cost'), f'DS{DS_i}-Theta_0'] = NSA_data['Repair_cost'][ + occ_type + ][DS_i - 1] # store metadata meta_dict.update({cmp_id: cmp_meta}) @@ -2300,15 +2267,9 @@ def create_Hazus_HU_fragility_db( 'Masonry, Engineered Residential Building, High-Rise (6+ Stories).' ), # ------------------------ - 'M.ECB.L': ( - 'Masonry, Engineered Commercial Building, Low-Rise (1-2 Stories).' - ), - 'M.ECB.M': ( - 'Masonry, Engineered Commercial Building, Mid-Rise (3-5 Stories).' - ), - 'M.ECB.H': ( - 'Masonry, Engineered Commercial Building, High-Rise (6+ Stories).' - ), + 'M.ECB.L': ('Masonry, Engineered Commercial Building, Low-Rise (1-2 Stories).'), + 'M.ECB.M': ('Masonry, Engineered Commercial Building, Mid-Rise (3-5 Stories).'), + 'M.ECB.H': ('Masonry, Engineered Commercial Building, High-Rise (6+ Stories).'), # ------------------------ 'C.ERB.L': ( 'Concrete, Engineered Residential Building, Low-Rise (1-2 Stories).' @@ -2643,7 +2604,6 @@ def find_class_type(entry: str) -> str | None: # for fragility_id in fragility_data['ID'].to_list(): - class_type = find_class_type(fragility_id) class_type_human_readable = class_types[class_type] diff --git a/pelicun/file_io.py b/pelicun/file_io.py index 0f099259f..3278d3995 100644 --- a/pelicun/file_io.py +++ b/pelicun/file_io.py @@ -157,7 +157,7 @@ def save_to_csv( unit conversions and reformatting applied. Otherwise, returns None after saving the data to a CSV file. """ - + if filepath is None: if log: log.msg('Preparing data ...', prepend_timestamp=False) @@ -166,13 +166,11 @@ def save_to_csv( log.msg(f'Saving data to {filepath}...', prepend_timestamp=False) if data is not None: - # make sure we do not modify the original data data = data.copy() # convert units and add unit information, if needed if units is not None: - if unit_conversion_factors is None: raise ValueError( 'When units is not None, ' @@ -190,7 +188,6 @@ def save_to_csv( labels_to_keep = [] for unit_name in units.unique(): - labels = units.loc[units == unit_name].index.values unit_factor = 1.0 / unit_conversion_factors[unit_name] @@ -237,17 +234,13 @@ def save_to_csv( data = base.convert_to_SimpleIndex(data, axis=1) if filepath is not None: - filepath = Path(filepath).resolve() if filepath.suffix == '.csv': - # save the contents of the DataFrame into a csv data.to_csv(filepath) if log: - log.msg( - 'Data successfully saved to file.', prepend_timestamp=False - ) + log.msg('Data successfully saved to file.', prepend_timestamp=False) else: raise ValueError( @@ -398,7 +391,6 @@ def load_data( # if there is information about units, separate that information # and optionally apply conversions to all numeric values if 'Units' in the_index: - units = data['Units'] if orientation == 1 else data.loc['Units'] data.drop('Units', axis=orientation, inplace=True) data = base.convert_dtypes(data) @@ -415,20 +407,18 @@ def load_data( conversion_factors = units.map( lambda unit: ( - 1.00 - if pd.isna(unit) - else unit_conversion_factors.get(unit, 1.00) + 1.00 if pd.isna(unit) else unit_conversion_factors.get(unit, 1.00) ) ) if orientation == 1: - data.loc[:, numeric_elements] = data.loc[ - :, numeric_elements - ].multiply(conversion_factors, axis=axis[orientation]) + data.loc[:, numeric_elements] = data.loc[:, numeric_elements].multiply( + conversion_factors, axis=axis[orientation] + ) else: - data.loc[numeric_elements, :] = data.loc[ - numeric_elements, : - ].multiply(conversion_factors, axis=axis[orientation]) + data.loc[numeric_elements, :] = data.loc[numeric_elements, :].multiply( + conversion_factors, axis=axis[orientation] + ) if log: log.msg('Unit conversion successful.', prepend_timestamp=False) @@ -501,12 +491,10 @@ def load_from_file(filepath, log=None): if not filepath.is_file(): raise FileNotFoundError( - f"The filepath provided does not point to an existing " - f"file: {filepath}" + f"The filepath provided does not point to an existing " f"file: {filepath}" ) if filepath.suffix == '.csv': - # load the contents of the csv into a DataFrame data = pd.read_csv( diff --git a/pelicun/model/asset_model.py b/pelicun/model/asset_model.py index 9cf1f3747..5f3c18006 100644 --- a/pelicun/model/asset_model.py +++ b/pelicun/model/asset_model.py @@ -303,6 +303,7 @@ def load_cmp_model(self, data_source): >>> model.load_cmp_model(data_dict) """ + def get_locations(loc_str): """ Parses a location string to determine specific sections of @@ -559,9 +560,7 @@ def get_attribute(attribute_str, dtype=float, default=np.nan): cmp_marginal_param_series = [] for col, cmp_marginal_param in cmp_marginal_param_dct.items(): cmp_marginal_param_series.append( - pd.Series( - cmp_marginal_param, dtype=dtypes[col], name=col, index=index - ) + pd.Series(cmp_marginal_param, dtype=dtypes[col], name=col, index=index) ) cmp_marginal_params = pd.concat(cmp_marginal_param_series, axis=1) @@ -592,9 +591,7 @@ def get_attribute(attribute_str, dtype=float, default=np.nan): self.cmp_marginal_params = cmp_marginal_params.drop('Units', axis=1) - self.log_msg( - "Model parameters successfully loaded.", prepend_timestamp=False - ) + self.log_msg("Model parameters successfully loaded.", prepend_timestamp=False) self.log_msg( "\nComponent model marginal distributions:\n" + str(cmp_marginal_params), @@ -621,8 +618,7 @@ def _create_cmp_RVs(self): uq.rv_class_map(family)( name=f'CMP-{cmp[0]}-{cmp[1]}-{cmp[2]}-{cmp[3]}', theta=[ - getattr(rv_params, f"Theta_{t_i}", np.nan) - for t_i in range(3) + getattr(rv_params, f"Theta_{t_i}", np.nan) for t_i in range(3) ], truncation_limits=[ getattr(rv_params, f"Truncate{side}", np.nan) @@ -659,7 +655,7 @@ def generate_cmp_sample(self, sample_size=None): generation, or if neither sample size is specified nor can be determined from the demand model. """ - + if self.cmp_marginal_params is None: raise ValueError( 'Model parameters have not been specified. Load' diff --git a/pelicun/model/damage_model.py b/pelicun/model/damage_model.py index aed601650..e5927da6a 100644 --- a/pelicun/model/damage_model.py +++ b/pelicun/model/damage_model.py @@ -127,9 +127,7 @@ def save_sample(self, filepath=None, save_units=False): self.log_msg('Saving damage sample...') cmp_units = self._asmnt.asset.cmp_units - qnt_units = pd.Series( - index=self.sample.columns, name='Units', dtype='object' - ) + qnt_units = pd.Series(index=self.sample.columns, name='Units', dtype='object') for cmp in cmp_units.index: qnt_units.loc[cmp] = cmp_units.loc[cmp] @@ -143,9 +141,7 @@ def save_sample(self, filepath=None, save_units=False): ) if filepath is not None: - self.log_msg( - 'Damage sample successfully saved.', prepend_timestamp=False - ) + self.log_msg('Damage sample successfully saved.', prepend_timestamp=False) return None # else: @@ -483,9 +479,7 @@ def map_ds(values, offset=int(ds_id + 1)): for PG in PGB.index: # determine demand capacity adjustment operation, if required cmp_loc_dir = '-'.join(PG[0:3]) - capacity_adjustment_operation = scaling_specification.get( - cmp_loc_dir, None - ) + capacity_adjustment_operation = scaling_specification.get(cmp_loc_dir, None) cmp_id = PG[0] blocks = PGB.loc[PG, 'Blocks'] @@ -697,9 +691,7 @@ def _generate_dmg_sample(self, sample_size, PGB, scaling_specification=None): # get the capacity and lsds samples capacity_sample = ( - pd.DataFrame(capacity_RVs.RV_sample) - .sort_index(axis=0) - .sort_index(axis=1) + pd.DataFrame(capacity_RVs.RV_sample).sort_index(axis=0).sort_index(axis=1) ) capacity_sample = base.convert_to_MultiIndex(capacity_sample, axis=1)['FRG'] capacity_sample.columns.names = ['cmp', 'loc', 'dir', 'uid', 'block', 'ls'] @@ -901,9 +893,7 @@ def _assemble_required_demand_data(self, EDP_req): # take the maximum of all available directions and scale it # using the nondirectional multiplier specified in the # self._asmnt.options (the default value is 1.2) - demand = ( - demand_source.loc[:, (EDP[0], EDP[1])].max(axis=1).values - ) + demand = demand_source.loc[:, (EDP[0], EDP[1])].max(axis=1).values demand = demand * self._asmnt.options.nondir_multi(EDP[0]) except KeyError: @@ -978,9 +968,7 @@ def _evaluate_damage_state( # Create a dataframe with demand values repeated for the # number of PGs and assign the columns as PG_cols demand_df.append( - pd.concat( - [pd.Series(demand_vals)] * len(PG_cols), axis=1, keys=PG_cols - ) + pd.concat([pd.Series(demand_vals)] * len(PG_cols), axis=1, keys=PG_cols) ) # Concatenate all demand dataframes into a single dataframe @@ -1211,7 +1199,6 @@ def _perform_dmg_task(self, task, ds_sample): # execute the events pres prescribed in the damage task for source_event, target_infos in events.items(): - # events can only be triggered by damage state occurrence if not source_event.startswith('DS'): raise ValueError( @@ -1227,7 +1214,6 @@ def _perform_dmg_task(self, task, ds_sample): target_infos = [target_infos] for target_info in target_infos: - # get the target component and event type target_cmp, target_event = target_info.split('_') @@ -1245,7 +1231,6 @@ def _perform_dmg_task(self, task, ds_sample): # trigger a damage state if target_event.startswith('DS'): - # get the ID of the damage state to switch the target # components to ds_target = int(target_event[2:]) @@ -1527,9 +1512,7 @@ def _complete_ds_cols(self, dmg_sample): # Get the header for the results that we can use to identify # cmp-loc-dir-uid sets - dmg_header = ( - dmg_sample.groupby(level=[0, 1, 2, 3], axis=1).first().iloc[:2, :] - ) + dmg_header = dmg_sample.groupby(level=[0, 1, 2, 3], axis=1).first().iloc[:2, :] # get the number of possible limit states ls_list = [col for col in DP.columns.unique(level=0) if 'LS' in col] @@ -1555,9 +1538,7 @@ def _complete_ds_cols(self, dmg_sample): else: # or if there are more than one, how many - ds_count += len( - cmp_data[(ls, 'DamageStateWeights')].split('|') - ) + ds_count += len(cmp_data[(ls, 'DamageStateWeights')].split('|')) # get the list of valid cmp-loc-dir-uid sets cmp_header = dmg_header.loc[ @@ -1657,7 +1638,6 @@ def calculate_internal( # for PG_i in self._asmnt.asset.cmp_sample.columns: ds_samples = [] for PGB_i in batches: - performance_group = pg_batch.loc[PGB_i] self.log_msg( diff --git a/pelicun/model/demand_model.py b/pelicun/model/demand_model.py index df3276a8a..8707cddc8 100644 --- a/pelicun/model/demand_model.py +++ b/pelicun/model/demand_model.py @@ -140,9 +140,7 @@ def save_sample(self, filepath=None, save_units=False): ) if filepath is not None: - self.log_msg( - 'Demand sample successfully saved.', prepend_timestamp=False - ) + self.log_msg('Demand sample successfully saved.', prepend_timestamp=False) return None # else: @@ -607,9 +605,7 @@ def get_filter_mask(lower_lims, upper_lims): distribution=cal_df.loc[:, 'Family'].values, censored_count=censored_count, detection_limits=cal_df.loc[:, ['CensorLower', 'CensorUpper']].values, - truncation_limits=cal_df.loc[ - :, ['TruncateLower', 'TruncateUpper'] - ].values, + truncation_limits=cal_df.loc[:, ['TruncateLower', 'TruncateUpper']].values, multi_fit=False, logger_object=self._asmnt.log, ) @@ -643,8 +639,7 @@ def get_filter_mask(lower_lims, upper_lims): self.marginal_params = model_params self.log_msg( - "\nCalibrated demand model marginal distributions:\n" - + str(model_params), + "\nCalibrated demand model marginal distributions:\n" + str(model_params), prepend_timestamp=False, ) @@ -654,8 +649,7 @@ def get_filter_mask(lower_lims, upper_lims): ) self.log_msg( - "\nCalibrated demand model correlation matrix:\n" - + str(self.correlation), + "\nCalibrated demand model correlation matrix:\n" + str(self.correlation), prepend_timestamp=False, ) @@ -1000,7 +994,7 @@ def generate_sample(self, config): # This will generate 1000 realizations of demand variables # with the specified configuration. """ - + if self.marginal_params is None: raise ValueError( 'Model parameters have not been specified. Either' diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index 471f5e210..6940ca2af 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -427,9 +427,7 @@ def _create_DV_RVs(self, case_list): # currently, we only support DMG-based loss calculations # but this will be extended in the very near future if driver_type != 'DMG': - raise ValueError( - f"Loss Driver type not recognized: " f"{driver_type}" - ) + raise ValueError(f"Loss Driver type not recognized: " f"{driver_type}") # load the parameters # TODO: remove specific DV_type references and make the code below @@ -466,8 +464,7 @@ def _create_DV_RVs(self, case_list): cost_family = cost_params_DS.get('Family', np.nan) cost_theta = [ - cost_params_DS.get(f"Theta_{t_i}", np.nan) - for t_i in range(3) + cost_params_DS.get(f"Theta_{t_i}", np.nan) for t_i in range(3) ] # If the first parameter is controlled by a function, we use @@ -485,8 +482,7 @@ def _create_DV_RVs(self, case_list): time_family = time_params_DS.get('Family', np.nan) time_theta = [ - time_params_DS.get(f"Theta_{t_i}", np.nan) - for t_i in range(3) + time_params_DS.get(f"Theta_{t_i}", np.nan) for t_i in range(3) ] # If the first parameter is controlled by a function, we use @@ -504,8 +500,7 @@ def _create_DV_RVs(self, case_list): carbon_family = carbon_params_DS.get('Family', np.nan) carbon_theta = [ - carbon_params_DS.get(f"Theta_{t_i}", np.nan) - for t_i in range(3) + carbon_params_DS.get(f"Theta_{t_i}", np.nan) for t_i in range(3) ] # If the first parameter is controlled by a function, we use @@ -523,8 +518,7 @@ def _create_DV_RVs(self, case_list): energy_family = energy_params_DS.get('Family', np.nan) energy_theta = [ - energy_params_DS.get(f"Theta_{t_i}", np.nan) - for t_i in range(3) + energy_params_DS.get(f"Theta_{t_i}", np.nan) for t_i in range(3) ] # If the first parameter is controlled by a function, we use @@ -553,9 +547,7 @@ def _create_DV_RVs(self, case_list): for loc, direction, uid in loc_dir_uid: # assign cost RV if pd.isna(cost_family) is False: - cost_rv_tag = ( - f'Cost-{loss_cmp_id}-{ds}-{loc}-{direction}-{uid}' - ) + cost_rv_tag = f'Cost-{loss_cmp_id}-{ds}-{loc}-{direction}-{uid}' RV_reg.add_RV( uq.rv_class_map(cost_family)( @@ -568,9 +560,7 @@ def _create_DV_RVs(self, case_list): # assign time RV if pd.isna(time_family) is False: - time_rv_tag = ( - f'Time-{loss_cmp_id}-{ds}-{loc}-{direction}-{uid}' - ) + time_rv_tag = f'Time-{loss_cmp_id}-{ds}-{loc}-{direction}-{uid}' RV_reg.add_RV( uq.rv_class_map(time_family)( @@ -624,16 +614,12 @@ def _create_DV_RVs(self, case_list): RV_reg.add_RV_set( uq.RandomVariableSet( f'DV-{loss_cmp_id}-{ds}-{loc}-{direction}-{uid}_set', - list( - RV_reg.RVs([cost_rv_tag, time_rv_tag]).values() - ), + list(RV_reg.RVs([cost_rv_tag, time_rv_tag]).values()), np.array([[1.0, rho], [rho, 1.0]]), ) ) - self.log_msg( - f"\n{rv_count} random variables created.", prepend_timestamp=False - ) + self.log_msg(f"\n{rv_count} random variables created.", prepend_timestamp=False) if rv_count > 0: return RV_reg @@ -678,7 +664,7 @@ def _calc_median_consequence(self, eco_qnt): recognized, or if the parameters are incomplete or unsupported. """ - + medians = {} DV_types = self.loss_params.index.unique(level=1) @@ -717,9 +703,7 @@ def _calc_median_consequence(self, eco_qnt): if ds_id == '0': continue - loss_params_DS = self.loss_params.loc[ - (loss_cmp_name, DV_type), ds - ] + loss_params_DS = self.loss_params.loc[(loss_cmp_name, DV_type), ds] # check if theta_0 is defined theta_0 = loss_params_DS.get('Theta_0', np.nan) @@ -936,9 +920,7 @@ def _generate_DV_sample(self, dmg_quantities, sample_size): res_list = [] key_list = [] - dmg_quantities.columns = dmg_quantities.columns.reorder_levels( - [0, 4, 1, 2, 3] - ) + dmg_quantities.columns = dmg_quantities.columns.reorder_levels([0, 4, 1, 2, 3]) dmg_quantities.sort_index(axis=1, inplace=True) DV_types = self.loss_params.index.unique(level=1) @@ -979,13 +961,11 @@ def _generate_DV_sample(self, dmg_quantities, sample_size): loc_list = [] for loc_id, loc in enumerate( - dmg_quantities.loc[:, (dmg_cmp_i, ds)].columns.unique( - level=0 - ) + dmg_quantities.loc[:, (dmg_cmp_i, ds)].columns.unique(level=0) ): - if ( - self._asmnt.options.eco_scale["AcrossFloors"] is True - ) and (loc_id > 0): + if (self._asmnt.options.eco_scale["AcrossFloors"] is True) and ( + loc_id > 0 + ): break if self._asmnt.options.eco_scale["AcrossFloors"] is True: @@ -1090,7 +1070,7 @@ def aggregate_losses(self): Each of these columns is summed or calculated based on the repair data available. """ - + self.log_div() self.log_msg("Aggregating repair consequences...") diff --git a/pelicun/model/pelicun_model.py b/pelicun/model/pelicun_model.py index 8a6a5d5ed..8a11acb78 100644 --- a/pelicun/model/pelicun_model.py +++ b/pelicun/model/pelicun_model.py @@ -189,9 +189,7 @@ def convert_marginal_params(self, marginal_params, units, arg_units=None): if arg_unit != '1 EA': # get the scale factor - arg_unit_factor = self._asmnt.calc_unit_scale_factor( - arg_unit - ) + arg_unit_factor = self._asmnt.calc_unit_scale_factor(arg_unit) # scale arguments, if needed for a_i, arg in enumerate(args): @@ -214,13 +212,11 @@ def convert_marginal_params(self, marginal_params, units, arg_units=None): ) # and update the values in the DF - marginal_params.loc[row_id, ['Theta_0', 'Theta_1', 'Theta_2']] = ( - theta - ) + marginal_params.loc[row_id, ['Theta_0', 'Theta_1', 'Theta_2']] = theta - marginal_params.loc[row_id, ['TruncateLower', 'TruncateUpper']] = ( - tr_limits - ) + marginal_params.loc[ + row_id, ['TruncateLower', 'TruncateUpper'] + ] = tr_limits # remove the added columns marginal_params = marginal_params[original_cols] diff --git a/pelicun/resources/auto/Hazus_Earthquake_IM.py b/pelicun/resources/auto/Hazus_Earthquake_IM.py index 90f720933..91f1e8535 100644 --- a/pelicun/resources/auto/Hazus_Earthquake_IM.py +++ b/pelicun/resources/auto/Hazus_Earthquake_IM.py @@ -409,10 +409,8 @@ def auto_populate(AIM): } }, "Options": { - "NonDirectionalMultipliers": { - "ALL": 1.0 - }, - } + "NonDirectionalMultipliers": {"ALL": 1.0}, + }, } elif assetType == "TransportationNetwork": @@ -447,10 +445,8 @@ def auto_populate(AIM): } }, "Options": { - "NonDirectionalMultipliers": { - "ALL": 1.0 - }, - } + "NonDirectionalMultipliers": {"ALL": 1.0}, + }, } elif inf_type == "HwyTunnel": @@ -482,10 +478,8 @@ def auto_populate(AIM): } }, "Options": { - "NonDirectionalMultipliers": { - "ALL": 1.0 - }, - } + "NonDirectionalMultipliers": {"ALL": 1.0}, + }, } elif inf_type == "Roadway": # get the road class @@ -515,16 +509,13 @@ def auto_populate(AIM): } }, "Options": { - "NonDirectionalMultipliers": { - "ALL": 1.0 - }, - } + "NonDirectionalMultipliers": {"ALL": 1.0}, + }, } else: print("subtype not supported in HWY") elif assetType == "WaterDistributionNetwork": - pipe_material_map = { "CI": "B", "AC": "B", @@ -703,7 +694,6 @@ def auto_populate(AIM): } elif wdn_element_type == "Tank": - tank_cmp_lines = { ("OG", "C", 1): {'PST.G.C.A.GS': ['ea', 1, 1, 1, 'N/A']}, ("OG", "C", 0): {'PST.G.C.U.GS': ['ea', 1, 1, 1, 'N/A']}, diff --git a/pelicun/resources/auto/Hazus_Earthquake_Story.py b/pelicun/resources/auto/Hazus_Earthquake_Story.py index 24fc1da64..74ff13465 100644 --- a/pelicun/resources/auto/Hazus_Earthquake_Story.py +++ b/pelicun/resources/auto/Hazus_Earthquake_Story.py @@ -267,10 +267,8 @@ def auto_populate(AIM): "Demands": {}, "Losses": {"Repair": repair_config}, "Options": { - "NonDirectionalMultipliers": { - "ALL": 1.0 - }, - } + "NonDirectionalMultipliers": {"ALL": 1.0}, + }, } else: diff --git a/pelicun/tests/reset_tests.py b/pelicun/tests/reset_tests.py index 1f9068877..9c7fc8512 100644 --- a/pelicun/tests/reset_tests.py +++ b/pelicun/tests/reset_tests.py @@ -91,8 +91,7 @@ def reset_all_test_data(restore=True, purge=False): cwd = os.path.basename(os.getcwd()) if cwd != 'pelicun': raise OSError( - 'Wrong directory. ' - 'See the docstring of `reset_all_test_data`. Aborting' + 'Wrong directory. ' 'See the docstring of `reset_all_test_data`. Aborting' ) # where the test result data are stored diff --git a/pelicun/tests/test_auto.py b/pelicun/tests/test_auto.py index 36458cb91..5bf2f34f0 100644 --- a/pelicun/tests/test_auto.py +++ b/pelicun/tests/test_auto.py @@ -109,9 +109,7 @@ def test_pelicun_default_path_replacement( assert modified_path.startswith(setup_expected_base_path) -def test_auto_population_script_execution( - setup_valid_config, setup_auto_script_path -): +def test_auto_population_script_execution(setup_valid_config, setup_auto_script_path): with patch('pelicun.base.pelicun_path', '/expected/path'), patch( 'os.path.exists', return_value=True ), patch('importlib.__import__') as mock_import: diff --git a/pelicun/tests/test_base.py b/pelicun/tests/test_base.py index f234188f7..57f3f6d5b 100644 --- a/pelicun/tests/test_base.py +++ b/pelicun/tests/test_base.py @@ -414,9 +414,7 @@ def test_convert_to_MultiIndex(): assert data.index.equals(pd.Index(('A-1', 'B-1', 'C-1'))) # Test a case where the index is already a MultiIndex - data_converted = base.convert_to_MultiIndex( - data_converted, axis=0, inplace=False - ) + data_converted = base.convert_to_MultiIndex(data_converted, axis=0, inplace=False) assert data_converted.index.equals(expected_index) # Test a case where the columns need to be converted to a MultiIndex @@ -428,9 +426,7 @@ def test_convert_to_MultiIndex(): assert data.columns.equals(pd.Index(('A-1', 'B-1'))) # Test a case where the columns are already a MultiIndex - data_converted = base.convert_to_MultiIndex( - data_converted, axis=1, inplace=False - ) + data_converted = base.convert_to_MultiIndex(data_converted, axis=1, inplace=False) assert data_converted.columns.equals(expected_columns) # Test an invalid axis parameter @@ -509,9 +505,7 @@ def test_describe(): # case 1: # passing a dataframe - df = pd.DataFrame( - ((1.00, 2.00, 3.00), (4.00, 5.00, 6.00)), columns=['A', 'B', 'C'] - ) + df = pd.DataFrame(((1.00, 2.00, 3.00), (4.00, 5.00, 6.00)), columns=['A', 'B', 'C']) desc = base.describe(df) assert np.all(desc.index == expected_idx) assert np.all(desc.columns == pd.Index(('A', 'B', 'C'), dtype='object')) diff --git a/pelicun/tests/test_file_io.py b/pelicun/tests/test_file_io.py index 5a141cef9..c98bd3afe 100644 --- a/pelicun/tests/test_file_io.py +++ b/pelicun/tests/test_file_io.py @@ -170,9 +170,7 @@ def test_load_data(): assert isinstance(data.columns, pd.core.indexes.multi.MultiIndex) assert data.columns.nlevels == 4 - _, units = file_io.load_data( - filepath, unit_conversion_factors, return_units=True - ) + _, units = file_io.load_data(filepath, unit_conversion_factors, return_units=True) for item in unit_conversion_factors: assert item in units.unique() diff --git a/pelicun/tests/test_model.py b/pelicun/tests/test_model.py index 23554ebc3..a0a6fea12 100644 --- a/pelicun/tests/test_model.py +++ b/pelicun/tests/test_model.py @@ -243,13 +243,10 @@ def test_estimate_RID(self, demand_model_with_sample): res = demand_model_with_sample.estimate_RID(demands, params) assert list(res.columns) == [('RID', '1', '1')] assert ( - demand_model_with_sample.estimate_RID(demands, params, method='xyz') - is None + demand_model_with_sample.estimate_RID(demands, params, method='xyz') is None ) - def test_calibrate_model( - self, calibrated_demand_model, demand_model_with_sample_C - ): + def test_calibrate_model(self, calibrated_demand_model, demand_model_with_sample_C): assert calibrated_demand_model.marginal_params['Family'].to_list() == [ 'normal', 'normal', @@ -372,9 +369,7 @@ def test_save_load_model_with_empirical( def test_generate_sample_exceptions(self, demand_model): # generating a sample from a non calibrated model should fail with pytest.raises(ValueError): - demand_model.generate_sample( - {"SampleSize": 3, 'PreserveRawOrder': False} - ) + demand_model.generate_sample({"SampleSize": 3, 'PreserveRawOrder': False}) def test_generate_sample(self, calibrated_demand_model): calibrated_demand_model.generate_sample( @@ -517,9 +512,7 @@ def test_convert_marginal_params(self, pelicun_model): ) units = pd.Series(['ea'], index=marginal_params.index) arg_units = None - res = pelicun_model.convert_marginal_params( - marginal_params, units, arg_units - ) + res = pelicun_model.convert_marginal_params(marginal_params, units, arg_units) # >>> res # Theta_0 @@ -557,9 +550,7 @@ def test_convert_marginal_params(self, pelicun_model): ) units = pd.Series(['ea', 'ft', 'in', 'in2'], index=marginal_params.index) arg_units = None - res = pelicun_model.convert_marginal_params( - marginal_params, units, arg_units - ) + res = pelicun_model.convert_marginal_params(marginal_params, units, arg_units) expected_df = pd.DataFrame( { @@ -595,9 +586,7 @@ def test_convert_marginal_params(self, pelicun_model): ) units = pd.Series(['test_three'], index=marginal_params.index) arg_units = pd.Series(['test_two'], index=marginal_params.index) - res = pelicun_model.convert_marginal_params( - marginal_params, units, arg_units - ) + res = pelicun_model.convert_marginal_params(marginal_params, units, arg_units) # >>> res # Theta_0 @@ -706,9 +695,7 @@ def test_load_cmp_model_1(self, asset_model): check_dtype=False, ) - expected_cmp_units = pd.Series( - data=['ea'], index=['component_a'], name='Units' - ) + expected_cmp_units = pd.Series(data=['ea'], index=['component_a'], name='Units') pd.testing.assert_series_equal( expected_cmp_units, @@ -1071,9 +1058,7 @@ def test_save_load_sample(self, damage_model_with_sample, assessment_instance): check_index_type=False, check_column_type=False, ) - _, units_from_variable = damage_model_with_sample.save_sample( - save_units=True - ) + _, units_from_variable = damage_model_with_sample.save_sample(save_units=True) assert np.all(units_from_variable.to_numpy() == 'ea') def test_load_damage_model(self, damage_model_model_loaded): @@ -1355,9 +1340,7 @@ def test__evaluate_damage_state_and_prepare_dmg_quantities( demand_dict, EDP_req, capacity_sample, lsds_sample ) - qnt_sample = damage_model._prepare_dmg_quantities( - ds_sample, dropzero=False - ) + qnt_sample = damage_model._prepare_dmg_quantities(ds_sample, dropzero=False) # note: the realized number of damage states is random, limiting # our assertions @@ -1370,7 +1353,6 @@ def test__evaluate_damage_state_and_prepare_dmg_quantities( assert list(qnt_sample.columns)[0] == ('B.10.31.001', '2', '2', '0', '0') def test__perform_dmg_task(self, assessment_instance): - damage_model = assessment_instance.damage # @@ -1913,18 +1895,10 @@ def test__create_DV_RVs(self, repair_model, loss_params_A): for rv in rvs: print(rv.theta) assert rv.distribution == 'normal' - np.testing.assert_array_equal( - rvs[0].theta, np.array((1.00, 0.390923, np.nan)) - ) - np.testing.assert_array_equal( - rvs[1].theta, np.array((1.00, 0.464027, np.nan)) - ) - np.testing.assert_array_equal( - rvs[2].theta, np.array((1.00, 0.390923, np.nan)) - ) - np.testing.assert_array_equal( - rvs[3].theta, np.array((1.00, 0.464027, np.nan)) - ) + np.testing.assert_array_equal(rvs[0].theta, np.array((1.00, 0.390923, np.nan))) + np.testing.assert_array_equal(rvs[1].theta, np.array((1.00, 0.464027, np.nan))) + np.testing.assert_array_equal(rvs[2].theta, np.array((1.00, 0.390923, np.nan))) + np.testing.assert_array_equal(rvs[3].theta, np.array((1.00, 0.464027, np.nan))) def test__calc_median_consequence(self, repair_model, loss_params_A): repair_model.loss_params = loss_params_A diff --git a/pelicun/tests/test_uq.py b/pelicun/tests/test_uq.py index eeed09545..3d4814a93 100644 --- a/pelicun/tests/test_uq.py +++ b/pelicun/tests/test_uq.py @@ -175,9 +175,7 @@ def test__get_theta(): def test__get_limit_probs(): # verify that it works for valid inputs - res = uq._get_limit_probs( - np.array((0.10, 0.20)), 'normal', np.array((0.15, 1.00)) - ) + res = uq._get_limit_probs(np.array((0.10, 0.20)), 'normal', np.array((0.15, 1.00))) assert np.allclose(res, np.array((0.4800611941616275, 0.5199388058383725))) res = uq._get_limit_probs( @@ -905,9 +903,7 @@ def test_LogNormalRandomVariable_cdf(): ) x = (-1.0, 0.0, 0.5, 1.0, 2.0) cdf = rv.cdf(x) - assert np.allclose( - cdf, (0.0, 0.0, 0.23597085, 0.49461712, 0.75326339), rtol=1e-5 - ) + assert np.allclose(cdf, (0.0, 0.0, 0.23597085, 0.49461712, 0.75326339), rtol=1e-5) # upper truncation rv = uq.LogNormalRandomVariable( @@ -917,9 +913,7 @@ def test_LogNormalRandomVariable_cdf(): ) x = (-1.0, 0.0, 0.5, 1.0, 2.0) cdf = rv.cdf(x) - assert np.allclose( - cdf, (0.00, 0.00, 0.25797755, 0.52840734, 0.79883714), rtol=1e-5 - ) + assert np.allclose(cdf, (0.00, 0.00, 0.25797755, 0.52840734, 0.79883714), rtol=1e-5) # no truncation rv = uq.LogNormalRandomVariable('test_rv', theta=(1.0, 1.0)) @@ -1054,9 +1048,7 @@ def test_UniformRandomVariable_inverse_transform(): def test_MultinomialRandomVariable(): # multinomial with invalid p values provided in the theta vector with pytest.raises(ValueError): - uq.MultinomialRandomVariable( - 'rv_invalid', np.array((0.20, 0.70, 0.10, 42.00)) - ) + uq.MultinomialRandomVariable('rv_invalid', np.array((0.20, 0.70, 0.10, 42.00))) def test_MultilinearCDFRandomVariable(): @@ -1159,9 +1151,7 @@ def test_DeterministicRandomVariable_inverse_transform(): rv = uq.DeterministicRandomVariable('test_rv', theta=np.array((0.00,))) rv.inverse_transform_sampling(4) inverse_transform = rv.sample - assert np.allclose( - inverse_transform, np.array((0.00, 0.00, 0.00, 0.00)), rtol=1e-5 - ) + assert np.allclose(inverse_transform, np.array((0.00, 0.00, 0.00, 0.00)), rtol=1e-5) def test_RandomVariable_Set(): diff --git a/pelicun/uq.py b/pelicun/uq.py index 472c2bf81..8e74952d5 100644 --- a/pelicun/uq.py +++ b/pelicun/uq.py @@ -1281,7 +1281,7 @@ class UtilityRandomVariable(BaseRandomVariable): @abstractmethod def __init__( self, - name, + name, f_map=None, anchor=None, ): @@ -1291,7 +1291,7 @@ def __init__( Parameters ---------- name: string - A unique string that identifies the random variable. + A unique string that identifies the random variable. f_map: function, optional A user-defined function that is applied on the realizations before returning a sample. @@ -1789,9 +1789,9 @@ def __init__( anchor=None, ): super().__init__( - name=name, + name=name, theta=raw_samples, - truncation_limits=truncation_limits, + truncation_limits=truncation_limits, f_map=f_map, anchor=anchor, ) @@ -1906,9 +1906,7 @@ def inverse_transform(self, sample_size): """ raw_sample_count = len(self._raw_samples) - new_sample = np.tile( - self._raw_samples, int(sample_size / raw_sample_count) + 1 - ) + new_sample = np.tile(self._raw_samples, int(sample_size / raw_sample_count) + 1) result = new_sample[:sample_size] return result From 630d09a3ac89e7c42757f4606b1fba8677f78bc7 Mon Sep 17 00:00:00 2001 From: Adam Zsarnoczay <33822153+zsarnoczay@users.noreply.github.com> Date: Tue, 7 May 2024 14:02:34 -0700 Subject: [PATCH 21/22] azs - apply black to the tools as well --- pelicun/tools/DL_calculation.py | 86 +++++++++------------------------ pelicun/tools/HDF_to_CSV.py | 3 -- pelicun/tools/export_DB.py | 4 -- 3 files changed, 24 insertions(+), 69 deletions(-) diff --git a/pelicun/tools/DL_calculation.py b/pelicun/tools/DL_calculation.py index d9f4488ba..10a04ca41 100644 --- a/pelicun/tools/DL_calculation.py +++ b/pelicun/tools/DL_calculation.py @@ -380,7 +380,6 @@ def run_pelicun( config_ap, CMP = auto_populate(config, auto_script_path) if config_ap['DL'] is None: - log_msg( "The prescribed auto-population script failed to identify " "a valid damage and loss configuration for this asset. " @@ -633,8 +632,7 @@ def run_pelicun( # save results if 'Demand' in config['DL']['Outputs']: out_reqs = [ - out if val else "" - for out, val in config['DL']['Outputs']['Demand'].items() + out if val else "" for out, val in config['DL']['Outputs']['Demand'].items() ] if np.any(np.isin(['Sample', 'Statistics'], out_reqs)): @@ -721,9 +719,7 @@ def run_pelicun( cmp_marginals.loc['excessive.coll.DEM', 'Units'] = 'ea' locs = demand_sample[coll_DEM].columns.unique(level=0) - cmp_marginals.loc['excessive.coll.DEM', 'Location'] = ','.join( - locs - ) + cmp_marginals.loc['excessive.coll.DEM', 'Location'] = ','.join(locs) dirs = demand_sample[coll_DEM].columns.unique(level=1) cmp_marginals.loc['excessive.coll.DEM', 'Direction'] = ','.join( @@ -802,8 +798,7 @@ def run_pelicun( ) out_reqs = [ - out if val else "" - for out, val in config['DL']['Outputs']['Asset'].items() + out if val else "" for out, val in config['DL']['Outputs']['Asset'].items() ] if np.any(np.isin(['Sample', 'Statistics'], out_reqs)): @@ -872,9 +867,7 @@ def run_pelicun( ): component_db = [ 'PelicunDefault/' - + default_DBs['fragility'][ - config['DL']['Asset']['ComponentDatabase'] - ], + + default_DBs['fragility'][config['DL']['Asset']['ComponentDatabase']], ] else: component_db = [] @@ -932,9 +925,9 @@ def run_pelicun( adf.loc[coll_CMP_name, ('Demand', 'Type')] = coll_DEM_name else: - adf.loc[coll_CMP_name, ('Demand', 'Type')] = ( - f'{coll_DEM_name}|{coll_DEM_spec}' - ) + adf.loc[ + coll_CMP_name, ('Demand', 'Type') + ] = f'{coll_DEM_name}|{coll_DEM_spec}' coll_DEM_unit = add_units( pd.DataFrame( @@ -990,9 +983,9 @@ def run_pelicun( # input file adf.loc['excessiveRID', ('Demand', 'Directional')] = 1 adf.loc['excessiveRID', ('Demand', 'Offset')] = 0 - adf.loc['excessiveRID', ('Demand', 'Type')] = ( - 'Residual Interstory Drift Ratio' - ) + adf.loc[ + 'excessiveRID', ('Demand', 'Type') + ] = 'Residual Interstory Drift Ratio' adf.loc['excessiveRID', ('Demand', 'Unit')] = 'unitless' adf.loc['excessiveRID', ('LS1', 'Theta_0')] = irrep_config[ @@ -1019,7 +1012,6 @@ def run_pelicun( # TODO: we can improve this by creating a water # network-specific assessment class if "Water" in config['DL']['Asset']['ComponentDatabase']: - # add a placeholder aggregate fragility that will never trigger # damage, but allow damage processes to aggregate the # various pipeline damages @@ -1086,9 +1078,7 @@ def run_pelicun( for target_val in target_vals: for cmp_type, cmp_id in cmp_map.items(): - if (cmp_type in target_val) and ( - cmp_id != '' - ): + if (cmp_type in target_val) and (cmp_id != ''): target_val = target_val.replace( cmp_type, cmp_id ) @@ -1113,9 +1103,7 @@ def run_pelicun( dmg_process = None else: - log_msg( - f"Prescribed Damage Process not recognized: " f"{dp_approach}" - ) + log_msg(f"Prescribed Damage Process not recognized: " f"{dp_approach}") # calculate damages PAL.damage.calculate(sample_size, dmg_process=dmg_process) @@ -1131,13 +1119,9 @@ def run_pelicun( ) is True ): - damage_units = damage_units.groupby( - level=[0, 1, 2, 4], axis=1 - ).first() + damage_units = damage_units.groupby(level=[0, 1, 2, 4], axis=1).first() - damage_groupby_uid = damage_sample.groupby( - level=[0, 1, 2, 4], axis=1 - ) + damage_groupby_uid = damage_sample.groupby(level=[0, 1, 2, 4], axis=1) damage_sample = damage_groupby_uid.sum().mask( damage_groupby_uid.count() == 0, np.nan @@ -1161,9 +1145,7 @@ def run_pelicun( damage_sample_s.to_csv( output_path / "DMG_sample.zip", index_label=damage_sample_s.columns.name, - compression=dict( - method='zip', archive_name='DMG_sample.csv' - ), + compression=dict(method='zip', archive_name='DMG_sample.csv'), ) output_files.append('DMG_sample.zip') @@ -1185,18 +1167,14 @@ def run_pelicun( ) is True ): - damage_groupby = damage_sample.groupby( - level=[0, 1, 3], axis=1 - ) + damage_groupby = damage_sample.groupby(level=[0, 1, 3], axis=1) damage_units = damage_units.groupby( level=[0, 1, 3], axis=1 ).first() else: - damage_groupby = damage_sample.groupby( - level=[0, 1, 4], axis=1 - ) + damage_groupby = damage_sample.groupby(level=[0, 1, 4], axis=1) damage_units = damage_units.groupby( level=[0, 1, 4], axis=1 @@ -1256,9 +1234,7 @@ def run_pelicun( grp_damage_s.to_csv( output_path / "DMG_grp.zip", index_label=grp_damage_s.columns.name, - compression=dict( - method='zip', archive_name='DMG_grp.csv' - ), + compression=dict(method='zip', archive_name='DMG_grp.csv'), ) output_files.append('DMG_grp.zip') @@ -1469,9 +1445,7 @@ def run_pelicun( adf.loc[rcarb, ('DS1', 'Theta_0')] = rCarbon_config["Median"] if pd.isna(rCarbon_config.get('Distribution', np.nan)) is False: - adf.loc[rcarb, ('DS1', 'Family')] = rCarbon_config[ - "Distribution" - ] + adf.loc[rcarb, ('DS1', 'Family')] = rCarbon_config["Distribution"] adf.loc[rcarb, ('DS1', 'Theta_1')] = rCarbon_config["Theta_1"] else: # add a default replacement carbon value as a placeholder @@ -1563,12 +1537,9 @@ def run_pelicun( drivers.append(f'DMG-{dmg_cmp}') loss_models.append(loss_cmp) - loss_map = pd.DataFrame( - loss_models, columns=['Repair'], index=drivers - ) + loss_map = pd.DataFrame(loss_models, columns=['Repair'], index=drivers) elif repair_config['MapApproach'] == "User Defined": - if repair_config.get('MapFilePath', False) is not False: loss_map_path = repair_config['MapFilePath'] @@ -1634,8 +1605,7 @@ def run_pelicun( ) out_reqs = [ - out if val else "" - for out, val in out_config_loss['Repair'].items() + out if val else "" for out, val in out_config_loss['Repair'].items() ] if np.any( @@ -1682,9 +1652,7 @@ def run_pelicun( if np.any( np.isin(['GroupedSample', 'GroupedStatistics'], out_reqs) ): - repair_groupby = repair_sample.groupby( - level=[0, 1, 2], axis=1 - ) + repair_groupby = repair_sample.groupby(level=[0, 1, 2], axis=1) repair_units = repair_units.groupby( level=[0, 1, 2], axis=1 @@ -1697,9 +1665,7 @@ def run_pelicun( if 'GroupedSample' in out_reqs: grp_repair_s = pd.concat([grp_repair, repair_units]) - grp_repair_s = convert_to_SimpleIndex( - grp_repair_s, axis=1 - ) + grp_repair_s = convert_to_SimpleIndex(grp_repair_s, axis=1) grp_repair_s.to_csv( output_path / "DV_repair_grp.zip", index_label=grp_repair_s.columns.name, @@ -1765,14 +1731,12 @@ def run_pelicun( damage_sample_s['irreparable'] = np.zeros(damage_sample_s.shape[0]) if 'Losses' in config['DL']: - if 'agg_repair' not in locals(): agg_repair = PAL.repair.aggregate_losses() agg_repair_s = convert_to_SimpleIndex(agg_repair, axis=1) else: - agg_repair_s = pd.DataFrame() summary = pd.concat( @@ -1816,8 +1780,7 @@ def run_pelicun( out_dict.update( { "Units": { - col: df_units.loc["Units", col] - for col in df_units.columns + col: df_units.loc["Units", col] for col in df_units.columns } } ) @@ -1841,7 +1804,6 @@ def run_pelicun( def main(): - args = sys.argv[1:] parser = argparse.ArgumentParser() diff --git a/pelicun/tools/HDF_to_CSV.py b/pelicun/tools/HDF_to_CSV.py index a72b66eb0..cb9e04acc 100644 --- a/pelicun/tools/HDF_to_CSV.py +++ b/pelicun/tools/HDF_to_CSV.py @@ -44,7 +44,6 @@ def convert_HDF(HDF_path): - HDF_ext = HDF_path.split('.')[-1] CSV_base = HDF_path[: -len(HDF_ext) - 1] @@ -53,14 +52,12 @@ def convert_HDF(HDF_path): store = pd.HDFStore(HDF_path) for key in store.keys(): - store[key].to_csv(f'{CSV_base}_{key[1:].replace("/","_")}.csv') store.close() if __name__ == '__main__': - args = sys.argv[1:] parser = argparse.ArgumentParser() diff --git a/pelicun/tools/export_DB.py b/pelicun/tools/export_DB.py index b4b8b0343..24d4563cf 100644 --- a/pelicun/tools/export_DB.py +++ b/pelicun/tools/export_DB.py @@ -59,7 +59,6 @@ def export_DB(data_path, target_dir): DB_df = pd.read_hdf(data_path, 'data') for row_id, row in DB_df.iterrows(): - row_dict = convert_Series_to_dict(row) with open(target_dir_data / f'{row_id}.json', 'w', encoding='utf-8') as f: @@ -68,13 +67,11 @@ def export_DB(data_path, target_dir): # add population if it exists try: - DB_df = pd.read_hdf(data_path, 'pop') pop_dict = {} for row_id, row in DB_df.iterrows(): - pop_dict.update({row_id: convert_Series_to_dict(row)}) with open(target_dir / 'population.json', 'w', encoding='utf-8') as f: @@ -85,7 +82,6 @@ def export_DB(data_path, target_dir): if __name__ == '__main__': - args = sys.argv[1:] parser = argparse.ArgumentParser() From bf8ccb5c1ea9dd7dd4ad41c49d44549c535850f2 Mon Sep 17 00:00:00 2001 From: Adam Zsarnoczay <33822153+zsarnoczay@users.noreply.github.com> Date: Tue, 7 May 2024 14:03:37 -0700 Subject: [PATCH 22/22] azs - DLC - fix typo --- pelicun/tools/DL_calculation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pelicun/tools/DL_calculation.py b/pelicun/tools/DL_calculation.py index 10a04ca41..07c944b36 100644 --- a/pelicun/tools/DL_calculation.py +++ b/pelicun/tools/DL_calculation.py @@ -449,7 +449,7 @@ def run_pelicun( ) if not sample_size_str: # give up - print('Sampling size not provided in config file.') + print('Sample size not provided in config file.') return -1 sample_size = int(sample_size_str)