Merge branch 'develop' into tennet_nl_2024_09_06_status

# Conflicts:
#	CHANGELOG.rst

.github/workflows/github_test_action.yml

@@ -314,7 +314,8 @@ jobs:
       - name: Check docs for Python ${{ matrix.python-version }}
         uses: e2nIEE/sphinx-action@master
         with:
-          pre-build-command: "python -m pip install --upgrade pip;
-                              python -m pip install .[docs];"
+          pre-build-command: "apt update && apt upgrade -y && apt install -y build-essential gfortran cmake pkg-config libopenblas-dev;
+            python -m pip install --upgrade pip;
+            python -m pip install .[docs];"
           build-command: "sphinx-build -b html . _build -W"
           docs-folder: "doc/"

CHANGELOG.rst

@@ -3,7 +3,10 @@ Change Log
 
 [upcoming release] - 2024-..-..
 -------------------------------
+- [ADDED] pandas series accessor for geo column
 - [FIXED] Increasing geojson precision as the default precision might cause problems with pandahub
+- [ADDED] Add GeographicalRegion and SubGeographicalRegion names and ids to bus df in cim converter
+- [CHANGED] Capitalize first letter of columns busbar_id, busbar_name and substation_id in bus df for cim converter
 
 [2.14.11] - 2024-07-08
 -------------------------------
@@ -19,6 +22,7 @@ Change Log
 
 [upcoming release] - 2024-..-..
 -------------------------------
+- [FIXED] passing literal json to 'read_json' is deprecated
 
 - [ADDED] pf2pp: possibility to convert a specific varaint and scenario
 - [ADDED] Static Var Compensator with Voltage Control
@@ -92,7 +96,9 @@ Change Log
 - [CHANGED] accelerate distributed slack power flow calculation by using sparse-aware operations in _subnetworks()
 - [CHANGED] Trafo Controllers can now be added to elements that are out of service, changed self.nothing_to_do()
 - [ADDED] Discrete shunt controller for local voltage regulation with shunt steps
+- [ADDED] fix lengths missmatch of output if ignore_zero_length is False in plotting utility function coords_from_node_geodata() and rename ignore_zero_length by ignore_no_geo_diff
 - [ADDED] cim2pp converter: Using lxml to parse XML files (better performance)
+- [ADDED] possibility to load JSON files with unknown object models and just store the models as dictionaries in the network
 
 [2.14.7] - 2024-06-14
 -------------------------------

pandapower/auxiliary.py

@@ -33,6 +33,7 @@
 from importlib.metadata import PackageNotFoundError
 from typing_extensions import deprecated
 
+from geojson import loads, GeoJSON
 import numpy as np
 import pandas as pd
 import scipy as sp
@@ -58,9 +59,16 @@
     import pandaplan.core.pplog as logging
 except ImportError:
     import logging
+try:
+    from geopandas import GeoSeries
+    from shapely import from_geojson
+    geopandas_available = True
+except ImportError:
+    geopandas_available = False
 
 logger = logging.getLogger(__name__)
 
+
 def log_to_level(msg, passed_logger, level):
     if level == "error":
         passed_logger.error(msg)
@@ -343,6 +351,85 @@ def __repr__(self):  # pragma: no cover
         return "\n".join(lines)
 
 
+@pd.api.extensions.register_series_accessor("geojson")
+class GeoAccessor:
+    """
+    pandas Series accessor for the geo column. It facilitates the use of geojson strings.
+    NaN entrys are dropped using the accessor!
+    """
+    def __init__(self, pandas_obj):
+        self._validate(pandas_obj)
+        self._obj = pandas_obj
+
+    @staticmethod
+    def _validate(obj):
+        try:
+            if not obj.dropna().apply(loads).apply(isinstance, args=(GeoJSON,)).all():
+                raise AttributeError("Can only use .geojson accessor with geojson string values!")
+        except Exception as e:
+            raise AttributeError(f"Can only use .geojson accessor with geojson string values!: {e}")
+        if not geopandas_available:
+            soft_dependency_error("GeoAccessor", "geopandas")
+
+    @staticmethod
+    def _extract_coords(x):
+        if x["type"] == "Point":
+            return np.array(x["coordinates"])
+        return [np.array(y) for y in x["coordinates"]]
+
+    @property
+    def _coords(self):
+        """
+        Extracts the geometry coordinates from the GeoJSON strings.
+        It is not recommended to use the standalone coordinates.
+        Important informations like the crs or latlon/lonlat are lost as a result.
+        """
+        return self._obj.dropna().apply(loads).apply(self._extract_coords)
+
+    @property
+    def as_geo_obj(self):
+        """
+        Loads the GeoJSON objects.
+        """
+        return self._obj.dropna().apply(loads)
+
+    @property
+    def type(self):
+        """
+        Extracts the geometry type of the GeoJSON string.
+        """
+        return self._obj.dropna().apply(loads).apply(lambda x: str(x["type"]))
+
+    @property
+    def as_shapely_obj(self):
+        """
+        Converts the GeoJSON strings to shapely geometrys.
+        """
+        return self._obj.dropna().apply(from_geojson)
+
+    @property
+    def as_geoseries(self):
+        """
+        Converts the PandasSeries to a GeoSeries with shapely geometrys.
+        """
+        return GeoSeries(self._obj.dropna().pipe(from_geojson), crs=4326, index=self._obj.dropna().index)
+
+    def __getattr__(self, item):
+        """
+        Enables access to all methods or attribute calls from a GeoSeries.
+        """
+        geoms = self.as_geoseries
+        if hasattr(geoms, item):
+            geoms_item = getattr(geoms, item)
+            if callable(geoms_item):
+                def wrapper(*args, **kwargs):
+                    return geoms_item(*args, **kwargs)
+                return wrapper
+            else:
+                return geoms_item
+        raise AttributeError(f"'GeoAccessor' object has no attribute '{item}'")
+
+
 def plural_s(number):
     return "" if number == 1 else "s"
 

pandapower/convert_format.py

@@ -11,8 +11,10 @@
 
 from pandapower._version import __version__, __format_version__
 from pandapower.create import create_empty_network, create_poly_cost
+from pandapower.plotting import geo
 from pandapower.results import reset_results
 from pandapower.control import TrafoController
+import pandapower.plotting.geo as geo
 
 try:
     import pandaplan.core.pplog as logging
@@ -22,14 +24,6 @@
 logger = logging.getLogger(__name__)
 
 
-def _compare_version(
-        net_version,
-        compare_version: Union[str, int],
-        compare: Callable[[Version, Version], bool] = lambda x, y: x < y
-) -> bool:
-    return compare(Version(str(net_version)), Version(str(compare_version)))
-
-
 def convert_format(net, elements_to_deserialize=None):
     """
     Converts old nets to new format to ensure consistency. The converted net is returned.
@@ -46,6 +40,7 @@ def convert_format(net, elements_to_deserialize=None):
     _add_missing_columns(net, elements_to_deserialize)
     _create_seperate_cost_tables(net, elements_to_deserialize)
     if Version(str(net.format_version)) < Version("3.0.0"):
+        _convert_geo_data(net, elements_to_deserialize)
         _convert_group_element_index(net)
         _convert_trafo_controller_parameter_names(net)
     if Version(str(net.format_version)) < Version("2.4.0"):
@@ -56,7 +51,7 @@ def convert_format(net, elements_to_deserialize=None):
         _convert_to_mw(net)
         _update_trafo_parameter_names(net, elements_to_deserialize)
         reset_results(net)
-    if isinstance(net.format_version, float) and net.format_version < 1.6:
+    if Version(str(net.format_version)) < Version("1.6"):
         set_data_type_of_columns_to_default(net)
     _convert_objects(net, elements_to_deserialize)
     _update_characteristics(net, elements_to_deserialize)
@@ -74,7 +69,7 @@ def _convert_geo_data(net, elements_to_deserialize=None):
         or (_check_elements_to_deserialize('line_geodata', elements_to_deserialize)
          and _check_elements_to_deserialize('line', elements_to_deserialize))):
         if hasattr(net, 'bus_geodata') or hasattr(net, 'line_geodata'):
-            if _compare_version(net.format_version, "1.6"):
+            if Version(str(net.format_version)) < Version("1.6"):
                 net.bus_geodata = pd.DataFrame.from_dict(net.bus_geodata)
                 net.line_geodata = pd.DataFrame.from_dict(net.line_geodata)
             geo.convert_geodata_to_geojson(net)

pandapower/converter/cim/cim2pp/converter_classes/connectivitynodes/connectivityNodesCim16.py

@@ -248,6 +248,18 @@ def _prepare_connectivity_nodes_cim16(self) -> Tuple[pd.DataFrame, pd.DataFrame]
         bb = bb.drop_duplicates(subset=['rdfId'], keep='first')
         connectivity_nodes = pd.merge(connectivity_nodes, bb, how='left', on='rdfId')
 
+        if "Substation" in connectivity_nodes.columns:
+            # add (sub) geographical regions
+            sgr = self.cimConverter.cim['eq']['SubGeographicalRegion'][['rdfId', 'name', 'Region']]
+            regions = pd.merge(self.cimConverter.cim['eq']['Substation'], sgr, left_on="Region", right_on="rdfId",
+                               suffixes=["_substation", "_SubGeographicalRegion"])
+            regions = pd.merge(self.cimConverter.cim['eq']['GeographicalRegion'], regions, left_on="rdfId", right_on="Region_SubGeographicalRegion")
+            regions = regions.rename(columns={'name': 'GeographicalRegion_name', 'name_SubGeographicalRegion': 'SubGeographicalRegion_name',
+                                              'rdfId': 'GeographicalRegion_id', 'rdfId_SubGeographicalRegion': 'SubGeographicalRegion_id'})
+            regions = regions.drop(columns=['name_substation', 'Region_substation', 'Region_SubGeographicalRegion'])
+            connectivity_nodes = pd.merge(connectivity_nodes, regions, how='left', left_on='Substation', right_on='rdfId_substation')
+            connectivity_nodes = connectivity_nodes.drop(columns=["rdfId_substation"])
+
         connectivity_nodes = connectivity_nodes.rename(columns={'rdfId': sc['o_id'], 'TopologicalNode': sc['ct'],
                                                                 'nominalVoltage': 'vn_kv', 'name_substation': 'zone'})
         connectivity_nodes['in_service'] = True

pandapower/converter/cim/cim2pp/converter_classes/impedance/seriesCompensatorsCim16.py

@@ -107,7 +107,7 @@ def _prepare_series_compensators_cim16(self) -> pd.DataFrame:
         ser_comp['gf_pu'] = 0.
         ser_comp['bf_pu'] = 0.
         ser_comp['gt_pu'] = 0.
-        ser_comp['bf_pu'] = 0.
+        ser_comp['bt_pu'] = 0.
         ser_comp['in_service'] = ser_comp.connected & ser_comp.connected2
         ser_comp = ser_comp.rename(columns={'rdfId_Terminal': sc['t_from'], 'rdfId_Terminal2': sc['t_to'],
                                             'rdfId': sc['o_id'], 'index_bus': 'from_bus', 'index_bus2': 'to_bus'})

pandapower/converter/cim/cim2pp/from_cim.py

@@ -103,6 +103,7 @@ def from_cim(file_list: List[str] = None, encoding: str = 'utf-8', convert_line_
              custom_converter_classes: Dict = None,
              cgmes_version: str = '2.4.15', **kwargs) -> \
         pandapower.auxiliary.pandapowerNet:
+    # Nur zum Testen, kann wieder gelöscht werden
     """
     Convert a CIM net to a pandapower net from XML files.
     Additional parameters for kwargs:

pandapower/converter/cim/cim_tools.py

@@ -24,7 +24,7 @@ def get_pp_net_special_columns_dict() -> Dict[str, str]:
                  'o_prf': 'origin_profile', 'ct': 'cim_topnode', 'tc': 'tapchanger_class', 'tc_id': 'tapchanger_id',
                  'pte_id': 'PowerTransformerEnd_id', 'pte_id_hv': 'PowerTransformerEnd_id_hv',
                  'pte_id_mv': 'PowerTransformerEnd_id_mv', 'pte_id_lv': 'PowerTransformerEnd_id_lv',
-                 'cnc_id': 'ConnectivityNodeContainer_id', 'sub_id': 'substation_id', 'src': 'source', 'name': 'name',
+                 'cnc_id': 'ConnectivityNodeContainer_id', 'sub_id': 'Substation_id', 'src': 'source', 'name': 'name',
                  'desc': 'description', 'a_id': 'analog_id'})
 
 
@@ -50,8 +50,9 @@ def extend_pp_net_cim(net: pandapowerNet, override: bool = True) -> pandapowerNe
     fill_dict: Dict[str, Dict[str, List[str]]] = dict()
 
     fill_dict['bus'] = dict()
-    fill_dict['bus'][np_str_type] = [sc['o_prf'], sc['ct'], sc['cnc_id'], sc['sub_id'], 'description', 'busbar_id',
-                                     'busbar_name']
+    fill_dict['bus'][np_str_type] = [sc['o_prf'], sc['ct'], sc['cnc_id'], sc['sub_id'], 'description', 'Busbar_id',
+                                     'Busbar_name', 'GeographicalRegion_id', 'GeographicalRegion_name',
+                                     'SubGeographicalRegion_id', 'SubGeographicalRegion_name']
 
     fill_dict['ext_grid'] = dict()
     fill_dict['ext_grid'][np_str_type] = [sc['t'], sc['sub'], 'description']

pandapower/file_io.py

@@ -238,7 +238,7 @@ def _from_excel_old(xls):
 
 def from_json(filename, convert=True, encryption_key=None, elements_to_deserialize=None,
               keep_serialized_elements=True, add_basic_std_types=False, replace_elements=None,
-              empty_dict_like_object=None):
+              empty_dict_like_object=None, ignore_unknown_objects=False):
     """
     Load a pandapower network from a JSON file.
     The index of the returned network is not necessarily in the same order as the original network.
@@ -270,6 +270,9 @@ def from_json(filename, convert=True, encryption_key=None, elements_to_deseriali
         the data of the json string. Give another dict-like object to start filling that alternative
         object with the json data.
 
+        **ignore_unknown_objects** (bool, False) - If set to True, ignore any objects that cannot be
+         deserialized instead of raising an error
+
     OUTPUT:
         **net** (dict) - The pandapower format network
 
@@ -294,12 +297,13 @@ def from_json(filename, convert=True, encryption_key=None, elements_to_deseriali
         keep_serialized_elements=keep_serialized_elements,
         add_basic_std_types=add_basic_std_types,
         replace_elements=replace_elements,
-        empty_dict_like_object=empty_dict_like_object)
+        empty_dict_like_object=empty_dict_like_object,
+        ignore_unknown_objects=ignore_unknown_objects)
 
 
 def from_json_string(json_string, convert=False, encryption_key=None, elements_to_deserialize=None,
                      keep_serialized_elements=True, add_basic_std_types=False,
-                     replace_elements=None, empty_dict_like_object=None):
+                     replace_elements=None, empty_dict_like_object=None, ignore_unknown_objects=False):
     """
     Load a pandapower network from a JSON string.
     The index of the returned network is not necessarily in the same order as the original network.
@@ -330,6 +334,9 @@ def from_json_string(json_string, convert=False, encryption_key=None, elements_t
         the data of the json string. Give another dict-like object to start filling that alternative
         object with the json data.
 
+        **ignore_unknown_objects** (bool, False) - If set to True, ignore any objects that cannot be
+         deserialized instead of raising an error
+
     OUTPUT:
         **net** (dict) - The pandapower format network
 
@@ -347,10 +354,12 @@ def from_json_string(json_string, convert=False, encryption_key=None, elements_t
 
     if elements_to_deserialize is None:
         net = json.loads(json_string, cls=io_utils.PPJSONDecoder,
-                         empty_dict_like_object=empty_dict_like_object)
+                         empty_dict_like_object=empty_dict_like_object,
+                         ignore_unknown_objects=ignore_unknown_objects)
     else:
         net = json.loads(json_string, cls=io_utils.PPJSONDecoder, deserialize_pandas=False,
-                         empty_dict_like_object=empty_dict_like_object)
+                         empty_dict_like_object=empty_dict_like_object,
+                         ignore_unknown_objects=ignore_unknown_objects)
         net_dummy = create_empty_network()
         if ('version' not in net.keys()) | (Version(net.version) < Version('2.1.0')):
             raise UserWarning('table selection is only possible for nets above version 2.0.1. '