Merge branch 'develop' into develop

.github/workflows/github_test_action.yml

@@ -314,7 +314,8 @@ jobs:
       - name: Check docs for Python ${{ matrix.python-version }}
         uses: e2nIEE/sphinx-action@master
         with:
-          pre-build-command: "python -m pip install --upgrade pip;
-                              python -m pip install .[docs];"
+          pre-build-command: "apt update && apt upgrade -y && apt install -y build-essential gfortran cmake pkg-config libopenblas-dev;
+            python -m pip install --upgrade pip;
+            python -m pip install .[docs];"
           build-command: "sphinx-build -b html . _build -W"
           docs-folder: "doc/"

CHANGELOG.rst

@@ -1,6 +1,15 @@
 Change Log
 =============
 
+[upcoming release] - 2024-..-..
+-------------------------------
+- [ADDED] pandas series accessor for geo column
+- [FIXED] Increasing geojson precision as the default precision might cause problems with pandahub
+- [ADDED] Add GeographicalRegion and SubGeographicalRegion names and ids to bus df in cim converter
+- [CHANGED] Capitalize first letter of columns busbar_id, busbar_name and substation_id in bus df for cim converter
+- [FIXED] Do not modify pandas options when importing pandapower
+- [FIXED] cim2pp: set default xml encoding to None to avoid error after changing to lxml
+
 [2.14.11] - 2024-07-08
 -------------------------------
 - [FIXED] Lightsim2grid version
@@ -15,6 +24,9 @@ Change Log
 
 [upcoming release] - 2024-..-..
 -------------------------------
+
+- [FIXED] copy array element to standard python scalar
+- [FIXED] passing literal json to 'read_json' is deprecated
 - [FIXED] replacing deprecated in1d with isin
 - [ADDED] A switch to disable updating the vk and vkr values for trafo3w
 - [FIXED] cast the column to the correct type before assigning values
@@ -81,8 +93,11 @@ Change Log
 - [ADDED] Add VSC element, dc buses, dc lines, and hybrid AC/DC power flow calculation
 - [CHANGED] accelerate _integrate_power_elements_connected_with_switch_buses() in get_equivalent()
 - [CHANGED] accelerate distributed slack power flow calculation by using sparse-aware operations in _subnetworks()
+- [CHANGED] Trafo Controllers can now be added to elements that are out of service, changed self.nothing_to_do()
 - [ADDED] Discrete shunt controller for local voltage regulation with shunt steps
+- [ADDED] fix lengths missmatch of output if ignore_zero_length is False in plotting utility function coords_from_node_geodata() and rename ignore_zero_length by ignore_no_geo_diff
 - [ADDED] cim2pp converter: Using lxml to parse XML files (better performance)
+- [ADDED] possibility to load JSON files with unknown object models and just store the models as dictionaries in the network
 
 [2.14.7] - 2024-06-14
 -------------------------------

pandapower/__init__.py

@@ -17,6 +17,9 @@
 from pandapower.runpm import *
 from pandapower.pf.runpp_3ph import runpp_3ph
 
+import geojson
+geojson.geometry.DEFAULT_PRECISION = 8
+
 import pandas as pd
 pd.options.mode.chained_assignment = None  # default='warn'
 

pandapower/auxiliary.py

@@ -33,6 +33,7 @@
 from importlib.metadata import PackageNotFoundError
 from typing_extensions import deprecated
 
+from geojson import loads, GeoJSON
 import numpy as np
 import pandas as pd
 import scipy as sp
@@ -58,9 +59,16 @@
     import pandaplan.core.pplog as logging
 except ImportError:
     import logging
+try:
+    from geopandas import GeoSeries
+    from shapely import from_geojson
+    geopandas_available = True
+except ImportError:
+    geopandas_available = False
 
 logger = logging.getLogger(__name__)
 
+
 def log_to_level(msg, passed_logger, level):
     if level == "error":
         passed_logger.error(msg)
@@ -343,6 +351,85 @@ def __repr__(self):  # pragma: no cover
         return "\n".join(lines)
 
 
+@pd.api.extensions.register_series_accessor("geojson")
+class GeoAccessor:
+    """
+    pandas Series accessor for the geo column. It facilitates the use of geojson strings.
+    NaN entrys are dropped using the accessor!
+    """
+    def __init__(self, pandas_obj):
+        self._validate(pandas_obj)
+        self._obj = pandas_obj
+
+    @staticmethod
+    def _validate(obj):
+        try:
+            if not obj.dropna().apply(loads).apply(isinstance, args=(GeoJSON,)).all():
+                raise AttributeError("Can only use .geojson accessor with geojson string values!")
+        except Exception as e:
+            raise AttributeError(f"Can only use .geojson accessor with geojson string values!: {e}")
+        if not geopandas_available:
+            soft_dependency_error("GeoAccessor", "geopandas")
+
+    @staticmethod
+    def _extract_coords(x):
+        if x["type"] == "Point":
+            return np.array(x["coordinates"])
+        return [np.array(y) for y in x["coordinates"]]
+
+    @property
+    def _coords(self):
+        """
+        Extracts the geometry coordinates from the GeoJSON strings.
+        It is not recommended to use the standalone coordinates.
+        Important informations like the crs or latlon/lonlat are lost as a result.
+        """
+        return self._obj.dropna().apply(loads).apply(self._extract_coords)
+
+    @property
+    def as_geo_obj(self):
+        """
+        Loads the GeoJSON objects.
+        """
+        return self._obj.dropna().apply(loads)
+
+    @property
+    def type(self):
+        """
+        Extracts the geometry type of the GeoJSON string.
+        """
+        return self._obj.dropna().apply(loads).apply(lambda x: str(x["type"]))
+
+    @property
+    def as_shapely_obj(self):
+        """
+        Converts the GeoJSON strings to shapely geometrys.
+        """
+        return self._obj.dropna().apply(from_geojson)
+
+    @property
+    def as_geoseries(self):
+        """
+        Converts the PandasSeries to a GeoSeries with shapely geometrys.
+        """
+        return GeoSeries(self._obj.dropna().pipe(from_geojson), crs=4326, index=self._obj.dropna().index)
+
+    def __getattr__(self, item):
+        """
+        Enables access to all methods or attribute calls from a GeoSeries.
+        """
+        geoms = self.as_geoseries
+        if hasattr(geoms, item):
+            geoms_item = getattr(geoms, item)
+            if callable(geoms_item):
+                def wrapper(*args, **kwargs):
+                    return geoms_item(*args, **kwargs)
+                return wrapper
+            else:
+                return geoms_item
+        raise AttributeError(f"'GeoAccessor' object has no attribute '{item}'")
+
+
 def plural_s(number):
     return "" if number == 1 else "s"
 

pandapower/build_branch.py

@@ -1304,7 +1304,8 @@ def _calculate_3w_tap_changers(t3, t2, sides):
         if any_at_star_point & np.any(mask_star_point := (tap_mask & at_star_point)): 
             t = tap_arrays["tap_step_percent"][side][mask_star_point] * np.exp(1j * np.deg2rad(tap_arrays["tap_step_degree"][side][mask_star_point]))
             tap_pos = tap_arrays["tap_pos"][side][mask_star_point]
-            t_corrected = 100 * t / (100 + (t * tap_pos))
+            tap_neutral = tap_arrays["tap_neutral"][side][mask_star_point]
+            t_corrected = 100 * t / (100 + (t * (tap_pos-tap_neutral)))
             tap_arrays["tap_step_percent"][side][mask_star_point] = np.abs(t_corrected)
             tap_arrays["tap_side"][side][mask_star_point] = "lv" if side == "hv" else "hv"
             tap_arrays["tap_step_degree"][side][mask_star_point] = np.rad2deg(np.angle(t_corrected))

pandapower/control/controller/trafo/ContinuousTapControl.py

@@ -50,18 +50,18 @@ def __init__(self, net, element_index, vm_set_pu, tol=1e-3, side="lv", element="
         self.vm_set_pu = vm_set_pu
 
     def _set_t_nom(self, net):
-        vn_hv_kv = read_from_net(net, self.element, self.controlled_element_index, 'vn_hv_kv', self._read_write_flag)
-        hv_bus = read_from_net(net, self.element, self.controlled_element_index, 'hv_bus', self._read_write_flag)
+        vn_hv_kv = read_from_net(net, self.element, self.element_index, 'vn_hv_kv', self._read_write_flag)
+        hv_bus = read_from_net(net, self.element, self.element_index, 'hv_bus', self._read_write_flag)
         vn_hv_bus_kv = read_from_net(net, "bus", hv_bus, 'vn_kv', self._read_write_flag)
 
         if self.element == "trafo3w" and self.side == "mv":
-            vn_mv_kv = read_from_net(net, self.element, self.controlled_element_index, 'vn_mv_kv', self._read_write_flag)
-            mv_bus = read_from_net(net, self.element, self.controlled_element_index, 'mv_bus', self._read_write_flag)
+            vn_mv_kv = read_from_net(net, self.element, self.element_index, 'vn_mv_kv', self._read_write_flag)
+            mv_bus = read_from_net(net, self.element, self.element_index, 'mv_bus', self._read_write_flag)
             vn_mv_bus_kv = read_from_net(net, "bus", mv_bus, 'vn_kv', self._read_write_flag)
             self.t_nom = vn_mv_kv / vn_hv_kv * vn_hv_bus_kv / vn_mv_bus_kv
         else:
-            vn_lv_kv = read_from_net(net, self.element, self.controlled_element_index, 'vn_lv_kv', self._read_write_flag)
-            lv_bus = read_from_net(net, self.element, self.controlled_element_index, 'lv_bus', self._read_write_flag)
+            vn_lv_kv = read_from_net(net, self.element, self.element_index, 'vn_lv_kv', self._read_write_flag)
+            lv_bus = read_from_net(net, self.element, self.element_index, 'lv_bus', self._read_write_flag)
             vn_lv_bus_kv = read_from_net(net, "bus", lv_bus, 'vn_kv', self._read_write_flag)
             self.t_nom = vn_lv_kv / vn_hv_kv * vn_hv_bus_kv / vn_lv_bus_kv
 
@@ -77,7 +77,7 @@ def control_step(self, net):
         if self.nothing_to_do(net):
             return
 
-        delta_vm_pu = read_from_net(net, "res_bus", self.controlled_bus, 'vm_pu', self._read_write_flag) - self.vm_set_pu
+        delta_vm_pu = read_from_net(net, "res_bus", self.trafobus, 'vm_pu', self._read_write_flag) - self.vm_set_pu
         tc = delta_vm_pu / self.tap_step_percent * 100 / self.t_nom
         self.tap_pos = self.tap_pos + tc * self.tap_side_coeff * self.tap_sign
         if self.check_tap_bounds:
@@ -87,7 +87,7 @@ def control_step(self, net):
         # necessary in case the dtype of the column is int
         if net[self.element].tap_pos.dtype != "float":
             net[self.element].tap_pos = net[self.element].tap_pos.astype(float)
-        write_to_net(net, self.element, self.controlled_element_index, "tap_pos", self.tap_pos, self._read_write_flag)
+        write_to_net(net, self.element, self.element_index, "tap_pos", self.tap_pos, self._read_write_flag)
 
     def is_converged(self, net):
         """
@@ -97,10 +97,10 @@ def is_converged(self, net):
         if self.nothing_to_do(net):
             return True
 
-        vm_pu = read_from_net(net, "res_bus", self.controlled_bus, "vm_pu", self._read_write_flag)
+        vm_pu = read_from_net(net, "res_bus", self.trafobus, "vm_pu", self._read_write_flag)
         # this is possible in case the trafo is set out of service by the connectivity check
         is_nan =  np.isnan(vm_pu)
-        self.tap_pos = read_from_net(net, self.element, self.controlled_element_index, "tap_pos", self._read_write_flag)
+        self.tap_pos = read_from_net(net, self.element, self.element_index, "tap_pos", self._read_write_flag)
         difference = 1 - self.vm_set_pu / vm_pu
 
         if self.check_tap_bounds:

pandapower/control/controller/trafo/DiscreteTapControl.py

@@ -108,9 +108,9 @@ def control_step(self, net):
         if self.nothing_to_do(net):
             return
 
-        vm_pu = read_from_net(net, "res_bus", self.controlled_bus, "vm_pu", self._read_write_flag)
+        vm_pu = read_from_net(net, "res_bus", self.trafobus, "vm_pu", self._read_write_flag)
         self.tap_pos = read_from_net(
-            net, self.element, self.controlled_element_index, "tap_pos", self._read_write_flag)
+            net, self.element, self.element_index, "tap_pos", self._read_write_flag)
 
         increment = np.where(
             self.tap_side_coeff * self.tap_sign == 1,
@@ -126,7 +126,7 @@ def control_step(self, net):
             self._hunting_taps = self._hunting_taps[1:, :]
 
         # WRITE TO NET
-        write_to_net(net, self.element, self.controlled_element_index, 'tap_pos',
+        write_to_net(net, self.element, self.element_index, 'tap_pos',
                      self.tap_pos, self._read_write_flag)
 
     def is_converged(self, net):
@@ -136,11 +136,11 @@ def is_converged(self, net):
         if self.nothing_to_do(net):
             return True
 
-        vm_pu = read_from_net(net, "res_bus", self.controlled_bus, "vm_pu", self._read_write_flag)
+        vm_pu = read_from_net(net, "res_bus", self.trafobus, "vm_pu", self._read_write_flag)
         # this is possible in case the trafo is set out of service by the connectivity check
         is_nan = np.isnan(vm_pu)
         self.tap_pos = read_from_net(
-            net, self.element, self.controlled_element_index, "tap_pos", self._read_write_flag)
+            net, self.element, self.element_index, "tap_pos", self._read_write_flag)
 
         reached_limit = np.where(self.tap_side_coeff * self.tap_sign == 1,
                                  (vm_pu < self.vm_lower_pu) & (self.tap_pos == self.tap_min) |

pandapower/control/controller/trafo_control.py

@@ -46,14 +46,16 @@ def __init__(self, net, element_index, side, tol, in_service, element, level=0,
 
         self._set_side(side)
         self._set_read_write_flag(net)
-        self._set_valid_controlled_index_and_bus(net)
+        # self._set_valid_controlled_index_and_bus(net)
         self._set_tap_parameters(net)
         self._set_tap_side_coeff(net)
 
         self.tol = tol
 
         self.set_recycle(net)
 
+        self.trafobus = read_from_net(net, self.element, self.element_index, self.side + '_bus', self._read_write_flag)
+
     def _set_read_write_flag(self, net):
         # if someone changes indices of the controller from single index to array and vice versa
         self._read_write_flag, _ = _detect_read_write_flag(net, self.element, self.element_index, "tap_pos")
@@ -67,25 +69,36 @@ def initialize_control(self, net):
         # update trafo tap parameters
         # we assume side does not change after the controller is created
         self._set_read_write_flag(net)
-        self._set_valid_controlled_index_and_bus(net)
+        # self._set_valid_controlled_index_and_bus(net)
         if self.nothing_to_do(net):
             return
         self._set_tap_parameters(net)
         self._set_tap_side_coeff(net)
 
     def nothing_to_do(self, net):
-        # if the controller shouldn't do anything, return True
-        if self.controlled_element_index is None or (
-                self._read_write_flag != 'single_index' and len(self.controlled_element_index) == 0):
-            return True
-        return False
+        element_in_service = read_from_net(net, self.element, self.element_index, 'in_service', self._read_write_flag)
+        ext_grid_bus = np.isin(self.trafobus, net.ext_grid.loc[net.ext_grid.in_service, 'bus'].values)
+        element_index_in_net = np.isin(self.element_index, net[self.element].index.values)
+        self.controlled = np.logical_and(np.logical_and(element_in_service, element_index_in_net), np.logical_not(ext_grid_bus))
+        if isinstance(self.element_index, np.int64) or isinstance(self.element_index, int):
+            # if the controller shouldn't do anything, return True
+            if not element_in_service or ext_grid_bus or not element_index_in_net or (
+                    self._read_write_flag != 'single_index' and len(self.element_index) == 0):
+                return True
+            return False
+        else:
+            # if the controller shouldn't do anything, return True
+            if np.all(~element_in_service[self.controlled]) or np.all(ext_grid_bus[self.controlled]) or np.all(~element_index_in_net[self.controlled]) or (
+                    self._read_write_flag != 'single_index' and len(self.element_index) == 0):
+                return True
+            return False
 
     def _set_tap_side_coeff(self, net):
-        tap_side = read_from_net(net, self.element, self.controlled_element_index, 'tap_side', self._read_write_flag)
+        tap_side = read_from_net(net, self.element, self.element_index, 'tap_side', self._read_write_flag)
         if (len(np.setdiff1d(tap_side, ['hv', 'lv'])) > 0 and self.element == "trafo") or \
             (len(np.setdiff1d(tap_side, ['hv', 'lv', 'mv'])) > 0 and self.element == "trafo3w"):
             raise ValueError("Trafo tap side (in net.%s) has to be either hv or lv, "
-                             "but received: %s for trafo %s" % (self.element, tap_side, self.controlled_element_index))
+                             "but received: %s for trafo %s" % (self.element, tap_side, self.element_index))
 
         if self._read_write_flag == "single_index":
             self.tap_side_coeff = 1 if tap_side == 'hv' else -1
@@ -113,29 +126,25 @@ def _set_side(self, side):
         self.side = side
 
     def _set_valid_controlled_index_and_bus(self, net):
-        self.trafobus = read_from_net(net, self.element, self.element_index, self.side + '_bus', self._read_write_flag)
         element_in_service = read_from_net(net, self.element, self.element_index, 'in_service', self._read_write_flag)
         ext_grid_bus = np.isin(self.trafobus, net.ext_grid.loc[net.ext_grid.in_service, 'bus'].values)
         element_index_in_net = np.isin(self.element_index, net[self.element].index.values)
-        controlled = np.logical_and(np.logical_and(element_in_service, element_index_in_net), np.logical_not(ext_grid_bus))
-        if self._read_write_flag == 'single_index':
-            self.controlled_element_index = self.element_index if controlled else None
-            self.controlled_bus = self.trafobus if controlled else None
-        else:
-            self.controlled_element_index = self.element_index[controlled]
-            self.controlled_bus = self.trafobus[controlled]
+        self.controlled = np.logical_and(np.logical_and(element_in_service, element_index_in_net), np.logical_not(ext_grid_bus))
+        if self._read_write_flag != 'single_index':
+            self.element_index = self.element_index[self.controlled]
+            self.trafobus = self.trafobus[self.controlled]
 
-        if np.all(~controlled):
+        if np.all(~self.controlled):
             logger.warning("All controlled buses are not valid: controller has no effect")
 
     def _set_tap_parameters(self, net):
-        self.tap_min = read_from_net(net, self.element, self.controlled_element_index, "tap_min", self._read_write_flag)
-        self.tap_max = read_from_net(net, self.element, self.controlled_element_index, "tap_max", self._read_write_flag)
-        self.tap_neutral = read_from_net(net, self.element, self.controlled_element_index, "tap_neutral", self._read_write_flag)
-        self.tap_step_percent = read_from_net(net, self.element, self.controlled_element_index, "tap_step_percent", self._read_write_flag)
-        self.tap_step_degree = read_from_net(net, self.element, self.controlled_element_index, "tap_step_degree", self._read_write_flag)
+        self.tap_min = read_from_net(net, self.element, self.element_index, "tap_min", self._read_write_flag)
+        self.tap_max = read_from_net(net, self.element, self.element_index, "tap_max", self._read_write_flag)
+        self.tap_neutral = read_from_net(net, self.element, self.element_index, "tap_neutral", self._read_write_flag)
+        self.tap_step_percent = read_from_net(net, self.element, self.element_index, "tap_step_percent", self._read_write_flag)
+        self.tap_step_degree = read_from_net(net, self.element, self.element_index, "tap_step_degree", self._read_write_flag)
 
-        self.tap_pos = read_from_net(net, self.element, self.controlled_element_index, "tap_pos", self._read_write_flag)
+        self.tap_pos = read_from_net(net, self.element, self.element_index, "tap_pos", self._read_write_flag)
         if self._read_write_flag == "single_index":
             self.tap_sign = 1 if np.isnan(self.tap_step_degree) else np.sign(np.cos(np.deg2rad(self.tap_step_degree)))
             if (self.tap_sign == 0) | (np.isnan(self.tap_sign)):
@@ -163,7 +172,7 @@ def set_recycle(self, net):
         net.controller.at[self.index, 'recycle'] = recycle
 
     # def timestep(self, net):
-    #     self.tap_pos = net[self.element].at[self.controlled_element_index, "tap_pos"]
+    #     self.tap_pos = net[self.element].at[self.element_index, "tap_pos"]
 
     def __repr__(self):
         s = '%s of %s %s' % (self.__class__.__name__, self.element, self.element_index)