Refactor tools.py: return 1 output df in timeseries Perez (#13)

* Hot fix for df_registries + clean up of plot functions

* Created function to generate df_outputs from df_registries

* Add pvrow_segment_index in surface registry to keep track of segmnts

* will be much easier to track in the surface registry
* need to make sure that the shaded surfaces are broken up after the discretization

* Now calculating interrow shading after discretizing surfaces

* this way can keep track of pvrow segments
* also now the unshaded surfaces of the prow are not splitted due to
direct shading as it used to be

* Updated tests as now no unecessary surface split for non shaded side

* so the array size changed in one test
* another test got order swapped, but should not matter

* Very small changes in expected values bc geometry cutting fn changed

* minor changes in the intersect point in interrow shading calculation
certainly caused the small changes in this case where direct shading happens

* Now can also calculate segment outputs from df_registries

* used v011 outputs to make sure it was correct

* Simplify outputs of timeseries Perez calculations

* fix one of the tests based on discretized segments

* Fixed functional test serial perez calculations

* Fixed all remaining tests from timeseries calc simplification

pvfactors/pvarray.py

@@ -213,15 +213,20 @@ def update_view_factors(self, solar_zenith, solar_azimuth, array_tilt,
         # --- Add edge points to geometries
         self.line_registry.pvgeometry.split_ground_geometry_from_edge_points(
             edge_points)
-        if self.has_direct_shading:
-            LOGGER.debug("...calculating interrow shading")
-            self.calculate_interrow_direct_shading()
 
         # ------- Surface creation: returning the surface registry, a line may
         # represent 2 surfaces (front and back)
         LOGGER.debug("...building surface registry")
         self.create_surface_registry()
 
+        # -------- Interrow shading
+        if self.has_direct_shading:
+            LOGGER.debug("...calculating interrow shading")
+            self.calculate_interrow_direct_shading()
+
+        # -------- Update the surface areas (/ lengths) after shading calculation
+        self.surface_registry.loc[:, 'area'] = self.surface_registry.pvgeometry.length
+
         # ------- View factors: define the surface views and calculate view
         # factors
         LOGGER.debug("...building view matrix and finding obstructions")
@@ -842,31 +847,40 @@ def calculate_interrow_direct_shading(self):
         """
         # Find the direction of shading
         # Direct shading calculation must be specific to the PVRow class
+        # Shading is said "forward" if the shadow of the pvrow is on the
+        # right side of the pvrow
         shading_is_forward = (self.pvrows[0].shadow['geometry']
                               .bounds[0] >=
                               self.pvrows[0].left_point.x)
+        # Determine if front or back surface has direct shading
+        if self.pvrows[0].is_front_side_illuminated(self.solar_2d_vector):
+            side_shaded = 'front'
+        else:
+            side_shaded = 'back'
+
         if shading_is_forward:
-            for i in range(1, self.n_pvrows):
-                pvrow = self.pvrows[i - 1]
-                adjacent_pvrow = self.pvrows[i]
+            for idx_pvrow in range(1, self.n_pvrows):
+                # idx_pvrow is the index of the shaded pvrow
+                pvrow = self.pvrows[idx_pvrow - 1]
+                adjacent_pvrow = self.pvrows[idx_pvrow]
                 # Shadows from left to right: find vector of shadow
                 top_point_vector = pvrow.highest_point
-                x1_shadow, x2_shadow = (pvrow
-                                        .get_shadow_bounds(
-                                            self.solar_2d_vector)
-                                        )
+                x1_shadow, x2_shadow = pvrow.get_shadow_bounds(
+                    self.solar_2d_vector)
                 ground_point = Point(x2_shadow, Y_GROUND)
                 linestring_shadow = LineString([top_point_vector, ground_point])
                 # FIXME: we do not want to create a line_registry object
-                self.line_registry.pvgeometry.split_pvrow_geometry(
-                    self.pvrows[i].line_registry_indices[0],
+                self.surface_registry.pvgeometry.split_pvrow_geometry(
+                    idx_pvrow,
                     linestring_shadow,
-                    adjacent_pvrow.highest_point
+                    adjacent_pvrow.highest_point,
+                    side_shaded
                 )
         else:
-            for i in range(self.n_pvrows - 2, -1, -1):
-                pvrow = self.pvrows[i + 1]
-                adjacent_pvrow = self.pvrows[i]
+            for idx_pvrow in range(self.n_pvrows - 2, -1, -1):
+                # idx_pvrow is the index of the shaded pvrow
+                pvrow = self.pvrows[idx_pvrow + 1]
+                adjacent_pvrow = self.pvrows[idx_pvrow]
                 # Shadows from right to left: find vector of shadow
                 top_point_vector = pvrow.highest_point
                 x1_shadow, x2_shadow = (pvrow
@@ -877,10 +891,11 @@ def calculate_interrow_direct_shading(self):
                 linestring_shadow = LineString(
                     [top_point_vector, ground_point])
                 # FIXME: we do not want to create a line_registry object
-                self.line_registry.pvgeometry.split_pvrow_geometry(
-                    self.pvrows[i].line_registry_indices[0],
+                self.surface_registry.pvgeometry.split_pvrow_geometry(
+                    idx_pvrow,
                     linestring_shadow,
-                    adjacent_pvrow.highest_point
+                    adjacent_pvrow.highest_point,
+                    side_shaded
                 )
 
 # ------- Surface creation
@@ -898,15 +913,11 @@ def create_surface_registry(self):
 
         front_surface_registry = copy.copy(self.line_registry)
         front_surface_registry['surface_side'] = 'front'
+        # Create pvrow back surfaces
         back_surface_registry = front_surface_registry.loc[
             front_surface_registry.line_type == 'pvrow'].copy()
         back_surface_registry.loc[:, 'surface_side'] = 'back'
-        # FIXME: this is not gonna work if not single line
-        if self.pvrows[0].is_front_side_illuminated(self.solar_2d_vector):
-            back_surface_registry.loc[:, 'shaded'] = False
-        else:
-            front_surface_registry.loc[front_surface_registry.line_type
-                                       == 'pvrow', 'shaded'] = False
+        # Merge two registries together
         self.surface_registry = front_surface_registry.append(
             back_surface_registry).reset_index(drop=False)
         # FIXME: needed to give special methods to surface_registry, but
@@ -918,14 +929,15 @@ def create_surface_registry(self):
         ).astype(int)
 
         # Discretize surfaces specified by user
+        self.surface_registry['pvrow_segment_index'] = np.nan
         self.discretize_surfaces()
 
         # Add columns that will be used during calculations
         self.surface_registry['irradiance_term'] = np.nan
         self.surface_registry['reflectivity'] = np.nan
         self.surface_registry['q0'] = np.nan
         self.surface_registry['qinc'] = np.nan
-        self.surface_registry['area'] = self.surface_registry.pvgeometry.length
+        self.surface_registry['area'] = np.nan
         self.surface_registry['index_pvrow_neighbor'] = np.nan
 
     def discretize_surfaces(self):
@@ -942,7 +954,8 @@ def discretize_surfaces(self):
             side = cut[2]
             self.pvrows[pvrow_index].calculate_cut_points(n_segments)
             self.surface_registry.pvgeometry.cut_pvrow_geometry(
-                self.pvrows[pvrow_index].cut_points, pvrow_index, side)
+                self.pvrows[pvrow_index].cut_points, pvrow_index, side,
+                count_segments=True)
 
 # ------- View matrix creation
     def create_view_matrix(self):

pvfactors/pvgeometry.py

@@ -187,7 +187,8 @@ def split_ground_geometry_from_edge_points(self, edge_points):
                 raise PVFactorsError("geoentry_to_break_up.shape[0] cannot be"
                                      " larger than 1")
 
-    def break_and_add_entries(self, geoentry_to_break_up, point):
+    def break_and_add_entries(self, geoentry_to_break_up, point,
+                              pvrow_segment_index=None):
         """
         Break up a surface geometry into two objects at a point location.
 
@@ -196,6 +197,8 @@ def break_and_add_entries(self, geoentry_to_break_up, point):
             column and ``pvgeometry`` extension
         :param point: point used to decide where to break up entry.
         :type point: :class:`shapely.Point` object
+        :param int pvrow_segment_index: (optional) index of segment being
+            broken up. Default is None.
         :return: None
         """
         # Get geometry
@@ -207,6 +210,10 @@ def break_and_add_entries(self, geoentry_to_break_up, point):
         self._obj.at[idx, 'geometry'] = geometry_1.values
         new_registry_entry = self._obj.loc[idx, :].copy()
         new_registry_entry['geometry'] = geometry_2.values
+        # Add a value to "pvrow segment index" if provided
+        if pvrow_segment_index is not None:
+            self._obj.at[idx, 'pvrow_segment_index'] = pvrow_segment_index
+            new_registry_entry['pvrow_segment_index'] = pvrow_segment_index + 1
         self._obj.loc[self._obj.shape[0], :] = new_registry_entry.values[0]
 
     @staticmethod
@@ -237,25 +244,30 @@ def cut_linestring(line, point):
                     LineString(coords[:i] + [(cp.x, cp.y)]),
                     LineString([(cp.x, cp.y)] + coords[i:])]
 
-    def split_pvrow_geometry(self, idx, line_shadow, pvrow_top_point):
+    def split_pvrow_geometry(self, idx_pvrow, line_shadow, pvrow_top_point,
+                             surface_side):
         """
         Break up pv row line into two pv row lines, a shaded one and an
         unshaded one. This function requires knowing the pv row line index in
         the registry, the "shadow line" that intersects with the pv row, and
         the top point of the pv row in order to decide which pv row line will
         be shaded or not after break up.
 
-        :param int idx: index of shaded pv row entry
+        :param int idx_pvrow: index of shaded pv row entry
         :param line_shadow: :class:`shapely.LineString` object representing the
             "shadow line" intersecting with the pv row line
         :param pvrow_top_point: the highest point of the pv row line (in the
             elevation direction)
         :param pvrow_top_point: :class:``shapely.Point`` object
+        :param str surface_side: surface side of the pvrow
         :return: None
         """
         # Define geometry to work on
-        geometry = self._obj.loc[idx, 'geometry']
-        # Find intersection point
+        df_pvrow = self._obj.loc[(self._obj.pvrow_index == idx_pvrow)
+                                 & (self._obj.surface_side == surface_side),
+                                 :]
+        geometry = df_pvrow.loc[:, 'geometry'].values[0]
+        # Find intersection point of line shadow and pvrow geometry
         point_intersect = geometry.intersection(line_shadow)
         # Check that the intersection is not too close to a boundary: if it
         # is it can create a "memory access error" it seems
@@ -268,26 +280,22 @@ def split_pvrow_geometry(self, idx, line_shadow, pvrow_top_point):
             pass
         else:
             # Cut geometry in two pieces
-            list_new_lines = self.cut_linestring(geometry, point_intersect)
-            # Add new geometry to index
-            new_registry_entry = self._obj.loc[idx, :].copy()
-            new_registry_entry['shaded'] = True
-            if pvrow_top_point in list_new_lines[0].boundary:
-                geometry_ill = pd.Series(list_new_lines[0])
-                geometry_shaded = pd.Series(list_new_lines[1])
-            elif pvrow_top_point in list_new_lines[1].boundary:
-                geometry_ill = pd.Series(list_new_lines[1])
-                geometry_shaded = pd.Series(list_new_lines[0])
-            else:
-                raise PVFactorsError("split_pvrow_geometry: "
-                                     "unknown error occured")
-
-            # Update registry
-            self._obj.at[idx, 'geometry'] = geometry_ill.values[0]
-            new_registry_entry['geometry'] = geometry_shaded.values[0]
-            self._obj.loc[self._obj.shape[0] + 1, :] = new_registry_entry
-
-    def cut_pvrow_geometry(self, list_points, pvrow_index, side):
+            self.cut_pvrow_geometry([point_intersect], idx_pvrow, surface_side)
+            # Find the geometries that should be marked as shaded
+            df_pvrow = self._obj.loc[(self._obj.pvrow_index == idx_pvrow)
+                                     & (self._obj.surface_side == surface_side),
+                                     :]
+            centroids = df_pvrow.pvgeometry.centroid.to_frame()
+            centroids.columns = ['geometry']
+            is_below_shading_interesect = (centroids.pvgeometry.bounds['miny']
+                                           < point_intersect.y)
+            self._obj.loc[(self._obj.pvrow_index == idx_pvrow)
+                          & (self._obj.surface_side == surface_side)
+                          & is_below_shading_interesect,
+                          'shaded'] = True
+
+    def cut_pvrow_geometry(self, list_points, pvrow_index, side,
+                           count_segments=False):
         """
         Break up pv row lines into multiple segments based on the list of
         points specified. This is the "discretization" of the pvrow segments.
@@ -300,17 +308,23 @@ def cut_pvrow_geometry(self, list_points, pvrow_index, side):
             registry.
         :param str side: only do it for one side of the selected PV row. This
             can only be 'front' or 'back'.
+        :param bool count_segments: (optional, default False) add pvrow segment idx
         :return: None
         """
         # TODO: is currently not able to work for other surfaces than pv rows..
-        for point in list_points:
+        for idx, point in enumerate(list_points):
+            if count_segments:
+                pvrow_segment_index = idx
+            else:
+                pvrow_segment_index = None
             df_selected = self._obj.loc[
                 (self._obj['pvrow_index'] == pvrow_index)
                 & (self._obj['surface_side'] == side), :]
             geoentry_to_break_up = df_selected.loc[
                 df_selected.pvgeometry.distance(point) < DISTANCE_TOLERANCE]
             if geoentry_to_break_up.shape[0] == 1:
-                self.break_and_add_entries(geoentry_to_break_up, point)
+                self.break_and_add_entries(geoentry_to_break_up, point,
+                                           pvrow_segment_index=pvrow_segment_index)
             elif geoentry_to_break_up.shape[0] > 1:
                 raise PVFactorsError("geoentry_to_break_up.shape[0] cannot be"
                                      "larger than 1")

pvfactors/tests/conftest.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+import pytest
+import os
+import pandas as pd
+
+DIR_TEST = os.path.dirname(__file__)
+DIR_TEST_DATA = os.path.join(DIR_TEST, 'test_files')
+
+
+@pytest.fixture(scope='function')
+def df_outputs():
+    """ Example of df_outputs to be used for tests """
+    fp = os.path.join(DIR_TEST_DATA, 'file_test_df_outputs.csv')
+    df_outputs = pd.read_csv(fp, header=[0, 1, 2], index_col=0)
+    df_outputs.index = pd.to_datetime(df_outputs.index)
+
+    yield df_outputs
+
+
+@pytest.fixture(scope='function')
+def df_registries():
+    """ Example of df_registries to be used for tests """
+    fp = os.path.join(DIR_TEST_DATA, 'file_test_df_registries.csv')
+    df_registries = pd.read_csv(fp, header=[0], parse_dates=['timestamps'])
+    yield df_registries
+
+
+@pytest.fixture(scope='function')
+def df_segments():
+    """ Example of df_segments to be used for tests """
+    fp = os.path.join(DIR_TEST_DATA, 'file_test_df_segments.csv')
+    df_segments = pd.read_csv(fp, header=[0, 1], index_col=0)
+    df_segments.index = pd.to_datetime(df_segments.index)
+    yield df_segments

pvfactors/tests/test_array_geometry.py

@@ -51,7 +51,7 @@ def test_plotting():
     is_ci = os.environ.get('CI', False)
     if not is_ci:
         import matplotlib.pyplot as plt
-        from pvfactors.tools import plot_line_registry
+        from pvfactors.tools import plot_pvarray
         # Create array where sun vector is in the direction of the modules
         arguments = {
             'n_pvrows': 3,
@@ -65,7 +65,7 @@ def test_plotting():
         }
         array = Array(**arguments)
         f, ax = plt.subplots(figsize=(10, 5))
-        _ = plot_line_registry(ax, array)
+        _ = plot_pvarray(ax, array)
 
         # Test with interrow forward shading
         arguments = {
@@ -80,7 +80,7 @@ def test_plotting():
         }
         array = Array(**arguments)
         f, ax = plt.subplots()
-        _ = plot_line_registry(ax, array)
+        _ = plot_pvarray(ax, array)
 
         # Test with interrow backward shading
         arguments = {
@@ -95,7 +95,7 @@ def test_plotting():
         }
         array = Array(**arguments)
         f, ax = plt.subplots()
-        _ = plot_line_registry(ax, array)
+        _ = plot_pvarray(ax, array)
 
     else:
         print("Not running 'test_plotting' in CI")
@@ -144,9 +144,9 @@ def test_interrow_shading():
     }
     array = Array(**arguments)
     # There should be 4 pvrows with direct shading
-    assert (array.line_registry.loc[
-        (array.line_registry.line_type == 'pvrow')
-        & array.line_registry.shaded]
+    assert (array.surface_registry.loc[
+        (array.surface_registry.line_type == 'pvrow')
+        & array.surface_registry.shaded]
         .shape[0] == 4)
 
     # Backward direct shading of the pvrows (sun in the back of the modules)
@@ -162,9 +162,9 @@ def test_interrow_shading():
     }
     array = Array(**arguments)
     # There should still be 4 pvrows with direct shading
-    assert (array.line_registry.loc[
-        (array.line_registry.line_type == 'pvrow')
-        & array.line_registry.shaded]
+    assert (array.surface_registry.loc[
+        (array.surface_registry.line_type == 'pvrow')
+        & array.surface_registry.shaded]
         .shape[0] == 4)
 
     print("Done.")

pvfactors/tests/test_circumsolar_shading.py

@@ -72,7 +72,6 @@ def test_serial_circumsolar_shading_calculation():
     array = Array(**arguments)
 
     # Run the calculation for functional testing
-    (df_outputs, df_bifacial, df_inputs_perez, df_outputs_segments,
-     df_registries) = (
+    df_registries, df_inputs_perez = (
         calculate_radiosities_serially_perez((arguments, df_inputs, save))
     )

pvfactors/tests/test_files/file_test_df_bifacial_gain.csv

@@ -0,0 +1,3 @@
+,0,1
+2017-04-15 10:26:00,0.13931705195219535,0.1281562651393122
+2017-04-15 10:27:00,0.13953385431646614,0.12830040096440057

pvfactors/tests/test_files/file_test_df_inputs_perez.csv

@@ -0,0 +1,3 @@
+,solar_zenith,solar_azimuth,array_tilt,dni,dhi,ghi,array_azimuth,poa_isotropic,poa_circumsolar,poa_horizon,vf_horizon,vf_circumsolar,vf_isotropic,luminance_horizon,luminance_circumsolar,luminance_isotropic,poa_total_diffuse
+2017-04-15 10:26:00,33.85139897,131.6267856,-25.6875,1028.370017,90.505481,920.948695,270.0,42.6681821006,51.0250378845,7.60195710557,0.433462489758,1.11849713283,0.950585804807,17.5377507517,45.6192835785,44.8861974215,101.295177091
+2017-04-15 10:27:00,33.69866897,131.9740324,-25.6925,1028.490326,90.557651,923.195471,270.0,42.6318473958,51.0146850523,7.58666529173,0.433541130166,1.11608053999,0.950566889722,17.4992976764,45.7087846479,44.8488663521,101.23319774

pvfactors/tests/test_files/file_test_df_outputs.csv

@@ -0,0 +1,5 @@
+pvrow,array_is_shaded,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
+side,,back,back,back,back,back,back,back,back,back,front,front,front,front,front,front,front,front,front,back,back,back,back,back,back,back,back,back,front,front,front,front,front,front,front,front,front
+term,,circumsolar_term,direct_term,horizon_band_shading_pct,horizon_term,irradiance_term,isotropic_term,q0,qinc,reflection_term,circumsolar_term,direct_term,horizon_band_shading_pct,horizon_term,irradiance_term,isotropic_term,q0,qinc,reflection_term,circumsolar_term,direct_term,horizon_band_shading_pct,horizon_term,irradiance_term,isotropic_term,q0,qinc,reflection_term,circumsolar_term,direct_term,horizon_band_shading_pct,horizon_term,irradiance_term,isotropic_term,q0,qinc,reflection_term
+4/15/2017 10:26,FALSE,0,0,0,7.601957106,7.601957106,2.366247585,7.410442474,148.2088495,138.2406448,51.02503788,955.2479674,0,7.601957106,1013.874962,41.81449994,31.9147256,1063.824187,8.134724361,0,0,50.43298874,3.768062934,3.768062934,1.727308046,6.83247544,136.6495088,131.1541378,51.02503788,955.2479674,0,7.601957106,1013.874962,42.81663696,31.98817677,1066.272559,9.580959761
+4/15/2017 10:27,FALSE,0,0,0,7.586665292,7.586665292,2.365154307,7.41991129,148.3982258,138.4464062,51.01468505,954.9966611,0,7.586665292,1013.598011,41.77857856,31.90585393,1063.528464,8.151874312,0,0,50.44237013,3.759771505,3.759771505,1.72653543,6.83828301,136.7656602,131.2793533,51.01468505,954.9966611,0,7.586665292,1013.598011,42.78020623,31.97939972,1065.979991,9.601772851