Create a python package of flexmeasures-entsoe

Makefile

@@ -0,0 +1,44 @@
+# Note: use tabs
+# actions which are virtual, i.e. not a script
+.PHONY: install install-for-dev install-deps install-flexmeasures-entsoe test freeze-deps upgrade-deps
+
+
+# ---- Development ---
+
+test:
+	make install-for-dev
+	pytest
+
+# ---- Installation ---
+
+install: install-deps install-flexmeasures-entsoe
+
+install-for-dev:
+	make freeze-deps
+	pip-sync requirements/app.txt requirements/dev.txt requirements/test.txt
+	make install-flexmeasures-entsoe
+	pre-commit install
+
+install-deps:
+	make install-pip-tools
+	make freeze-deps
+	pip-sync requirements/app.txt
+
+install-flexmeasures-entsoe:
+	python setup.py develop
+
+install-pip-tools:
+	pip3 install -q "pip-tools>=6.2"
+
+freeze-deps:
+	make install-pip-tools
+	pip-compile -o requirements/app.txt requirements/app.in
+	pip-compile -o requirements/test.txt requirements/test.in
+	pip-compile -o requirements/dev.txt requirements/dev.in
+
+upgrade-deps:
+	make install-pip-tools
+	pip-compile --upgrade -o requirements/app.txt requirements/app.in
+	pip-compile --upgrade -o requirements/test.txt requirements/test.in
+	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in
+	make test

README.md

@@ -26,8 +26,7 @@ Use ``--help`` to learn more usage details.
 
 First of all, this is a FlexMeasures plugin. Consult the FlexMeasures documentation for setup.
 
-1. Add the path to this directory to your FlexMeasures (>v0.4.0) config file,
-using the `FLEXMEASURES_PLUGINS` setting.
+1. Add the plugin to [the `FLEXMEASURES_PLUGINS` setting](https://flexmeasures.readthedocs.io/en/latest/configuration.html#plugin-config). Either use `/path/to/flexmeasures-entsoe/flexmeasures_entsoe` or `flexmeasures_entsoe` if you installed this as a package locally (see below).
 
 2. Add `ENTSOE_AUTH_TOKEN` to your FlexMeasures config (e.g. ~/.flexmeasures.cfg).
 You can generate this token after you made an account at ENTSO-E, read more [here](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html#_authentication_and_authorisation). 
@@ -41,7 +40,7 @@ You can generate this token after you made an account at ENTSO-E, read more [her
    The `ENTSOE_DERIVED_DATA_SOURCE` option is used to name the source of data that this plugin derives from ENTSO-E data, like a CO₂ signal.
    Original ENTSO-E data is reported as being sourced by `"ENTSO-E"`.
 
-3. `pip install entsoe-py`
+3. To install this plugin locally as a package, try `pip install .`.
 
 
 ## Testing
@@ -61,4 +60,11 @@ To keep our code quality high, we use pre-commit:
 
     pip install pre-commit black flake8 mypy
     pre-commit install
+
+or:
+
+    make install-for-dev
+
+Try it:
+
     pre-commit run --all-files --show-diff-on-failure

flexmeasures_entsoe/generation/__init__.py

@@ -0,0 +1,10 @@
+from datetime import timedelta
+
+# sensor_name, unit, event_resolution, data sourced directly by ENTSO-E or not (i.e. derived)
+generation_sensors = (
+    ("Scheduled generation", "MW", timedelta(minutes=15), True),
+    ("Solar", "MW", timedelta(hours=1), True),
+    ("Wind Onshore", "MW", timedelta(hours=1), True),
+    ("Wind Offshore", "MW", timedelta(hours=1), True),
+    ("CO₂ intensity", "kg/MWh", timedelta(minutes=15), False),
+)

generation/day_ahead.py → flexmeasures_entsoe/generation/day_ahead.py

@@ -4,25 +4,20 @@
 import click
 from flask.cli import with_appcontext
 from flask import current_app
-import entsoe
-from entsoe import EntsoePandasClient
 
 # from entsoe.entsoe import URL
 import pandas as pd
 from flexmeasures.data.transactional import task_with_status_report
 
 from .. import (
     entsoe_data_bp,
-    DEFAULT_COUNTRY_CODE,
-    DEFAULT_COUNTRY_TIMEZONE,
 )  # noqa: E402
 from . import generation_sensors
 from ..utils import (
     create_entsoe_client,
     ensure_country_code_and_timezone,
     ensure_data_source,
     ensure_data_source_for_derived_data,
-    get_auth_token_from_config_and_set_server_url,
     abort_if_data_empty,
     parse_from_and_to_dates_default_today_and_tomorrow,
     save_entsoe_series,
@@ -100,19 +95,22 @@ def import_day_ahead_generation(
     when tomorrow's prices are announced.
     """
     # Set up FlexMeasures data structure
-    country_code, country_timezone = ensure_country_code_and_timezone(country_code, country_timezone)
+    country_code, country_timezone = ensure_country_code_and_timezone(
+        country_code, country_timezone
+    )
     entsoe_data_source = ensure_data_source()
     derived_data_source = ensure_data_source_for_derived_data()
     sensors = ensure_sensors(generation_sensors, country_code, country_timezone)
-
     # Parse CLI options (or set defaults)
     from_time, until_time = parse_from_and_to_dates_default_today_and_tomorrow(
         from_date, to_date, country_timezone
     )
 
     # Start import
     client = create_entsoe_client()
-    log, now = start_import_log("day-ahead generation", from_time, until_time, country_code, country_timezone)
+    log, now = start_import_log(
+        "day-ahead generation", from_time, until_time, country_code, country_timezone
+    )
 
     log.info("Getting scheduled generation ...")
     # We assume that the green (solar & wind) generation is not included in this (it is not scheduled)

prices/day_ahead.py → flexmeasures_entsoe/prices/day_ahead.py

@@ -3,18 +3,13 @@
 
 import click
 from flask.cli import with_appcontext
-from flask import current_app
 import pandas as pd
 
-import entsoe
-from entsoe import EntsoePandasClient
 from flexmeasures.data.transactional import task_with_status_report
 
 from . import pricing_sensors
 from .. import (
     entsoe_data_bp,
-    DEFAULT_COUNTRY_CODE,
-    DEFAULT_COUNTRY_TIMEZONE,
 )  # noqa: E402
 from ..utils import (
     create_entsoe_client,
@@ -23,7 +18,6 @@
     parse_from_and_to_dates_default_today_and_tomorrow,
     ensure_sensors,
     save_entsoe_series,
-    get_auth_token_from_config_and_set_server_url,
     abort_if_data_empty,
     start_import_log,
 )
@@ -74,7 +68,9 @@ def import_day_ahead_prices(
     when tomorrow's prices are announced.
     """
     # Set up FlexMeasures data structure
-    country_code, country_timezone = ensure_country_code_and_timezone(country_code, country_timezone)
+    country_code, country_timezone = ensure_country_code_and_timezone(
+        country_code, country_timezone
+    )
     sensors = ensure_sensors(pricing_sensors, country_code, country_timezone)
     entsoe_data_source = ensure_data_source()
     # For now, we only have one pricing sensor ...
@@ -88,7 +84,9 @@ def import_day_ahead_prices(
 
     # Start import
     client = create_entsoe_client()
-    log, now = start_import_log("day-ahead price", from_time, until_time, country_code, country_timezone)
+    log, now = start_import_log(
+        "day-ahead price", from_time, until_time, country_code, country_timezone
+    )
 
     log.info("Getting prices ...")
     prices: pd.Series = client.query_day_ahead_prices(
@@ -100,4 +98,6 @@ def import_day_ahead_prices(
     if not dryrun:
         log.info(f"Saving {len(prices)} beliefs for Sensor {pricing_sensor.name} ...")
         prices.name = "event_value"  # required by timely_beliefs, TODO: check if that still is the case, see https://github.com/SeitaBV/timely-beliefs/issues/64
-        save_entsoe_series(prices, pricing_sensor, entsoe_data_source, country_timezone, now)
+        save_entsoe_series(
+            prices, pricing_sensor, entsoe_data_source, country_timezone, now
+        )

requirements/Readme.md

@@ -0,0 +1,21 @@
+# Requirements
+
+All FlexMeasures requirements are specified in this directory.
+We separate by use case:
+
+- app: All requirements for running the FlexMeasures platform
+- test: Additional requirements used for running automated tests 
+- dev: Additional requirements used for developers (this includes testing)
+
+Also note the following distinction:
+
+
+## .in files
+
+Here, we describe the requirements. We give the name of a requirement or even a range (e.g. `>=1.0.`).
+
+## .txt files
+
+These files are not to be edited by hand. They are created by `pip-compile` (or `make freeze-deps`).
+
+Each requirement is pinned to a specific version in these files. The great benefit is reproducibility across environments (local dev as well as staging or production).

requirements/app.in

@@ -0,0 +1,2 @@
+flexmeasures>=0.9.3
+entsoe-py