Skip to content

Commit

Permalink
Preparation of new release
Browse files Browse the repository at this point in the history
  • Loading branch information
@weibezahn
weibezahn committed Dec 20, 2018
1 parent a139ba2 commit 4d45b31
Show file tree
Hide file tree
Showing 4 changed files with 197 additions and 128 deletions.
10 changes: 5 additions & 5 deletions checksums.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
conventional_power_plants_DE.csv,755ab91a138d133e1c4312a38799eda50617103a39587a39ee56d2a4c66c5749
conventional_power_plants_DE.xlsx,eae9c3675d8bb0c8ec655f705fff5465bbdd9fa5bc20a8b933fae3e450666b42
conventional_power_plants_EU.csv,aad5f15cef3acd3b3e4783aa92645e3b50ab9fc0d7f1667e24ba5b6f0f12b975
conventional_power_plants_EU.xlsx,b2403a19cbe9af953a889ce4c311d9d63b7b64db691bfddbe78a0448950256cf
conventional_power_plants.sqlite,e3c301c3d5b65f427b57e81899f0bc876a04b13647d5fec5fe7917b880275b43
conventional_power_plants_DE.csv,bda10d8d5eab6898620d1f974e7996784e5722003b065c1dcf06f59a7485e591
conventional_power_plants_DE.xlsx,a73fabb01b6fc872f4d945a7167493751a44288a5bde78ff949c1cd4acd12801
conventional_power_plants_EU.csv,0c4f9fb7ca2ef2fde265ae96d28dcda24b12773ebb2cce023b1028d60018f2d4
conventional_power_plants_EU.xlsx,08c13d0d44a85b35adcd3e003d313bb41272b57f598cf25d7950c90b76a4dcac
conventional_power_plants.sqlite,67d4b84472a3e368db596fc8b8423c164d1c33882880aeac3c833cd2610d74d8
43 changes: 28 additions & 15 deletions download_and_process_DE.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@
"outputs": [],
"source": [
"download_from = 'original_sources'\n",
"#download_from = 'opsd_server' "
"#download_from = 'opsd_server'"
]
},
{
Expand All @@ -162,10 +162,10 @@
" url_bnetza = ('http://www.bundesnetzagentur.de/SharedDocs/Downloads/DE/'\n",
" 'Sachgebiete/Energie/Unternehmen_Institutionen/Versorgungssicherheit/'\n",
" 'Erzeugungskapazitaeten/Kraftwerksliste/Kraftwerksliste_CSV.csv'\n",
" '?__blob=publicationFile&v=10')\n",
" '?__blob=publicationFile&v=16')\n",
"\n",
" # UBA Power plant list\n",
" url_uba = ('https://www.umweltbundesamt.de/sites/default/files/medien/372/dokumente/kraftwerke-de-ab-100-mw.xls')"
" url_uba = ('https://www.umweltbundesamt.de/sites/default/files/medien/372/dokumente/kraftwerke_de_ab_100_mw_0.xls')"
]
},
{
Expand All @@ -181,13 +181,13 @@
" # Specify direction to original_data folder on the opsd data server\n",
" # BNetzA Power plant list\n",
" url_bnetza = 'http://data.open-power-system-data.org/conventional_power_plants/'\n",
" url_bnetza = url_bnetza + '2018-02-27'\n",
" url_bnetza = url_bnetza + '2018-12-20'\n",
" url_bnetza = url_bnetza +'/original_data/Kraftwerksliste_CSV.csv'\n",
" \n",
" # UBA Power plant list\n",
" url_uba = 'http://data.open-power-system-data.org/conventional_power_plants/'\n",
" url_uba = url_uba + '2018-02-27'\n",
" url_uba = url_uba +'/original_data/kraftwerke-de-ab-100-mw.xls'"
" url_uba = url_uba + '2018-12-20'\n",
" url_uba = url_uba +'/original_data/kraftwerke_de_ab_100_mw_0.xls'"
]
},
{
Expand Down Expand Up @@ -232,6 +232,10 @@
"outputs": [],
"source": [
"plantlist = getbnetzalist(url_bnetza)\n",
"\n",
"# clean unnamed columns\n",
"plantlist.drop([c for c in plantlist.columns if 'Unnamed:' in c], axis=1, inplace=True)\n",
"\n",
"plantlist.head()"
]
},
Expand Down Expand Up @@ -284,7 +288,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"scrolled": false
},
"outputs": [],
Expand All @@ -307,7 +310,7 @@
" 'Blockname':\n",
" 'block',\n",
" ('Aufnahme der kommerziellen Stromerzeugung der derzeit in Betrieb '\n",
" 'befindlichen Erzeugungseinheit\\n(Jahr)'):\n",
" 'befindlichen Erzeugungseinheit\\n(Datum/Jahr)'):\n",
" 'commissioned',\n",
" ('Kraftwerksstatus \\n(in Betrieb/\\nvorläufig stillgelegt/\\nsaisonale '\n",
" 'Konservierung\\nReservekraftwerk/\\nSonderfall)'):\n",
Expand Down Expand Up @@ -379,7 +382,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"scrolled": false
},
"outputs": [],
Expand All @@ -395,17 +397,19 @@
" 'Erdgas': 'Natural gas',\n",
" 'Braunkohle': 'Lignite',\n",
" 'Kernenergie': 'Nuclear',\n",
" 'Pumpspeicher': 'Hydro',\n",
" 'Pumpspeicher': 'Hydro PSP',\n",
" 'Biomasse': 'Biomass and biogas',\n",
" 'Mineralölprodukte': 'Oil',\n",
" 'Laufwasser': 'Hydro',\n",
" 'Sonstige Energieträger (nicht erneuerbar) ': 'Other fuels',\n",
" 'Abfall': 'Waste',\n",
" 'Speicherwasser (ohne Pumpspeicher)': 'Hydro',\n",
" 'Speicherwasser (ohne Pumpspeicher)': 'Hydro reservoir',\n",
" 'Unbekannter Energieträger (nicht erneuerbar)': 'Other fuels',\n",
" 'Sonstige Energieträger (nicht erneuerbar)': 'Other fuels',\n",
" 'Mehrere Energieträger (nicht erneuerbar)': 'Mixed fossil fuels',\n",
" 'Deponiegas': 'Sewage and landfill gas',\n",
" 'Windenergie (Onshore-Anlage)': 'Onshore',\n",
" 'Windenergie (Onshore-Anlage)neu': 'Onshore',\n",
" 'Windenergie (Offshore-Anlage)': 'Offshore',\n",
" 'Solare Strahlungsenergie': 'Solar',\n",
" 'Klärgas': 'Sewage and landfill gas',\n",
Expand Down Expand Up @@ -469,8 +473,10 @@
" 'Vorläufig Stillgelegt (ohne StA)': 'shutdown_temporary',\n",
" 'Endgültig Stillgelegt 2016 (ohne StA)': 'shutdown_2016',\n",
" 'Endgültig Stillgelegt 2017 (mit StA)' : 'shutdown_2017',\n",
" 'Endgültig Stillgelegt 2017 (ohne StA)': 'shutdown_2017',\n",
" 'Endgültig Stillgelegt 2018 (mit StA)' : 'shutdown_2018',\n",
" 'Endgültig Stillgelegt 2018 (ohne StA)': 'shutdown_2018',\n",
" 'gesetzlich an Stilllegung gehindert' : 'operating',\n",
" 'Engültig Stillgelegt 2017 (ohne StA)' : 'shutdown_2017',\n",
" 'Netzreserve' : 'reserve',\n",
"} \n",
"plantlist['status'].replace(dict_plantstatus, inplace=True) \n",
Expand Down Expand Up @@ -1082,6 +1088,8 @@
"\n",
"dict_technology = {\n",
" 'GT': 'Gas turbine',\n",
" 'DT': 'Steam turbine',\n",
" 'GT / DT': 'Combined cycle',\n",
" 'GuD': 'Combined cycle',\n",
" 'DKW': 'Steam turbine',\n",
" 'LWK': 'Run-of-river',\n",
Expand Down Expand Up @@ -1125,6 +1133,8 @@
" 'HKW / SSA': 'CHP',\n",
" 'IKW / HKW': 'CHP',\n",
" 'GT': '',\n",
" 'DT': '',\n",
" 'GT / DT': '',\n",
" 'GuD': '',\n",
" 'DKW': '',\n",
" 'LWK': '',\n",
Expand Down Expand Up @@ -1174,8 +1184,12 @@
" plantlist['technology'].isnull())), 'technology'] = 'Steam turbine'\n",
"plantlist.loc[(plantlist['fuel'] == 'Hydro') & ((plantlist['technology'] == '') | (\n",
" plantlist['technology'].isnull())), 'technology'] = 'Run-of-river'\n",
"plantlist.loc[(plantlist['fuel'] == 'Hydro') & ((plantlist[\n",
" 'technology'] == '') | (plantlist['technology'].isnull())), 'technology'] = 'Pumped storage'\n",
"plantlist.loc[(plantlist['fuel'] == 'Hydro PSP') & ((plantlist['technology'] == '') | (\n",
" plantlist['technology'].isnull())), 'technology'] = 'Pumped storage'\n",
"plantlist.loc[(plantlist['fuel'] == 'Hydro PSP'), 'fuel'] = 'Hydro'\n",
"plantlist.loc[(plantlist['fuel'] == 'Hydro reservoir') & ((plantlist['technology'] == '') | (\n",
" plantlist['technology'].isnull())), 'technology'] = 'RES'\n",
"plantlist.loc[(plantlist['fuel'] == 'Hydro reservoir'), 'fuel'] = 'Hydro'\n",
"plantlist.loc[(plantlist['fuel'] == 'reservoir') & ((plantlist['technology'] == '') | (\n",
" plantlist['technology'].isnull())), 'technology'] = 'RES'\n",
"\n",
Expand Down Expand Up @@ -1914,7 +1928,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"scrolled": false
},
"outputs": [],
Expand Down
62 changes: 36 additions & 26 deletions download_and_process_EU.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,8 @@
"import sqlite3\n",
"import zipfile\n",
"import io\n",
"from io import BytesIO\n",
"from PIL import Image\n",
"import requests\n",
"import csv\n",
"\n",
Expand Down Expand Up @@ -252,9 +254,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"conf = \"\"\"\n",
Expand All @@ -268,62 +268,67 @@
" '2017-03-03': ProductionParkOverview\n",
" '2017-07-03': ProductionParkOverview\n",
" '2018-02-27': ProductionParkOverview\n",
" '2018-12-20': ProductionParkOverview\n",
" filetype: xls\n",
" sheetname: 'ProductionParkOverview'\n",
" skiprows: 1\n",
" \n",
" NL: \n",
" Tennet_Q1:\n",
" url_initial: http://www.tennet.org/english/operational_management/export_data.aspx\n",
" url_template: http://www.tennet.org/english/operational_management/export_data.aspx?exporttype=installedcapacity&format=csv&quarter=2017-1&submit=3\n",
" filename: export_Q12017\n",
" url_template: http://www.tennet.org/english/operational_management/export_data.aspx?exporttype=installedcapacity&format=csv&quarter=2018-1&submit=3\n",
" filename: export_Q12018\n",
" filename_opsd:\n",
" '2016-10-27': export_Q12015\n",
" '2017-03-03': export_Q12016\n",
" '2017-07-03': export_Q12016\n",
" '2018-02-27': export_Q12017\n",
" '2018-12-20': export_Q12018\n",
" filetype: csv\n",
" sep: ','\n",
" skiprows: 0\n",
" decimal: '.'\n",
" encoding: 'utf-8'\n",
" Tennet_Q2:\n",
" url_initial: http://www.tennet.org/english/operational_management/export_data.aspx\n",
" url_template: http://www.tennet.org/english/operational_management/export_data.aspx?exporttype=installedcapacity&format=csv&quarter=2017-2&submit=3\n",
" filename: export_Q22017\n",
" url_template: http://www.tennet.org/english/operational_management/export_data.aspx?exporttype=installedcapacity&format=csv&quarter=2018-2&submit=3\n",
" filename: export_Q22018\n",
" filename_opsd:\n",
" '2016-10-27': export_Q22015\n",
" '2017-03-03': export_Q22016\n",
" '2017-07-03': export_Q22016\n",
" '2018-02-27': export_Q22017\n",
" '2018-12-20': export_Q22018\n",
" filetype: csv\n",
" sep: ','\n",
" skiprows: 0\n",
" decimal: '.'\n",
" encoding: 'utf-8'\n",
" Tennet_Q3:\n",
" url_initial: http://www.tennet.org/english/operational_management/export_data.aspx\n",
" url_template: http://www.tennet.org/english/operational_management/export_data.aspx?exporttype=installedcapacity&format=csv&quarter=2017-3&submit=3\n",
" filename: export_Q32017\n",
" url_template: http://www.tennet.org/english/operational_management/export_data.aspx?exporttype=installedcapacity&format=csv&quarter=2018-3&submit=3\n",
" filename: export_Q32018\n",
" filename_opsd:\n",
" '2016-10-27': export_Q32015\n",
" '2017-03-03': export_Q32016\n",
" '2017-07-03': export_Q32016\n",
" '2018-02-27': export_Q32017\n",
" '2018-12-20': export_Q32018\n",
" filetype: csv\n",
" sep: ','\n",
" skiprows: 0\n",
" decimal: '.'\n",
" encoding: 'utf-8'\n",
" Tennet_Q4:\n",
" url_initial: http://www.tennet.org/english/operational_management/export_data.aspx\n",
" url_template: http://www.tennet.org/english/operational_management/export_data.aspx?exporttype=installedcapacity&format=csv&quarter=2017-4&submit=3\n",
" filename: export_Q42017\n",
" url_template: http://www.tennet.org/english/operational_management/export_data.aspx?exporttype=installedcapacity&format=csv&quarter=2018-4&submit=3\n",
" filename: export_Q42018\n",
" filename_opsd:\n",
" '2016-10-27': export_Q42015\n",
" '2017-03-03': export_Q42016\n",
" '2017-07-03': export_Q42016\n",
" '2018-02-27': export_Q42017\n",
" '2018-12-20': export_Q42018\n",
" filetype: csv\n",
" sep: ','\n",
" skiprows: 0\n",
Expand All @@ -333,13 +338,14 @@
" IT:\n",
" Terna: \n",
" url_initial: http://www.terna.it/it-it/sistemaelettrico/transparencyreport/generation/installedgenerationcapacity.aspx\n",
" url_template: http://download.terna.it/terna/0000/0216/18.XLSX \n",
" url_template: http://download.terna.it/terna/0000/0216/16.XLSX \n",
" filename: 16\n",
" filename_opsd:\n",
" '2016-10-27': 16\n",
" '2017-03-03': 16\n",
" '2017-07-03': 16\n",
" '2018-02-27': 18\n",
" '2018-02-27': 16\n",
" '2018-12-20': 16\n",
" filetype: xls\n",
" sheetname: 'UPR PmaxOver 100MW'\n",
" skiprows: 0\n",
Expand All @@ -354,6 +360,7 @@
" '2017-03-03': Centrales_production_reference\n",
" '2017-07-03': Centrales_production_reference\n",
" '2018-02-27': Centrales_production_reference\n",
" '2018-12-20': Centrales_production_reference\n",
" filetype: zip\n",
" sep: '\\t'\n",
" skiprows: 2\n",
Expand All @@ -370,6 +377,7 @@
" '2017-03-03': ElectraExp\n",
" '2017-07-03': ElectraExp\n",
" '2018-02-27': ElectraExp\n",
" '2018-12-20': ElectraExp\n",
" filetype: zip\n",
" sep: ';'\n",
" skiprows: 0\n",
Expand All @@ -378,14 +386,15 @@
"\n",
" FI: \n",
" EnergyAuthority:\n",
" url_initial: http://www.energiavirasto.fi/en/web/energy-authority/power-plant-register\n",
" url_template: http://www.energiavirasto.fi/documents/10191/0/Energiaviraston+voimalaitosrekisteri+01022018.xlsx\n",
" filename: Energiaviraston+voimalaitosrekisteri+01022018\n",
" url_initial: https://www.energiavirasto.fi/en/web/energy-authority/power-plant-register\n",
" url_template: https://www.energiavirasto.fi/documents/10191/0/Energiaviraston+voimalaitosrekisteri+29102018.xlsx/4b4b87f5-9eee-409d-836f-5b0040dba7de\n",
" filename: Energiaviraston+voimalaitosrekisteri+29102018\n",
" filename_opsd:\n",
" '2016-10-27': Energiaviraston+Voimalaitosrekisteri+040316\n",
" '2017-03-03': Energiaviraston+Voimalaitosrekisteri+010117\n",
" '2017-07-03': Energiaviraston+voimalaitosrekisteri+06072017\n",
" '2018-02-27': Energiaviraston+voimalaitosrekisteri+01022018\n",
" '2018-12-20': Energiaviraston+voimalaitosrekisteri+29102018\n",
" filetype: xlsx\n",
" sheetname: 'English'\n",
" skiprows: 1\n",
Expand All @@ -400,6 +409,7 @@
" '2017-03-03': units_list\n",
" '2017-07-03': units_list\n",
" '2018-02-27': units_list\n",
" '2018-12-20': units_list\n",
" filetype: csv \n",
" sep: ';'\n",
" skiprows: 0\n",
Expand All @@ -424,7 +434,7 @@
" CEPS:\n",
" url_initial: http://www.ceps.cz/en/all-data#AvailableCapacity\n",
" url_template: http://www.ceps.cz/en/all-data#AvailableCapacity\n",
" url_request1: http://www.ceps.cz/en/all-data?do=loadGraphData&method=AvailableCapacity&filter_data={\"dateFrom\":\"2018-01-01 00:00:00\",\"dateTo\":\"2018-12-28 23:59:59\",\"version\":\"YF\",\"agregation\":\"MI\",\"function\":\"AVG\"}&graph_id=1026&move_graph=year&download=xls\n",
" url_request1: http://www.ceps.cz/en/all-data?do=loadGraphData&method=AvailableCapacity&filter_data={\"dateFrom\":\"2018-01-01 00:00:00\",\"dateTo\":\"2018-12-20 23:59:59\",\"version\":\"YF\",\"agregation\":\"MI\",\"function\":\"AVG\"}&graph_id=1026&move_graph=year&download=xls\n",
" url_request2: http://www.ceps.cz/download-data/?format=txt\n",
" params:\n",
" format: txt\n",
Expand All @@ -450,6 +460,7 @@
" '2017-03-03': Data\n",
" '2017-07-03': Data\n",
" '2018-02-27': data\n",
" '2018-12-20': data\n",
" filetype: csv\n",
" sep: ','\n",
" skiprows: 0\n",
Expand All @@ -459,18 +470,19 @@
" CH:\n",
" BFE:\n",
" url_initial: http://www.bfe.admin.ch/themen/00490/00491/index.html?lang=de&dossier_id=01049\n",
" url_template: http://www.bfe.admin.ch/php/modules/publikationen/stream.php?extlang=de&name=de_880983817.zip&endung=Statistik%20der%20Wasserkraftanlagen%20der%20Schweiz\n",
" filename: '2016 Statistik der Wasserkraftanlagen der Schweiz 1.1.2017_N'\n",
" url_template: http://www.bfe.admin.ch/php/modules/publikationen/stream.php?extlang=de&name=de_496108515.zip&endung=Statistik%20der%20Wasserkraftanlagen%20der%20Schweiz\n",
" filename: 'Statistik der Wasserkraftanlagen der Schweiz 1.1.2018'\n",
" filename_opsd:\n",
" '2016-10-27': 'Statistik der Wasserkraftanlagen der Schweiz 1.1.2016'\n",
" '2017-03-03': 'Statistik der Wasserkraftanlagen der Schweiz 1.1.2016'\n",
" '2017-07-03': 'Statistik der Wasserkraftanlagen der Schweiz 1.1.2017'\n",
" '2018-02-27': 'Statistik der Wasserkraftanlagen der Schweiz 1.1.2017'\n",
" '2018-12-20': 'Statistik der Wasserkraftanlagen der Schweiz 1.1.2018'\n",
" filetype: zip\n",
" sheetname: 'Vorschau'\n",
" skiprows: 0\n",
" \n",
" \"\"\"\n",
"\"\"\"\n",
"conf = yaml.load(conf)"
]
},
Expand Down Expand Up @@ -2579,9 +2591,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"data_ES = importdata('ES', 'SEDE')"
Expand Down Expand Up @@ -3044,6 +3054,7 @@
" 'Coal / biomass': 'Mixed fossil fuels',\n",
" 'Coal / biomass / gas / waste derived fuel': 'Mixed fossil fuels',\n",
" 'Coal / oil': 'Mixed fossil fuels',\n",
" 'Coal/oil': 'Mixed fossil fuels',\n",
" 'Diesel': 'Oil',\n",
" 'Gas': 'Natural gas',\n",
" 'Gas / oil': 'Mixed fossil fuels',\n",
Expand All @@ -3056,6 +3067,7 @@
" 'Nuclear': 'Nuclear',\n",
" 'OCGT': 'Natural gas',\n",
" 'Oil': 'Oil',\n",
" 'Light oil ': 'Oil',\n",
" 'Pumped storage': 'Hydro',\n",
" 'Straw': 'Biomass and biogas',\n",
" 'Waste': 'Non-renewable waste',\n",
Expand Down Expand Up @@ -3310,9 +3322,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"dataframes = [#data_BE,\n",
Expand Down
Loading

0 comments on commit 4d45b31

Please sign in to comment.