diff --git a/Dockerfile.postgis b/Dockerfile.postgis new file mode 100644 index 00000000..65381fb9 --- /dev/null +++ b/Dockerfile.postgis @@ -0,0 +1,4 @@ +FROM postgres:11 + +RUN apt-get update +RUN apt-get install -y postgresql-11-postgis-2.5 diff --git a/README.rst b/README.rst index 2740d6be..526cc5e6 100644 --- a/README.rst +++ b/README.rst @@ -51,3 +51,44 @@ Run ~$ ego_data_processing ``` +Local database with docker +========================== + +You need to have `docker-compose` installed. And some more tools, see requirements_system.txt. + +### Spin up database with docker + +``` +docker-compose up -d --build +``` + +### Database credentials + +| database | dp | +| port | 54321 | +| user | oeuser | +| password | egon | + + +Usage +===== + + +Pre-processing +************** + +Execute `eGo_import_and_preprocessing.py` for starting pre-processing from scratch doing data import and a pre-processing of imported tables. + +Once completed, export required schemas to SQL dump + +_You might have to install `postgresql-client-11` from additional source to have a compatible client with the v11 dockered PostgreSQL database._ + +``` + /usr/lib/postgresql/11/bin/pg_dump -d dp -U oeuser -p 54321 -h localhost -Fc -n model_draft -n openstreetmap -n boundaries -n society > `date -I`_eGo_data_pre-processing_bremen.backup +``` + +Import a dump by + +``` +/usr/lib/postgresql/11/bin/pg_restore -d dp -U oeuser -p 54321 -h localhost 2020-03-28_eGo_data_pre-processing_deu.backup +``` diff --git a/dataprocessing/tools/osm2pgsql/osm2oedb.txt b/dataprocessing/tools/osm2pgsql/osm2oedb.txt deleted file mode 100644 index bc5ecaac..00000000 --- a/dataprocessing/tools/osm2pgsql/osm2oedb.txt +++ /dev/null @@ -1 +0,0 @@ -osm2pgsql --create --slim --hstore-all --number-processes 2 --cache 5120 -H localhost -P 5432 -d oedb -U -W -p osm_deu -S oedb.style germany-161001.osm.pbf \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..e4ae1841 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3.5' +services: + data_processing_db: + image: postgres:11-postgis + container_name: data_processing + restart: unless-stopped + build: + context: . + dockerfile: Dockerfile.postgis + shm_size: '1gb' + ports: + - "127.0.0.1:54321:5432" + environment: + POSTGRES_DB: dp + POSTGRES_USER: oeuser + POSTGRES_PASSWORD: egon + volumes: + - $HOME/docker/volumes/postgres/data_processing:/var/lib/postgresql/data + - ./utilities/entrypoints:/docker-entrypoint-initdb.d/ diff --git a/documentation/bpmn/ego_dp_bpmn.graphml b/documentation/bpmn/archiv/ego_dp_bpmn.graphml similarity index 100% rename from documentation/bpmn/ego_dp_bpmn.graphml rename to documentation/bpmn/archiv/ego_dp_bpmn.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml index 7d47a0a5..c35f9baf 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml @@ -17,7 +17,7 @@ - + eGo Pre-Processing - bkg_vg250 @@ -53,13 +53,13 @@ - + - + @@ -75,7 +75,7 @@ - + ego_pp_vg250_import.sql @@ -93,7 +93,7 @@ - + boundaries.bkg_vg250_1_sta @@ -115,10 +115,10 @@ boundaries.bkg_vg250_6_gem< - + - ? + 1.6 s @@ -131,7 +131,7 @@ boundaries.bkg_vg250_6_gem< - + @@ -154,7 +154,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -178,7 +178,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -202,7 +202,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + Geodatenzugangsgesetz (GeoZG) @@ -219,7 +219,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + ego_dp_structure_boundaries_vg250.sql @@ -237,7 +237,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + boundaries.bkg_vg250_1_sta @@ -275,8 +275,8 @@ society.destatis_zensus_population_per_ha_mview - - + + boundaries.bkg_vg250_1_sta_mview boundaries.bkg_vg250_1_sta_error_geom_mview @@ -304,10 +304,10 @@ boundaries.bkg_vg250_statistics_view - + - ? + 3 min 42 s @@ -318,12 +318,154 @@ boundaries.bkg_vg250_statistics_view + + + + + + import_vg250.py + + + + + + + + + + + + + + + + + + boundaries.bkg_vg250_sta_YYYY-MM-DD +boundaries.bkg_vg250_lan_YYYY-MM-DD +boundaries.bkg_vg250_rbz_YYYY-MM-DD +boundaries.bkg_vg250_krs_YYYY-MM-DD +boundaries.bkg_vg250_vwg_YYYY-MM-DD +boundaries.bkg_vg250_gem_YYYY-MM-DD + + + + + + + + + + + + + + + + + 15 s + + + + + + + + + + - + + + + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) + + + + + + + + + + + + + - boundaries.vg250_*_YYYY-MM-DD + vg250_01-01.gk3.shape.ebenen.zip + + + + + + + + + + + + + + + + + ego_pp_vg250_mview.sql + + + + + + + + + + + + + + + + + + boundaries.bkg_vg250_2_lan_nuts_view + + + + + + + + + + + + + + + + + 1.8 s + + + + + + + + + + + + + + + + + boundaries.bkg_vg250_2_lan @@ -334,6 +476,113 @@ boundaries.bkg_vg250_statistics_view + + + + + + + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) + + + + + + + + + + + + + + + + ego_pp_destatis_zensus_insidevg250.sql + + + + + + + + + + + + + + + + + + society.destatis_zensus_population_per_ha_mview +model_draft.destatis_zensus_population_per_ha_inside +model_draft.destatis_zensus_population_per_ha_invg_mview +model_draft.destatis_zensus_population_per_ha_outvg_mview + + + + + + + + + + + + + + + + + 46 s + + + + + + + + + + + + + + + + + society.destatis_zensus_population_per_ha +boundaries.bkg_vg250_1_sta_union_mview + + + + + + + + + + + + + + + + + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) + + + + + + + + + @@ -372,19 +621,7 @@ boundaries.bkg_vg250_statistics_view - - - - - - - - - - - - - + @@ -394,11 +631,11 @@ boundaries.bkg_vg250_statistics_view - + - + @@ -406,54 +643,57 @@ boundaries.bkg_vg250_statistics_view - + - - + + - + - - + + - + - + - - + + + + - + - + - + - - + + + + - - + - + @@ -466,6 +706,155 @@ boundaries.bkg_vg250_statistics_view + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml new file mode 100644 index 00000000..c086fb01 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml @@ -0,0 +1,272 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - cosmoclmgrid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + import_cosmoclmgrid.py + + + + + + + + + + + + + + + + + + climate.cosmoclmgrid + + + + + + + + + + + + + + + + + 7 s + + + + + + + + + + + + + + + + cosmoclmgrid.zip + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <no metadata available> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml new file mode 100644 index 00000000..b48a5076 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml @@ -0,0 +1,347 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - demand federal states + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_demand_setup.sql + + + + + + + + + + + + + + + + + + demand.ego_demand_federalstate + + + + + + + + + + + + + + + + + Open Data Commons Open Database License 1.0 + + + + + + + + + + + + + + + + regional statistical offices + + + + + + + + + + + + + + + + + + destatis_gva_per_district.sql + + + + + + + + + + + + + + + + + + VGR der Länder 2011 + + + + + + + + + + + + + + + + + economy.destatis_gva_per_district + + + + + + + + + + + + + + + + + "Open Data Commons Open Database License 1.0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml new file mode 100644 index 00000000..d6821a1b --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml @@ -0,0 +1,744 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - openstreetmap + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_osm_deu_import.sql + + + + + + + + + + + + + + + + + + openstreetmap.osm_deu_line +openstreetmap.osm_deu_nodes +openstreetmap.osm_deu_point +openstreetmap.osm_deu_polygon +openstreetmap.osm_deu_rels +openstreetmap.osm_deu_roads +openstreetmap.osm_deu_ways + + + + + + + + + + + + + + + + + 1 h 12 min + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + openstreetmap (osm2pgsql) + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + ego_pp_osm_line_street_mview.sql + + + + + + + + + + + + + + + + + + openstreetmap.osm_deu_line_street_mview + + + + + + + + + + + + + + + + + 3 min + + + + + + + + + + + + + + + + openstreetmap.osm_deu_line + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + ego_pp_osm_polygon_building_mview.sql + + + + + + + + + + + + + + + + + + openstreetmap.osm_deu_polygon_building_mview + + + + + + + + + + + + + + + + + 17 min + + + + + + + + + + + + + + + + openstreetmap.osm_deu_polygon + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + osm2postgres.py + + + + + + + + + + + + + + + + + + public.osm_deu_line +public.osm_deu_nodes +public.osm_deu_point +public.osm_deu_polygon +public.osm_deu_rels +public.osm_deu_roads +public.osm_deu_ways + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + 4 h + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_osmtgmod.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_osmtgmod.graphml new file mode 100644 index 00000000..c877738f --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_osmtgmod.graphml @@ -0,0 +1,236 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - osmTGmod + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + osmTGmod - ego_otg.py + + + + + + + + + + + + + + + + + + grid.otg_ehvhv_branch_data +grid.otg_ehvhv_bus_data +grid.otg_ehvhv_dcline_data +grid-otg_ehvhv_results_metadata + + + + + + + + + + + + + + + + + OSM Import + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_renpassGIS.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_renpassGIS.graphml new file mode 100644 index 00000000..2396d869 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_renpassGIS.graphml @@ -0,0 +1,407 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - renpassG!S + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + simple_feedin.py + + + + + + + + + + + + + + + + + + model_draft.ego_power_class + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + + model_draft.ego_supply_renewable_bnetza_full_attribute +model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + scenario_import.py + + + + + + + + + + + + + + + + + + calc_renpass_gis.renpass_gis_scenario +calc_renpass_gis.renpass_gis_linear_transformer +calc_renpass_gis.renpass_gis_source +calc_renpass_gis.renpass_gis_sink +calc_renpass_gis.renpass_gis_storage +calc_renpass_gis.renpass_gis_result + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + NODE_DATA.csv +SEQ_DATA.csv +RESULTS_DATA.csv + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml new file mode 100644 index 00000000..ea5d5619 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml @@ -0,0 +1,393 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - scenario_log + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + function_scenario_log.sql + + + + + + + + + + + + + + + + + + 0.01 s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_scenario.sql + + + + + + + + + + + + + + + + + + model_draft.ego_scenario + + + + + + + + + + + + + + + + + 0.01 s + + + + + + + + + + + + + + + + Creative Commons Zero v1.0 Universal +Reiner Lemoine Institut; CC0-1.0 + + + + + + + + + + + + + + + + ego_pp_scenario_log.sql + + + + + + + + + + + + + + + + + + 0.01 s + + + + + + + + + + + + + + + + model_draft.scenario_log + + + + + + + + + + + + + + + + + Creative Commons Zero v1.0 Universal +Reiner Lemoine Institut; CC0-1.0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_slp.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_slp.graphml new file mode 100644 index 00000000..2504ad6a --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_slp.graphml @@ -0,0 +1,244 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - slp + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pre_slp_parameters.sql + + + + + + + + + + + + + + + + + + scenario.ego_slp_parameters + + + + + + + + + + + + + + + + + 0.01s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_slp_parameters.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_slp_parameters.graphml new file mode 100644 index 00000000..362b8fc6 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_slp_parameters.graphml @@ -0,0 +1,273 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - slp_parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pre_slp_parameters.sql + + + + + + + + + + + + + + + + + + scenario.ego_slp_parameters + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml new file mode 100644 index 00000000..3af490e1 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml @@ -0,0 +1,1215 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - supply + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + preprocessing/python_scripts/ +renpass_gis/simple_feedin/simple_feedin.py + + + + + + + + + + + + + + + + + + model_draft.ego_renewable_feedin +model_draft.ego_power_class + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_res_rea_by_scenario.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + + model_draft.ego_small_chp_plant_germany +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_supply_scenario_capacities +boundaries.bkg_vg250_2_lan_nuts_view +boundaries.bkg_vg250_6_gem_rs_mview + + + + + + + + + + + + + + + + + model_draft.ego_supply_conv_powerplant_2035 + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + + ego_dp_conv_by_scenario.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_supply_conv_powerplant_2035 + + + + + + + + + + + + + + + + + ego_dp_supply_add_coastdat_gid.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant +coastdat.cosmoclmgrid + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.bnetza_eeganlagenstammdaten_wind_classification +model_draft.ego_supply_renewable_bnetza_full_attribute + + + + + + + + + + + + + + + + + ego_dp_preprocessing_conv_powerplant.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_supply_conv_nep2035_temp + + + + + + + + + + + + + + + + + ego_dp_preprocessing_res_powerplant.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + + supply.ego_renewable_powerplant + + + + + + + + + + + + + + + + + supply.ego_conventional_powerplant + + + + + + + + + + + + + + + + + bkg_vg250 + + + + + + + + + + + + + + + + + ego_nep_2015_scenario_capacities.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_scenario_capacities + + + + + + + + + + + + + + + + + NEP 2015 +FlEnS + + + + + + + + + + + + + + + + + utility/ego_dp_preprocessing_conv_powerplant_2035.sql + + + + + + + + + + + + + + + + + + coastdat + + + + + + + + + + + + + + + + + model_draft.ego_neighbours_offshore_point +model_draft.ego_weather_measurement_point + + + + + + + + + + + + + + + + + climate.cosmoclmgrid +boundaries.bkg_vg250_1_sta +model_draft.renpass_gis_parameter_region +model_draft.ego_grid_hv_electrical_neighbours_bus + + + + + + + + + + + + + + + + + model_draft.bnetza_eeganlagenstammdaten_wind_classification +model_draft.ego_supply_renewable_bnetza_full_attribute + + + + + + + + + + + + + + + + + model_draft.ego_small_chp_plant_germany + + + + + + + + + + + + + + + + + ego-simple-feedin_per_scenario.sql + + + + + + + + + + + + + + + + + + model_draft.renpass_gis_parameter_region +model_draft.ego_grid_hv_electrical_neighbours_bus (DP) + + + + + + + + + + + + + + + + + supply.ego_renewable_powerplant + + + + + + + + + + + + + + + + + supply.ego_conventional_powerplant + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_wind_potential_areas.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_wind_potential_areas.graphml new file mode 100644 index 00000000..65920424 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_wind_potential_areas.graphml @@ -0,0 +1,378 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - wind potential areas + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + vernetzen_wind_potential_area_v1.3.sql + + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + import_vernetzen_wind_potential_areas.py + + + + + + + + + + + + + + + + + + supply.vernetzen_wind_potential_area + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + vernetzen_wind_potential_areas.zip + + + + + + + + + + + + + + + + + <no license> + + + + + + + + + + + + + + + + supply.vernetzen_wind_potential_area + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml new file mode 100644 index 00000000..3f242694 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml @@ -0,0 +1,568 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - zensus + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_destatis_zensus_import.sql + + + + + + + + + + + + + + + + + + society.destatis_zensus_population_per_ha + + + + + + + + + + + + + + + + + 5 min + + + + + + + + + + + + + + + + + tmp_destatis.zensus_population_per_ha +tmp_destatis.zensus_population_per_ha_grid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + destatis_zensus_raw_import.sql + + + + + + + + + + + + + + + + + + tmp_destatis.zensus_2011_pop_per_ha + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5 min + + + + + + + + + + + + + + + + Datenlizenz Deutschland – Namensnennung – Version 2.0 +Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 + + + + + + + + + + + + + + + + destatis_zensus_import_csv.py + + + + + + + + + + + + + + + + + + tmp_destatis.zensus_2011_pop_per_ha + + + + + + + + + + + + + + + + + 6 min + + + + + + + + + + + + + + + + Zensus_Bevoelkerung_100m-Gitter.zip + + + + + + + + + + + + + + + + + Datenlizenz Deutschland – Namensnennung – Version 2.0 +Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 + + + + + + + + + + + + + + + + tmp_destatis.zensus_population_per_ha +tmp_destatis.zensus_population_per_ha_grid + + + + + + + + + + + + + + + + + Datenlizenz Deutschland – Namensnennung – Version 2.0 +Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml b/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml index 32514ce6..69f59134 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml @@ -17,11 +17,11 @@ - + - eGo Data-PreProcessing - + eGo Data-PreProcessing + @@ -53,18 +53,18 @@ - + - + - + @@ -75,7 +75,7 @@ - + zensus @@ -93,10 +93,10 @@ - + - bkg_vg250 + zensus_vg250 @@ -109,30 +109,9 @@ - - - - - - boundaries.bkg_vg250_1_sta_union_mview -model_draft.ego_boundaries_bkg_vg250_6_gem_clean -political_boundary.bkg_vg250_4_krs_mview -boundaries.bkg_vg250_6_gem -political_boundary.bkg_vg250_1_sta_union_mview - - - - - - - - - - - - + @@ -153,13 +132,13 @@ political_boundary.bkg_vg250_1_sta_union_mview - + - + - + @@ -177,85 +156,396 @@ political_boundary.bkg_vg250_1_sta_union_mview + + + + + + + vg250 + + + + + + + + + + + - - - - + + + + + openstreetmap + + + + + + + + + + + + + + + + + + scenario_log + + + + + + + + + + + + + + + + + + SLP + + + + + + + + + + + + + + + + + + cosmoclmgrid + + + + + + + + + + + + + + + + + + wind potential areas + + + + + + + + + + + + + + + + + + renpassG!S + + + + + + + + + + + + + + + + + + supply_powerplant + + + + + + + + + + + + + + + + + + demand_federalstate - + - + + + + + + + + + + + + + osmTGmod + + + + + + + - + - - - - - + + - + - + - + - - - - + + - + - + - - - - + + - + - + - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_REA.graphml b/documentation/bpmn/ego_dp_bpmn_section_REA.graphml new file mode 100644 index 00000000..0a94dba3 --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_section_REA.graphml @@ -0,0 +1,1875 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - REA + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_500m + + + + + + + + + + + + + + + + + + rea/ego_dp_lattice_500m.sql + + + + + + + + + + + + + + + + + + boundaries.bkg_vg250_1_sta_union_mview + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + vernetzen + + + + + + + + + + + + + + + + + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) + + + + + + + + + + + + + + + + bkg_vg250 + + + + + + + + + + + + + + + + + model_draft.ego_lattice_500m + + + + + + + + + + + + + + + + + rea/ego_dp_lattice_50m.sql + + + + + + + + + + + + + + + + + + boundaries.bkg_vg250_1_sta_union_mview + + + + + + + + + + + + + + + + + model_draft.ego_lattice_50m + + + + + + + + + + + + + + + + + + rea/ego_dp_rea_wpa_per_mvgd.sql + + + + + + + + + + + + + + + + + + supply.vernetzen_wind_potential_area +model_draft.ego_grid_mv_griddistrict + + + + + + + + + + + + + + + + + model_draft.ego_supply_wpa_per_mvgd + + + + + + + + + + + + + + + + + rea/ego_dp_rea_lattice_per_area_500m.sql + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_500m +model_draft.ego_grid_mv_griddistrict +model_draft.ego_supply_wpa_per_mvgd +model_draft.ego_demand_loadarea + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_500m_wpa_mview +model_draft.ego_lattice_500m_la_mview +model_draft.ego_lattice_500m_x_mview +model_draft.ego_lattice_500m_out_mview + + + + + + + + + + + + + + + + + rea/ego_dp_rea_lattice_per_area_50m.sql + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_50m +model_draft.ego_grid_mv_griddistrict +model_draft.ego_demand_loadarea + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_50m_la_mview + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_50m + + + + + + + + + + + + + + + + + rea/ego_dp_rea_setup.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_osm_sector_per_griddistrict_4_agricultural +model_draft.ego_grid_mv_griddistrict +model_draft.ego_grid_hvmv_substation + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant_out_mview +model_draft.ego_dp_supply_res_powerplant_out_nn + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_osm_sector_per_griddistrict_4_agricultural + + + + + + + + + + + + + + + + + rea/ego_dp_rea_m1.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_m1_1_a_mview +model_draft.ego_supply_rea_m1_1_dea_temp +model_draft.ego_supply_rea_m1_1_osm_temp +model_draft.ego_supply_rea_m1_1_jnt_temp +model_draft.ego_supply_rea_m1_1_mview +model_draft.ego_supply_rea_m1_1_rest_mview +model_draft.ego_supply_rea_m1_2_a_mview +model_draft.ego_supply_rea_m1_2_dea_temp +model_draft.ego_supply_rea_m1_2_osm_temp +model_draft.ego_supply_rea_m1_2_jnt_temp +model_draft.ego_supply_rea_m1_2_mview +model_draft.ego_supply_rea_m1_2_rest_mview + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_osm_sector_per_griddistrict_4_agricultural + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + rea/ego_dp_rea_m2.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_m2_a_mview +model_draft.ego_supply_rea_m2_windfarm +model_draft.ego_supply_rea_m2_farm_temp +model_draft.ego_supply_rea_m2_wpa_temp +model_draft.ego_supply_rea_m2_jnt_temp +model_draft.ego_supply_rea_m2_mview +model_draft.ego_supply_rea_m2_rest_mview + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_grid_mv_griddistrict +model_draft.ego_supply_wpa_per_mvgd + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + rea/ego_dp_rea_m3.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_m3_a_mview +model_draft.ego_supply_rea_m3_dea_temp +model_draft.ego_supply_rea_m3_grid_wpa_temp +model_draft.ego_supply_rea_m3_jnt_temp +model_draft.ego_supply_rea_m3_mview +model_draft.ego_supply_rea_m3_rest_mview + + + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_lattice_500m_wpa_mview + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + rea/ego_dp_rea_m4.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_m4_a_mview +model_draft.ego_supply_rea_m4_dea_temp +model_draft.ego_supply_rea_m4_grid_wpa_temp +model_draft.ego_supply_rea_m4_jnt_temp +model_draft.ego_supply_rea_m4_mview +model_draft.ego_supply_rea_m4_rest_mview + + + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_lattice_500m_wpa_mview + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + rea/ego_dp_rea_m5.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_m5_a_mview +model_draft.ego_supply_rea_m5_dea_temp +model_draft.ego_supply_rea_m5_grid_la_temp +model_draft.ego_supply_rea_m5_jnt_temp +model_draft.ego_supply_rea_m5_rest_mview +model_draft.ego_supply_rea_m5_rest_2_mview +model_draft.ego_supply_rea_m5_mview + + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_demand_loadarea +model_draft.ego_grid_lv_griddistrict + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + rea/ego_dp_rea_results.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_per_gentype_and_voltlevel +model_draft.ego_supply_rea_per_loadarea +model_draft.ego_supply_rea_per_method + + + + + + + + + + + + + + + + + + model_draft.ego_grid_mv_griddistrict +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_demand_loadarea + + + + + + + + + + + + + + + + + model_draft.ego_grid_mv_griddistrict +model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + DP griddistrict + + + + + + + + + + + + + + + + + DP loadarea + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + powerplants + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml b/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml index 72c3f0f9..e4c0a044 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml @@ -1,6 +1,6 @@ - + @@ -13,15 +13,15 @@ - + - + - eGo DataProcessing - griddistrict - + eGo DataProcessing - griddistrict + @@ -53,18 +53,18 @@ - + - + - + @@ -78,7 +78,7 @@ - ego_dp_mv_griddistrict.sql + ego_dp_mv_griddistrict.sql @@ -96,18 +96,12 @@ - ego_political_boundary_hvmv_subst_per_gem_1_mview -ego_political_boundary_hvmv_subst_per_gem_2_mview -ego_political_boundary_hvmv_subst_per_gem_3_mview -ego_grid_hvmv_substation_voronoi_mview -ego_grid_mv_griddistrict -ego_grid_mv_griddistrict_dump - - - - - - + model_draft.ego_grid_mv_griddistrict +model_draft.ego_boundaries_hvmv_subst_per_gem_1_mview +model_draft.ego_boundaries_hvmv_subst_per_gem_2_mview +model_draft.ego_boundaries_hvmv_subst_per_gem_3_mview +model_draft.ego_grid_hvmv_substation_voronoi_mview +model_draft.ego_grid_mv_griddistrict_dump @@ -119,42 +113,15 @@ ego_grid_mv_griddistrict_dump - - - - - - 2 min - - - - - - - - - - - - - - - - - model_draft.vg250_6_gem_clean_mview + + model_draft.ego_boundaries_bkg_vg250_6_gem_clean model_draft.ego_grid_hvmv_substation -model_draft.ego_grid_hvmv_substation_voronoi - - - - - - +model_draft.ego_grid_hvmv_substation_voronoi @@ -165,7 +132,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + @@ -189,10 +156,10 @@ model_draft.ego_grid_hvmv_substation_voronoi - + - + @@ -213,20 +180,14 @@ model_draft.ego_grid_hvmv_substation_voronoi - + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -236,19 +197,13 @@ model_draft.ego_grid_hvmv_substation_voronoi - + - political_boundary.bkg_vg250 - - - - - - + bkg_vg250 @@ -259,43 +214,31 @@ model_draft.ego_grid_hvmv_substation_voronoi - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + DP substation + - + - + - + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -305,93 +248,71 @@ model_draft.ego_grid_hvmv_substation_voronoi - + - - + + - model_draft.ego_grid_mv_griddistrict -grid.ego_dp_mv_griddistrict - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) - - + - + - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + + + + + ego_dp_substation_id_to_generator.sql + - + + - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_mv_griddistrict +model_draft.ego_grid_ehv_substation_voronoi + + - - - - - - - - - - - - - - - - model_draft.ego_grid_hvmv_substation -grid.ego_hvmv_substation + + - + - + @@ -399,67 +320,45 @@ grid.ego_hvmv_substation - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant + + - + - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + supply_powerplant + - + - - - - - - - - - - - - - + @@ -471,7 +370,7 @@ grid.ego_hvmv_substation - + @@ -483,7 +382,7 @@ grid.ego_hvmv_substation - + @@ -495,19 +394,19 @@ grid.ego_hvmv_substation - + - - - + + + - + - + @@ -519,28 +418,76 @@ grid.ego_hvmv_substation - + - + - + - - - - + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml b/documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml new file mode 100644 index 00000000..42e7bf67 --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml @@ -0,0 +1,196 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - input data check + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_structure_input_verification.sql + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_scenario_input + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml b/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml index e31e6455..a852c905 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml @@ -1,6 +1,6 @@ - + @@ -13,15 +13,15 @@ - + - + - eGo DataProcessing - substation - + eGo DataProcessing - loadarea + @@ -53,18 +53,18 @@ - + - + - + @@ -75,10 +75,10 @@ - + - ego_dp_loadarea_landuse.sql + ego_dp_loadarea_landuse.sql @@ -93,26 +93,24 @@ - + - openstreetmap.osm_deu_polygon_urban -osm_deu_polygon_urban_sector_1_residential_mview -osm_deu_polygon_urban_sector_2_retail_mview -osm_deu_polygon_urban_sector_3_industrial_mview -osm_deu_polygon_urban_sector_4_agricultural_mview - - - - - - + openstreetmap.osm_deu_polygon_urban_sector_1_residential_mview +openstreetmap.osm_deu_polygon_urban_sector_2_retail_mview +openstreetmap.osm_deu_polygon_urban_sector_3_industrial_mview +openstreetmap.osm_deu_polygon_urban_sector_4_agricultural_mview +openstreetmap.osm_deu_polygon_urban +openstreetmap.osm_deu_polygon_urban_error_geom_vg250_mview +openstreetmap.osm_deu_polygon_urban_vg250_cut_mview +openstreetmap.osm_deu_polygon_urban_vg250_clean_cut_multi_mview +openstreetmap.osm_deu_polygon_urban_vg250_clean_cut_mview + - @@ -120,24 +118,18 @@ osm_deu_polygon_urban_sector_4_agricultural_mview - + - model_draft.ego_demand_la_zensus + model_draft.ego_demand_la_zensus model_draft.ego_demand_la_zensus_cluster -model_draft.zensus_population_per_load_area_stats_mview - - - - - - +model_draft.ego_society_zensus_per_la_mview + - @@ -145,10 +137,10 @@ model_draft.zensus_population_per_load_area_stats_mview - + - ego_dp_loadarea_loadmelt.sql + ego_dp_loadarea_loadmelt.sql @@ -163,24 +155,18 @@ model_draft.zensus_population_per_load_area_stats_mview - + - model_draft.ego_demand_load_collect + model_draft.ego_demand_load_collect model_draft.ego_demand_load_collect_buffer100_mview -model_draft.ego_demand_load_melt - - - - - - +model_draft.ego_demand_load_melt + - @@ -188,10 +174,10 @@ model_draft.ego_demand_load_melt - + - ego_dp_loadarea_loadcut_griddistrict.sql + ego_dp_loadarea_loadcut_griddistrict.sql @@ -206,29 +192,23 @@ model_draft.ego_demand_load_melt - + - model_draft.ego_osm_sector_per_griddistrict_1_residential + model_draft.ego_osm_sector_per_griddistrict_1_residential model_draft.ego_osm_sector_per_griddistrict_2_retail model_draft.ego_osm_sector_per_griddistrict_3_industrial model_draft.ego_osm_sector_per_griddistrict_4_agricultural -model_draft.ego_demand_loadarea_centre_mview model_draft.ego_demand_loadarea_smaller100m2_mview model_draft.ego_demand_loadarea_error_noags_mview -model_draft.ego_demand_loadarea - - - - - - +model_draft.ego_demand_loadarea +openstreetmap.osm_deu_polygon_urban_sector_3_industrial_nolargescale_mview + - @@ -236,21 +216,17 @@ model_draft.ego_demand_loadarea - + - 20 sec - - - - - - + openstreetmap.osm_deu_polygon +boundaries.bkg_vg250_1_sta_union_mview + - + @@ -258,43 +234,36 @@ model_draft.ego_demand_loadarea - + - 30 min - - - - - - + model_draft.destatis_zensus_population_per_ha_invg_mview +society.destatis_zensus_population_per_ha_mview +model_draft.ego_demand_la_osm + - + - - - - - 20 min - - - - - - + + + + + ego_dp_loadarea_industry_consumer.sql + - + + @@ -302,21 +271,19 @@ model_draft.ego_demand_loadarea - - + + - 30 min - - - - - - + model_draft.ego_landuse_industry +model_draft.ego_demand_hv_largescaleconsumer +model_draft.ego_demand_per_district +model_draft.ego_demand_per_gva + - + @@ -324,58 +291,34 @@ model_draft.ego_demand_loadarea - + - openstreetmap.osm_deu_polygon -political_bkg_vg250_1_sta_union_mview - - - - - - + demand.ego_demand_federalstate +economic.destatis_gva_per_districts +political_boundary.bkg_vg250_4_krs_mview +model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_grid_ehv_substation_voronoi +model_draft.ego_grid_hvmv_substation +openstreetmap.osm_deu_polygon_urban_sector_3_industrial_mview + - - - - - - - - - - - - - - social.destatis_zensus_population_per_ha_mview -model_draft.ego_demand_la_osm - - - - - - - - - - + - + - ego_dp_loadarea_industry_consumer.sql + ego_dp_loadarea_loads.sql @@ -387,88 +330,68 @@ model_draft.ego_demand_la_osm - + - + - model_draft.ego_landuse_industry -model_draft.ego_demand_hv_largescaleconsumer - - - - - - + model_draft.ego_demand_la_osm +model_draft.osm_deu_polygon_urban_buffer100_mview + + - - + - + - demand.ego_demand_federalstate -economic.destatis_gva_per_districts -political_boundary.bkg_vg250_4_krs_mview -supply.ego_conv_powerplant -model_draft.ego_grid_ehv_substation_voronoi -model_draft.ego_grid_hvmv_substation - - - - - - - + model_draft.ego_demand_hv_largescaleconsumer +openstreetmap.osm_deu_polygon_urban + - - + - + - 10 sec - - - - - - + model_draft.ego_deu_loads_osm +model_draft.ego_social_zensus_load_cluster + - + - + - + - ego_dp_loadarea_loads.sql + ego_dp_loadarea_consumption.sql @@ -480,125 +403,113 @@ model_draft.ego_grid_hvmv_substation - + - - + + - model_draft.ego_demand_la_osm - - - - - - + model_draft.ego_demand_loadarea +model_draft.ego_demand_per_district + - - + - + - 45 min - - - - - - + model_draft.ego_demand_per_district +model_draft.ego_demand_loadarea +demand.ego_demand_federalstate +model_draft.ego_grid_hvmv_substation + - + - + - + - model_draft.ego_demand_hv_largescaleconsumer -openstreetmap.osm_deu_polygon_urban - - - - - - + model_draft.ego_demand_load_melt +model_draft.ego_grid_mv_griddistrict +society.destatis_zensus_population_per_ha_mview +openstreetmap.osm_deu_polygon_urban_sector_1_residential_mview +openstreetmap.osm_deu_polygon_urban_sector_2_retail_mview +openstreetmap.osm_deu_polygon_urban_sector_4_agricultural_mview +openstreetmap.osm_deu_polygon_urban +model_draft.ego_demand_hv_largescaleconsumer +model_draft.ego_boundaries_bkg_vg250_6_gem_clean + + - - + - - + + - model_draft.ego_deu_loads_osm -model_draft.ego_social_zensus_load_cluster - - - - - - + Datenlizenz Deutschland – Namensnennung – Version 2.0 +Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - + - + - + - ego_dp_loadarea_consumption.sql + ego_dp_loadarea_census.sql - + - + - - - - - model_draft.ego_demand_loadarea -demand.ego_loadarea + + + + + + @@ -607,27 +518,22 @@ demand.ego_loadarea + - - + - + - - + + - 16 sec - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -637,76 +543,48 @@ demand.ego_loadarea - + - - + + - model_draft.ego_demand_per_district -model_draft.ego_demand_loadarea -demand.ego_demand_federalstate - - - - - - + openstreetmap + - - + - - + + - model_draft.ego_demand_load_melt -model_draft.ego_grid_mv_griddistrict -social.destatis_zensus_population_per_ha_mview -economic.ioer_urban_share_industrial_centroid -openstreetmap.osm_deu_polygon_urban_sector_1_residential_mview -openstreetmap.osm_deu_polygon_urban_sector_2_retail_mview -openstreetmap.osm_deu_polygon_urban_sector_3_industrial_mview -openstreetmap.osm_deu_polygon_urban_sector_4_agricultural_mview -model_draft.ego_political_boundary_bkg_vg250_6_gem_clean - - - - - - + destatis gva + - - + - + - Datenlizenz Deutschland – Namensnennung – Version 2.0 -Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -716,30 +594,29 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - - - - - ego_dp_loadarea_census.sql + + + + + bkg_vg250 - + - - + - + - + - + @@ -749,34 +626,45 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - + - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + zensus + - + + + + + + + + + + + + DP griddistrict + + + + + + @@ -784,16 +672,16 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - + + - openstreetmap.osm_deu + DP substation + - @@ -801,16 +689,16 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - economic.destatis_gva_per_districts + demand federalstate + - @@ -818,22 +706,27 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_mv_griddistrict +model_draft.ego_grid_mv_griddistrict_type1 +model_draft.ego_grid_mv_griddistrict_type2 +model_draft.ego_grid_mv_griddistrict_type3 +model_draft.ego_demand_loadarea +model_draft.ego_boundaries_bkg_vg250_6_gem_clean +model_draft.ego_boundaries_hvmv_subst_per_gem_1_mview +model_draft.ego_boundaries_hvmv_subst_per_gem_3_mview +model_draft.ego_grid_hvmv_substation_voronoi_cut +boundaries.bkg_vg250_6_gem +boundaries.bkg_vg250_1_sta_mview + - + @@ -841,80 +734,72 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - + + - political_boundary.bkg_vg250 + boundaries.bkg_vg250_6_gem_pts +boundaries.bkg_vg250_statistics_mview + + - - - - - - - - - - - - - + + + + + ego_dp_loadarea_statistic.sql - + - + + - - - - - social.destatis_zensus_population_per_ha + + + + + ego_dp_loadarea_loadcut_voronoi.sql + - - + + - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + + + + + ego_dp_loadarea_peakload.py + - + + @@ -922,22 +807,17 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + model_draft.ego_demand_loadarea +scenario.ego_slp_parameters + - + @@ -945,45 +825,34 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + model_draft.ego_demand_loadarea + - + - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + + + + + ego_dp_loadarea_griddistrict_results.sql + - + + @@ -991,17 +860,12 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Vervielfältigung und Verbreitung, auch +auszugsweise, mit Quellenangabe gestattet. +© Statistisches Landesamt Baden-Württemberg @@ -1014,22 +878,17 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + model_draft.ego_demand_loadarea_voi +model_draft.ego_demand_loadarea_voi_smaller100m2_mview + - + @@ -1037,22 +896,19 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - + + - Datenlizenz Deutschland – Namensnennung – Version 2.0 -Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - - - - - + model_draft.ego_demand_load_melt +model_draft.ego_grid_hvmv_substation_voronoi +model_draft.ego_boundaries_bkg_vg250_6_gem_clean +model_draft.ego_grid_mv_griddistrict + - + @@ -1060,10 +916,10 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - supply.ego_conv_powerplant + BDEW slp @@ -1077,11 +933,11 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - + + - model_draft.ego_grid_ehv_substation -grid.ego_ehv_substation + model_draft.ego_grid_mv_griddistrict + @@ -1095,206 +951,19 @@ grid.ego_ehv_substation - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - - - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - - - - - - - - - - - - - - - - - demand.ego_demand_federalstate - - - - - - - - - - - - - - - - - model_draft.ego_grid_hvmv_substation -model_draft.ego_grid_mv_griddistrict -model_draft.ego_demand_loadarea - - - - - - - - - - - - - - - - - - - - + - 5 min - - - - - - - - - - - - - - - - - - - - - - model_draft.ego_data_processing_results -model_draft.ego_data_processing_results_mvgd -boundaries.bkg_vg250_statistics_mview - - - - - - - - - - - - - - - - - - - - - - - ego_dp_loadarea_statistic.sql - - - - - - - - - - - - - - - - - - ego_dp_loadarea_loadcut_voronoi.sql - - - - - - - - - - - - - - - - - - ego_dp_loadarea_peakload.py - - - - - - - - - - - - - - - - - - model_draft.ego_demand_loadarea -scenario.ego_slp_parameters - - - - - - + model_draft.ego_grid_mv_griddistrict +boundaries.bkg_vg250_6_gem +model_draft.ego_grid_mv_griddistrict_type1 +model_draft.ego_grid_mv_griddistrict_type2 +model_draft.ego_grid_mv_griddistrict_type3 +society.destatis_zensus_population_per_ha_mview +model_draft.ego_boundaries_hvmv_subst_per_gem_1_mview +model_draft.ego_boundaries_hvmv_subst_per_gem_3_mview +model_draft.ego_grid_hvmv_substation_voronoi_cut +model_draft.ego_demand_loadarea @@ -1305,19 +974,14 @@ scenario.ego_slp_parameters - + - - + + - model_draft.ego_demand_loadarea - - - - - - + model_draft.ego_grid_mv_griddistrict + @@ -1328,72 +992,9 @@ scenario.ego_slp_parameters - - - - - - - 1 h - - - - - - - - - - - - - - - - - - - - - - ego_dp_loadarea_griddistrict_results.sql - - - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - + @@ -1405,7 +1006,7 @@ scenario.ego_slp_parameters - + @@ -1417,55 +1018,7 @@ scenario.ego_slp_parameters - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1477,31 +1030,7 @@ scenario.ego_slp_parameters - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1513,7 +1042,7 @@ scenario.ego_slp_parameters - + @@ -1525,7 +1054,7 @@ scenario.ego_slp_parameters - + @@ -1537,7 +1066,7 @@ scenario.ego_slp_parameters - + @@ -1549,7 +1078,7 @@ scenario.ego_slp_parameters - + @@ -1561,7 +1090,7 @@ scenario.ego_slp_parameters - + @@ -1573,7 +1102,7 @@ scenario.ego_slp_parameters - + @@ -1585,7 +1114,7 @@ scenario.ego_slp_parameters - + @@ -1597,7 +1126,7 @@ scenario.ego_slp_parameters - + @@ -1609,7 +1138,7 @@ scenario.ego_slp_parameters - + @@ -1621,7 +1150,7 @@ scenario.ego_slp_parameters - + @@ -1633,7 +1162,7 @@ scenario.ego_slp_parameters - + @@ -1645,43 +1174,31 @@ scenario.ego_slp_parameters - - - - - - - - - - - - - + - - + + - + - + - - + + - + - + @@ -1693,7 +1210,7 @@ scenario.ego_slp_parameters - + @@ -1705,7 +1222,7 @@ scenario.ego_slp_parameters - + @@ -1717,7 +1234,7 @@ scenario.ego_slp_parameters - + @@ -1729,7 +1246,7 @@ scenario.ego_slp_parameters - + @@ -1741,7 +1258,7 @@ scenario.ego_slp_parameters - + @@ -1753,7 +1270,7 @@ scenario.ego_slp_parameters - + @@ -1765,7 +1282,7 @@ scenario.ego_slp_parameters - + @@ -1777,7 +1294,7 @@ scenario.ego_slp_parameters - + @@ -1789,7 +1306,7 @@ scenario.ego_slp_parameters - + @@ -1801,7 +1318,7 @@ scenario.ego_slp_parameters - + @@ -1813,7 +1330,7 @@ scenario.ego_slp_parameters - + @@ -1825,7 +1342,7 @@ scenario.ego_slp_parameters - + @@ -1837,7 +1354,7 @@ scenario.ego_slp_parameters - + @@ -1849,7 +1366,7 @@ scenario.ego_slp_parameters - + @@ -1861,7 +1378,7 @@ scenario.ego_slp_parameters - + @@ -1873,7 +1390,7 @@ scenario.ego_slp_parameters - + @@ -1885,19 +1402,7 @@ scenario.ego_slp_parameters - - - - - - - - - - - - - + @@ -1909,7 +1414,7 @@ scenario.ego_slp_parameters - + @@ -1921,7 +1426,7 @@ scenario.ego_slp_parameters - + @@ -1933,37 +1438,27 @@ scenario.ego_slp_parameters - + - + - + - - - - - - - - - - - + - + @@ -1975,7 +1470,7 @@ scenario.ego_slp_parameters - + @@ -1987,7 +1482,7 @@ scenario.ego_slp_parameters - + @@ -1999,7 +1494,7 @@ scenario.ego_slp_parameters - + @@ -2011,7 +1506,7 @@ scenario.ego_slp_parameters - + @@ -2023,6 +1518,283 @@ scenario.ego_slp_parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_lowvoltage.graphml b/documentation/bpmn/ego_dp_bpmn_section_lowvoltage.graphml index 19a0afcf..1622457e 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_lowvoltage.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_lowvoltage.graphml @@ -1,6 +1,6 @@ - + @@ -13,15 +13,15 @@ - + - + - eGo DataProcessing - lowvoltage - + eGo DataProcessing - lowvoltage + @@ -31,29 +31,29 @@ + + + - - - - - + + - + @@ -64,7 +64,7 @@ - + @@ -74,186 +74,192 @@ - - + + - - ego_dp_lv_substation.sql + + + + + + + + + - + - - + - - - - - 9 min + + + + + + - + + - + - - - - - ego_dp_lv_substation_voronoi.sql + + + + + model_draft.ego_lattice_360m_lv +model_draft.ego_grid_mvlv_substation +model_draft.ego_grid_lv_loadarea_rest - + - - + + - - + + - 2 h - - - - - - + model_draft.ego_demand_loadarea +model_draft.ego_grid_mv_griddistrict + - + - - - - - - - - - - - - + + + + + model_draft.ego_grid_mvlv_substation_voronoi - + - + - - - - - - - - - - - - + + + + + model_draft.ego_grid_lv_griddistrict_cut +model_draft.ego_grid_lv_griddistrict_cut_1subst +model_draft.ego_grid_lv_griddistrict_cut_0subst +model_draft.ego_grid_lv_griddistrict_cut_xsubst +model_draft.ego_grid_lv_griddistrict_cut_nn +model_draft.ego_grid_lv_griddistrict_cut_nn_collect + - + - + + - - + + - Open Data Commons Open Database License 1.0 - - - - - - + model_draft.ego_grid_lv_griddistrict_cut_nn_collect +model_draft.ego_grid_mvlv_substation +society.destatis_zensus_population_per_ha_mview +model_draft.ego_osm_sector_per_griddistrict_1_residential +model_draft.ego_osm_sector_per_griddistrict_2_retail +model_draft.ego_osm_sector_per_griddistrict_3_industrial +model_draft.ego_osm_sector_per_griddistrict_4_agricultural + + + - + + - - - - - ego_dp_lv_loadcut.sql + + + + + model_draft.ego_grid_lv_griddistrict +model_draft.ego_demand_loadarea +scenario.ego_slp_parameters + - + - - + + - - - - - ego_dp_lv_griddistrict.sql + + + + + model_draft.ego_grid_hvmv_substation_dummy +model_draft.ego_grid_mvlv_substation - + - - + @@ -261,10 +267,10 @@ - - + + - ego_dp_lv_consumption_peakload.sql + ego_dp_lv_substation.sql @@ -278,115 +284,92 @@ - - - - - model_draft.ego_grid_mvlv_substation - - - - - - + + + + + ego_dp_lv_substation_voronoi.sql + - - + + - - - - - - model_draft.ego_demand_loadarea -model_draft.ego_grid_mv_griddistrict - - - - - - + + + + + ego_dp_lv_loadcut.sql + - - + + - - - - - Open Data Commons Open Database License 1.0 - - - - - - + + + + + ego_dp_lv_griddistrict.sql + - + + - - - - - model_draft.ego_grid_lv_griddistrict - - - - - - + + + + + ego_dp_lv_consumption_peakload.sql + - - + + + - - + + - Open Data Commons Open Database License 1.0 - - - - - - + model_draft.ego_demand_loadarea +model_draft.ego_grid_mvlv_substation +model_draft.ego_grid_mvlv_substation_voronoi + - + @@ -394,21 +377,21 @@ model_draft.ego_grid_mv_griddistrict - - + + - 20 min - - - - - - + model_draft.ego_grid_lv_griddistrict +model_draft.ego_osm_sector_per_lvgd_1_residential +model_draft.ego_osm_sector_per_lvgd_2_retail +model_draft.ego_osm_sector_per_lvgd_3_industrial +model_draft.ego_osm_sector_per_lvgd_4_agricultural + - + + @@ -416,21 +399,16 @@ model_draft.ego_grid_mv_griddistrict - - + + - 40 min - - - - - - + model_draft.ego_grid_lv_griddistrict - + + @@ -438,21 +416,16 @@ model_draft.ego_grid_mv_griddistrict - - + + - 5 min - - - - - - + zensus - + + @@ -460,16 +433,10 @@ model_draft.ego_grid_mv_griddistrict - - + + - model_draft.ego_grid_mvlv_substation_voronoi - - - - - - + DP loadarea @@ -483,16 +450,10 @@ model_draft.ego_grid_mv_griddistrict - - + + - model_draft.ego_grid_lv_griddistrict_cut_nn_collect - - - - - - + DP griddistrict @@ -504,25 +465,18 @@ model_draft.ego_grid_mv_griddistrict - - + - model_draft.ego_grid_mvlv_substation - - - - - - + DP substation - + @@ -530,39 +484,10 @@ model_draft.ego_grid_mv_griddistrict - - - - Open Data Commons Open Database License 1.0 - - - - - - - - - - - - - - - - - - - - - + + - scenario.ego_slp_parameters - - - - - - + BDEW SLP @@ -573,337 +498,324 @@ model_draft.ego_grid_mv_griddistrict - - - - - - - Open Data Commons Open Database License 1.0 - - - - - - - - - - - - - - - - + - + - - - - - - + + + + - + - + - + - - - - + + - + - + - - - - - - + + + + - + - + - + - - - - + + - + - + - - - - + + - + - + - - - - + + - + - + - - - - - - + + + + - + - + - - - - - - + + + + - + - + - + - - - - - - - + + - + - + - + - - - - - - - + + - + - + - - - - - - + + + + - + - + - - - - - - + + + + - + + - + - - - - - - + + + + - + + - + - + - - - - - - - + + - + + - + - + - - - - + + - + + - + - + - - - - + + - + + - + - + - - - - + + - + + - + - + - - - - - - - + + - + + - + - + - - - - - - - + + - + + - + - + - - - - - - - + + - + + - + - + - - - - + + - + + - + - + - - - - - - - + + - + + - + - + - - - - + + - + + - + - + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_postprocessing.graphml b/documentation/bpmn/ego_dp_bpmn_section_postprocessing.graphml new file mode 100644 index 00000000..4abb7506 --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_section_postprocessing.graphml @@ -0,0 +1,483 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - postprocessing + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_nep2035_grid_variations.sql + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_extension_temp_resolution +model_draft.ego_grid_pf_hv_extension_source +model_draft.ego_grid_pf_hv_extension_storage +model_draft.ego_grid_pf_hv_extension_storage_pq_set +model_draft.ego_grid_pf_hv_extension_bus +model_draft.ego_grid_pf_hv_extension_line +model_draft.ego_grid_pf_hv_extension_link +model_draft.ego_grid_pf_hv_extension_transformer +model_draft.ego_grid_pf_hv_extension_load +model_draft.ego_grid_pf_hv_extension_load_pq_set +model_draft.ego_grid_pf_hv_extension_generator +model_draft.ego_grid_pf_hv_extension_generator_pq_set +calc_renpass_gis.extension_translate_to_pf + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_temp_resolution +model_draft.ego_grid_pf_hv_source +model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_transformer +model_draft.ego_grid_pf_hv_link +model_draft.scn_nep2035_b2_line +grid.bnetza_vorhabenpunkte_bbplg +grid.bnetza_vorhabenpunkte_enlag +model_draft.opsd_hourly_timeseries +calc_renpass_gis.renpass_gis_linear_transformer +calc_renpass_gis.renpass_gis_source +grid.otg_ehvhv_bus_data +model_draft.ego_grid_line_expansion_costs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NEP 2035 B2 + + + + + + + + + + + + + + + + + DP powerflow + + + + + + + + + + + + + + + + + BBPlG + + + + + + + + + + + + + + + + + EnLAG + + + + + + + + + + + + + + + + + OPSD + + + + + + + + + + + + + + + + + renpassGIS + + + + + + + + + + + + + + + + + osmTGmod + + + + + + + + + + + + + + + + + expansion costs +NEP/dena + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml b/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml new file mode 100644 index 00000000..6a84e0ec --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml @@ -0,0 +1,2323 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - powerflow + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_fix_ehv_subnetworks.sql + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_scenario_settings +model_draft.ego_grid_pf_hv_source +model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_busmap +model_draft.ego_grid_pf_hv_generator +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_load +model_draft.ego_grid_pf_hv_storage +model_draft.ego_grid_pf_hv_temp_resolution +model_draft.ego_grid_pf_hv_transformer +model_draft.ego_grid_pf_hv_bus_v_mag_set +model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_grid_pf_hv_load_pq_set +model_draft.ego_grid_pf_hv_storage_pq_set +model_draft.ego_grid_pf_hv_results_bus +model_draft.ego_grid_pf_hv_results_bus_t +model_draft.ego_grid_pf_hv_results_generator +model_draft.ego_grid_pf_hv_results_generator_t +model_draft.ego_grid_pf_hv_results_line +model_draft.ego_grid_pf_hv_results_line_t +model_draft.ego_grid_pf_hv_results_load +model_draft.ego_grid_pf_hv_results_load_t +model_draft.ego_grid_pf_hv_results_meta +model_draft.ego_grid_pf_hv_results_storage +model_draft.ego_grid_pf_hv_results_storage_t +model_draft.ego_grid_pf_hv_results_transformer +model_draft.ego_grid_pf_hv_results_transformer_t +model_draft.ego_grid_pf_hv_link + + + + + + + + + + + + + + + + + ego_dp_powerflow_hv_setup.sql + + + + + + + + + + + + + + + + + + ego_dp_powerflow_osmtgmod_to_pypsa.sql + + + + + + + + + + + + + + + + + + grid.otg_ehvhv_bus_data +grid.otg_ehvhv_branch_data +grid.otg_ehvhv_branch_data +model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_transformer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_electrical_neighbour.sql + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_grid_future_scenarios.sql + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_transformer + + + + + + + + + + + + + + + + + ego_dp_powerflow_assignment_otgid.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_assignment_unid.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_create_pp_mviews.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_assignment_generator.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_assignment_load.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_assignment_storage.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_timeseries_generator.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_griddistrict_demand.py + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_timeseries_demand.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_lopf_data.sql + + + + + + + + + + + + + + + + + + + ego_dp_datacheck.sql + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_hv_electrical_neighbours_bus +model_draft.ego_grid_hv_electrical_neighbours_line +model_draft.ego_grid_hv_electrical_neighbours_transformer +model_draft.ego_grid_hv_electrical_neighbours_link + + + + + + + + + + + + + + + + + model_draft.ego_grid_pp_entsoe_bus +model_draft.ego_grid_pp_entsoe_line +model_draft.ego_grid_pf_hv_line +grid.otg_ehvhv_branch_data +grid.otg_ehvhv_bus_data + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_transformer +grid.otg_ehvhv_bus_data + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_transformer + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_link +model_draft.ego_grid_pf_hv_transformer + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_link +model_draft.ego_grid_pf_hv_transformer + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_ehv_substation +model_draft.ego_grid_pf_hv_bus +climate.cosmoclmgrid + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + cosmoclmgrid + + + + + + + + + + + + + + + + + DP griddistrict + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + model_draft.ego_supply_generator + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + model_draft.ego_supply_conv_powerplant_sq_mview +model_draft.ego_supply_conv_powerplant_nep2035_mview +model_draft.ego_supply_conv_powerplant_ego100_mview +model_draft.ego_supply_res_powerplant_sq_mview +model_draft.ego_supply_res_powerplant_nep2035_mview +model_draft.ego_supply_res_powerplant_ego100_mview + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + model_draft.ego_supply_conv_powerplant_sq_mview +model_draft.ego_supply_res_powerplant_sq_mview +model_draft.ego_supply_conv_powerplant_nep2035_mview +model_draft.ego_supply_res_powerplant_nep2035_mview +model_draft.ego_supply_conv_powerplant_ego100_mview +model_draft.ego_supply_res_powerplant_ego100_mview +model_draft.ego_grid_pf_hv_source +model_draft.ego_power_class +model_draft.ego_grid_pf_hv_generator + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_generator + + + + + + + + + + + + + + + + + model_draft.ego_supply_pf_generator_single +model_draft.ego_supply_aggr_weather + + + + + + + + + + + + + + + + + + model_draft.ego_demand_loadarea +model_draft.ego_demand_hv_largescaleconsumer +model_draft.ego_grid_pf_hv_load + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_load + + + + + + + + + + + + + + + + + model_draft.ego_demand_loads +model_draft.ego_demand_pf_load_single + + + + + + + + + + + + + + + + + model_draft.ego_supply_conv_powerplant_sq_mview +model_draft.ego_grid_pf_hv_source +model_draft.ego_grid_pf_hv_storage +model_draft.ego_grid_hv_electrical_neighbours_bus +calc_renpass_gis.renpass_gis_storage + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_storage + + + + + + + + + + + + + + + + + model_draft.ego_demand_pf_storage_single + + + + + + + + + + + + + + + + + model_draft.ego_supply_pf_generator_single +model_draft.ego_grid_pf_hv_generator +model_draft.ego_renewable_feedin +model_draft.ego_grid_hv_electrical_neighbours_bus +calc_renpass_gis.renpass_gis_linear_transformer +calc_renpass_gis.renpass_gis_source +calc_renpass_gis.renpass_gis_results + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_grid_pf_hv_generator +model_draft.ego_grid_pf_hv_load + + + + + + + + + + + + + + + + + + calc_renpass_gis.pf_pp_by_source_aggr_id +calc_renpass_gis.pp_feedin_by_pf_source +model_draft.ren_feedin_by_gen_id +calc_renpass_gis.translate_to_pf +model_draft.ren_feedin_foreign +model_draft.offshore_feedin_foreign + + + + + + + + + + + + + + + + + renpassGIS + + + + + + + + + + + + + + + + + ego_setup_renewable_feedin.sql + + + + + + + + + + + + + + + + + DP loadarea + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_load_pq_set + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_hvmv_substation +model_draft.ego_demand_hvmv_demand +model_draft.ego_grid_hv_electrical_neighbours_bus +calc_renpass_gis.renpass_gis_results +model_draft.ego_grid_pf_hv_load +model_draft.ego_grid_pf_hv_load_pq_set + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_generator +model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_grid_pf_hv_storage + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_generator +model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_grid_pf_hv_storage +model_draft.ego_grid_hvmv_substation +model_draft.ego_storage_h2_areas_de + + + + + + + + + + + + + + + + + storage h2 areas + + + + + + + + + + + + + + + + + model_draft_ego_grid_pf_hv_data_check + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_transformer +model_draft.ego_grid_pf_hv_load +model_draft.ego_grid_pf_hv_load_pq_set +model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_pf_hv_generator +model_draft.ego_supply_pf_generator_single +model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_grid_mv_griddistrict + + + + + + + + + + + + + + + + + osmTGmod /otg + + + + + + + + + + + + + + + + + model_draft.ego_demand_hvmv_demand + + + + + + + + + + + + + + + + + model_draft.ego_demand_hv_largescaleconsumer +model_draft.ego_demand_loadarea +BDEW SLP + + + + + + + + + + + + + + + + + BDEW SLP + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_substation.graphml b/documentation/bpmn/ego_dp_bpmn_section_substation.graphml index 3888de0e..0d86ab5e 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_substation.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_substation.graphml @@ -1,6 +1,6 @@ - + @@ -13,15 +13,15 @@ - + - + - eGo DataProcessing - substation - + eGo DataProcessing - substation + @@ -31,40 +31,40 @@ + + + - - - - - + + - + - + - + @@ -78,7 +78,7 @@ - ego_dp_substation_hvmv_voronoi.sql + ego_dp_substation_hvmv_voronoi.sql @@ -93,102 +93,65 @@ - + - model_draft.ego_grid_hvmv_substation_dummy -model_draft.ego_grid_hvmv_substation_voronoi - - - - - - + model_draft.ego_grid_hvmv_substation_dummy +model_draft.ego_grid_hvmv_substation_voronoi + + + - - - - - - - 6 min - - - - - - - - - - - - - - - - - model_draft.ego_grid_hvmv_substation - - - - - - + model_draft.ego_grid_hvmv_substation +model_draft.ego_boundaries_bkg_vg250_6_gem_clean + - - + - + - model_draft.ego_grid_hvmv_substation - - - - - - + model_draft.ego_grid_hvmv_substation + - - + - + ego_dp_substation_hvmv.sql @@ -202,13 +165,13 @@ ego_dp_substation_hvmv.sql - + - + ego_dp_substation_ehv.sql @@ -222,128 +185,69 @@ ego_dp_substation_ehv.sql - - - - - - - 32 min - - - - - - - - - - - - - - - - - - - - - - 3 min - - - - - - - - - - - - - - - - + - openstreetmap.osm_deu_ways + openstreetmap.osm_deu_ways openstreetmap.osm_deu_polygon +openstreetmap.osm_deu_line openstreetmap.osm_deu_nodes -openstreetmap.osm_deu_rels -political_boundary.bkg_vg250_1_sta_union_mview - - - - - - +openstreetmap.osm_deu_point +boundaries.bkg_vg250_1_sta_union_mview + - - + - openstreetmap.osm_deu_ways + openstreetmap.osm_deu_ways openstreetmap.osm_deu_polygon openstreetmap.osm_deu_nodes +openstreetmap.osm_deu_line +openstreetmap.osm_deu_point openstreetmap.osm_deu_rels -political_bkg_vg250_1_sta_union_mview - - - - - - +boundaries.bkg_vg250_1_sta_union_mview + - - + - + - model_draft.ego_grid_ehv_substation - - - - - - + model_draft.ego_grid_ehv_substation + - - + @@ -358,8 +262,8 @@ political_bkg_vg250_1_sta_union_mview - + @@ -367,388 +271,71 @@ political_bkg_vg250_1_sta_union_mview - + - + ego_dp_substation_otg.sql - - - - - - - - - - - - - - - - 1 sec - - - - - - - - - - - - - - - - - - - - - - grid.otg_ehvhv_bus_data -model_draft.ego_grid_ehv_substation -model_draft.ego_grid_hvmv_substation - - - - - - - - - - - - - - - - - - - - - - - model_draft.ego_grid_ehv_substation -model_draft.ego_grid_hvmv_substation - - - - - - - - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - openstreetmap.osm_deu - - - - - - - - - - - - - - - - - - - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - - - - - - - - - - - - - - - - - political_boundary.bkg_vg250 - - - - - - - - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - + - + + - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + grid.otg_ehvhv_bus_data +model_draft.ego_grid_ehv_substation +model_draft.ego_grid_hvmv_substation + - + - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + model_draft.ego_grid_ehv_substation +model_draft.ego_grid_hvmv_substation + - + - + - + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -758,67 +345,55 @@ model_draft.ego_grid_hvmv_substation - + - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) + + + + + + - + + - + - + - - + + - model_draft.ego_grid_hvmv_substation -grid.ego_hvmv_substation - - - - - - + openstreetmap + - - + - + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -828,36 +403,30 @@ grid.ego_hvmv_substation - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + bkg_vg250 + - + - + - ego_dp_substation_ehv_voronoi.sql + ego_dp_substation_ehv_voronoi.sql @@ -870,158 +439,68 @@ grid.ego_hvmv_substation - + - + - model_draft.ego_grid_ehv_substation_voronoi - - - - - - + model_draft.ego_grid_ehv_substation_voronoi + + - - - - - - - - - - - - - - ? - - - - - - - - - + - + - model_draft.ego_grid_ehv_substation - - - - - - + model_draft.ego_grid_hvmv_substation_dummy +model_draft.ego_grid_ehv_substation - - - - - - - - - - - - - - model_draft.ego_grid_ehv_substation -grid.ego_ehv_substation - - - - - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + osmTGmod /otg + - + - + - + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -1031,79 +510,43 @@ grid.ego_ehv_substation - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + model_draft.ego_grid_hvmv_substation + - + - + - - + + - grid.otg_ehvhv_bus_data - - - - - - + model_draft.ego_grid_ehv_substation + - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1115,7 +558,7 @@ grid.ego_ehv_substation - + @@ -1127,7 +570,7 @@ grid.ego_ehv_substation - + @@ -1139,7 +582,7 @@ grid.ego_ehv_substation - + @@ -1151,7 +594,7 @@ grid.ego_ehv_substation - + @@ -1163,19 +606,7 @@ grid.ego_ehv_substation - - - - - - - - - - - - - + @@ -1187,19 +618,7 @@ grid.ego_ehv_substation - - - - - - - - - - - - - + @@ -1211,7 +630,7 @@ grid.ego_ehv_substation - + @@ -1223,7 +642,7 @@ grid.ego_ehv_substation - + @@ -1235,7 +654,7 @@ grid.ego_ehv_substation - + @@ -1247,7 +666,7 @@ grid.ego_ehv_substation - + @@ -1259,7 +678,7 @@ grid.ego_ehv_substation - + @@ -1271,7 +690,7 @@ grid.ego_ehv_substation - + @@ -1283,19 +702,7 @@ grid.ego_ehv_substation - - - - - - - - - - - - - + @@ -1307,7 +714,7 @@ grid.ego_ehv_substation - + @@ -1319,7 +726,7 @@ grid.ego_ehv_substation - + @@ -1331,7 +738,7 @@ grid.ego_ehv_substation - + @@ -1343,7 +750,7 @@ grid.ego_ehv_substation - + @@ -1355,7 +762,7 @@ grid.ego_ehv_substation - + @@ -1367,7 +774,7 @@ grid.ego_ehv_substation - + @@ -1379,7 +786,7 @@ grid.ego_ehv_substation - + @@ -1391,7 +798,7 @@ grid.ego_ehv_substation - + @@ -1403,7 +810,7 @@ grid.ego_ehv_substation - + @@ -1415,7 +822,7 @@ grid.ego_ehv_substation - + @@ -1427,7 +834,7 @@ grid.ego_ehv_substation - + @@ -1439,28 +846,44 @@ grid.ego_ehv_substation - + - + - - - - - - + + + + - + - + + + + + + + + + + + - - - - + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_versioning.graphml b/documentation/bpmn/ego_dp_bpmn_section_versioning.graphml new file mode 100644 index 00000000..931fe27b --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_section_versioning.graphml @@ -0,0 +1,514 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - versioning + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_versioning.sql + + + + + + + + + + + + + + + + + + grid.ego_dp_ehv_substation +grid.ego_dp_hvmv_substation +grid.ego_dp_mvlv_substation +grid.ego_dp_ehv_griddistrict +grid.ego_dp_mv_griddistrict +grid.ego_dp_lv_griddistrict +demand.ego_dp_loadarea +supply.ego_dp_conv_powerplant +supply.ego_dp_res_powerplant +grid.ego_pf_hv_bus +grid.ego_pf_hv_generator +grid.ego_pf_hv_generator_pq_set +grid.ego_pf_hv_line +grid.ego_pf_hv_link +grid.ego_pf_hv_load +grid.ego_pf_hv_load_pq_set +grid.ego_pf_hv_source +grid.ego_pf_hv_storage +grid.ego_pf_hv_storage_pq_set +grid.ego_pf_hv_temp_resolution +grid.ego_pf_hv_transformer +grid.ego_pf_hv_extension_bus +grid.ego_pf_hv_extension_generator +grid.ego_pf_hv_extension_generator_pq_set +grid.ego_pf_hv_extension_line +grid.ego_pf_hv_extension_link +grid.ego_pf_hv_extension_load +grid.ego_pf_hv_extension_load_pq_set +grid.ego_pf_hv_extension_source +grid.ego_pf_hv_extension_storage +grid.ego_pf_hv_extension_storage_pq_set +grid.ego_pf_hv_extension_temp_resolution +grid.ego_pf_hv_extension_transformer +grid.ego_line_expansion_costs +supply.ego_renewable_feedin +supply.ego_power_class +supply.ego_aggr_weather +grid.ego_pf_hv_data_check + + + + + + + + + + + + + + + + + + model_draft.ego_grid_ehv_substation +model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_mvlv_substation +model_draft.ego_grid_ehv_substation_voronoi +model_draft.ego_grid_mv_griddistrict +model_draft.ego_grid_lv_griddistrict +model_draft.ego_demand_loadarea +model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_generator +model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_link +model_draft.ego_grid_pf_hv_load +model_draft.ego_grid_pf_hv_load_pq_set +model_draft.ego_grid_pf_hv_source +model_draft.ego_grid_pf_hv_storage +model_draft.ego_grid_pf_hv_storage_pq_set +model_draft.ego_grid_pf_hv_temp_resolution +model_draft.ego_grid_pf_hv_transformer +model_draft.ego_grid_pf_hv_extension_bus +model_draft.ego_grid_pf_hv_extension_generator +model_draft.ego_grid_pf_hv_extension_generator_pq_set +model_draft.ego_grid_pf_hv_extension_line +model_draft.ego_grid_pf_hv_extension_link +model_draft.ego_grid_pf_hv_extension_load +model_draft.ego_grid_pf_hv_extension_load_pq_set +model_draft.ego_grid_pf_hv_extension_source +model_draft.ego_grid_pf_hv_extension_storage +model_draft.ego_grid_pf_hv_extension_storage_pq_set +model_draft.ego_grid_pf_hv_extension_temp_resolution +model_draft.ego_grid_pf_hv_extension_transformer +model_draft.ego_grid_line_expansion_costs +model_draft.ego_renewable_feedin +model_draft.ego_power_class +model_draft.ego_supply_aggr_weather +model_draft.ego_grid_pf_hv_data_check + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_versioning_mviews.sql + + + + + + + + + + + + + + + + + + supply.ego_dp_conv_powerplant_sq_mview +supply.ego_dp_conv_powerplant_nep2035_mview +supply.ego_dp_conv_powerplant_ego100_mview +supply.ego_dp_res_powerplant_sq_mview +supply.ego_dp_res_powerplant_nep2035_mview +supply.ego_dp_res_powerplant_ego100_mview + + + + + + + + + + + + + + + + + + supply.ego_dp_conv_powerplant +supply.ego_dp_res_powerplant + + + + + + + + + + + + + + + + + DP griddistrict + + + + + + + + + + + + + + + + + DP powerflow + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + DP loadarea + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_sections.graphml b/documentation/bpmn/ego_dp_bpmn_sections.graphml index 96caf4a3..c4b152da 100644 --- a/documentation/bpmn/ego_dp_bpmn_sections.graphml +++ b/documentation/bpmn/ego_dp_bpmn_sections.graphml @@ -1,6 +1,6 @@ - + @@ -13,15 +13,15 @@ - + - + - eGo DataProcessing - + eGo DataProcessing + @@ -53,18 +53,18 @@ - + - + - + @@ -73,42 +73,13 @@ - - - - - - demand.ego_loadarea - - - - - - - - - - - - - - - - - - + - INPUT DATA -CHECK - - - - - - + INPUT DATA +CHECK @@ -120,19 +91,13 @@ CHECK - + - + - GRIDDISTRICT - - - - - - + GRIDDISTRICT @@ -144,19 +109,13 @@ CHECK - + - + - SUBSTATION - - - - - - + SUBSTATION @@ -168,34 +127,10 @@ CHECK - - - - - - - grid.ego_hvmv_substation -grid.ego_ehv_substation - - - - - - - - - - - - - - - - - + - + @@ -216,33 +151,10 @@ grid.ego_ehv_substation - - - - - - - grid.ego_mv_griddistrict - - - - - - - - - - - - - - - - - + - + @@ -263,19 +175,13 @@ grid.ego_ehv_substation - + - + - POWERFLOW - - - - - - + POWERFLOW @@ -287,19 +193,13 @@ grid.ego_ehv_substation - + - + - LOADAREA - - - - - - + LOADAREA @@ -311,89 +211,31 @@ grid.ego_ehv_substation - - - - - - - model_draft.ego_scenario_input - - - - - - - - - - - - - - - - - - - - - - - ? - - - - - - - - - - - - - - - - - + - - - - - grid.ego_dp_mvlv_substation -grid.ego_dp_lv_griddistrict - - - - - - + + + + + LOWVOLTAGE + - - + + - + - + - LOWVOLTAGE - - - - - - + REA @@ -405,43 +247,32 @@ grid.ego_dp_lv_griddistrict - + - - - - - grid.ego_dp_mvlv_substation -grid.ego_dp_lv_griddistrict - - - - - - + + + + + VERSIONING + - - + + - + - + - REA - - - - - - + POST- +PROCESSING @@ -455,22 +286,7 @@ grid.ego_dp_lv_griddistrict - - - - - - - - - - - - - - - - + @@ -482,7 +298,7 @@ grid.ego_dp_lv_griddistrict - + @@ -494,7 +310,7 @@ grid.ego_dp_lv_griddistrict - + @@ -506,7 +322,7 @@ grid.ego_dp_lv_griddistrict - + @@ -518,7 +334,7 @@ grid.ego_dp_lv_griddistrict - + @@ -530,7 +346,7 @@ grid.ego_dp_lv_griddistrict - + @@ -542,82 +358,19 @@ grid.ego_dp_lv_griddistrict - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - + + - + - + @@ -629,31 +382,24 @@ grid.ego_dp_lv_griddistrict - + - + - - - - - - - - - + + + + - + - + - - - - + + diff --git a/documentation/bpmn/ego_dp_bpmn_sections_input.graphml b/documentation/bpmn/ego_dp_bpmn_sections_input.graphml new file mode 100644 index 00000000..614ec84e --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_sections_input.graphml @@ -0,0 +1,783 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + INPUT DATA +CHECK + + + + + + + + + + + + + + + + + + GRIDDISTRICT + + + + + + + + + + + + + + + + + + SUBSTATION + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + POWERFLOW + + + + + + + + + + + + + + + + + + LOADAREA + + + + + + + + + + + + + + + + + + LOWVOLTAGE + + + + + + + + + + + + + + + + + + REA + + + + + + + + + + + + + + + + + + VERSIONING + + + + + + + + + + + + + + + + + + POST- +PROCESSING + + + + + + + + + + + + + + + + + + openstreetmap +vg250 +osmTGmod + + + + + + + + + + + + + + + + + vg250 +conv_powerplant +res_powerplant + + + + + + + + + + + + + + + + + openstreetmap +vg250 +VGR / gva +zensus +conv_powerplant +demand_federalstate + + + + + + + + + + + + + + + + + + slp_parameter + + + + + + + + + + + + + + + + + DP loadarea +DP griddistrict +DP substation + + + + + + + + + + + + + + + + + vg250 +wind_potential_area +res_powerplant + + + + + + + + + + + + + + + + + DP loadarea +DP griddistrict + + + + + + + + + + + + + + + + + osmTGmod +cosmoclmgrid +renpassGIS +renewable_feedin +storage h2 areas + + + + + + + + + + + + + + + + + DP griddistrict +DP loadarea +DP substation + + + + + + + + + + + + + + + + + bnetza_bbplg +bnetza_enlag +renpassGIS + + + + + + + + + + + + + + + + + DP powerflow + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/preprocessing/__init__.py b/preprocessing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/preprocessing/cosmoclmgrid/import_cosmoclmgrid.py b/preprocessing/cosmoclmgrid/import_cosmoclmgrid.py new file mode 100644 index 00000000..844ea2df --- /dev/null +++ b/preprocessing/cosmoclmgrid/import_cosmoclmgrid.py @@ -0,0 +1,121 @@ +import os +import zipfile +import geopandas as gpd +import sqlalchemy +from geoalchemy2 import Geometry +import time +from dataprocessing.tools import metadata +import json + + +CRS = 4326 + + +# Function to generate WKB hex +def wkb_hexer(line): + return line.wkb_hex + + +def import_cosmoclmgrid(file, path, db, **kwds): + + if not db.dialect.has_schema(db, kwds["schema"]): + db.execute(sqlalchemy.schema.CreateSchema(kwds["schema"])) + + shp_file = file.replace(".zip", ".shp") + subdir = file.replace(".zip", "") + os.makedirs(os.path.join(path, subdir), exist_ok=True) + + zf = zipfile.ZipFile(os.path.join(path, file)) + zf.extractall(path=os.path.join(path, subdir)) + + # read with pandas + cosmoclmgrid = gpd.read_file(os.path.join(path, subdir, shp_file)) + + cosmoclmgrid['geom'] = cosmoclmgrid['geometry'].apply(wkb_hexer) + cosmoclmgrid.drop("geometry", axis=1, inplace=True) + + # create table from geopandas dataframe + cosmoclmgrid.to_sql(kwds["table"], + db, + kwds["schema"], + index=False, + if_exists="replace", + chunksize=10000, + method="multi", + dtype={'geom': Geometry('POLYGON')} + ) + + create_pkey_constraint = "ALTER TABLE {schema}.{table} ADD CONSTRAINT " \ + "{table}_pkey PRIMARY KEY (gid);".format(**kwds) + db.execution_options(autocommit=True).execute(create_pkey_constraint) + + # create metadata json str + metadata_dict = {"name": "Spatial grid coastDat-2 re-analysis data set", + "title": "COSMO CLM grid", + "description": "This spatial grid provides reference polygons for coastDat-2 " + "re-analysis data.", + "language": ["EN"], + "spatial": { + "location": "0.22 ° x 0.22 °", + "extent": "Europe", + "resolution": "0.22 ° x 0.22 °" + }, + "temporal": { + "referenceDate": "2014", + "timeseries": { + "start": "", + "end": "", + "resolution": "", + "alignment": "", + "aggregationType": "" + } + }, + "sources": [ + { + "title": "coastDat-2 re-analysis data set", + "description": "coastDat-2 re-analysis data is a long-term backcasted " + "climate data set based on the regional weather forceasting " + "model COSMO CLM", + "path": "https://www.coastdat.de/about_us/index.php.en", + "licenses": [ + { + "name": "", + "title": "", + "path": "", + "instruction": "", + "attribution": "" + } + ] + } + ], + "licenses": [ + { + "name": "Open Data Commons Open Database License 1.0", + "title": "", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "Guido Pleßmann", + "email": "http://github.com/gplssm", + "date": time.strftime("%Y-%m-%d"), + "object": "", + "comment": "Only uploaded the data" + } + ], + "metaMetadata": { + "metadataVersion": "OEP-1.4.0", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + } + + json_str = "'" + json.dumps(metadata_dict) + "'" + + metadata.submit_comment(db, json_str, kwds["schema"], kwds["table"]) diff --git a/preprocessing/eGo_import_and_preprocessing.py b/preprocessing/eGo_import_and_preprocessing.py new file mode 100644 index 00000000..c45af395 --- /dev/null +++ b/preprocessing/eGo_import_and_preprocessing.py @@ -0,0 +1,176 @@ +""" +eGo PreProcessing (eGoPP) + +This script opens an oedb database connection and executes different parts of eGo. +Reads python and SQL scripts and gives logging infos during the execution. +Also see corresponding BPML diagram. + +This file is part of project "open_eGo DataProcessing" (https://github.com/openego/data_processing/). +It's copyrighted by the contributors recorded in the version control history: +openego/data_processing/preprocessing/eGo_PreProcessing.py + +SPDX-License-Identifier: AGPL-3.0-or-later +""" + +__copyright__ = "Reiner Lemoine Institut" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://www.gnu.org/licenses/agpl-3.0.en.html" +__author__ = "gplssm, Ludee" + +import logging +import time +import os +import subprocess +from dataprocessing.tools import io +from egoio.tools import db +from sqlalchemy import create_engine +import yaml +from urllib.request import urlretrieve +import importlib + +# Configure logging +logger = logging.getLogger('EEEE') +logger.setLevel(logging.INFO) +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) +formatter = logging.Formatter('%(asctime)s %(message)s', + datefmt='%Y-%m-%d %I:%M:%S') +ch.setFormatter(formatter) +logger.addHandler(ch) + +SCENARIOLOG = True +DOWNLOADDIR = os.path.join(os.path.expanduser("~"), ".egon-pre-processing-cached/") +FNULL = open(os.devnull, 'w') + +db_info = { + "host": "localhost", + "port": 54321, + "user": "oeuser", + "password": "egon", + "database": "dp" +} + + +def create_data_dir(dir=DOWNLOADDIR): + """ + Parameters + ---------- + dir: str + Define alternative dir for chached data + """ + + os.makedirs(dir, exist_ok=True) + logger.info('Cached data directory {} created (or already existed).'.format(dir)) + + +def download_data(url, filename): + + file = os.path.join(DOWNLOADDIR, filename) + + if not os.path.isfile(file): + urlretrieve(url, file) + + +def preprocessing(): + + # get current time and inform about start + total_time = time.time() + logger.info('ego preprocessing started...') + + # create directory for cached downloaded data + create_data_dir() + + # list of sql- and python-snippets that process the data in correct order + script_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__))) + + snippets = [ + 'ego_pre_voltage_level.sql', + 'ego_pre_slp_parameters.sql' + ] + datasets = yaml.load(open("import.yml"), Loader=yaml.SafeLoader) + + # get database connection + conn_oedb = db.connection(readonly=True).connect() + # engine_local = create_engine('postgresql+psycopg2://oeuser:egon@localhost:54321/dp') + engine_local = create_engine('postgresql+psycopg2://{user}:{password}@{host}:{port}/{database}'.format(**db_info)) + conn = engine_local.connect() + + # iterate over data sets + for key, dataset in datasets.items(): + for download in dataset.get("required_data", []): + logger.info("Downloading '{}' ...".format(download["filename"])) + snippet_time = time.time() + download_data(download["url"], download["filename"]) + logger.info('...successfully done in {:.2f} seconds.'.format( + time.time() - snippet_time)) + + for script in dataset.get("scripts", []): + + # timing and logging + snippet_time = time.time() + logger.info("Execute '{}' ...".format(script["script"])) + if script["language"] == 'SQL': + if SCENARIOLOG is True: + snippet_str = open(os.path.join(script_dir, key, script["script"])).read() + elif SCENARIOLOG is False: + snippet_str = "".join( + [l for l in open(os.path.join(script_dir, key, script["script"])).readlines() + if not l.startswith("SELECT scenario_log") and not l.startswith("SELECT ego_scenario_log")]) + + # execute desired sql snippet + conn.execution_options(autocommit=True).execute(snippet_str) + elif script["language"] == 'python': + if len(script["script"].split("::")) > 1: + module, func = script["script"].split("::") + + args = script.get("args", {}) + + if script["db"] == "info": + db_details = db_info + elif script["db"] == "conn": + db_details = engine_local + else: + "" + + mod = importlib.import_module(module) + func_to_call = getattr(mod, func) + func_to_call(script["filename"], DOWNLOADDIR, db_details, **args) + else: + filename = os.path.join(script_dir, key, script["script"]) + script_str = open(filename, "rb").read() + + # execute desired sql snippet + exec(compile(script_str, filename, 'exec')) + elif script["language"] == 'bash': + filename = os.path.join(script_dir, key, script["script"]) + script_str = script["script"] + + if "filename" in script: + script_str += script["filename"] + + # execute desired bash script + # rc (script_str) + subprocess.run(script_str, + shell=True, + env={"PGPASSWORD": "egon"}, + stdout=FNULL, + cwd= os.path.join(script_dir, key)) + else: + raise NameError('{} is neither a python nor a sql script (at least it ' + 'has not the right extension). Please add an extension ' + 'to the script name (.py or .sql)'.format(script["script"])) + + # inform the user + logger.info('...successfully done in {:.2f} seconds.'.format( + time.time() - snippet_time)) + + # close database connection + conn.close() + + logger.info('eGo PreProcessing script successfully executed in {:.2f} seconds'.format( + time.time() - total_time)) + + +if __name__ == '__main__': + preprocessing() diff --git a/preprocessing/eGo_preprocessing.py b/preprocessing/eGo_preprocessing.py deleted file mode 100644 index 000ff365..00000000 --- a/preprocessing/eGo_preprocessing.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -eGo PreProcessing (eGoPP) - -This script opens an oedb database connection and executes different parts of eGo. -Reads python and SQL scripts and gives logging infos during the execution. -Also see corresponding BPML diagram. - -This file is part of project "open_eGo DataProcessing" (https://github.com/openego/data_processing/). -It's copyrighted by the contributors recorded in the version control history: -openego/data_processing/preprocessing/eGo_PreProcessing.py - -SPDX-License-Identifier: AGPL-3.0-or-later -""" - -__copyright__ = "Reiner Lemoine Institut" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ "https://www.gnu.org/licenses/agpl-3.0.en.html" -__author__ = "gplssm, Ludee" - -import pandas as pd -import logging -import time -import os -import codecs -from tools import io - -def preprocessing(): - # Configure logging - logger = logging.getLogger('EEEE') - logger.setLevel(logging.INFO) - ch = logging.StreamHandler() - ch.setLevel(logging.INFO) - formatter = logging.Formatter('%(asctime)s %(message)s', - datefmt='%Y-%m-%d %I:%M:%S') - ch.setFormatter(formatter) - logger.addHandler(ch) - - # get current time and inform about start - total_time = time.time() - logger.info('ego preprocessing started...') - - # list of sql- and python-snippets that process the data in correct order - snippet_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), - 'preprocessing')) - script_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), - 'python_scripts')) - - snippets = [ - 'ego_pre_voltage_level.sql', - 'ego_pre_slp_parameters.sql' - ] - - # get database connection - conn = io.oedb_session(section='oedb') - - # iterate over list of sql- and python-snippets and execute them - for snippet in snippets: - # timing and logging - snippet_time = time.time() - logger.info("Execute '{}' ...".format(snippet)) - if os.path.splitext(snippet)[1] == '.sql': - snippet_str = open(os.path.join(snippet_dir, snippet)).read() - - # execute desired sql snippet - conn.execution_options(autocommit=True).execute(snippet_str) - elif os.path.splitext(snippet)[1] == '.py': - filename = os.path.join(script_dir, snippet) - script_str = open(filename, "rb").read() - - # execute desired sql snippet - exec(compile(script_str, filename, 'exec')) - else: - raise NameError('{} is neither a python nor a sql script (at least it ' - 'has not the right extension). Please add an extension ' - 'to the script name (.py or .sql)'.format(snippet)) - - # inform the user - logger.info('...successfully done in {:.2f} seconds.'.format( - time.time() - snippet_time)) - - # close database connection - conn.close() - - logger.info('eGo PreProcessing script successfully executed in {:.2f} seconds'.format( - time.time() - total_time)) - - -if __name__ == '__main__': - preprocessing() diff --git a/preprocessing/import.yml b/preprocessing/import.yml new file mode 100644 index 00000000..ee8effbd --- /dev/null +++ b/preprocessing/import.yml @@ -0,0 +1,106 @@ +scenario_log: + scripts: + - script: function_scenario_log.sql + language: SQL + - script: ego_pp_scenario.sql + language: SQL + - script: ego_pp_scenario_log.sql + language: SQL +openstreetmap: + required_data: +# - url: https://download.geofabrik.de/europe/germany/bremen-latest.osm.pbf + - url: https://download.geofabrik.de/europe/germany-latest.osm.pbf + filename: deu-latest.osm.pbf + scripts: + - script: preprocessing.openstreetmap.import_osm::osm2postgres + language: python + filename: deu-latest.osm.pbf + db: info + - script: ego_pp_osm_deu_import.sql + language: SQL + - script: ego_pp_osm_line_street_mview.sql + language: SQL + - script: ego_pp_osm_polygon_building_mview.sql + language: SQL +zensus: + required_data: + - url: https://next.rl-institut.de/s/Asoxps5oWGkGrZi/download + filename: Zensus_Bevoelkerung_100m-Gitter.zip + scripts: + - script: preprocessing.zensus.destatis_zensus_import_csv::import_zensus2011 + language: python + filename: Zensus_Bevoelkerung_100m-Gitter.zip + db: conn + args: + table: zensus_2011_pop_per_ha + schema: tmp_destatis + - script: destatis_zensus_raw_import.sql + language: SQL + - script: ego_pp_destatis_zensus_import.sql + language: SQL +vg250: + required_data: + - url: https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/aktuell/vg250_01-01.gk3.shape.ebenen.zip + filename: vg250_01-01.gk3.shape.ebenen.zip + scripts: + - script: create_boundaries_schema.sql + language: SQL + - script: preprocessing.vg250.import_vg250::import_vg250 + language: python + filename: vg250_01-01.gk3.shape.ebenen.zip + db: info + - script: ego_pp_vg250_import.sql + language: SQL + - script: ego_pp_vg250_metadata.sql + language: SQL + - script: ego_pp_vg250_mview.sql + language: SQL + - script: ego_dp_structure_boundaries_vg250.sql + language: SQL +zensus_vg250: + scripts: + - script: ego_pp_destatis_zensus_insidevg250.sql + language: SQL + - script: ego_pp_destatis_zensus_metadata.sql + language: SQL +cosmoclmgrid: + required_data: + - url: https://next.rl-institut.de/s/jX5ci9kimgjdXAM/download + filename: cosmoclmgrid.zip + scripts: + - script: preprocessing.cosmoclmgrid.import_cosmoclmgrid::import_cosmoclmgrid + language: python + filename: cosmoclmgrid.zip + db: conn + args: + schema: "climate" + table: "cosmoclmgrid" +supply: + required_data: + - url: https://next.rl-institut.de/s/TsjxR5P2Q6Z8Y7X/download + filename: vernetzen_wind_potential_areas.zip + scripts: + - script: preprocessing.supply.import_vernetzen_wind_potential_areas::import_vernetzen_wind_potential_areas + language: python + filename: vernetzen_wind_potential_areas.zip + db: conn + args: + schema: "supply" + table: "vernetzen_wind_potential_area" + - script: vernetzen_wind_potential_area_v1.3.sql + language: SQL +slp: + scripts: + - script: ego_pre_slp_parameters.sql + language: SQL +osmtgmod: + required_data: + - url: https://next.rl-institut.de/s/zb4yWoxHpKACw7Q/download + filename: otg_ehvhv_data.zip + scripts: + - script: preprocessing.osmtgmod.import_osmtgmod::import_osmtgmod + language: python + filename: otg_ehvhv_data.zip + db: conn + args: + schema: "grid" diff --git a/dataprocessing/tools/osm2pgsql/default.style b/preprocessing/openstreetmap/default.style similarity index 100% rename from dataprocessing/tools/osm2pgsql/default.style rename to preprocessing/openstreetmap/default.style diff --git a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_metadata.sql b/preprocessing/openstreetmap/ego_pp_osm_deu_import.sql similarity index 67% rename from preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_metadata.sql rename to preprocessing/openstreetmap/ego_pp_osm_deu_import.sql index 0332f8bc..f59ea8d6 100644 --- a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_metadata.sql +++ b/preprocessing/openstreetmap/ego_pp_osm_deu_import.sql @@ -1,5 +1,13 @@ -/* -Metadata for OpenStreetMap tables +/* +Preprocess OpenStreetMap tables after import in public schema with osm2pgsql +Basic operations: +0. grant +1. remove index +2. add column 'gid' serial +3. add primary keys +4. rename geo-column 'way' to 'geom' +6. add index +7. move tables to openstreetmap schema __copyright__ = "Reiner Lemoine Institut" __license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" @@ -7,17 +15,181 @@ __url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" __author__ = "Ludee" */ +CREATE SCHEMA IF NOT EXISTS openstreetmap; + +-- ego scenario log (version,io,schema_name,table_name,script_name,comment) +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_line','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_point','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_polygon','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_rels','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_roads','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_ways','ego_pp_osm_deu_import.sql','verification'); + + +-- 0. grant oeuser +DO +$$ +DECLARE + row record; +BEGIN + FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'osm_deu_%%' + LOOP + EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' OWNER TO oeuser;'; + END LOOP; +END; +$$; + +-- 1. remove wrong index +-- 1.1 remove all wrong 'pkey' (_line, _point, _polygon, _roads) +DO +$$ +DECLARE + row record; +BEGIN + FOR row IN SELECT indexname FROM pg_indexes + WHERE (schemaname='public' + AND indexname LIKE '%%_pkey') + AND (tablename LIKE '%%_line' + OR tablename LIKE '%%_point' + OR tablename LIKE '%%_polygon' + OR tablename LIKE '%%_roads') + LOOP + EXECUTE 'DROP INDEX public.' || quote_ident(row.indexname) || ';'; + END LOOP; +END; +$$; + +-- 1.2 remove all wrong other indexes (_line, _point, _polygon, _roads) +DO +$$ +DECLARE + row record; +BEGIN + FOR row IN SELECT indexname FROM pg_indexes + WHERE (schemaname='public' + AND indexname LIKE '%%_index') + LOOP + EXECUTE 'DROP INDEX public.' || quote_ident(row.indexname) || ';'; + END LOOP; +END; +$$; + + +-- 2. add column 'gid' serial +-- 3. add primary keys +-- 4. rename geo-column 'way' to 'geom' +DO +$$ +DECLARE + row record; +BEGIN + FOR row IN SELECT tablename FROM pg_tables + WHERE schemaname='public' + AND (tablename LIKE '%%_line' + OR tablename LIKE '%%_point' + OR tablename LIKE '%%_polygon' + OR tablename LIKE '%%_roads') + LOOP + EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' ADD gid SERIAL;'; + EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' ADD PRIMARY KEY (gid);'; + EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' RENAME COLUMN way TO geom;'; + END LOOP; +END; +$$; + + +-- 5. Set geometry and SRID (original!) + +-- ALTER TABLE public.osm_deu_line +-- ALTER COLUMN geom TYPE geometry(LineString,3857) +-- USING ST_SetSRID(geom,3857); +-- +-- ALTER TABLE public.osm_deu_point +-- ALTER COLUMN geom TYPE geometry(Point,3857) +-- USING ST_SetSRID(geom,3857); +-- +-- ALTER TABLE public.osm_deu_polygon +-- ALTER COLUMN geom TYPE geometry(MultiPolygon, 3857) +-- USING ST_SetSRID(ST_MULTI(geom),3857); +-- +-- ALTER TABLE public.osm_deu_roads +-- ALTER COLUMN geom TYPE geometry(LineString,3857) +-- USING ST_SetSRID(geom,3857); + + +-- 6. add index +-- 6.1 add indexes GIST (geom) +DO +$$ +DECLARE + row record; +BEGIN + FOR row IN SELECT tablename FROM pg_tables + WHERE schemaname='public' + AND (tablename LIKE '%%_line' + OR tablename LIKE '%%_point' + OR tablename LIKE '%%_polygon' + OR tablename LIKE '%%_roads') + LOOP + EXECUTE 'CREATE INDEX ' || quote_ident(row.tablename) || '_geom_idx ON public.' || quote_ident(row.tablename) || ' USING gist (geom);'; + END LOOP; +END; +$$; + +-- 6.2 Add indexes GIN (tags) +DO +$$ +DECLARE + row record; +BEGIN + FOR row IN SELECT tablename FROM pg_tables + WHERE schemaname='public' + AND (tablename LIKE '%%_line' + OR tablename LIKE '%%_point' + OR tablename LIKE '%%_polygon' + OR tablename LIKE '%%_roads') + LOOP + EXECUTE 'CREATE INDEX ' || quote_ident(row.tablename) || '_tags_idx ON public.' || quote_ident(row.tablename) || ' USING GIN (tags);'; + END LOOP; +END; +$$; + + + +-- 7. move tables to openstreetmap schema +DO +$$ +DECLARE + row record; +BEGIN + FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'osm_deu_%%' + LOOP + EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' SET SCHEMA openstreetmap;'; + END LOOP; +END; +$$; + +-- ego scenario log (version,io,schema_name,table_name,script_name,comment) +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_line','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_nodes','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_point','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_polygon','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_rels','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_roads','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_ways','ego_pp_osm_deu_import.sql','setup osm tables'); + + -- metadata COMMENT ON TABLE openstreetmap.osm_deu_line IS '{ "title": "OpenStreetMap (OSM) - Germany - Line", "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", "language": [ "eng", "ger" ], - "spatial": + "spatial": {"location": "none", "extent": "Germany", "resolution": "vector"}, - "temporal": + "temporal": {"reference_date": "2016-10-01", "start": "none", "end": "none", @@ -28,7 +200,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_line IS '{ "url": "http://download.geofabrik.de/europe/germany.html", "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", "copyright": "© OpenStreetMap contributors"} ], - "license": + "license": {"id": "ODbL-1.0", "name": "Open Data Commons Open Database License 1.0", "version": "1.0", @@ -40,7 +212,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_line IS '{ {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], "resources": [ - {"name": "openstreetmap.osm_deu_line", + {"name": "openstreetmap.osm_deu_line", "format": "PostgreSQL", "fields": [ {"name": "gid", "description": "Unique identifier", "unit": "none" }, @@ -53,11 +225,11 @@ COMMENT ON TABLE openstreetmap.osm_deu_nodes IS '{ "title": "OpenStreetMap (OSM) - Germany - Nodes", "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", "language": [ "eng", "ger" ], - "spatial": + "spatial": {"location": "none", "extent": "Germany", "resolution": "none"}, - "temporal": + "temporal": {"reference_date": "2016-10-01", "start": "none", "end": "none", @@ -68,7 +240,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_nodes IS '{ "url": "http://download.geofabrik.de/europe/germany.html", "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", "copyright": "© OpenStreetMap contributors"} ], - "license": + "license": {"id": "ODbL-1.0", "name": "Open Data Commons Open Database License 1.0", "version": "1.0", @@ -80,7 +252,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_nodes IS '{ {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], "resources": [ - {"name": "openstreetmap.osm_deu_nodes", + {"name": "openstreetmap.osm_deu_nodes", "format": "PostgreSQL", "fields": [ {"name": "id", "description": "Unique identifier", "unit": "none" }, @@ -93,11 +265,11 @@ COMMENT ON TABLE openstreetmap.osm_deu_point IS '{ "title": "OpenStreetMap (OSM) - Germany - Point", "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", "language": [ "eng", "ger" ], - "spatial": + "spatial": {"location": "none", "extent": "Germany", "resolution": "none"}, - "temporal": + "temporal": {"reference_date": "2016-10-01", "start": "none", "end": "none", @@ -108,7 +280,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_point IS '{ "url": "http://download.geofabrik.de/europe/germany.html", "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", "copyright": "© OpenStreetMap contributors"} ], - "license": + "license": {"id": "ODbL-1.0", "name": "Open Data Commons Open Database License 1.0", "version": "1.0", @@ -120,7 +292,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_point IS '{ {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], "resources": [ - {"name": "openstreetmap.osm_deu_point", + {"name": "openstreetmap.osm_deu_point", "format": "PostgreSQL", "fields": [ {"name": "gid", "description": "Unique identifier", "unit": "none" }, @@ -133,11 +305,11 @@ COMMENT ON TABLE openstreetmap.osm_deu_polygon IS '{ "title": "OpenStreetMap (OSM) - Germany - Polygon", "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", "language": [ "eng", "ger" ], - "spatial": + "spatial": {"location": "none", "extent": "Germany", "resolution": "none"}, - "temporal": + "temporal": {"reference_date": "2016-10-01", "start": "none", "end": "none", @@ -148,7 +320,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_polygon IS '{ "url": "http://download.geofabrik.de/europe/germany.html", "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", "copyright": "© OpenStreetMap contributors"} ], - "license": + "license": {"id": "ODbL-1.0", "name": "Open Data Commons Open Database License 1.0", "version": "1.0", @@ -160,7 +332,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_polygon IS '{ {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], "resources": [ - {"name": "openstreetmap.osm_deu_polygon", + {"name": "openstreetmap.osm_deu_polygon", "format": "PostgreSQL", "fields": [ {"name": "gid", "description": "Unique identifier", "unit": "none" }, @@ -173,11 +345,11 @@ COMMENT ON TABLE openstreetmap.osm_deu_rels IS '{ "title": "OpenStreetMap (OSM) - Germany - Rels", "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", "language": [ "eng", "ger" ], - "spatial": + "spatial": {"location": "none", "extent": "Germany", "resolution": "none"}, - "temporal": + "temporal": {"reference_date": "2016-10-01", "start": "none", "end": "none", @@ -188,7 +360,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_rels IS '{ "url": "http://download.geofabrik.de/europe/germany.html", "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", "copyright": "© OpenStreetMap contributors"} ], - "license": + "license": {"id": "ODbL-1.0", "name": "Open Data Commons Open Database License 1.0", "version": "1.0", @@ -200,7 +372,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_rels IS '{ {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], "resources": [ - {"name": "openstreetmap.osm_deu_rels", + {"name": "openstreetmap.osm_deu_rels", "format": "PostgreSQL", "fields": [ {"name": "id", "description": "Unique identifier", "unit": "none" }, @@ -216,11 +388,11 @@ COMMENT ON TABLE openstreetmap.osm_deu_roads IS '{ "title": "OpenStreetMap (OSM) - Germany - Roads", "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", "language": [ "eng", "ger" ], - "spatial": + "spatial": {"location": "none", "extent": "Germany", "resolution": "none"}, - "temporal": + "temporal": {"reference_date": "2016-10-01", "start": "none", "end": "none", @@ -231,7 +403,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_roads IS '{ "url": "http://download.geofabrik.de/europe/germany.html", "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", "copyright": "© OpenStreetMap contributors"} ], - "license": + "license": {"id": "ODbL-1.0", "name": "Open Data Commons Open Database License 1.0", "version": "1.0", @@ -243,7 +415,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_roads IS '{ {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], "resources": [ - {"name": "openstreetmap.osm_deu_roads", + {"name": "openstreetmap.osm_deu_roads", "format": "PostgreSQL", "fields": [ {"name": "gid", "description": "Unique identifier", "unit": "none" }, @@ -256,11 +428,11 @@ COMMENT ON TABLE openstreetmap.osm_deu_ways IS '{ "title": "OpenStreetMap (OSM) - Germany - Ways", "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", "language": [ "eng", "ger" ], - "spatial": + "spatial": {"location": "none", "extent": "Germany", "resolution": "none"}, - "temporal": + "temporal": {"reference_date": "2016-10-01", "start": "none", "end": "none", @@ -271,7 +443,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_ways IS '{ "url": "http://download.geofabrik.de/europe/germany.html", "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", "copyright": "© OpenStreetMap contributors"} ], - "license": + "license": {"id": "ODbL-1.0", "name": "Open Data Commons Open Database License 1.0", "version": "1.0", @@ -283,7 +455,7 @@ COMMENT ON TABLE openstreetmap.osm_deu_ways IS '{ {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], "resources": [ - {"name": "openstreetmap.osm_deu_ways", + {"name": "openstreetmap.osm_deu_ways", "format": "PostgreSQL", "fields": [ {"name": "id", "description": "Unique identifier", "unit": "none" }, @@ -293,10 +465,10 @@ COMMENT ON TABLE openstreetmap.osm_deu_ways IS '{ "metadata_version": "1.3"}'; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_line','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_nodes','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_point','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_polygon','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_rels','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_roads','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_ways','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_line','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_nodes','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_point','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_polygon','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_rels','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_roads','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_ways','ego_pp_osm_deu_metadata.sql','metadata'); \ No newline at end of file diff --git a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_line_street_mview.sql b/preprocessing/openstreetmap/ego_pp_osm_line_street_mview.sql similarity index 93% rename from preprocessing/sql_snippets/openstreetmap/ego_pp_osm_line_street_mview.sql rename to preprocessing/openstreetmap/ego_pp_osm_line_street_mview.sql index daf8d37d..d1a9d7df 100644 --- a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_line_street_mview.sql +++ b/preprocessing/openstreetmap/ego_pp_osm_line_street_mview.sql @@ -9,7 +9,7 @@ __author__ = "jong42, Ludee" -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','input','openstreetmap','osm_deu_line','ego_pp_osm_line_street_mview.sql',''); +SELECT scenario_log('eGo_PP','PP1','input','openstreetmap','osm_deu_line','ego_pp_osm_line_street_mview.sql','setup osm tables'); -- street DROP MATERIALIZED VIEW IF EXISTS openstreetmap.osm_deu_line_street_mview; @@ -84,4 +84,4 @@ COMMENT ON MATERIALIZED VIEW openstreetmap.osm_deu_line_street_mview IS '{ "metadata_version": "1.3"}'; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','output','openstreetmap','osm_deu_line_street_mview','ego_pp_osm_line_street_mview.sql','Streets'); +SELECT scenario_log('eGo_PP','PP1','output','openstreetmap','osm_deu_line_street_mview','ego_pp_osm_line_street_mview.sql','setup osm tables'); diff --git a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_polygon_building_mview.sql b/preprocessing/openstreetmap/ego_pp_osm_polygon_building_mview.sql similarity index 92% rename from preprocessing/sql_snippets/openstreetmap/ego_pp_osm_polygon_building_mview.sql rename to preprocessing/openstreetmap/ego_pp_osm_polygon_building_mview.sql index 70dd1dbe..1697e5e9 100644 --- a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_polygon_building_mview.sql +++ b/preprocessing/openstreetmap/ego_pp_osm_polygon_building_mview.sql @@ -1,4 +1,4 @@ -/* +/* Extracted OSM buildings from polygon __copyright__ = "Reiner Lemoine Institut" @@ -9,7 +9,7 @@ __author__ = "Ludee" -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','input','openstreetmap','osm_deu_polygon','ego_pp_osm_polygon_building_mview.sql',''); +SELECT scenario_log('eGo_PP','PP1','input','openstreetmap','osm_deu_polygon','ego_pp_osm_polygon_building_mview.sql','setup osm tables'); -- building DROP MATERIALIZED VIEW IF EXISTS openstreetmap.osm_deu_polygon_building_mview; @@ -78,4 +78,4 @@ COMMENT ON MATERIALIZED VIEW openstreetmap.osm_deu_polygon_building_mview IS '{ "metadata_version": "1.3"}'; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','output','openstreetmap','osm_deu_polygon_building_mview','ego_pp_osm_polygon_building_mview.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','openstreetmap','osm_deu_polygon_building_mview','ego_pp_osm_polygon_building_mview.sql','setup osm tables'); diff --git a/dataprocessing/tools/osm2pgsql/empty.style b/preprocessing/openstreetmap/empty.style similarity index 100% rename from dataprocessing/tools/osm2pgsql/empty.style rename to preprocessing/openstreetmap/empty.style diff --git a/preprocessing/openstreetmap/import_osm.py b/preprocessing/openstreetmap/import_osm.py new file mode 100644 index 00000000..2a4195c6 --- /dev/null +++ b/preprocessing/openstreetmap/import_osm.py @@ -0,0 +1,20 @@ +import subprocess +import os + + +NUM_PROCESSES = 4 +CACHE_SIZE = 4096 + + +def osm2postgres(file, path, db_info): + + stylefile = os.path.join(os.path.dirname(os.path.abspath(__file__)), "oedb.style") + pbf_file = os.path.join(path, file) + + db_info.update({"stylefile": stylefile, "pbffile": pbf_file}) + + cmd = "osm2pgsql --create --slim --hstore-all --number-processes NUM_PROCESSES --cache CACHE_SIZE -H {host} -P {port} -d {database} -U {user} -p osm_deu -S {stylefile} {pbffile}".format(**db_info) + + subprocess.run(cmd, shell=True, env={"PGPASSWORD": "egon"}) + + diff --git a/dataprocessing/tools/osm2pgsql/oedb.style b/preprocessing/openstreetmap/oedb.style similarity index 100% rename from dataprocessing/tools/osm2pgsql/oedb.style rename to preprocessing/openstreetmap/oedb.style diff --git a/preprocessing/osmtgmod/import_osmtgmod.py b/preprocessing/osmtgmod/import_osmtgmod.py new file mode 100644 index 00000000..f7a6e594 --- /dev/null +++ b/preprocessing/osmtgmod/import_osmtgmod.py @@ -0,0 +1,68 @@ +import os +import zipfile +import sqlalchemy +import geopandas as gpd +import pandas as pd +from preprocessing.utility.utils import wkb_hexer +from geoalchemy2 import Geometry +from shapely.wkt import loads as wkt_loads + + +def certain_file_types(directory, extension): + return (f for f in os.listdir(directory) if f.endswith('.' + extension)) + + +def import_osmtgmod(file, path, db, **kwds): + + # Create schema if it does not exist + if not db.dialect.has_schema(db, kwds["schema"]): + db.execute(sqlalchemy.schema.CreateSchema(kwds["schema"])) + + # TODO: download and add metadata to osmtgmod zip + # TODO: write metadata to local table (would this be easier when ORMs are used locally?) + # TODO: Add primary keys to table + + # File names and paths + file_base, file_ext = os.path.splitext(os.path.join(path, file)) + extracted_path = os.path.join(path, file_base) + + # Extract data from zip archive + zip = zipfile.ZipFile(os.path.join(path, file)) + zip.extractall(path=extracted_path) + + # iterate through shapefiles and import to database + for shp_file in certain_file_types(extracted_path, "shp"): + df = gpd.read_file(os.path.join(extracted_path, shp_file)) + + # Store geometries in WKB hex format + df['geom'] = df['geometry'].apply(wkb_hexer) + df.drop("geometry", axis=1, inplace=True) + if "topo" in df.columns: + df['topo'] = df['topo'].apply(wkt_loads) + df['topo'] = df['topo'].apply(wkb_hexer) + + table = os.path.splitext(shp_file)[0] + + df.to_sql(table, + db, + kwds["schema"], + index=False, + if_exists="replace", + chunksize=10000, + method="multi", + dtype={'geom': Geometry(), + 'topo': Geometry()} + ) + + for csv_file in certain_file_types(extracted_path, "csv"): + df = pd.read_csv(os.path.join(extracted_path, csv_file), + parse_dates=["osm_date", "abstraction_date"]) + + table = os.path.splitext(csv_file)[0] + + df.to_sql(table, + db, + kwds["schema"], + index=False, + if_exists="replace" + ) diff --git a/preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst b/preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst new file mode 100644 index 00000000..9a6be09d --- /dev/null +++ b/preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst @@ -0,0 +1,27 @@ +============================= +Scenario import of renpassG!S +============================= + + + +Installation +============ + +.. code-block:: + + $ virtualenv renpass_importer --clear -p python3.5 + $ source bin/activate + $ pip3 install -r requirements.txt + + + +How to import scnario csv files to the oedb +=========================================== + +.. code-block:: + + $ python3 scenario_import.py "/PATH/status_quo.csv" "/PATH/status_quo_seq.csv" "/PATH/results/scenario_status_quo_-dd-dd_results_complete.csv" -n "status quo" -c "scenario of sensitivity" + + + + diff --git a/preprocessing/python_scripts/renpass_gis/scenario_import/db.py b/preprocessing/python_scripts/renpass_gis/scenario_import/db.py index 3eb2b3da..a5b41654 100644 --- a/preprocessing/python_scripts/renpass_gis/scenario_import/db.py +++ b/preprocessing/python_scripts/renpass_gis/scenario_import/db.py @@ -14,6 +14,7 @@ from sqlalchemy.ext.automap import automap_base import configparser as cp import os.path as path +import oedialect # read configuration file FILENAME = 'config.ini' @@ -29,7 +30,7 @@ password=cfg.get(section, 'password'), host=cfg.get(section, 'host'), port=cfg.get(section, 'port'), - db=cfg.get(section, 'db'))) + db=cfg.get(section, 'database'))) print("Connected to database.") diff --git a/preprocessing/python_scripts/renpass_gis/scenario_import/requirements.txt b/preprocessing/python_scripts/renpass_gis/scenario_import/requirements.txt new file mode 100644 index 00000000..805d6242 --- /dev/null +++ b/preprocessing/python_scripts/renpass_gis/scenario_import/requirements.txt @@ -0,0 +1,16 @@ +certifi==2018.8.24 +chardet==3.0.4 +configparser==3.5.0 +docopt==0.6.2 +idna==2.7 +numpy==1.15.1 +oedialect==0.0.2.dev0 +pandas==0.20.1 +pathlib==1.0.1 +psycopg2==2.7.5 +python-dateutil==2.7.3 +pytz==2018.5 +requests==2.19.1 +six==1.11.0 +SQLAlchemy==1.2.0 +urllib3==1.23 diff --git a/preprocessing/scenario_log/ego_pp_scenario.sql b/preprocessing/scenario_log/ego_pp_scenario.sql index 000b97c8..ccc300bc 100644 --- a/preprocessing/scenario_log/ego_pp_scenario.sql +++ b/preprocessing/scenario_log/ego_pp_scenario.sql @@ -18,11 +18,12 @@ __author__ = "Ludee" -- scenario list +CREATE SCHEMA IF NOT EXISTS model_draft; DROP TABLE IF EXISTS model_draft.ego_scenario CASCADE; CREATE TABLE model_draft.ego_scenario ( id serial, model text, - version text, + version text UNIQUE, version_name text, "release" boolean, "comment" text, @@ -99,5 +100,3 @@ COMMENT ON TABLE model_draft.ego_scenario IS '{ {"name": "timestamp", "description": "timestamp (Berlin)", "unit": "none" } ] } ], "metadata_version": "1.3"}'; --- scenario log (project,version,io,schema_name,table_name,script_name,comment) -SELECT scenario_log('eGo_PP','PP1','setup','model_draft','ego_scenario','ego_pp_scenario.sql','Update scenario list'); diff --git a/preprocessing/scenario_log/ego_pp_scenario_log.sql b/preprocessing/scenario_log/ego_pp_scenario_log.sql index 327924b6..8711ff5d 100644 --- a/preprocessing/scenario_log/ego_pp_scenario_log.sql +++ b/preprocessing/scenario_log/ego_pp_scenario_log.sql @@ -24,7 +24,6 @@ GROUP BY version ORDER BY version; */ -/* -- scenario log table DROP TABLE IF EXISTS model_draft.scenario_log CASCADE; CREATE TABLE model_draft.scenario_log ( @@ -36,7 +35,7 @@ CREATE TABLE model_draft.scenario_log ( table_name text, script_name text, entries integer, - status text, + comment text, user_name text, "timestamp" timestamp, meta_data text, @@ -49,7 +48,6 @@ ALTER TABLE model_draft.scenario_log -- access rights ALTER TABLE model_draft.scenario_log OWNER TO oeuser; -*/ -- metadata COMMENT ON TABLE model_draft.scenario_log IS '{ diff --git a/preprocessing/scenario_log/function_scenario_log.sql b/preprocessing/scenario_log/function_scenario_log.sql index 515a2996..0fe3ed70 100644 --- a/preprocessing/scenario_log/function_scenario_log.sql +++ b/preprocessing/scenario_log/function_scenario_log.sql @@ -1,4 +1,4 @@ -/* +/* Scenario Log Function (Multiproject version) Function to add an entry to the Scenario Log table: @@ -33,45 +33,38 @@ __author__ = "Ludwig Hülk" * SPDX-License-Identifier: AGPL-3.0-or-later */ - --- Scenario Log function -DROP FUNCTION IF EXISTS scenario_log(text,text,text,text,text,text,text); +DROP FUNCTION IF EXISTS scenario_log(text,text,text,text,text,text,text); CREATE OR REPLACE FUNCTION scenario_log(text,text,text,text,text,text,text) - RETURNS text AS - $$ - DECLARE - _project ALIAS FOR $1; - _version ALIAS FOR $2; - _io ALIAS FOR $3; - _schema_name ALIAS FOR $4; - _table_name ALIAS FOR $5; - _script_name ALIAS FOR $6; - _comment ALIAS FOR $7; - BEGIN - EXECUTE 'INSERT INTO model_draft.scenario_log ' || - '(project,version,io,schema_name,table_name,script_name,entries,comment,user_name,timestamp,meta_data) - SELECT ' || quote_literal(_project) || ',' || - quote_literal(_version) || ',' || - quote_literal(_io) || ',' || - quote_literal(_schema_name) || ',' || - quote_literal(_table_name) || ',' || - quote_literal(_script_name) || ',' || - 'COUNT(*),' || - quote_literal(_comment) || ',' || - 'session_user,' || - E'NOW() AT TIME ZONE \'Europe/Berlin\' ,' || - E'obj_description(\'' || _schema_name || '.' || _table_name || E'\' ::regclass) ::json ' || - 'FROM ' || _schema_name || '.' || _table_name || ';' ; - RETURN 'inserted'; - END; - $$ LANGUAGE plpgsql; + RETURNS text AS + $$ + DECLARE + _project ALIAS FOR $1; + _version ALIAS FOR $2; + _io ALIAS FOR $3; + _schema_name ALIAS FOR $4; + _table_name ALIAS FOR $5; + _script_name ALIAS FOR $6; + _comment ALIAS FOR $7; + BEGIN + EXECUTE 'INSERT INTO model_draft.scenario_log ' || + '(project,version,io,schema_name,table_name,script_name,entries,comment,user_name,timestamp,meta_data) + SELECT ' || quote_literal(_project) || ',' || + quote_literal(_version) || ',' || + quote_literal(_io) || ',' || + quote_literal(_schema_name) || ',' || + quote_literal(_table_name) || ',' || + quote_literal(_script_name) || ',' || + 'COUNT(*),' || + quote_literal(_comment) || ',' || + 'session_user,' || + E'NOW() AT TIME ZONE \'Europe/Berlin\' ,' || + E'obj_description(\'' || _schema_name || '.' || _table_name || E'\' ::regclass) ::json ' || + 'FROM ' || _schema_name || '.' || _table_name || ';' ; + RETURN 'inserted'; + END; + $$ LANGUAGE plpgsql; -- access rights ALTER FUNCTION scenario_log(text,text,text,text,text,text,text) OWNER TO oeuser; GRANT EXECUTE ON FUNCTION scenario_log(text,text,text,text,text,text,text) TO oeuser; --- scenario log (project,version,io,schema_name,table_name,script_name,comment) -SELECT scenario_log('eGo_PP','PP1','setup','model_draft','scenario_log','function_scenario_log.sql','Function testing'); - --- select latest entry -SELECT * FROM model_draft.scenario_log ORDER BY timestamp DESC LIMIT 1; diff --git a/preprocessing/sql_snippets/ego_pre_slp_parameters.sql b/preprocessing/slp/ego_pre_slp_parameters.sql similarity index 95% rename from preprocessing/sql_snippets/ego_pre_slp_parameters.sql rename to preprocessing/slp/ego_pre_slp_parameters.sql index 5dc6b424..b9cd6e68 100644 --- a/preprocessing/sql_snippets/ego_pre_slp_parameters.sql +++ b/preprocessing/slp/ego_pre_slp_parameters.sql @@ -10,6 +10,7 @@ __author__ = "gplssm" */ -- create empty parameters table +CREATE SCHEMA IF NOT EXISTS scenario; DROP TABLE IF EXISTS scenario.ego_slp_parameters CASCADE; CREATE TABLE scenario.ego_slp_parameters ( parameter text, @@ -72,4 +73,4 @@ SELECT obj_description('scenario.ego_slp_parameters' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.8','input','scenario','ego_slp_parameters','ego_slp_parameters.sql','Create SLP parameter table'); +SELECT scenario_log('eGo_PP','PP1','input','scenario','ego_slp_parameters','ego_slp_parameters.sql','Create SLP parameter table'); diff --git a/preprocessing/sql_snippets/ego_dp_preprocessing_conv_powerplant.sql b/preprocessing/sql_snippets/ego_dp_preprocessing_conv_powerplant.sql index 03eab239..6e19474e 100644 --- a/preprocessing/sql_snippets/ego_dp_preprocessing_conv_powerplant.sql +++ b/preprocessing/sql_snippets/ego_dp_preprocessing_conv_powerplant.sql @@ -176,7 +176,7 @@ SELECT obj_description('model_draft.ego_dp_supply_conv_powerplant'::regclass)::j INSERT INTO model_draft.ego_dp_supply_conv_powerplant SELECT 'v0.3.0'::text as preversion, - id, + gid, bnetza_id, company, name, @@ -208,10 +208,10 @@ INSERT INTO model_draft.ego_dp_supply_conv_powerplant lon, comment, geom, - voltage_level, - subst_id, - otg_id, - un_id, + NULL as voltage_level, + NULL as subst_id, + NULL as otg_id, + NULL as un_id, NULL::int as la_id, 'Status Quo'::text as scenario, NULL::text as flag, diff --git a/preprocessing/sql_snippets/ego_dp_preprocessing_res_powerplant.sql b/preprocessing/sql_snippets/ego_dp_preprocessing_res_powerplant.sql index ed5db82b..a9be9cff 100644 --- a/preprocessing/sql_snippets/ego_dp_preprocessing_res_powerplant.sql +++ b/preprocessing/sql_snippets/ego_dp_preprocessing_res_powerplant.sql @@ -195,21 +195,22 @@ Insert into model_draft.ego_dp_supply_res_powerplant source || ' ego_dp' as source, comment || ' geom changes by rea' as comment , ST_Transform(geom,3035) as geom, - subst_id, - otg_id, - un_id, - voltage_level, - la_id, - mvlv_subst_id, - rea_sort, - rea_flag, + NULL as subst_id, + NULL as otg_id, + NULL as un_id, + NULL as voltage_level, + NULL as la_id, + NULL as mvlv_subst_id, + NULL as rea_sort, + NULL as rea_flag, NULL as rea_geom_line, Null as rea_geom_new, 'Status Quo'::text as scenario, 'constantly'::text as flag, - Null as nuts + NULL as nuts, + NULL as w-id FROM - model_draft.ego_supply_res_powerplant + supply.ego_renewable_powerplant WHERE geom is not NULL; diff --git a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_import.sql b/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_import.sql deleted file mode 100644 index 268bfee8..00000000 --- a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_import.sql +++ /dev/null @@ -1,200 +0,0 @@ -/* -Preprocess OpenStreetMap tables after import in public schema with osm2pgsql -Basic operations: -0. grant -1. remove index -2. add column 'gid' serial -3. add primary keys -4. rename geo-column 'way' to 'geom' -6. add index -7. move tables to openstreetmap schema - -__copyright__ = "Reiner Lemoine Institut" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "Ludee" -*/ - - --- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_line','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_nodes','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_point','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_polygon','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_rels','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_roads','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_ways','ego_pp_osm_deu_import.sql','verification'); - -/* --- Tests for the loops --- Select all tables with OSM in public -SELECT * -FROM information_schema.tables -WHERE table_schema = 'public' - AND table_name LIKE 'osm_deu_%'; - --- Select all tables with OSM in public -SELECT * -FROM pg_tables -WHERE schemaname='public' - AND tablename LIKE 'osm_deu_%'; - --- Select all indexes with OSM in public -SELECT * -FROM pg_indexes -WHERE schemaname='public' - AND tablename LIKE 'osm_deu_%' - AND indexname LIKE '%_pkey'; -*/ - --- 0. grant oeuser -DO -$$ -DECLARE - row record; -BEGIN - FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'osm_deu_%' - LOOP - EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' OWNER TO oeuser;'; - END LOOP; -END; -$$; - - --- 1. remove wrong index --- 1.1 remove all wrong 'pkey' (_line, _point, _polygon, _roads) -DO -$$ -DECLARE - row record; -BEGIN - FOR row IN SELECT indexname FROM pg_indexes - WHERE (schemaname='public' - AND indexname LIKE '%_pkey') - AND (tablename LIKE '%_line' - OR tablename LIKE '%_point' - OR tablename LIKE '%_polygon' - OR tablename LIKE '%_roads') - LOOP - EXECUTE 'DROP INDEX public.' || quote_ident(row.indexname) || ';'; - END LOOP; -END; -$$; - --- 1.2 remove all wrong other indexes (_line, _point, _polygon, _roads) -DO -$$ -DECLARE - row record; -BEGIN - FOR row IN SELECT indexname FROM pg_indexes - WHERE (schemaname='public' - AND indexname LIKE '%_index') - LOOP - EXECUTE 'DROP INDEX public.' || quote_ident(row.indexname) || ';'; - END LOOP; -END; -$$; - --- 2. add column 'gid' serial --- 3. add primary keys --- 4. rename geo-column 'way' to 'geom' -DO -$$ -DECLARE - row record; -BEGIN - FOR row IN SELECT tablename FROM pg_tables - WHERE schemaname='public' - AND (tablename LIKE '%_line' - OR tablename LIKE '%_point' - OR tablename LIKE '%_polygon' - OR tablename LIKE '%_roads') - LOOP - EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' ADD gid SERIAL;'; - EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' ADD PRIMARY KEY (gid);'; - EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' RENAME COLUMN way TO geom;'; - END LOOP; -END; -$$; - - --- 5. Set geometry and SRID (original!) - --- ALTER TABLE public.osm_deu_line --- ALTER COLUMN geom TYPE geometry(LineString,3857) --- USING ST_SetSRID(geom,3857); --- --- ALTER TABLE public.osm_deu_point --- ALTER COLUMN geom TYPE geometry(Point,3857) --- USING ST_SetSRID(geom,3857); --- --- ALTER TABLE public.osm_deu_polygon --- ALTER COLUMN geom TYPE geometry(MultiPolygon, 3857) --- USING ST_SetSRID(ST_MULTI(geom),3857); --- --- ALTER TABLE public.osm_deu_roads --- ALTER COLUMN geom TYPE geometry(LineString,3857) --- USING ST_SetSRID(geom,3857); - - --- 6. add index --- 6.1 add indexes GIST (geom) -DO -$$ -DECLARE - row record; -BEGIN - FOR row IN SELECT tablename FROM pg_tables - WHERE schemaname='public' - AND (tablename LIKE '%_line' - OR tablename LIKE '%_point' - OR tablename LIKE '%_polygon' - OR tablename LIKE '%_roads') - LOOP - EXECUTE 'CREATE INDEX ' || quote_ident(row.tablename) || '_geom_idx ON public.' || quote_ident(row.tablename) || ' USING gist (geom);'; - END LOOP; -END; -$$; - --- 6.2 Add indexes GIN (tags) -DO -$$ -DECLARE - row record; -BEGIN - FOR row IN SELECT tablename FROM pg_tables - WHERE schemaname='public' - AND (tablename LIKE '%_line' - OR tablename LIKE '%_point' - OR tablename LIKE '%_polygon' - OR tablename LIKE '%_roads') - LOOP - EXECUTE 'CREATE INDEX ' || quote_ident(row.tablename) || '_tags_idx ON public.' || quote_ident(row.tablename) || ' USING GIN (tags);'; - END LOOP; -END; -$$; - - - --- 7. move tables to openstreetmap schema -DO -$$ -DECLARE - row record; -BEGIN - FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'osm_deu_%' - LOOP - EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' SET SCHEMA openstreetmap;'; - END LOOP; -END; -$$; - --- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_line','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_nodes','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_point','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_polygon','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_rels','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_roads','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_ways','ego_pp_osm_deu_import.sql','setup osm tables'); diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_nodes.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_nodes.sql deleted file mode 100644 index fcf7a619..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_nodes.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_nodes IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_nodes", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_point.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_point.sql deleted file mode 100644 index 01b8364a..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_point.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_point IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_point", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_polygon.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_polygon.sql deleted file mode 100644 index cf40ba5b..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_polygon.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_polygon IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_polygon", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_rels.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_rels.sql deleted file mode 100644 index d9d76176..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_rels.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_rels IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_rels", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_roads.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_roads.sql deleted file mode 100644 index de7245da..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_roads.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_roads IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_roads", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_ways.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_ways.sql deleted file mode 100644 index 446152fd..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_ways.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_ways IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_ways", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/supply/import_vernetzen_wind_potential_areas.py b/preprocessing/supply/import_vernetzen_wind_potential_areas.py new file mode 100644 index 00000000..a4ff35e9 --- /dev/null +++ b/preprocessing/supply/import_vernetzen_wind_potential_areas.py @@ -0,0 +1,58 @@ +import os +import zipfile +import geopandas as gpd +import sqlalchemy +from geoalchemy2 import Geometry, WKTElement +from shapely.geometry import MultiPolygon + + +CRS = 4326 + + +# Function to generate WKB hex +def wkb_hexer(line): + return line.wkb_hex + + +def force_to_multipolygon(value): + + if isinstance(value, MultiPolygon): + return value + else: + return MultiPolygon([value]) + + +def import_vernetzen_wind_potential_areas(file, path, db, **kwds): + + if not db.dialect.has_schema(db, kwds["schema"]): + db.execute(sqlalchemy.schema.CreateSchema(kwds["schema"])) + + shp_file = file.replace(".zip", ".shp") + subdir = file.replace(".zip", "") + os.makedirs(os.path.join(path, subdir), exist_ok=True) + + + zf = zipfile.ZipFile(os.path.join(path, file)) + zf.extractall(path=os.path.join(path, subdir)) + + # read with pandas + wind_area = gpd.read_file(os.path.join(path, subdir, shp_file)) + + wind_area["geometry"] =wind_area["geometry"].apply(force_to_multipolygon) + wind_area['geom'] = wind_area['geometry'].apply(wkb_hexer) + wind_area.drop("geometry", axis=1, inplace=True) + + # create table from geopandas dataframe + wind_area.to_sql(kwds["table"], + db, kwds["schema"], + index=False, + if_exists="replace", + chunksize=10000, + method="multi", + dtype={'geom': Geometry('MULTIPOLYGON')} + ) + + create_pkey_constraint = "ALTER TABLE {schema}.{table} ADD CONSTRAINT " \ + "{table}_pkey PRIMARY KEY (region_key);".format(**kwds) + db.execution_options(autocommit=True).execute(create_pkey_constraint) + diff --git a/preprocessing/sql_snippets/vernetzen_wind_potential_area_v1.3.sql b/preprocessing/supply/vernetzen_wind_potential_area_v1.3.sql similarity index 100% rename from preprocessing/sql_snippets/vernetzen_wind_potential_area_v1.3.sql rename to preprocessing/supply/vernetzen_wind_potential_area_v1.3.sql diff --git a/preprocessing/utility/__init__.py b/preprocessing/utility/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/preprocessing/utility/cosmoclmgrid_to_shp.py b/preprocessing/utility/cosmoclmgrid_to_shp.py new file mode 100644 index 00000000..9658a9ef --- /dev/null +++ b/preprocessing/utility/cosmoclmgrid_to_shp.py @@ -0,0 +1,26 @@ +from egoio.db_tables.climate import Cosmoclmgrid +from preprocessing.utility.download_oedb_data_to_shp import oedbtable2shp +from sqlalchemy import func +from sqlalchemy.orm import sessionmaker +from egoio.tools import db + + +CRS = 4326 + + +def cosmoclmgrid2shp(): + + filename = "cosmoclmgrid.shp" + + engine_oedb = db.connection(readonly=True) + session = sessionmaker(bind=engine_oedb)() + + query = session.query(Cosmoclmgrid.gid, + func.ST_AsText(func.ST_Transform(Cosmoclmgrid.geom, CRS)).label("geom")) + + oedbtable2shp(query, filename, CRS, "geom", "gid") + + + +if __name__== "__main__": + cosmoclmgrid2shp() \ No newline at end of file diff --git a/preprocessing/utility/download_oedb_data_to_shp.py b/preprocessing/utility/download_oedb_data_to_shp.py new file mode 100644 index 00000000..b6fcbda2 --- /dev/null +++ b/preprocessing/utility/download_oedb_data_to_shp.py @@ -0,0 +1,42 @@ +import pandas as pd +from egoio.tools import db +from sqlalchemy.orm import sessionmaker +import geopandas as gdp +from shapely import wkt +import zipfile +import os +from preprocessing.utility.utils import DOWNLOADDIR, zipdir + + +def oedbtable2shp(table_query, filename, crs, geometry_col, index_col, zip_it=True): + + file_basename = os.path.basename(os.path.splitext(filename)[0]) + shp_dir = os.path.join(DOWNLOADDIR, file_basename) + os.makedirs(shp_dir, exist_ok=True) + shp_filename = os.path.join(shp_dir, filename) + zip_filename = os.path.join(DOWNLOADDIR, file_basename + ".zip") + + engine_oedb = db.connection(readonly=True) + session = sessionmaker(bind=engine_oedb)() + + table_df = pd.read_sql_query(table_query.statement, + session.bind, + index_col=index_col) + + if geometry_col: + table_df[geometry_col] = table_df[geometry_col].apply(wkt.loads) + table_gdf = gdp.GeoDataFrame(table_df, + geometry=geometry_col, + crs=crs) + + table_gdf.to_file(shp_filename) + else: + table_df.to_csv(shp_filename) + + if zip_it: + zf = zipfile.ZipFile( + zip_filename, + mode='w', + compression=zipfile.ZIP_DEFLATED) + zipdir(shp_dir, zf) + zf.close() \ No newline at end of file diff --git a/preprocessing/sql_snippets/ego_dp_supply_add_coastdat_gid.sql b/preprocessing/utility/ego_dp_supply_add_coastdat_gid.sql similarity index 100% rename from preprocessing/sql_snippets/ego_dp_supply_add_coastdat_gid.sql rename to preprocessing/utility/ego_dp_supply_add_coastdat_gid.sql diff --git a/preprocessing/sql_snippets/ego_weather_points_per_scenario.sql b/preprocessing/utility/ego_weather_points_per_scenario.sql similarity index 100% rename from preprocessing/sql_snippets/ego_weather_points_per_scenario.sql rename to preprocessing/utility/ego_weather_points_per_scenario.sql diff --git a/preprocessing/utility/oedbtodb.py b/preprocessing/utility/oedbtodb.py new file mode 100644 index 00000000..ca650c2b --- /dev/null +++ b/preprocessing/utility/oedbtodb.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- +""" + + +Sources +------- +https://github.com/OpenEnergyPlatform/oedialect/blob/master/doc/example/oedialec +https://stackoverflow.com/questions/21770829/sqlalchemy-copy-schema-and-data-of-subquery-to-another-database +https://geoalchemy-2.readthedocs.io/en/latest/elements.html + +""" + +import oedialect + +import sqlalchemy as sa + +from egoio.tools import Base +from importlib import import_module +from sqlalchemy import event +from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm.session import make_transient +from geoalchemy2.elements import WKBElement + + +def meta_url(schema, table): + + base = "https://openenergy-platform.org/api/v0/" + return base + "schema/" + schema + "/tables/" + table + "/meta/" + + +def create_session(url): + """ Returns SQLAlchemy session object + + Parameters + ---------- + url : string + Database dialect and connection arguments + """ + engine = sa.create_engine(url) + return sessionmaker(bind=engine)() + + +def unique_schema(qualified): + """ Returns unique schema names + + Parameters + ---------- + qualified : list + List of qualified names in `schema.table` format + """ + + schemas = list(set([schema for schema, table in [i.split(".") for i in qualified]])) + + return schemas + + +def map_name_table(names, base): + """ Create map from `schema.table` to associated table object + + Parameters + ---------- + names : list + List of qualified names in `schema.table` format + base : `sqlalchemy.ext.declarative.api.Base` + + """ + mapper = {} + + for name, obj in base.metadata.tables.items(): + if name in names: + mapper[name] = obj + return mapper + + +def map_name_class(names, base): + """ Create map from `schema.table` to class + + Parameters + ---------- + names : list + List of qualified names in `schema.table` format + base : `sqlalchemy.ext.declarative.api.Base` + + Sources + ------- + https://stackoverflow.com/questions/11668355/sqlalchemy-get-model-from-table-name-this-may-imply-appending-some-function-to + + """ + mapper = {} + + for cls in Base._decl_class_registry.values(): + if hasattr(cls, "__table__"): + name = cls.__table__.schema + "." + cls.__tablename__ + if name in names: + mapper[name] = cls + return mapper + + +def main(): + """ """ + + LOCAL = "postgresql+psycopg2://user:password@localhost:5432/database" + REMOTE = "postgresql+oedialect://openenergy-platform.org" + + INPUT = { + "environment.dlm250_geb01_f": None, + "model_draft.wn_abw_ego_dp_hvmv_substation": {"version": "v0.3.0"}, + } + + source_session = create_session(REMOTE) + target_session = create_session(LOCAL) + + schemas = unique_schema(INPUT) + + # https://stackoverflow.com/questions/41678073/import-class-from-module-dynamically + + # by importing a submodule the class registry + # of `egoio.tools.Base` is extended with + # all classes of the imported submodule + for s in schemas: + packagename = "egoio.db_tables" + import_module(packagename + "." + s) + + class_map = map_name_class(INPUT.keys(), Base) + table_map = map_name_table(INPUT.keys(), Base) + + # create schema if not exists + # maybe as event hook? + # https://github.com/sqlalchemy/sqlalchemy/issues/3982 + for schema in schemas: + query = "CREATE SCHEMA IF NOT EXISTS {schema}".format(schema=schema) + target_session.execute(query) + target_session.commit() + + # create subset of database model in local database + metadata = Base.metadata + metadata.bind = target_session.connection().engine + + # https://stackoverflow.com/questions/19175311/how-to-create-only-one-table-with-sqlalchemy + metadata.create_all(tables=table_map.values(), checkfirst=True) + + # retrieve data from remote database + results = [] + + for name, v in INPUT.items(): + query = source_session.query(class_map[name]) + if v: + if "version" in v: + query = query.filter(class_map[name].version == v["version"]) + for ds in query: + make_transient(ds) + results.append(ds) + + # create WKBElement from WKBElement ;) + # otherwise database error about SRID being 0 + for r in results: + for k, v in vars(r).items(): + if isinstance(v, WKBElement): + wkb = WKBElement(v.data, srid=v.srid) + setattr(r, k, wkb) + target_session.add(r) + + target_session.commit() + source_session.close() + target_session.close() + + +if __name__ == "__main__": + main() diff --git a/preprocessing/utility/osmTGmod_to_shp.py b/preprocessing/utility/osmTGmod_to_shp.py new file mode 100644 index 00000000..6cf529dc --- /dev/null +++ b/preprocessing/utility/osmTGmod_to_shp.py @@ -0,0 +1,130 @@ +from egoio.db_tables.grid import OtgEhvhvBusDatum, OtgEhvhvBranchDatum, OtgEhvhvDclineDatum, OtgEhvhvResultsMetadatum +import os +from preprocessing.utility.download_oedb_data_to_shp import oedbtable2shp +from preprocessing.utility.utils import DOWNLOADDIR, zipdir +from sqlalchemy import func +from sqlalchemy.orm import sessionmaker +from egoio.tools import db +import zipfile + + +CRS = 4326 + + +def osmTGmod2shp(): + + filename_zip = "otg_ehvhv_data.zip" + filename_bus = "otg_ehvhv_bus_data.shp" + filename_branch = "otg_ehvhv_branch_data.shp" + filename_dcline = "otg_ehvhv_dcline_data.shp" + filename_results = "otg_ehvhv_results_metadata.csv" + + engine_oedb = db.connection(readonly=True) + session = sessionmaker(bind=engine_oedb)() + + bus_query = session.query( + OtgEhvhvBusDatum.result_id, + OtgEhvhvBusDatum.view_id, + OtgEhvhvBusDatum.bus_i, + OtgEhvhvBusDatum.bus_type, + OtgEhvhvBusDatum.pd, + OtgEhvhvBusDatum.qd, + OtgEhvhvBusDatum.gs, + OtgEhvhvBusDatum.bs, + OtgEhvhvBusDatum.bus_area, + OtgEhvhvBusDatum.vm, + OtgEhvhvBusDatum.va, + OtgEhvhvBusDatum.base_kv, + OtgEhvhvBusDatum.zone, + OtgEhvhvBusDatum.vmax, + OtgEhvhvBusDatum.vmin, + OtgEhvhvBusDatum.osm_substation_id, + OtgEhvhvBusDatum.cntr_id, + OtgEhvhvBusDatum.frequency, + OtgEhvhvBusDatum.osm_name, + OtgEhvhvBusDatum.result, + func.ST_AsText(func.ST_Transform(OtgEhvhvBusDatum.geom, CRS)).label("geom")) + + branch_query = session.query( + OtgEhvhvBranchDatum.result_id, + OtgEhvhvBranchDatum.view_id, + OtgEhvhvBranchDatum.branch_id, + OtgEhvhvBranchDatum.f_bus, + OtgEhvhvBranchDatum.t_bus, + OtgEhvhvBranchDatum.br_r, + OtgEhvhvBranchDatum.br_x, + OtgEhvhvBranchDatum.br_b, + OtgEhvhvBranchDatum.rate_a, + OtgEhvhvBranchDatum.rate_b, + OtgEhvhvBranchDatum.rate_c, + OtgEhvhvBranchDatum.tap, + OtgEhvhvBranchDatum.shift, + OtgEhvhvBranchDatum.br_status, + OtgEhvhvBranchDatum.link_type, + OtgEhvhvBranchDatum.branch_voltage, + OtgEhvhvBranchDatum.cables, + OtgEhvhvBranchDatum.frequency, + OtgEhvhvBranchDatum.result, + func.ST_AsText(func.ST_Transform(OtgEhvhvBranchDatum.geom, CRS)).label("geom"), + func.ST_AsText(func.ST_Transform(OtgEhvhvBranchDatum.topo, CRS)).label("topo") + ) + + dcline_query = session.query( + OtgEhvhvDclineDatum.result_id, + OtgEhvhvDclineDatum.view_id, + OtgEhvhvDclineDatum.dcline_id, + OtgEhvhvDclineDatum.f_bus, + OtgEhvhvDclineDatum.t_bus, + OtgEhvhvDclineDatum.br_status, + OtgEhvhvDclineDatum.pf, + OtgEhvhvDclineDatum.pt, + OtgEhvhvDclineDatum.qf, + OtgEhvhvDclineDatum.qt, + OtgEhvhvDclineDatum.vf, + OtgEhvhvDclineDatum.vt, + OtgEhvhvDclineDatum.pmin, + OtgEhvhvDclineDatum.pmax, + OtgEhvhvDclineDatum.qminf, + OtgEhvhvDclineDatum.qmaxf, + OtgEhvhvDclineDatum.qmint, + OtgEhvhvDclineDatum.qmaxt, + OtgEhvhvDclineDatum.loss0, + OtgEhvhvDclineDatum.loss1, + OtgEhvhvDclineDatum.link_type, + OtgEhvhvDclineDatum.branch_voltage, + OtgEhvhvDclineDatum.cables, + OtgEhvhvDclineDatum.frequency, + func.ST_AsText(func.ST_Transform(OtgEhvhvDclineDatum.geom, CRS)).label("geom"), + func.ST_AsText(func.ST_Transform(OtgEhvhvDclineDatum.topo, CRS)).label("topo"), + OtgEhvhvDclineDatum.result + ) + + results_query = session.query( + OtgEhvhvResultsMetadatum.id, + OtgEhvhvResultsMetadatum.osm_date, + OtgEhvhvResultsMetadatum.abstraction_date, + OtgEhvhvResultsMetadatum.applied_plans, + OtgEhvhvResultsMetadatum.applied_year, + OtgEhvhvResultsMetadatum.user_comment, + ) + + oedbtable2shp(bus_query, filename_bus, CRS, "geom", "view_id", zip_it=False) + oedbtable2shp(branch_query, filename_branch, CRS, "geom", "view_id", zip_it=False) + oedbtable2shp(dcline_query, filename_dcline, CRS, "geom", "view_id", zip_it=False) + oedbtable2shp(results_query, filename_results, CRS, None, "id", zip_it=False) + + # zip all files into a single archive + directories = [os.path.join(DOWNLOADDIR, os.path.splitext(f)[0]) for f in + [filename_bus, filename_branch, filename_dcline, filename_results] + ] + with zipfile.ZipFile( + os.path.join(DOWNLOADDIR, filename_zip), + mode='w', + compression=zipfile.ZIP_DEFLATED) as zf: + + for directory in directories: + zipdir(directory, zf) + zf.close() + +if __name__== "__main__": + osmTGmod2shp() diff --git a/preprocessing/utility/utils.py b/preprocessing/utility/utils.py new file mode 100644 index 00000000..b1fb33c8 --- /dev/null +++ b/preprocessing/utility/utils.py @@ -0,0 +1,15 @@ +import os + + +DOWNLOADDIR = os.path.join(os.path.expanduser("~"), ".egon-pre-processing-cached/") + + +def zipdir(path, ziph): + # ziph is zipfile handle + for root, dirs, files in os.walk(path): + for file in files: + ziph.write(os.path.join(root, file), arcname=file) + + +def wkb_hexer(line): + return line.wkb_hex \ No newline at end of file diff --git a/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py b/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py new file mode 100644 index 00000000..a53bfde8 --- /dev/null +++ b/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py @@ -0,0 +1,28 @@ +from egoio.db_tables.supply import VernetzenWindPotentialArea +import os +from preprocessing.utility.download_oedb_data_to_shp import oedbtable2shp +from sqlalchemy import func +from sqlalchemy.orm import sessionmaker +from egoio.tools import db + + + +CRS = 4326 + + +def vernetzen_windpotential_areas2shp(): + + filename = "vernetzen_wind_potential_areas.shp" + + engine_oedb = db.connection(readonly=True) + session = sessionmaker(bind=engine_oedb)() + + table_query = session.query( + VernetzenWindPotentialArea.region_key, + func.ST_AsText(func.ST_Transform(VernetzenWindPotentialArea.geom, CRS)).label("geom")) + + oedbtable2shp(table_query, filename, CRS, "geom", "region_key") + + +if __name__== "__main__": + vernetzen_windpotential_areas2shp() \ No newline at end of file diff --git a/preprocessing/vg250/__init__.py b/preprocessing/vg250/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/preprocessing/vg250/create_boundaries_schema.sql b/preprocessing/vg250/create_boundaries_schema.sql new file mode 100644 index 00000000..38a91000 --- /dev/null +++ b/preprocessing/vg250/create_boundaries_schema.sql @@ -0,0 +1 @@ +CREATE SCHEMA IF NOT EXISTS boundaries; \ No newline at end of file diff --git a/preprocessing/sql_snippets/ego_dp_structure_boundaries_vg250.sql b/preprocessing/vg250/ego_dp_structure_boundaries_vg250.sql similarity index 92% rename from preprocessing/sql_snippets/ego_dp_structure_boundaries_vg250.sql rename to preprocessing/vg250/ego_dp_structure_boundaries_vg250.sql index a8329a9b..49016094 100644 --- a/preprocessing/sql_snippets/ego_dp_structure_boundaries_vg250.sql +++ b/preprocessing/vg250/ego_dp_structure_boundaries_vg250.sql @@ -14,24 +14,24 @@ __author__ = "Ludee" -- 1. Nationalstaat (sta) - country (cntry) -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','input','boundaries','bkg_vg250_1_sta','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','input','boundaries','bkg_vg250_1_sta','ego_dp_structure_boundaries_vg250.sql',' '); -- 1. country - mview with tiny buffer because of intersection (in official data) DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_1_sta_mview CASCADE; CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_mview AS SELECT vg.reference_date ::text, - vg.id ::integer, + vg.gid ::integer, vg.bez ::text, vg.gf ::double precision, ST_AREA(ST_TRANSFORM(vg.geom, 3035)) / 10000 ::double precision AS area_ha, ST_MULTI(ST_BUFFER(ST_TRANSFORM(vg.geom,3035),-0.001)) ::geometry(MultiPolygon,3035) AS geom FROM boundaries.bkg_vg250_1_sta AS vg WHERE vg.reference_date = '2016-01-01' - ORDER BY vg.id; + ORDER BY vg.gid; -- index (id) -CREATE UNIQUE INDEX bkg_vg250_1_sta_mview_id_idx - ON boundaries.bkg_vg250_1_sta_mview (id); +CREATE UNIQUE INDEX bkg_vg250_1_sta_mview_gid_idx + ON boundaries.bkg_vg250_1_sta_mview (gid); -- index GIST (geom) CREATE INDEX bkg_vg250_1_sta_mview_geom_idx @@ -112,18 +112,18 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_1_sta_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_1_sta_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_1_sta_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 1. country - error geom DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_1_sta_error_geom_mview CASCADE; CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_error_geom_mview AS - SELECT sub.id AS id, + SELECT sub.gid AS gid, sub.error AS error, sub.error_reason AS error_reason, ST_SETSRID(location(ST_IsValidDetail(sub.geom)),3035) ::geometry(Point,3035) AS geom FROM ( - SELECT source.id AS id, -- PK + SELECT source.gid AS gid, -- PK ST_IsValid(source.geom) AS error, reason(ST_IsValidDetail(source.geom)) AS error_reason, source.geom AS geom @@ -132,9 +132,9 @@ CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_error_geom_mview AS ) AS sub WHERE sub.error = FALSE; --- index (id) -CREATE UNIQUE INDEX bkg_vg250_1_sta_error_geom_mview_id_idx - ON boundaries.bkg_vg250_1_sta_error_geom_mview (id); +-- index (gid) +CREATE UNIQUE INDEX bkg_vg250_1_sta_error_geom_mview_gid_idx + ON boundaries.bkg_vg250_1_sta_error_geom_mview (gid); -- index GIST (geom) CREATE INDEX bkg_vg250_1_sta_error_geom_mview_geom_idx @@ -157,7 +157,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_error_geom_mview IS '{ "Extend": "Germany; Nationalstaat (sta) - country (cntry)" }], "Description": ["Errors in country border"], "Column":[ - {"Name": "id", "Description": "Unique identifier", "Unit": " " }, + {"Name": "gid", "Description": "Unique identifier", "Unit": " " }, {"Name": "error", "Description": "Error", "Unit": " " }, {"Name": "error_reason", "Description": "Error reason", "Unit": " " }, {"Name": "geom", "Description": "Geometry", "Unit": " " } ], @@ -177,14 +177,14 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_error_geom_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_1_sta_error_geom_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_1_sta_error_geom_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_1_sta_error_geom_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 1. country - union DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_1_sta_union_mview CASCADE; CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_union_mview AS SELECT '2016-01-01' ::text AS reference_date, - '1' ::integer AS id, + '1' ::integer AS gid, 'Bundesrepublik' ::text AS bez, ST_AREA(un.geom) / 10000 ::double precision AS area_ha, un.geom ::geometry(MultiPolygon,3035) AS geom @@ -193,9 +193,9 @@ CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_union_mview AS WHERE vg.bez = 'Bundesrepublik' AND reference_date = '2016-01-01' ) AS un; --- index (id) -CREATE UNIQUE INDEX bkg_vg250_1_sta_union_mview_id_idx - ON boundaries.bkg_vg250_1_sta_union_mview (id); +-- index (gid) +CREATE UNIQUE INDEX bkg_vg250_1_sta_union_mview_gid_idx + ON boundaries.bkg_vg250_1_sta_union_mview (gid); -- index GIST (geom) CREATE INDEX bkg_vg250_1_sta_union_mview_geom_idx @@ -219,7 +219,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_union_mview IS '{ "Description": ["Geometry union"], "Column":[ {"Name": "reference_date", "Description": "Reference Year", "Unit": " " }, - {"Name": "id", "Description": "Unique identifier", "Unit": " " }, + {"Name": "gid", "Description": "Unique identifier", "Unit": " " }, {"Name": "bez", "Description": "Bezeichnung der Verwaltungseinheit", "Unit": " " }, {"Name": "area_ha", "Description": "Area in ha", "Unit": "ha" }, {"Name": "geom", "Description": "Geometry", "Unit": " " } ], @@ -239,14 +239,14 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_union_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_1_sta_union_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_1_sta_union_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_1_sta_union_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 1. state borders - bounding box DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_1_sta_bbox_mview CASCADE; CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_bbox_mview AS SELECT '2016-01-01' ::text AS reference_date, - '1' ::integer AS id, + '1' ::integer AS gid, 'Bundesrepublik' ::text AS bez, ST_AREA(un.geom) / 10000 ::double precision AS area_ha, un.geom ::geometry(Polygon,3035) AS geom @@ -254,9 +254,9 @@ CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_bbox_mview AS FROM boundaries.bkg_vg250_1_sta_union_mview AS vg ) AS un; --- index (id) -CREATE UNIQUE INDEX bkg_vg250_1_sta_bbox_mview_id_idx - ON boundaries.bkg_vg250_1_sta_bbox_mview (id); +-- index (gid) +CREATE UNIQUE INDEX bkg_vg250_1_sta_bbox_mview_gid_idx + ON boundaries.bkg_vg250_1_sta_bbox_mview (gid); -- index GIST (geom) CREATE INDEX bkg_vg250_1_sta_bbox_mview_geom_idx @@ -280,7 +280,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_bbox_mview IS '{ "Description": ["Geometry bounding box"], "Column":[ {"Name": "reference_date", "Description": "Reference Year", "Unit": " " }, - {"Name": "id", "Description": "Unique identifier", "Unit": " " }, + {"Name": "gid", "Description": "Unique identifier", "Unit": " " }, {"Name": "bez", "Description": "Bezeichnung der Verwaltungseinheit", "Unit": " " }, {"Name": "area_ha", "Description": "Area in ha", "Unit": "ha" }, {"Name": "geom", "Description": "Geometry", "Unit": " " } ], @@ -300,13 +300,13 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_bbox_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_1_sta_bbox_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_1_sta_bbox_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_1_sta_bbox_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 2. Bundesland (lan) - federal state (fst) -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','input','boundaries','bkg_vg250_2_lan','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','input','boundaries','bkg_vg250_2_lan','ego_dp_structure_boundaries_vg250.sql',' '); -- 2. federal state - mview with tiny buffer because of intersection (in official data) DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_2_lan_mview CASCADE; @@ -324,7 +324,7 @@ CREATE MATERIALIZED VIEW boundaries.bkg_vg250_2_lan_mview AS GROUP BY lan.ags_0,lan.gen ORDER BY lan.ags_0; --- index (id) +-- index (gid) CREATE UNIQUE INDEX bkg_vg250_2_lan_mview_ags_0_idx ON boundaries.bkg_vg250_2_lan_mview (ags_0); @@ -370,13 +370,13 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_2_lan_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_2_lan_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_2_lan_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_2_lan_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 4. Landkreis (krs) - district (dist) -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','input','boundaries','bkg_vg250_4_krs','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','input','boundaries','bkg_vg250_4_krs','ego_dp_structure_boundaries_vg250.sql',' '); -- 4. district - mview DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_4_krs_mview CASCADE; @@ -394,7 +394,7 @@ CREATE MATERIALIZED VIEW boundaries.bkg_vg250_4_krs_mview AS WHERE vg.reference_date = '2016-01-01' ORDER BY vg.id; --- index (id) +-- index (gid) CREATE UNIQUE INDEX bkg_vg250_4_krs_mview_id_idx ON boundaries.bkg_vg250_4_krs_mview (id); @@ -444,12 +444,11 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_4_krs_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_4_krs_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_4_krs_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_4_krs_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 6. Gemeinde (gem) - municipality (mun) -/* -- census data per municipality DROP TABLE IF EXISTS society.destatis_zensus_population_per_bkg_vg250_6_gem; CREATE TABLE society.destatis_zensus_population_per_bkg_vg250_6_gem ( @@ -468,8 +467,40 @@ INSERT INTO society.destatis_zensus_population_per_bkg_vg250_6_gem (reference_da FROM boundaries.bkg_vg250_6_gem AS vg ORDER BY vg.id; --- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','input','society','destatis_zensus_population_per_ha_mview','ego_dp_structure_boundaries_vg250.sql',' '); + +-- zensus points with population (includes zensus points outside borders vg250) +DROP MATERIALIZED VIEW IF EXISTS society.destatis_zensus_population_per_ha_mview CASCADE; +CREATE MATERIALIZED VIEW society.destatis_zensus_population_per_ha_mview AS + SELECT a.gid, + a.population, + a.geom_point, + a.geom + FROM society.destatis_zensus_population_per_ha AS a + WHERE a.population >= 0 + ORDER BY a.gid; + +-- index +CREATE UNIQUE INDEX destatis_zensus_population_per_ha_mview_gid_idx + ON society.destatis_zensus_population_per_ha_mview (gid); + +CREATE INDEX destatis_zensus_population_per_ha_mview_geom_point_idx + ON society.destatis_zensus_population_per_ha_mview USING GIST (geom_point); + +CREATE INDEX destatis_zensus_population_per_ha_mview_geom_idx + ON society.destatis_zensus_population_per_ha_mview USING GIST (geom); + +-- access rights +ALTER TABLE society.destatis_zensus_population_per_ha_mview OWNER TO oeuser; + +-- metadata +COMMENT ON MATERIALIZED VIEW society.destatis_zensus_population_per_ha_mview IS '{ + "comment": "eGoPP - Temporary table", + "version": "PP1", + "published": "none" }'; + +-- scenario log (project,version,io,schema_name,table_name,script_name,comment) +SELECT scenario_log('eGo_PP','PP1','preprocessing','society','destatis_zensus_population_per_ha_mview','ego_pp_destatis_zensus_insidevg250.sql','none'); + -- zensus 2011 population UPDATE society.destatis_zensus_population_per_bkg_vg250_6_gem AS t1 @@ -501,10 +532,10 @@ UPDATE society.destatis_zensus_population_per_bkg_vg250_6_gem AS ce UPDATE society.destatis_zensus_population_per_bkg_vg250_6_gem AS ce SET census_density = '0' WHERE census_density IS NULL; - */ + -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','input','boundaries','bkg_vg250_6_gem','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','input','boundaries','bkg_vg250_6_gem','ego_dp_structure_boundaries_vg250.sql',' '); -- 6. municipality - mview DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_6_gem_mview CASCADE; @@ -578,7 +609,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_6_gem_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_6_gem_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_6_gem_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_6_gem_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 6. municipality - error geom @@ -643,7 +674,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_6_gem_error_geom_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_6_gem_error_geom_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_6_gem_error_geom_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_6_gem_error_geom_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 6. municipality - dump @@ -723,7 +754,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_6_gem_dump_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_6_gem_dump_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_6_gem_dump_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_6_gem_dump_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- ego @@ -840,7 +871,7 @@ COMMENT ON MATERIALIZED VIEW model_draft.ego_boundaries_bkg_vg250_6_gem_hole_mvi SELECT obj_description('model_draft.ego_boundaries_bkg_vg250_6_gem_hole_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_boundaries_bkg_vg250_6_gem_hole_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','model_draft','ego_boundaries_bkg_vg250_6_gem_hole_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- update holes @@ -903,7 +934,7 @@ COMMENT ON TABLE model_draft.ego_boundaries_bkg_vg250_6_gem_clean IS '{ SELECT obj_description('model_draft.ego_boundaries_bkg_vg250_6_gem_clean' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_boundaries_bkg_vg250_6_gem_clean','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','model_draft','ego_boundaries_bkg_vg250_6_gem_clean','ego_dp_structure_boundaries_vg250.sql',' '); -- validation @@ -947,4 +978,4 @@ COMMENT ON VIEW boundaries.bkg_vg250_statistics_view IS '{ "version": "v0.3.0" }' ; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_statistics_view','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_statistics_view','ego_dp_structure_boundaries_vg250.sql',' '); diff --git a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_import.sql b/preprocessing/vg250/ego_pp_vg250_import.sql similarity index 89% rename from preprocessing/sql_snippets/boundaries/ego_pp_vg250_import.sql rename to preprocessing/vg250/ego_pp_vg250_import.sql index d0c4edc0..12e6fd5a 100644 --- a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_import.sql +++ b/preprocessing/vg250/ego_pp_vg250_import.sql @@ -1,4 +1,4 @@ -/* +/* Import vg250 tables __copyright__ = "Reiner Lemoine Institut" @@ -51,7 +51,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_sta%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_sta%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_1_sta SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -106,7 +106,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_lan%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_lan%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_2_lan SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -161,7 +161,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_rbz%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_rbz%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_3_rbz SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -216,7 +216,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_krs%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_krs%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_4_krs SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -271,7 +271,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_vwg%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_vwg%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_5_vwg SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -326,7 +326,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_gem%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_gem%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_6_gem SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -340,12 +340,12 @@ $$; /* -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_1_sta','ego_pp_vg250_import.sql','setup vg250 tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_2_lan','ego_pp_vg250_import.sql','setup vg250 tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_3_rbz','ego_pp_vg250_import.sql','setup vg250 tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_4_krs','ego_pp_vg250_import.sql','setup vg250 tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_5_vwg','ego_pp_vg250_import.sql','setup vg250 tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_6_gem','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_1_sta','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_2_lan','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_3_rbz','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_4_krs','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_5_vwg','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_6_gem','ego_pp_vg250_import.sql','setup vg250 tables'); */ -- drop tables @@ -354,7 +354,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_%%' LOOP EXECUTE E'DROP TABLE IF EXISTS boundaries.' || quote_ident(vg250.tablename) || ' ;'; END LOOP; diff --git a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_metadata.sql b/preprocessing/vg250/ego_pp_vg250_metadata.sql similarity index 98% rename from preprocessing/sql_snippets/boundaries/ego_pp_vg250_metadata.sql rename to preprocessing/vg250/ego_pp_vg250_metadata.sql index 69c7286e..f6d77d50 100644 --- a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_metadata.sql +++ b/preprocessing/vg250/ego_pp_vg250_metadata.sql @@ -455,9 +455,9 @@ COMMENT ON TABLE boundaries.bkg_vg250_6_gem IS '{ -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_1_sta','ego_pp_vg250_metadata.sql',''); -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_2_lan','ego_pp_vg250_metadata.sql',''); -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_3_rbz','ego_pp_vg250_metadata.sql',''); -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_4_krs','ego_pp_vg250_metadata.sql',''); -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_5_vwg','ego_pp_vg250_metadata.sql',''); -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_6_gem','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_1_sta','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_2_lan','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_3_rbz','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_4_krs','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_5_vwg','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_6_gem','ego_pp_vg250_metadata.sql',''); diff --git a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_mview.sql b/preprocessing/vg250/ego_pp_vg250_mview.sql similarity index 92% rename from preprocessing/sql_snippets/boundaries/ego_pp_vg250_mview.sql rename to preprocessing/vg250/ego_pp_vg250_mview.sql index 72017d1e..a5addf5f 100644 --- a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_mview.sql +++ b/preprocessing/vg250/ego_pp_vg250_mview.sql @@ -1,4 +1,4 @@ --- Create MView with NUTS IDs +-- Create MView with NUTS IDs CREATE MATERIALIZED VIEW boundaries.bkg_vg250_2_lan_nuts_view AS SELECT lan.ags_0, diff --git a/preprocessing/vg250/import_vg250.py b/preprocessing/vg250/import_vg250.py new file mode 100644 index 00000000..a379bd47 --- /dev/null +++ b/preprocessing/vg250/import_vg250.py @@ -0,0 +1,28 @@ +import os +import zipfile +import subprocess + + +def certain_file_types(directory, extension): + return (f for f in os.listdir(directory) if f.endswith('.' + extension)) + + +def import_vg250(file, path, db_info, year=2019): + FNULL = open(os.devnull, 'w') + + zip = zipfile.ZipFile(os.path.join(path, file)) + zip.extractall(path=path) + file_stripped, _ = os.path.splitext(file) + file_splitted = file_stripped.split("_") + dir1 = file_splitted[0] + "_" + str(year) + "-" + "_".join(file_splitted[1:]) + vg250_shp_path = os.path.join(path, dir1, "vg250_ebenen") + table_date = "_" + "_".join(reversed(open(os.path.join(path, dir1, "aktualitaet.txt")).read().strip().split("."))) + + for f in certain_file_types(vg250_shp_path, "shp"): + table_base_name, ext = os.path.splitext(f) + tablename = table_base_name + table_date + ext + cmd_shp = 'shp2pgsql "' + os.path.join(vg250_shp_path, f) + '" boundaries.{}'.format(tablename) + cmd_pgql = ' | psql -h {host} -d {database} -U {user} -p {port}'.format(**db_info) + cmd = cmd_shp + cmd_pgql + + subprocess.run(cmd, shell=True, env={"PGPASSWORD": "egon"}, stdout=FNULL) diff --git a/preprocessing/zensus/destatis_zensus_import_csv.py b/preprocessing/zensus/destatis_zensus_import_csv.py new file mode 100644 index 00000000..72ad39e6 --- /dev/null +++ b/preprocessing/zensus/destatis_zensus_import_csv.py @@ -0,0 +1,28 @@ +import os +import zipfile +import pandas as pd +import sqlalchemy + + +def import_zensus2011(file, path, db, year=2019, **kwds): + + if not db.dialect.has_schema(db, kwds["schema"]): + db.execute(sqlalchemy.schema.CreateSchema(kwds["schema"])) + + csv_file = file.replace("zip", "csv") + + zip = zipfile.ZipFile(os.path.join(path, file)) + zip.extractall(path=path) + + # read with pandas + zensus_raw = pd.read_csv(os.path.join(path, file), delimiter=";") + zensus_raw.columns =["grid_id", "x_mp_100m", "y_mp_100m", "population"] + # zensus_raw = zensus_raw.iloc[:20000] + + # create table from geopandas dataframe + zensus_raw.to_sql(kwds["table"], + db, kwds["schema"], + index=False, + if_exists="replace", + chunksize=10000, + method="multi") diff --git a/preprocessing/zensus/destatis_zensus_raw_import.sql b/preprocessing/zensus/destatis_zensus_raw_import.sql new file mode 100644 index 00000000..2e667abc --- /dev/null +++ b/preprocessing/zensus/destatis_zensus_raw_import.sql @@ -0,0 +1,57 @@ +-- 01 +-- create table and insert (OK!) +DROP TABLE IF EXISTS tmp_destatis.zensus_population_per_ha; +CREATE TABLE tmp_destatis.zensus_population_per_ha ( + gid SERIAL NOT NULL, + grid_id character varying(254) NOT NULL, + x_mp numeric(10,0), + y_mp numeric(10,0), + population numeric(10,0), + geom geometry(Point,3035), + CONSTRAINT zensus_population_per_ha_pkey PRIMARY KEY (gid)); + +-- 02 +-- Insert (OK!) +INSERT INTO tmp_destatis.zensus_population_per_ha (grid_id,x_mp,y_mp,population,geom)( + SELECT + zensus.grid_id AS grid_id, + zensus.x_mp_100m AS x_mp, + zensus.y_mp_100m AS y_mp, + zensus.population AS population, + ST_SetSRID(ST_MakePoint(zensus.x_mp_100m, zensus.y_mp_100m),3035) AS geom + FROM tmp_destatis.zensus_2011_pop_per_ha AS zensus); + +-- 03 +-- Create index (OK!) +CREATE INDEX zensus_population_per_ha_geom_idx + ON tmp_destatis.zensus_population_per_ha + USING gist (geom); + +-- 04 +-- CREATE TABLE (OK!) +DROP TABLE IF EXISTS tmp_destatis.zensus_population_per_ha_grid; +CREATE TABLE tmp_destatis.zensus_population_per_ha_grid ( + gid INT, + grid_id character varying(254) NOT NULL, + x_mp numeric(10,0), + y_mp numeric(10,0), + population INT, + geom geometry(Polygon, 3035), + CONSTRAINT zensus_population_per_ha_grid_pkey PRIMARY KEY (gid)); + +-- 05 +-- INSERT GRID (OK!) +INSERT INTO tmp_destatis.zensus_population_per_ha_grid +SELECT pts.gid AS gid, + pts.grid_id AS grid_id, + pts.x_mp AS x_mp, + pts.y_mp AS y_mp, + pts.population AS population, + ST_SetSRID((ST_MakeEnvelope(pts.x_mp-50,pts.y_mp-50,pts.x_mp+50,pts.y_mp+50)),3035) AS geom +FROM tmp_destatis.zensus_population_per_ha AS pts; + +-- 06 +-- CREATE INDEX GIST (OK!) +CREATE INDEX zensus_population_per_ha_grid_geom_idx + ON tmp_destatis.zensus_population_per_ha_grid + USING GIST (geom); \ No newline at end of file diff --git a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_import.sql b/preprocessing/zensus/ego_pp_destatis_zensus_import.sql similarity index 88% rename from preprocessing/sql_snippets/society/ego_pp_destatis_zensus_import.sql rename to preprocessing/zensus/ego_pp_destatis_zensus_import.sql index 3de541c6..3ebd2654 100644 --- a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_import.sql +++ b/preprocessing/zensus/ego_pp_destatis_zensus_import.sql @@ -7,6 +7,7 @@ __url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" __author__ = "Ludee" */ +CREATE SCHEMA IF NOT EXISTS society; -- zensus CREATE TABLE society.destatis_zensus_population_per_ha ( @@ -64,8 +65,10 @@ CREATE INDEX destatis_zensus_population_per_ha_geom_point_idx -- insert data INSERT INTO society.destatis_zensus_population_per_ha SELECT pnt.gid,pnt.grid_id,pnt.x_mp,pnt.y_mp,pnt.population,pnt.geom AS geom_point,poly.geom - FROM orig_destatis.zensus_population_per_ha AS pnt - JOIN orig_destatis.zensus_population_per_ha_grid AS poly ON (pnt.gid=poly.gid); + FROM tmp_destatis.zensus_population_per_ha AS pnt + JOIN tmp_destatis.zensus_population_per_ha_grid AS poly ON (pnt.gid=poly.gid); -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','preprocessing','society','destatis_zensus_population_per_ha','ego_pp_destatis_zensus_import.sql','setup zensus table'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','society','destatis_zensus_population_per_ha','ego_pp_destatis_zensus_import.sql','setup zensus table'); + +DROP SCHEMA tmp_destatis CASCADE; \ No newline at end of file diff --git a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_insidevg250.sql b/preprocessing/zensus_vg250/ego_pp_destatis_zensus_insidevg250.sql similarity index 98% rename from preprocessing/sql_snippets/society/ego_pp_destatis_zensus_insidevg250.sql rename to preprocessing/zensus_vg250/ego_pp_destatis_zensus_insidevg250.sql index c94fcb40..93c0a621 100644 --- a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_insidevg250.sql +++ b/preprocessing/zensus_vg250/ego_pp_destatis_zensus_insidevg250.sql @@ -47,8 +47,7 @@ COMMENT ON MATERIALIZED VIEW society.destatis_zensus_population_per_ha_mview IS "published": "none" }'; -- scenario log (project,version,io,schema_name,table_name,script_name,comment) -SELECT scenario_log('eGo_PP','PP1','output','society','destatis_zensus_population_per_ha_mview','ego_pp_destatis_zensus_insidevg250.sql','none'); - +SELECT scenario_log('eGo_PP','PP1','preprocessing','society','destatis_zensus_population_per_ha_mview','ego_pp_destatis_zensus_insidevg250.sql','none'); -- zensus points inside Germany (vg250) DROP TABLE IF EXISTS model_draft.destatis_zensus_population_per_ha_inside CASCADE; diff --git a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_metadata.sql b/preprocessing/zensus_vg250/ego_pp_destatis_zensus_metadata.sql similarity index 96% rename from preprocessing/sql_snippets/society/ego_pp_destatis_zensus_metadata.sql rename to preprocessing/zensus_vg250/ego_pp_destatis_zensus_metadata.sql index 1996b084..6cd9c03c 100644 --- a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_metadata.sql +++ b/preprocessing/zensus_vg250/ego_pp_destatis_zensus_metadata.sql @@ -58,4 +58,4 @@ COMMENT ON TABLE society.destatis_zensus_population_per_ha IS '{ "metadata_version": "1.3"}'; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','preprocessing','society','destatis_zensus_population_per_ha','ego_pp_destatis_zensus_import.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','society','destatis_zensus_population_per_ha','ego_pp_destatis_zensus_import.sql','metadata'); diff --git a/setup.py b/setup.py index 089d6c5d..1c676eb7 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,11 @@ 'workalendar', 'oemof.db', 'demandlib', - 'ego.io >=0.0.1rc4, <= 0.0.2', - 'geoalchemy2' + "pyyaml", + "geopandas" + 'egoio @ git+https://github.com/openego/ego.io.git@features/use-one-Base-definition', + 'oedialect @ git+https://github.com/OpenEnergyPlatform/oedialect.git@4957e63ca46a976e35eecf84036466cb624e6bfe', + 'geoalchemy2 < 0.7.0', ], extras_require={ 'docs': [ diff --git a/utilities/entrypoints/00_create_extensions.sql b/utilities/entrypoints/00_create_extensions.sql new file mode 100644 index 00000000..78b98ad4 --- /dev/null +++ b/utilities/entrypoints/00_create_extensions.sql @@ -0,0 +1,2 @@ +create extension hstore; +CREATE EXTENSION postgis; \ No newline at end of file