Split preprocessing

src/egon/data/airflow/dags/pipeline.py

@@ -145,11 +145,8 @@
         dependencies=[osm, zensus_miscellaneous]
     )
     osm_buildings_streets.insert_into(pipeline)
-    osm_buildings_streets_preprocessing = tasks[
-        "osm_buildings_streets.preprocessing"
-    ]
-    osm_buildings_streets_residential_census_mapping = tasks[
-        "osm_buildings_streets.create-buildings-residential-zensus-mapping"
+    osm_buildings_streets_extract_amenities = tasks[
+        "osm_buildings_streets.extract-amenities"
     ]
 
     saltcavern_storage = SaltcavernData(dependencies=[data_bundle, vg250])
@@ -284,8 +281,7 @@
             map_zensus_grid_districts,
             zensus_vg250,
             demandregio,
-            osm_buildings_streets_preprocessing,
-            osm_buildings_streets_residential_census_mapping,
+            osm_buildings_streets_extract_amenities,
         ],
         tasks=(
             hh_profiles.houseprofiles_in_census_cells,

src/egon/data/datasets/osm_buildings_streets/__init__.py

@@ -75,15 +75,25 @@ def execute_sql_script(script):
 
 
 def preprocessing():
-    print("Preprocessing buildings and shops...")
+    print("Extracting buildings, amenities and shops...")
     sql_scripts = [
         "osm_amenities_shops_preprocessing.sql",
-        "osm_buildings_filter.sql",
+        "osm_buildings_extract.sql",
     ]
     for script in sql_scripts:
         execute_sql_script(script)
 
 
+def filter_buildings():
+    print("Filter buildings...")
+    execute_sql_script("osm_buildings_filter.sql")
+
+
+def filter_buildings_residential():
+    print("Filter residential buildings...")
+    execute_sql_script("osm_buildings_filter_residential.sql")
+
+
 def create_buildings_filtered_zensus_mapping():
     print("Create census mapping table for filtered buildings...")
     execute_sql_script("osm_buildings_filtered_zensus_mapping.sql")
@@ -136,6 +146,10 @@ def __init__(self, dependencies):
             dependencies=dependencies,
             tasks=(
                 preprocessing,
+                {
+                    filter_buildings,
+                    filter_buildings_residential
+                },
                 {
                     create_buildings_filtered_zensus_mapping,
                     create_buildings_residential_zensus_mapping

src/egon/data/datasets/osm_buildings_streets/osm_buildings_extract.sql

@@ -0,0 +1,27 @@
+/*
+ * Original Autor: IsGut (johnrobert@t-online.de)
+ * Adapted by: nesnoj (jonathan.amme@rl-institut.de)
+*/
+
+-------------------------------------------------------------
+-- extract all buildings, calculate area, create centroids --
+-------------------------------------------------------------
+DROP TABLE IF EXISTS openstreetmap.osm_buildings;
+CREATE TABLE openstreetmap.osm_buildings AS
+    SELECT
+        poly.osm_id,
+        poly.amenity,
+        poly.building,
+        poly.name,
+        ST_TRANSFORM(poly.geom, 3035) AS geom,
+        ST_AREA(ST_TRANSFORM(poly.geom, 3035)) AS area,
+        ST_TRANSFORM(ST_CENTROID(poly.geom), 3035) AS geom_point,
+        poly.tags
+    FROM openstreetmap.osm_polygon poly
+    WHERE poly.building IS NOT NULL;
+
+-- add PK as some osm ids are not unique
+ALTER TABLE openstreetmap.osm_buildings ADD COLUMN id SERIAL PRIMARY KEY;
+
+CREATE INDEX ON openstreetmap.osm_buildings USING gist (geom);
+CREATE INDEX ON openstreetmap.osm_buildings USING gist (geom_point);

src/egon/data/datasets/osm_buildings_streets/osm_buildings_filter.sql

@@ -3,29 +3,6 @@
  * Adapted by: nesnoj (jonathan.amme@rl-institut.de)
 */
 
--------------------------------------------------------------
--- extract all buildings, calculate area, create centroids --
--------------------------------------------------------------
-DROP TABLE IF EXISTS openstreetmap.osm_buildings;
-CREATE TABLE openstreetmap.osm_buildings AS
-    SELECT
-        poly.osm_id,
-        poly.amenity,
-        poly.building,
-        poly.name,
-        ST_TRANSFORM(poly.geom, 3035) AS geom,
-        ST_AREA(ST_TRANSFORM(poly.geom, 3035)) AS area,
-        ST_TRANSFORM(ST_CENTROID(poly.geom), 3035) AS geom_point,
-        poly.tags
-    FROM openstreetmap.osm_polygon poly
-    WHERE poly.building IS NOT NULL;
-
--- add PK as some osm ids are not unique
-ALTER TABLE openstreetmap.osm_buildings ADD COLUMN id SERIAL PRIMARY KEY;
-
-CREATE INDEX ON openstreetmap.osm_buildings USING gist (geom);
-CREATE INDEX ON openstreetmap.osm_buildings USING gist (geom_point);
-
 -----------------------------------------------------------------------
 -- extract specific buildings only, calculate area, create centroids --
 -----------------------------------------------------------------------
@@ -159,25 +136,3 @@ ALTER TABLE openstreetmap.osm_buildings_filtered
 
 CREATE INDEX ON openstreetmap.osm_buildings_filtered USING gist (geom);
 CREATE INDEX ON openstreetmap.osm_buildings_filtered USING gist (geom_point);
-
---------------------------------------------------------------------------
--- extract residential buildings only, calculate area, create centroids --
---------------------------------------------------------------------------
-DROP TABLE if exists openstreetmap.osm_buildings_residential;
-CREATE TABLE openstreetmap.osm_buildings_residential as
-    select *
-    from openstreetmap.osm_buildings bld
-    where
-        bld.building like 'yes'
-        or bld.building like 'apartments'
-        or bld.building like 'detached'
-        or bld.building like 'farm'
-        or bld.building like 'house'
-        or bld.building like 'residential'
-        or bld.building like 'semidetached_house';
-
-ALTER TABLE openstreetmap.osm_buildings_residential
-    ADD CONSTRAINT osm_buildings_residential_id_pkey PRIMARY KEY (id);
-
-CREATE INDEX ON openstreetmap.osm_buildings_residential USING gist (geom);
-CREATE INDEX ON openstreetmap.osm_buildings_residential USING gist (geom_point);

src/egon/data/datasets/osm_buildings_streets/osm_buildings_filter_residential.sql

@@ -0,0 +1,26 @@
+/*
+ * Original Autor: IsGut (johnrobert@t-online.de)
+ * Adapted by: nesnoj (jonathan.amme@rl-institut.de)
+*/
+
+--------------------------------------------------------------------------
+-- extract residential buildings only, calculate area, create centroids --
+--------------------------------------------------------------------------
+DROP TABLE if exists openstreetmap.osm_buildings_residential;
+CREATE TABLE openstreetmap.osm_buildings_residential as
+    select *
+    from openstreetmap.osm_buildings bld
+    where
+        bld.building like 'yes'
+        or bld.building like 'apartments'
+        or bld.building like 'detached'
+        or bld.building like 'farm'
+        or bld.building like 'house'
+        or bld.building like 'residential'
+        or bld.building like 'semidetached_house';
+
+ALTER TABLE openstreetmap.osm_buildings_residential
+    ADD CONSTRAINT osm_buildings_residential_id_pkey PRIMARY KEY (id);
+
+CREATE INDEX ON openstreetmap.osm_buildings_residential USING gist (geom);
+CREATE INDEX ON openstreetmap.osm_buildings_residential USING gist (geom_point);