Replace materialized view of water by tables with diff update (openmaptiles#853)

frodrigo · web-flow · commit 97216c5c191b · 2020-05-20T13:14:22.000-04:00
Replacing materialized view by a tables with update from trigger on change only. Start with the most simple cases. Just replicate the change on: * `osm_water_polygon` to `osm_water_lakeline`, * `osm_water_polygon` to `osm_water_point`. Use a view to factorize the `osm_water_lakeline` and `osm_water_point_view` definition and reuse it in the trigger. The update of `osm_important_waterway_linestring` is more complex, as it is a merge of `osm_waterway_linestring`. It not done in the same way. At the end of the transaction we remove impacted and recompute them. The goal is to update more quickly the content of derivated table by just updating the changing content. It replaces the update of materialized view because their need a full recompute (with lock issue). Note, an advanced version of differential update over materialized view as already implemented in the building cluster PR openmaptiles#725. It addresses openmaptiles#814 and a part of openmaptiles#809.
diff --git a/layers/water_name/update_water_lakeline.sql b/layers/water_name/update_water_lakeline.sql
@@ -1,11 +1,8 @@
-DROP TRIGGER IF EXISTS trigger_flag_line ON osm_water_polygon;
-DROP TRIGGER IF EXISTS trigger_refresh ON water_lakeline.updates;
+DROP TRIGGER IF EXISTS trigger_delete_line ON osm_water_polygon;
+DROP TRIGGER IF EXISTS trigger_update_line ON osm_water_polygon;
+DROP TRIGGER IF EXISTS trigger_insert_line ON osm_water_polygon;
 
--- etldoc:  osm_water_polygon ->  osm_water_lakeline
--- etldoc:  lake_centerline  ->  osm_water_lakeline
-DROP MATERIALIZED VIEW IF EXISTS osm_water_lakeline CASCADE;
-
-CREATE MATERIALIZED VIEW osm_water_lakeline AS (
+CREATE OR REPLACE VIEW osm_water_lakeline_view AS
 	SELECT wp.osm_id,
 		ll.wkb_geometry AS geometry,
 		name, name_en, name_de,
@@ -15,39 +12,67 @@ CREATE MATERIALIZED VIEW osm_water_lakeline AS (
     FROM osm_water_polygon AS wp
     INNER JOIN lake_centerline ll ON wp.osm_id = ll.osm_id
     WHERE wp.name <> '' AND ST_IsValid(wp.geometry)
-) /* DELAY_MATERIALIZED_VIEW_CREATION */;
+;
+
+-- etldoc:  osm_water_polygon ->  osm_water_lakeline
+-- etldoc:  lake_centerline  ->  osm_water_lakeline
+CREATE TABLE IF NOT EXISTS osm_water_lakeline AS
+SELECT * FROM osm_water_lakeline_view;
+DO $$
+    BEGIN
+        ALTER TABLE osm_water_lakeline ADD CONSTRAINT osm_water_lakeline_pk PRIMARY KEY (osm_id);
+    EXCEPTION WHEN others then
+        RAISE NOTICE 'primary key osm_water_lakeline_pk already exists in osm_water_lakeline.';
+    END;
+$$;
 CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline USING gist(geometry);
 
 -- Handle updates
 
 CREATE SCHEMA IF NOT EXISTS water_lakeline;
 
-CREATE TABLE IF NOT EXISTS water_lakeline.updates(id serial primary key, t text, unique (t));
-CREATE OR REPLACE FUNCTION water_lakeline.flag() RETURNS trigger AS $$
+CREATE OR REPLACE FUNCTION water_lakeline.delete() RETURNS trigger AS $BODY$
 BEGIN
-    INSERT INTO water_lakeline.updates(t) VALUES ('y')  ON CONFLICT(t) DO NOTHING;
+    DELETE FROM osm_water_lakeline
+    WHERE osm_water_lakeline.osm_id = OLD.osm_id ;
+
     RETURN null;
 END;
-$$ language plpgsql;
-
-CREATE OR REPLACE FUNCTION water_lakeline.refresh() RETURNS trigger AS
-  $BODY$
-  BEGIN
-    RAISE LOG 'Refresh water_lakeline';
-    REFRESH MATERIALIZED VIEW osm_water_lakeline;
-    DELETE FROM water_lakeline.updates;
+$BODY$ language plpgsql;
+
+CREATE OR REPLACE FUNCTION water_lakeline.update() RETURNS trigger AS $BODY$
+BEGIN
+    UPDATE osm_water_lakeline
+    SET (osm_id, geometry, name, name_en, name_de, tags, area, is_intermittent) =
+        (SELECT * FROM osm_water_lakeline_view WHERE osm_water_lakeline_view.osm_id = NEW.osm_id)
+    WHERE osm_water_lakeline.osm_id = NEW.osm_id;
+
     RETURN null;
-  END;
-  $BODY$
-language plpgsql;
-
-CREATE TRIGGER trigger_flag_line
-    AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon
-    FOR EACH STATEMENT
-    EXECUTE PROCEDURE water_lakeline.flag();
-
-CREATE CONSTRAINT TRIGGER trigger_refresh
-    AFTER INSERT ON water_lakeline.updates
-    INITIALLY DEFERRED
+END;
+$BODY$ language plpgsql;
+
+CREATE OR REPLACE FUNCTION water_lakeline.insert() RETURNS trigger AS $BODY$
+BEGIN
+    INSERT INTO osm_water_lakeline
+    SELECT *
+    FROM osm_water_lakeline_view
+    WHERE osm_water_lakeline_view.osm_id = NEW.osm_id;
+
+    RETURN null;
+END;
+$BODY$ language plpgsql;
+
+CREATE TRIGGER trigger_delete_line
+    AFTER DELETE ON osm_water_polygon
+    FOR EACH ROW
+    EXECUTE PROCEDURE water_lakeline.delete();
+
+CREATE TRIGGER trigger_update_line
+    AFTER UPDATE ON osm_water_polygon
+    FOR EACH ROW
+    EXECUTE PROCEDURE water_lakeline.update();
+
+CREATE TRIGGER trigger_insert_line
+    AFTER INSERT ON osm_water_polygon
     FOR EACH ROW
-    EXECUTE PROCEDURE water_lakeline.refresh();
+    EXECUTE PROCEDURE water_lakeline.insert();
diff --git a/layers/water_name/update_water_point.sql b/layers/water_name/update_water_point.sql
@@ -1,11 +1,8 @@
-DROP TRIGGER IF EXISTS trigger_flag_point ON osm_water_polygon;
-DROP TRIGGER IF EXISTS trigger_refresh ON water_point.updates;
+DROP TRIGGER IF EXISTS trigger_delete_point ON osm_water_polygon;
+DROP TRIGGER IF EXISTS trigger_update_point ON osm_water_polygon;
+DROP TRIGGER IF EXISTS trigger_insert_point ON osm_water_polygon;
 
--- etldoc:  osm_water_polygon ->  osm_water_point
--- etldoc:  lake_centerline ->  osm_water_point
-DROP MATERIALIZED VIEW IF EXISTS  osm_water_point CASCADE;
-
-CREATE MATERIALIZED VIEW osm_water_point AS (
+CREATE OR REPLACE VIEW osm_water_point_view AS
     SELECT
         wp.osm_id, ST_PointOnSurface(wp.geometry) AS geometry,
         wp.name, wp.name_en, wp.name_de,
@@ -15,39 +12,67 @@ CREATE MATERIALIZED VIEW osm_water_point AS (
     FROM osm_water_polygon AS wp
     LEFT JOIN lake_centerline ll ON wp.osm_id = ll.osm_id
     WHERE ll.osm_id IS NULL AND wp.name <> ''
-) /* DELAY_MATERIALIZED_VIEW_CREATION */;
+;
+
+-- etldoc:  osm_water_polygon ->  osm_water_point
+-- etldoc:  lake_centerline ->  osm_water_point
+CREATE TABLE IF NOT EXISTS osm_water_point AS
+SELECT * FROM osm_water_point_view;
+DO $$
+    BEGIN
+        ALTER TABLE osm_water_point ADD CONSTRAINT osm_water_point_pk PRIMARY KEY (osm_id);
+    EXCEPTION WHEN others then
+        RAISE NOTICE 'primary key osm_water_point_pk already exists in osm_water_point.';
+    END;
+$$;
 CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING gist (geometry);
 
 -- Handle updates
 
 CREATE SCHEMA IF NOT EXISTS water_point;
 
-CREATE TABLE IF NOT EXISTS water_point.updates(id serial primary key, t text, unique (t));
-CREATE OR REPLACE FUNCTION water_point.flag() RETURNS trigger AS $$
+CREATE OR REPLACE FUNCTION water_point.delete() RETURNS trigger AS $BODY$
 BEGIN
-    INSERT INTO water_point.updates(t) VALUES ('y')  ON CONFLICT(t) DO NOTHING;
+    DELETE FROM osm_water_point
+    WHERE osm_water_point.osm_id = OLD.osm_id ;
+
     RETURN null;
 END;
-$$ language plpgsql;
-
-CREATE OR REPLACE FUNCTION water_point.refresh() RETURNS trigger AS
-  $BODY$
-  BEGIN
-    RAISE LOG 'Refresh water_point';
-    REFRESH MATERIALIZED VIEW osm_water_point;
-    DELETE FROM water_point.updates;
+$BODY$ language plpgsql;
+
+CREATE OR REPLACE FUNCTION water_point.update() RETURNS trigger AS $BODY$
+BEGIN
+    UPDATE osm_water_point
+    SET (osm_id, geometry, name, name_en, name_de, tags, area, is_intermittent) =
+        (SELECT * FROM osm_water_point_view WHERE osm_water_point_view.osm_id = NEW.osm_id)
+    WHERE osm_water_point.osm_id = NEW.osm_id;
+
     RETURN null;
-  END;
-  $BODY$
-language plpgsql;
-
-CREATE TRIGGER trigger_flag_point
-    AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon
-    FOR EACH STATEMENT
-    EXECUTE PROCEDURE water_point.flag();
-
-CREATE CONSTRAINT TRIGGER trigger_refresh
-    AFTER INSERT ON water_point.updates
-    INITIALLY DEFERRED
+END;
+$BODY$ language plpgsql;
+
+CREATE OR REPLACE FUNCTION water_point.insert() RETURNS trigger AS $BODY$
+BEGIN
+    INSERT INTO osm_water_point
+    SELECT *
+    FROM osm_water_point_view
+    WHERE osm_water_point_view.osm_id = NEW.osm_id;
+
+    RETURN null;
+END;
+$BODY$ language plpgsql;
+
+CREATE TRIGGER trigger_delete_point
+    AFTER DELETE ON osm_water_polygon
+    FOR EACH ROW
+    EXECUTE PROCEDURE water_point.delete();
+
+CREATE TRIGGER trigger_update_point
+    AFTER UPDATE ON osm_water_polygon
+    FOR EACH ROW
+    EXECUTE PROCEDURE water_point.update();
+
+CREATE TRIGGER trigger_insert_point
+    AFTER INSERT ON osm_water_polygon
     FOR EACH ROW
-    EXECUTE PROCEDURE water_point.refresh();
+    EXECUTE PROCEDURE water_point.insert();
diff --git a/layers/waterway/update_important_waterway.sql b/layers/waterway/update_important_waterway.sql
