Remove support for creating CAggs with old format

Timescale 2.7 released a new version of Continuous Aggregate (#4269) that store the final aggregation state instead of the byte array of the partial aggregate state, offering multiple opportunities of optimizations as well a more compact form. In 2.10.0 released on February 2023 the Continuous Aggregate old format deprecation was announced. With this PR the ability of creating Continuous Aggregate in the old format was removed, but we still support migrate from the old to the new format by running the `cagg_migrate` procedure. This is the continuation of the PR #5977 started by @pdipesh02. References: https://docs.timescale.com/api/latest/continuous-aggregates/cagg_migrate/ https://github.com/timescale/timescaledb/releases/tag/2.10.0 https://github.com/timescale/timescaledb/releases/tag/2.7.0 #5977
timescale · Dec 13, 2023 · 78490c4 · 78490c4
1 parent 13c4f08
commit 78490c4
Show file tree

Hide file tree

Showing 49 changed files with 1,265 additions and 6,957 deletions.
diff --git a/.unreleased/pr_6360 b/.unreleased/pr_6360
@@ -0,0 +1,3 @@
+Implements: #6360 Remove support for creating Continuous Aggregates with old format
+
+Thanks: @pdipesh02 for working on removing the old Continuous Aggregate format
diff --git a/scripts/test_functions.inc b/scripts/test_functions.inc
@@ -5,7 +5,7 @@ SCRIPT_DIR=$(dirname $0)
 # Run tests given as arguments.
 #
 # Options:
-#   -r    Run repair tests as a separate pass (optional)
+#   -r    Run repair tests (optional)
 #   -k    Keep temporary directory
 #   -vN   Use version N of the update tests (required)
 run_tests() (
@@ -17,24 +17,28 @@ run_tests() (
     while getopts "kv:r" opt;
     do
         case $opt in
-             v)
-                 TEST_VERSION=v$OPTARG
-                 ;;
-	     k)
-		 DO_CLEANUP=false
-		 ;;
-             r)
-                 TEST_REPAIR=true
-                 ;;
+            v)
+                TEST_VERSION=v$OPTARG
+                ;;
+            k)
+                DO_CLEANUP=false
+                ;;
+            r)
+                TEST_REPAIR=true
+                ;;
+            *)
+                exit 1
+                ;;
         esac
     done
 
     shift $((OPTIND-1))
 
     export TAGS="$@"
-    bash ${SCRIPT_DIR}/test_updates.sh
     if [[ "$TEST_REPAIR" = "true" ]]; then
         bash ${SCRIPT_DIR}/test_repairs.sh
+    else
+        bash ${SCRIPT_DIR}/test_updates.sh
     fi
     EXIT_CODE=$?
     if [ $EXIT_CODE -ne 0 ]; then

diff --git a/scripts/test_updates_pg13.sh b/scripts/test_updates_pg13.sh
@@ -10,11 +10,16 @@ source ${SCRIPT_DIR}/test_functions.inc
 run_tests "$@" -v7 \
           2.1.0-pg13 2.1.1-pg13 2.2.0-pg13 2.2.1-pg13 2.3.0-pg13 2.3.1-pg13 \
           2.4.0-pg13 2.4.1-pg13 2.4.2-pg13
+
 run_tests "$@" -v8 \
           2.5.0-pg13 2.5.1-pg13 2.5.2-pg13 2.6.0-pg13 2.6.1-pg13 2.7.0-pg13 2.7.1-pg13 2.7.2-pg13 \
           2.8.0-pg13 2.8.1-pg13 2.9.0-pg13 2.9.1-pg13 2.9.2-pg13 2.9.3-pg13
 
-# Also run repair tests for >=2.10.x versions due to PR #5441
+run_tests "$@" -v8 \
+          2.10.0-pg13 2.10.1-pg13 2.10.2-pg13 2.10.3-pg13 2.11.0-pg13 2.11.1-pg13 2.11.2-pg13 \
+          2.12.0-pg13 2.12.1-pg13 2.12.2-pg13 2.13.0-pg13
+
+# Run repair tests for >= 2.10.x versions due to PR #5441
 run_tests "$@" -r -v8 \
           2.10.0-pg13 2.10.1-pg13 2.10.2-pg13 2.10.3-pg13 2.11.0-pg13 2.11.1-pg13 2.11.2-pg13 \
           2.12.0-pg13 2.12.1-pg13 2.12.2-pg13 2.13.0-pg13

diff --git a/scripts/test_updates_pg14.sh b/scripts/test_updates_pg14.sh
@@ -9,11 +9,16 @@ source ${SCRIPT_DIR}/test_functions.inc
 
 run_tests "$@" -v7 \
           2.5.0-pg14 2.5.1-pg14
+
 run_tests "$@" -v8 \
           2.5.0-pg14 2.5.1-pg14 2.5.2-pg14 2.6.0-pg14 2.6.1-pg14 2.7.0-pg14 2.7.1-pg14 2.7.2-pg14 \
           2.8.0-pg14 2.8.1-pg14 2.9.0-pg14 2.9.1-pg14 2.9.2-pg14 2.9.3-pg14
 
-# Also run repair tests for >=2.10.x versions due to PR #5441
+run_tests "$@" -v8 \
+          2.10.0-pg14 2.10.1-pg14 2.10.2-pg14 2.10.3-pg14 2.11.0-pg14 2.11.1-pg14 2.11.2-pg14 \
+          2.12.0-pg14 2.12.1-pg14 2.12.2-pg14 2.13.0-pg14
+
+# Run repair tests for >=2.10.x versions due to PR #5441
 run_tests "$@" -r -v8 \
           2.10.0-pg14 2.10.1-pg14 2.10.2-pg14 2.10.3-pg14 2.11.0-pg14 2.11.1-pg14 2.11.2-pg14 \
           2.12.0-pg14 2.12.1-pg14 2.12.2-pg14 2.13.0-pg14

diff --git a/scripts/test_updates_pg15.sh b/scripts/test_updates_pg15.sh
@@ -10,7 +10,11 @@ source ${SCRIPT_DIR}/test_functions.inc
 run_tests "$@" -v8 \
           2.9.0-pg15 2.9.1-pg15 2.9.2-pg15 2.9.3-pg15
 
-# Also run repair tests for >=2.10.x versions due to PR #5441
+run_tests "$@" -v8 \
+          2.10.0-pg15 2.10.1-pg15 2.10.2-pg15 2.10.3-pg15 2.11.0-pg15 2.11.1-pg15 \
+          2.11.2-pg15 2.12.0-pg15 2.12.1-pg15 2.12.2-pg15 2.13.0-pg15
+
+# Run repair tests for >=2.10.x versions due to PR #5441
 run_tests "$@" -r -v8 \
           2.10.0-pg15 2.10.1-pg15 2.10.2-pg15 2.10.3-pg15 2.11.0-pg15 2.11.1-pg15 \
           2.11.2-pg15 2.12.0-pg15 2.12.1-pg15 2.12.2-pg15 2.13.0-pg15

diff --git a/scripts/test_updates_pg16.sh b/scripts/test_updates_pg16.sh
@@ -7,7 +7,10 @@ SCRIPT_DIR=$(dirname $0)
 # shellcheck source=scripts/test_functions.inc
 source ${SCRIPT_DIR}/test_functions.inc
 
-# Also run repair tests for >=2.10.x versions due to PR #5441
+run_tests "$@" -v8 \
+          2.13.0-pg16
+
+# Run repair tests for >=2.10.x versions due to PR #5441
 run_tests "$@" -r -v8 \
           2.13.0-pg16
 
diff --git a/sql/updates/post-update.sql b/sql/updates/post-update.sql
@@ -1,67 +1,50 @@
 -- needed post 1.7.0 to fixup continuous aggregates created in 1.7.0 ---
-DO $$
+DO
+$$
 DECLARE
  vname regclass;
+ mat_ht_id INTEGER;
  materialized_only bool;
- ts_version TEXT;
+ finalized bool;
+ ts_major INTEGER;
+ ts_minor INTEGER;
 BEGIN
-    SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
-    IF ts_version >= '2.7.0' THEN
-            CREATE PROCEDURE _timescaledb_internal.post_update_cagg_try_repair(
-                cagg_view REGCLASS, force_rebuild boolean
-            ) AS '@MODULE_PATHNAME@', 'ts_cagg_try_repair' LANGUAGE C;
+    -- procedures with SET clause cannot execute transaction
+    -- control so we adjust search_path in procedure body
+    SET LOCAL search_path TO pg_catalog, pg_temp;
+
+    SELECT ((string_to_array(extversion,'.'))[1])::int, ((string_to_array(extversion,'.'))[2])::int
+    INTO ts_major, ts_minor
+    FROM pg_extension WHERE extname = 'timescaledb';
+
+    IF ts_major >= 2 AND ts_minor >= 7 THEN
+      CREATE PROCEDURE _timescaledb_functions.post_update_cagg_try_repair(
+        cagg_view REGCLASS, force_rebuild BOOLEAN
+      ) AS '@MODULE_PATHNAME@', 'ts_cagg_try_repair' LANGUAGE C;
     END IF;
-    FOR vname, materialized_only IN select format('%I.%I', cagg.user_view_schema, cagg.user_view_name)::regclass, cagg.materialized_only from _timescaledb_catalog.continuous_agg cagg
+
+    FOR vname, mat_ht_id, materialized_only, finalized IN
+      SELECT format('%I.%I', cagg.user_view_schema, cagg.user_view_name)::regclass, cagg.mat_hypertable_id, cagg.materialized_only, cagg.finalized
+      FROM _timescaledb_catalog.continuous_agg cagg
     LOOP
-        -- the cast from oid to text returns
-        -- quote_qualified_identifier (see regclassout).
-        --
-        -- We use the if statement to handle pre-2.0 as well as
-        -- post-2.0.  This could be turned into a procedure if we want
-        -- to have something more generic, but right now it is just
-        -- this case.
-        IF ts_version < '2.0.0' THEN
-            EXECUTE format('ALTER VIEW %s SET (timescaledb.materialized_only=%L) ', vname::text, materialized_only);
-        ELSIF ts_version < '2.7.0' THEN
-            EXECUTE format('ALTER MATERIALIZED VIEW %s SET (timescaledb.materialized_only=%L) ', vname::text, materialized_only);
-        ELSE
-            SET log_error_verbosity TO VERBOSE;
-            CALL _timescaledb_internal.post_update_cagg_try_repair(vname, false);
-        END IF;
+      IF ts_major < 2 THEN
+        EXECUTE format('ALTER VIEW %s SET (timescaledb.materialized_only=%L) ', vname::text, materialized_only);
+
+      ELSIF ts_major = 2 AND ts_minor < 7 THEN
+        EXECUTE format('ALTER MATERIALIZED VIEW %s SET (timescaledb.materialized_only=%L) ', vname::text, materialized_only);
+
+      ELSIF ts_major = 2 AND ts_minor >= 7 THEN
+        SET log_error_verbosity TO VERBOSE;
+        CALL _timescaledb_functions.post_update_cagg_try_repair(vname, false);
+
+      END IF;
     END LOOP;
-    IF ts_version >= '2.7.0' THEN
-            DROP PROCEDURE IF EXISTS _timescaledb_internal.post_update_cagg_try_repair;
-    END IF;
-    EXCEPTION WHEN OTHERS THEN RAISE;
-END
-$$;
 
--- For tsdb >= v2.10.0 apply the cagg repair when necessary
-DO $$
-DECLARE
- vname regclass;
- materialized_only bool;
- ts_version TEXT;
-BEGIN
-    SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
-     IF ts_version >= '2.10.0' THEN
-	        CREATE PROCEDURE _timescaledb_internal.post_update_cagg_try_repair(
-	            cagg_view REGCLASS, force_rebuild BOOLEAN
-	        ) AS '@MODULE_PATHNAME@', 'ts_cagg_try_repair' LANGUAGE C;
-
-	        FOR vname, materialized_only IN select format('%I.%I', cagg.user_view_schema, cagg.user_view_name)::regclass, cagg.materialized_only from _timescaledb_catalog.continuous_agg cagg
-	        LOOP
-	            IF ts_version >= '2.10.0' THEN
-	                SET log_error_verbosity TO VERBOSE;
-	                CALL _timescaledb_internal.post_update_cagg_try_repair(vname, true);
-	            END IF;
-	        END LOOP;
-
-	        DROP PROCEDURE IF EXISTS _timescaledb_internal.post_update_cagg_try_repair(REGCLASS, BOOLEAN);
-	    END IF;
-    EXCEPTION WHEN OTHERS THEN RAISE;
+    IF ts_major >= 2 AND ts_minor >= 7 THEN
+      DROP PROCEDURE IF EXISTS _timescaledb_functions.post_update_cagg_try_repair(REGCLASS, BOOLEAN);
+    END IF;
 END
-$$;
+$$ LANGUAGE PLPGSQL;
 
 -- can only be dropped after views have been rebuilt
 DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_watermark(oid);
@@ -152,12 +135,12 @@ BEGIN
 END $$;
 
 -- Create dimension partition information for existing space-partitioned hypertables
-CREATE FUNCTION _timescaledb_internal.update_dimension_partition(hypertable REGCLASS) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_dimension_partition_update' LANGUAGE C VOLATILE;
-SELECT _timescaledb_internal.update_dimension_partition(format('%I.%I', h.schema_name, h.table_name))
+CREATE FUNCTION _timescaledb_functions.update_dimension_partition(hypertable REGCLASS) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_dimension_partition_update' LANGUAGE C VOLATILE;
+SELECT _timescaledb_functions.update_dimension_partition(format('%I.%I', h.schema_name, h.table_name))
 FROM _timescaledb_catalog.hypertable h
 INNER JOIN _timescaledb_catalog.dimension d ON (d.hypertable_id = h.id)
 WHERE d.interval_length IS NULL;
-DROP FUNCTION _timescaledb_internal.update_dimension_partition;
+DROP FUNCTION _timescaledb_functions.update_dimension_partition;
 
 -- Report warning when partial aggregates are used
 DO $$

diff --git a/test/sql/updates/post.repair.sql b/test/sql/updates/post.repair.sql
@@ -2,13 +2,15 @@
 -- Please see the included NOTICE for copyright information and
 -- LICENSE-APACHE for a copy of the license.
 
-SELECT extversion < '2.10.0' AS test_repair_dimension
-FROM pg_extension
-WHERE extname = 'timescaledb' \gset
+SELECT
+     split_part(extversion, '.', 1)::int * 100000 +
+     split_part(extversion, '.', 2)::int *    100 AS extversion_num
+FROM
+     pg_extension WHERE extname = 'timescaledb' \gset
 
-SELECT extversion >= '2.10.0' AND :'TEST_VERSION' >= 'v8' AS test_repair_cagg_joins
-FROM pg_extension
-WHERE extname = 'timescaledb' \gset
+SELECT
+     :extversion_num <  201000 AS test_repair_dimension,
+     :extversion_num >= 201000 AND :'TEST_VERSION' >= 'v8' AS test_repair_cagg_joins \gset
 
 \if :test_repair_dimension
     -- Re-add the dropped foreign key constraint that was dropped for

diff --git a/test/sql/updates/setup.continuous_aggs.sql b/test/sql/updates/setup.continuous_aggs.sql
@@ -7,9 +7,13 @@
 -- we keep them separate anyway so that we can do additional checking
 -- if necessary.
 SELECT
-	extversion < '2.0.0' AS has_refresh_mat_view
-  FROM pg_extension
- WHERE extname = 'timescaledb' \gset
+     split_part(extversion, '.', 1)::int * 100000 +
+     split_part(extversion, '.', 2)::int *    100 AS extversion_num
+FROM
+     pg_extension WHERE extname = 'timescaledb' \gset
+
+SELECT
+     :extversion_num <  200000 AS has_refresh_mat_view \gset
 
 CREATE TYPE custom_type AS (high int, low int);
 

diff --git a/test/sql/updates/setup.continuous_aggs.v2.sql b/test/sql/updates/setup.continuous_aggs.v2.sql
@@ -19,9 +19,7 @@ SELECT
   :ts_major < 2 AS has_ignore_invalidations_older_than,
   :ts_major < 2 AS has_max_interval_per_job,
   :ts_major >= 2 AS has_create_mat_view,
-  :ts_major >= 2 AS has_continuous_aggs_policy,
-  :ts_major = 2 AND :ts_minor >= 7 AS has_continuous_aggs_finals_form,
-  :ts_major = 2 AND :ts_minor IN (7,8) AS has_continuous_aggs_finalized_option
+  :ts_major >= 2 AS has_continuous_aggs_policy
   FROM pg_extension
  WHERE extname = 'timescaledb' \gset
 
@@ -63,11 +61,7 @@ SELECT generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timest
     -- we had a bug related to that and need to verify if compression can be
     -- enabled on such a view
     CREATE MATERIALIZED VIEW rename_cols
-    \if :has_continuous_aggs_finalized_option
-    WITH (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.finalized=false) AS
-    \else
     WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS
-    \endif
 \endif
     SELECT time_bucket('1 week', timec) AS bucket,
            location,
@@ -85,11 +79,7 @@ SELECT generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timest
     WITH ( timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.refresh_lag='-30 day', timescaledb.max_interval_per_job ='1000 day')
 \else
     CREATE MATERIALIZED VIEW IF NOT EXISTS mat_before
-    \if :has_continuous_aggs_finalized_option
-    WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
-    \else
     WITH ( timescaledb.continuous, timescaledb.materialized_only=true)
-    \endif
 \endif
     AS
       SELECT time_bucket('1week', timec) as bucket,
@@ -165,11 +155,7 @@ CREATE SCHEMA cagg;
     WITH ( timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.refresh_lag='-30 day', timescaledb.max_interval_per_job ='1000 day')
 \else
     CREATE MATERIALIZED VIEW IF NOT EXISTS cagg.realtime_mat
-    \if :has_continuous_aggs_finalized_option
-    WITH (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.finalized=false)
-    \else
     WITH ( timescaledb.continuous, timescaledb.materialized_only=false)
-    \endif
 \endif
     AS
       SELECT time_bucket('1week', timec) as bucket,
@@ -236,11 +222,7 @@ CALL refresh_continuous_aggregate('cagg.realtime_mat',NULL,NULL);
            timescaledb.max_interval_per_job = '100000 days')
 \else
     CREATE MATERIALIZED VIEW IF NOT EXISTS  mat_ignoreinval
-    \if :has_continuous_aggs_finalized_option
-    WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
-    \else
     WITH ( timescaledb.continuous, timescaledb.materialized_only=true)
-    \endif
 \endif
     AS
       SELECT time_bucket('1 week', timec) as bucket,
@@ -277,11 +259,7 @@ SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-20 00:00'::timest
            timescaledb.max_interval_per_job='100000 days' )
 \else
     CREATE MATERIALIZED VIEW mat_inval
-    \if :has_continuous_aggs_finalized_option
-    WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
-    \else
     WITH ( timescaledb.continuous, timescaledb.materialized_only=true )
-    \endif
 \endif
     AS
       SELECT time_bucket('10 minute', time) as bucket, location, min(temperature) as min_temp,
@@ -325,11 +303,7 @@ INSERT INTO int_time_test VALUES
            timescaledb.refresh_interval='12 hours')
 \else
     CREATE MATERIALIZED VIEW mat_inttime
-    \if :has_continuous_aggs_finalized_option
-    WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
-    \else
     WITH ( timescaledb.continuous, timescaledb.materialized_only=true )
-    \endif
 \endif
     AS
       SELECT time_bucket( 2, timeval), COUNT(col1)
@@ -347,11 +321,7 @@ INSERT INTO int_time_test VALUES
            timescaledb.refresh_interval='12 hours')
 \else
     CREATE MATERIALIZED VIEW mat_inttime2
-    \if :has_continuous_aggs_finalized_option
-    WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
-    \else
     WITH ( timescaledb.continuous, timescaledb.materialized_only=true )
-    \endif
 \endif
     AS
       SELECT time_bucket( 2, timeval), COUNT(col1)
@@ -385,11 +355,7 @@ SELECT create_hypertable('conflict_test', 'time', chunk_time_interval => INTERVA
            timescaledb.refresh_interval='12 hours' )
 \else
     CREATE MATERIALIZED VIEW mat_conflict
-    \if :has_continuous_aggs_finalized_option
-    WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
-    \else
     WITH ( timescaledb.continuous, timescaledb.materialized_only=true )
-    \endif
 \endif
     AS
       SELECT time_bucket('10 minute', time) as bucket, location, min(temperature) as min_temp,
@@ -448,9 +414,6 @@ WITH (
 \if :has_max_interval_per_job
      timescaledb.refresh_lag='-30 day',
      timescaledb.max_interval_per_job ='1000 day',
-\endif
-\if :has_continuous_aggs_finalized_option
-     timescaledb.finalized = false,
 \endif
      timescaledb.continuous
 ) AS
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		Implements: #6360 Remove support for creating Continuous Aggregates with old format

		Thanks: @pdipesh02 for working on removing the old Continuous Aggregate format