Merge pull request #250 from ScalefreeCOM/create-ghost-records-only-w…

…hen-not-incremental only create ghost records when it is not an incremental run
ScalefreeCOM · Sep 5, 2024 · 846a926 · 846a926
2 parents edceed4 + 34098fd
commit 846a926
Show file tree

Hide file tree

Showing 6 changed files with 12 additions and 20 deletions.
diff --git a/macros/staging/bigquery/stage.sql b/macros/staging/bigquery/stage.sql
@@ -438,7 +438,7 @@ hashed_columns AS (
 {%- endif -%}
 {%- endif -%}
 
-{% if enable_ghost_records %}
+{%- if enable_ghost_records and not is_incremental() %}
 {# Creating Ghost Record for unknown case, based on datatype #}
 unknown_values AS (
 
@@ -597,7 +597,7 @@ columns_to_select AS (
 
     FROM {{ last_cte }}
 
-{%- if enable_ghost_records %}
+{%- if enable_ghost_records and not is_incremental() %}
     UNION ALL
 
     SELECT

diff --git a/macros/staging/exasol/stage.sql b/macros/staging/exasol/stage.sql
@@ -426,7 +426,7 @@ hashed_columns AS (
 {%- endif -%}
 {%- endif -%}
 
-{% if enable_ghost_records %}
+{%- if enable_ghost_records and not is_incremental() %}
 {# Creating Ghost Record for unknown case, based on datatype #}
 unknown_values AS (
     SELECT
@@ -580,7 +580,7 @@ columns_to_select AS (
 
     FROM {{ last_cte }}
 
-{%- if enable_ghost_records %}
+{%- if enable_ghost_records and not is_incremental() %}
     UNION ALL
     SELECT
 

diff --git a/macros/staging/postgres/stage.sql b/macros/staging/postgres/stage.sql
@@ -438,7 +438,7 @@ hashed_columns AS (
 {%- endif -%}
 {%- endif -%}
 
-{%- if enable_ghost_records -%}
+{%- if enable_ghost_records and not is_incremental() %}
 {# Creating Ghost Record for unknown case, based on datatype #}
 unknown_values AS (
 
@@ -597,7 +597,7 @@ columns_to_select AS (
 
     FROM {{ last_cte }}
 
-{% if enable_ghost_records %}
+{%- if enable_ghost_records and not is_incremental() %}
     UNION ALL
 
     SELECT

diff --git a/macros/staging/redshift/stage.sql b/macros/staging/redshift/stage.sql
@@ -447,7 +447,7 @@ hashed_columns AS (
 {%- endif -%}
 {%- endif -%}
 
-{%- if enable_ghost_records -%}
+{%- if enable_ghost_records and not is_incremental() %}
 {# Creating Ghost Record for unknown case, based on datatype #}
 unknown_values AS (
 
@@ -606,7 +606,7 @@ columns_to_select AS (
 
     FROM {{ last_cte }}
 
-{%- if enable_ghost_records %}
+{%- if enable_ghost_records and not is_incremental() %}
     UNION ALL
 
     SELECT

diff --git a/macros/staging/snowflake/stage.sql b/macros/staging/snowflake/stage.sql
@@ -389,8 +389,7 @@ hashed_columns AS (
 {%- endif -%}
 {%- endif -%}
 
-{% if enable_ghost_records %}
-{% if not is_incremental() %}
+{%- if enable_ghost_records and not is_incremental() %}
 {# Creating Ghost Record for unknown case, based on datatype #}
 unknown_values AS (
 
@@ -535,7 +534,6 @@ ghost_records AS (
     SELECT * FROM error_values
 ),
 {%- endif %}
-{%- endif %}
 
 {%- if not include_source_columns -%}
   {% set source_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=source_columns_to_select, exclude_columns_list=derived_column_names) %}
@@ -551,16 +549,14 @@ columns_to_select AS (
 
     FROM {{ last_cte }}
 
-{% if enable_ghost_records %}
-  {% if not is_incremental() %}
+{%- if enable_ghost_records and not is_incremental() %}
     UNION ALL
 
     SELECT
 
     {{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }}
 
     FROM ghost_records
-  {% endif %}
 {% endif %}
 )
 

diff --git a/macros/staging/synapse/stage.sql b/macros/staging/synapse/stage.sql
@@ -439,8 +439,7 @@ hashed_columns AS (
 {% set processed_hash_columns = datavault4dbt.process_hash_column_excludes(hashed_columns) -%}
 {%- endif -%}
 
-{% if enable_ghost_records %}
-{% if not is_incremental() %}
+{%- if enable_ghost_records and not is_incremental() %}
 {# Creating Ghost Record for unknown case, based on datatype #}
 unknown_values AS (
 
@@ -585,7 +584,6 @@ ghost_records AS (
     SELECT * FROM error_values
 ),
 {%- endif %}
-{%- endif %}
 
 {%- if not include_source_columns -%}
   {% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %}
@@ -600,16 +598,14 @@ columns_to_select AS (
 
     FROM {{ last_cte }}
 
-{% if enable_ghost_records %}
-  {% if not is_incremental() %}
+{%- if enable_ghost_records and not is_incremental() %}
     UNION ALL
 
     SELECT
 
     {{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }}
 
     FROM ghost_records
-  {% endif %}
 {% endif %}
 )