From 0d0139b4d68d1faddf9e2e77423e4490c4694c0b Mon Sep 17 00:00:00 2001 From: Theo Kiehn <162969167+tkiehn@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:13:09 +0200 Subject: [PATCH 1/3] only create ghost records when it is not an incremental run --- macros/staging/bigquery/stage.sql | 4 ++-- macros/staging/exasol/stage.sql | 4 ++-- macros/staging/postgres/stage.sql | 4 ++-- macros/staging/redshift/stage.sql | 4 ++-- macros/staging/snowflake/stage.sql | 8 ++------ macros/staging/synapse/stage.sql | 7 ++----- 6 files changed, 12 insertions(+), 19 deletions(-) diff --git a/macros/staging/bigquery/stage.sql b/macros/staging/bigquery/stage.sql index e8eb5789..8c94c387 100644 --- a/macros/staging/bigquery/stage.sql +++ b/macros/staging/bigquery/stage.sql @@ -438,7 +438,7 @@ hashed_columns AS ( {%- endif -%} {%- endif -%} -{% if enable_ghost_records %} +{%- if enable_ghost_records and not is_incremental() %} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( @@ -597,7 +597,7 @@ columns_to_select AS ( FROM {{ last_cte }} -{%- if enable_ghost_records %} +{%- if enable_ghost_records and not is_incremental() %} UNION ALL SELECT diff --git a/macros/staging/exasol/stage.sql b/macros/staging/exasol/stage.sql index 3935599d..50dd35d3 100644 --- a/macros/staging/exasol/stage.sql +++ b/macros/staging/exasol/stage.sql @@ -426,7 +426,7 @@ hashed_columns AS ( {%- endif -%} {%- endif -%} -{% if enable_ghost_records %} +{%- if enable_ghost_records and not is_incremental() %} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( SELECT @@ -580,7 +580,7 @@ columns_to_select AS ( FROM {{ last_cte }} -{%- if enable_ghost_records %} +{%- if enable_ghost_records and not is_incremental() %} UNION ALL SELECT diff --git a/macros/staging/postgres/stage.sql b/macros/staging/postgres/stage.sql index d5a3d0ad..9edd3a38 100644 --- a/macros/staging/postgres/stage.sql +++ b/macros/staging/postgres/stage.sql @@ -438,7 +438,7 @@ hashed_columns AS ( {%- endif -%} {%- endif -%} -{%- if enable_ghost_records -%} +{%- if enable_ghost_records and not is_incremental() %} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( @@ -597,7 +597,7 @@ columns_to_select AS ( FROM {{ last_cte }} -{% if enable_ghost_records %} +{%- if enable_ghost_records and not is_incremental() %} UNION ALL SELECT diff --git a/macros/staging/redshift/stage.sql b/macros/staging/redshift/stage.sql index 016b5e76..467cdace 100644 --- a/macros/staging/redshift/stage.sql +++ b/macros/staging/redshift/stage.sql @@ -447,7 +447,7 @@ hashed_columns AS ( {%- endif -%} {%- endif -%} -{%- if enable_ghost_records -%} +{%- if enable_ghost_records and not is_incremental() %} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( @@ -606,7 +606,7 @@ columns_to_select AS ( FROM {{ last_cte }} -{%- if enable_ghost_records %} +{%- if enable_ghost_records and not is_incremental() %} UNION ALL SELECT diff --git a/macros/staging/snowflake/stage.sql b/macros/staging/snowflake/stage.sql index 2a04f34e..956c632e 100644 --- a/macros/staging/snowflake/stage.sql +++ b/macros/staging/snowflake/stage.sql @@ -389,8 +389,7 @@ hashed_columns AS ( {%- endif -%} {%- endif -%} -{% if enable_ghost_records %} -{% if not is_incremental() %} +{%- if enable_ghost_records and not is_incremental() %} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( @@ -535,7 +534,6 @@ ghost_records AS ( SELECT * FROM error_values ), {%- endif %} -{%- endif %} {%- if not include_source_columns -%} {% set source_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=source_columns_to_select, exclude_columns_list=derived_column_names) %} @@ -551,8 +549,7 @@ columns_to_select AS ( FROM {{ last_cte }} -{% if enable_ghost_records %} - {% if not is_incremental() %} +{%- if enable_ghost_records and not is_incremental() %} UNION ALL SELECT @@ -560,7 +557,6 @@ columns_to_select AS ( {{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }} FROM ghost_records - {% endif %} {% endif %} ) diff --git a/macros/staging/synapse/stage.sql b/macros/staging/synapse/stage.sql index 027f604b..935ac2ed 100644 --- a/macros/staging/synapse/stage.sql +++ b/macros/staging/synapse/stage.sql @@ -439,8 +439,7 @@ hashed_columns AS ( {% set processed_hash_columns = datavault4dbt.process_hash_column_excludes(hashed_columns) -%} {%- endif -%} -{% if enable_ghost_records %} -{% if not is_incremental() %} +{%- if enable_ghost_records and not is_incremental() %} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( @@ -600,8 +599,7 @@ columns_to_select AS ( FROM {{ last_cte }} -{% if enable_ghost_records %} - {% if not is_incremental() %} +{%- if enable_ghost_records and not is_incremental() %} UNION ALL SELECT @@ -609,7 +607,6 @@ columns_to_select AS ( {{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }} FROM ghost_records - {% endif %} {% endif %} ) From e18090cf2083d9dc7eb51bba4265ecc4928744e7 Mon Sep 17 00:00:00 2001 From: Theo Kiehn <162969167+tkiehn@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:09:10 +0200 Subject: [PATCH 2/3] fix additional endif --- macros/staging/synapse/stage.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/macros/staging/synapse/stage.sql b/macros/staging/synapse/stage.sql index 935ac2ed..84edee88 100644 --- a/macros/staging/synapse/stage.sql +++ b/macros/staging/synapse/stage.sql @@ -584,7 +584,6 @@ ghost_records AS ( SELECT * FROM error_values ), {%- endif %} -{%- endif %} {%- if not include_source_columns -%} {% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %} From 34098fd6809e32cfe088230698067f05918d2209 Mon Sep 17 00:00:00 2001 From: Theo Kiehn <162969167+tkiehn@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:50:40 +0200 Subject: [PATCH 3/3] Trigger Pipeline