From f2aac7a2856791c73f0cb9a4167330e829dcc469 Mon Sep 17 00:00:00 2001 From: Theo Kiehn <162969167+tkiehn@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:11:05 +0200 Subject: [PATCH] fix redshift ghost records --- .../supporting/ghost_record_per_datatype.sql | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/macros/supporting/ghost_record_per_datatype.sql b/macros/supporting/ghost_record_per_datatype.sql index 7fec0dd3..cf0ebd7a 100644 --- a/macros/supporting/ghost_record_per_datatype.sql +++ b/macros/supporting/ghost_record_per_datatype.sql @@ -349,25 +349,30 @@ {%- set beginning_of_all_times = datavault4dbt.beginning_of_all_times() -%} {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%} {%- set timestamp_format = datavault4dbt.timestamp_format() -%} + {%- set beginning_of_all_times_date = var('datavault4dbt.beginning_of_all_times_date', '0001-01-01') -%} {%- set end_of_all_times_date = var('datavault4dbt.end_of_all_times_date', '8888-12-31') -%} {%- set date_format = var('datavault4dbt.date_format', 'YYYY-mm-dd') -%} + {%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%} {%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%} + {%- set hash = datavault4dbt.hash_method() -%} {%- set hash_default_values = datavault4dbt.hash_default_values(hash_function=hash) -%} {%- set hash_alg= hash_default_values['hash_alg'] -%} {%- set unknown_value__HASHTYPE = hash_default_values['unknown_key'] -%} {%- set error_value__HASHTYPE = hash_default_values['error_key'] -%} + {%- set datatype = datatype | string | upper | trim -%} {%- if ghost_record_type == 'unknown' -%} {%- if 'TIMESTAMP' in datatype %}{{ datavault4dbt.string_to_timestamp(timestamp_format, beginning_of_all_times_date) }} AS {{ alias }} {%- elif datatype == 'TIMETZ' %} CAST('00:00:01 UTC' as TIMETZ) as {{ alias }} {%- elif datatype == 'TIME' %} CAST('00:00:01' as TIME) as {{ alias }} - {%- elif datatype == 'DATE'-%} TO_DATE('{{ beginning_of_all_times_date }}', '{{ date_format }}' ) as "{{ alias }}" - {%- elif datatype in ['CHAR', 'CHARACTER', 'NCHAR', 'BPCHAR', 'VARCHAR', 'CHARACTER VARYING', 'NVARCHAR', 'TEXT'] %} '{{unknown_value__STRING}}' as {{ alias }} - {%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8', 'DECIMAL', 'NUMERIC'] %} CAST(0 as {{ datatype }}) as {{ alias }} + {%- elif datatype == 'DATE'-%} TO_DATE('{{ beginning_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }} + {%- elif 'CHAR' in datatype or datatype == 'TEXT' %} '{{unknown_value__STRING}}' as {{ alias }} + {%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST(0 as {{ datatype }}) as {{ alias }} + {%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST(0 as {{ datatype }}) as {{ alias }} {%- elif datatype in ['BOOLEAN', 'BOOL'] %} CAST('TRUE' as BOOLEAN) as {{ alias }} {%- elif datatype in ['VARBYTE', 'VARBINARY', 'BINARY VARYING'] %} CAST('{{ unknown_value__HASHTYPE }}' as {{ datatype }}) as {{ alias }} {%- elif datatype == 'GEOMETRY' %} CAST(ST_POINT(0, 90) as {{ datatype }}) as {{ alias }} @@ -377,9 +382,10 @@ {%- if 'TIMESTAMP' in datatype %}{{ datavault4dbt.string_to_timestamp(timestamp_format, end_of_all_times) }} AS {{ alias }} {%- elif datatype == 'TIMETZ' %} CAST('23:59:59 UTC' as TIMETZ) as {{ alias }} {%- elif datatype == 'TIME' %} CAST('23:59:59' as TIME) as {{ alias }} - {%- elif datatype == 'DATE'-%} TO_DATE('{{ end_of_all_times_date }}', '{{ date_format }}' ) as "{{ alias }}" - {%- elif datatype in ['CHAR', 'CHARACTER', 'NCHAR', 'BPCHAR', 'VARCHAR', 'CHARACTER VARYING', 'NVARCHAR', 'TEXT'] %} '{{error_value__STRING}}' as {{ alias }} - {%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8', 'DECIMAL', 'NUMERIC'] %} CAST(-1 as {{ datatype }}) as {{ alias }} + {%- elif datatype == 'DATE'-%} TO_DATE('{{ end_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }} + {%- elif 'CHAR' in datatype or datatype == 'TEXT' %} '{{error_value__STRING}}' as {{ alias }} + {%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST(-1 as {{ datatype }}) as {{ alias }} + {%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST(-1 as {{ datatype }}) as {{ alias }} {%- elif datatype in ['BOOLEAN', 'BOOL'] %} CAST('FALSE' as BOOLEAN) as {{ alias }} {%- elif datatype in ['VARBYTE', 'VARBINARY', 'BINARY VARYING'] %} CAST('{{ error_value__HASHTYPE }}' as {{ datatype }}) as {{ alias }} {%- elif datatype == 'GEOMETRY' %} CAST(ST_POINT(0, 90) as {{ datatype }}) as {{ alias }} @@ -390,5 +396,4 @@ {{ exceptions.raise_compiler_error("Invalid Ghost Record Type. Accepted are 'unknown' and 'error'.") }} {%- endif %} {%- endif -%} - {%- endmacro -%}