Models are failing to compile with v1.5.27
models % cat snow_kafka_users_info.sql
{{ config(
materialized='incremental',
on_schema_change='append_new_columns',
sync = True,
source = 'KAFKA',
target_type = 'SNOWFLAKE',
target_connection = 'snowflake_target_conn',
target_schema = 'ROYON',
target_table_alias = 'USER_INFO_TBL',
options = {
'topic': 'user_info',
'EXCLUDE_COLUMNS': ['password'],
'COLUMN_TRANSFORMATIONS': {'credit_card':'MD5(credit_card)'},
'DEDUPLICATE_WITH': {'COLUMNS': ['user_id'], 'WINDOW': '1 HOURS'},
'WRITE_INTERVAL': '1 MINUTES',
'CREATE_TABLE_IF_MISSING': True,
'COMPUTE_CLUSTER': 'Default Compute (Free)',
'START_FROM': 'NOW'
},
primary_key=[{'field':'user_id', 'type':'string'}]
)
}}
SELECT * FROM {{ ref('upsolver_kafka_samples') }}
Core:
- installed: 1.5.2
- latest: 1.6.3 - Update available!
Your version of dbt-core is out of date!
You can find instructions for upgrading here:
https://docs.getdbt.com/docs/installation
Plugins:
- upsolver: 1.5.24 - Update available!
18:03:08 On model.kafka_to_snowflake_direct.snow_kafka_users_info: /* {"app": "dbt", "dbt_version": "1.5.2", "profile_name": "kafka_to_snowflake_direct", "target_name": "dev", "node_id": "model.kafka_to_snowflake_direct.snow_kafka_users_info"} */
CREATE SYNC JOB snow_kafka_users_info_job
COMPUTE_CLUSTER = "Default Compute (Free)"
START_FROM = NOW
EXCLUDE_COLUMNS = ('password')
CREATE_TABLE_IF_MISSING = True
DEDUPLICATE_WITH = (WINDOW=1 HOURS ,COLUMNS=(user_id))
WRITE_INTERVAL = 1 MINUTES
COLUMN_TRANSFORMATIONS = (credit_card=MD5(credit_card))
AS COPY FROM KAFKA upsolver_kafka_samples
topic = 'user_info'
INTO snowflake snowflake_target_conn.ROYON.USER_INFO_TBL
WITH EXPECTATION not_null__user_id EXPECT user_id IS NOT NULL ON VIOLATION WARN
WITH EXPECTATION not_null__address EXPECT address IS NOT NULL ON VIOLATION WARN
WITH EXPECTATION not_null__first_name EXPECT first_name IS NOT NULL ON VIOLATION WARN
WITH EXPECTATION not_null__last_name EXPECT last_name IS NOT NULL ON VIOLATION WARN
WITH EXPECTATION not_null__phone EXPECT phone IS NOT NULL ON VIOLATION WARN
18:03:08 Opening a new connection, currently in state closed
18:03:10 SQL status: OK -1 in 2.0 seconds
18:03:10 Timing info for model.kafka_to_snowflake_direct.snow_kafka_users_info (execute): 14:03:08.799776 => 14:03:10.937171
18:03:10 On model.kafka_to_snowflake_direct.snow_kafka_users_info: Close
18:03:10 Sending event: {'category': 'dbt', 'action': 'run_model', 'label': '2b47dc7c-9a17-4542-8650-7e85c28c79f0', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x104944070>]}
18:03:10 1 of 1 OK created sql incremental model ROYON.snow_kafka_users_info ............ [OK -1 in 2.15s]
18:03:10 Finished running node model.kafka_to_snowflake_direct.snow_kafka_users_info
18:03:10 Connection 'master' was properly closed.
18:03:10 Connection 'model.kafka_to_snowflake_direct.snow_kafka_users_info' was properly closed.
18:03:10
18:03:10 Finished running 1 incremental model in 0 hours 0 minutes and 4.19 seconds (4.19s).
18:03:10 Command end result
18:03:10
18:03:10 Completed successfully
18:03:10
18:03:10 Done. PASS=1 WARN=0 ERROR=0 SKIP=0 TOTAL=1
18:03:10 Command `dbt build` succeeded at 14:03:10.952687 after 4.46 seconds
18:03:10 Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1031c5bd0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10485e740>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1044d6770>]}
18:03:10 Flushing usage events
Same model is failing to compile with v1.5.27
18:08:23 On model.kafka_to_snowflake_direct.snow_kafka_users_info: /* {"app": "dbt", "dbt_version": "1.5.2", "profile_name": "kafka_to_snowflake_direct", "target_name": "dev", "node_id": "model.kafka_to_snowflake_direct.snow_kafka_users_info"} */
CREATE SYNC JOB snow_kafka_users_info_job
EXCLUDE_COLUMNS = ('password')
COMPUTE_CLUSTER = "Default Compute (Free)"
DEDUPLICATE_WITH = (WINDOW=1 HOURS ,COLUMNS=(user_id))
START_FROM = NOW
COLUMN_TRANSFORMATIONS = (credit_card=MD5(credit_card))
EXCLUDE_COLUMNS = ('password')
CREATE_TABLE_IF_MISSING = True
DEDUPLICATE_WITH = (WINDOW=1 HOURS ,COLUMNS=(user_id))
COLUMN_TRANSFORMATIONS = (credit_card=MD5(credit_card))
AS COPY FROM KAFKA upsolver_kafka_samples
topic = 'user_info'
INTO snowflake DEMO_DB.ROYON.snow_kafka_users_info
18:08:23 Opening a new connection, currently in state closed
18:08:23 Timing info for model.kafka_to_snowflake_direct.snow_kafka_users_info (execute): 14:08:23.727022 => 14:08:23.931617
18:08:23 On model.kafka_to_snowflake_direct.snow_kafka_users_info: Close
18:08:23 Database Error in model snow_kafka_users_info (models/snow_kafka_users_info.sql)
Wrong SQL query: Syntax Error : Duplicate option found: EXCLUDE_COLUMNS, each option may appear at most once [request_id=6447e2b6-3878-491d-92e0-3760d6d5bdcc]
compiled Code at target/run/kafka_to_snowflake_direct/models/snow_kafka_users_info.sql
18:08:23 Sending event: {'category': 'dbt', 'action': 'run_model', 'label': '17a50420-0e94-44f3-b577-cf9f1ce5a416', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x108612770>]}
18:08:23 1 of 1 ERROR creating sql incremental model ROYON.snow_kafka_users_info ........ [ERROR in 0.22s]
18:08:23 Finished running node model.kafka_to_snowflake_direct.snow_kafka_users_info
18:08:23 Connection 'master' was properly closed.
18:08:23 Connection 'model.kafka_to_snowflake_direct.snow_kafka_users_info' was properly closed.
18:08:23
18:08:23 Finished running 1 incremental model in 0 hours 0 minutes and 1.36 seconds (1.36s).
18:08:23 Command end result
18:08:23
18:08:23 Completed with 1 error and 0 warnings:
18:08:23
18:08:23 Database Error in model snow_kafka_users_info (models/snow_kafka_users_info.sql)
18:08:23 Wrong SQL query: Syntax Error : Duplicate option found: EXCLUDE_COLUMNS, each option may appear at most once [request_id=6447e2b6-3878-491d-92e0-3760d6d5bdcc]
18:08:23 compiled Code at target/run/kafka_to_snowflake_direct/models/snow_kafka_users_info.sql
18:08:23
18:08:23 Done. PASS=0 WARN=0 ERROR=1 SKIP=0 TOTAL=1
Models are failing to compile with v1.5.27
Same model is failing to compile with v1.5.27