Skip to content

Commit

Permalink
fix(json): determine the data is null or not (#746)
Browse files Browse the repository at this point in the history
  • Loading branch information
@grieve54706
grieve54706 authored Aug 15, 2024
1 parent 3f9358b commit a109f91
Show file tree
Hide file tree
Showing 8 changed files with 79 additions and 15 deletions.
23 changes: 12 additions & 11 deletions ibis-server/app/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,18 @@ def _to_json_obj(df: pd.DataFrame) -> dict:
data = df.to_dict(orient="split", index=False)

def default(d):
match d:
case decimal.Decimal():
return float(d)
case pd.Timestamp():
return d.value // 10**6
case datetime.datetime():
return int(d.timestamp())
case datetime.date():
return calendar.timegm(d.timetuple()) * 1000
case _:
raise d
if pd.isnull(d):
return None
if isinstance(d, decimal.Decimal):
return float(d)
elif isinstance(d, pd.Timestamp):
return d.value // 10**6
elif isinstance(d, datetime.datetime):
return int(d.timestamp())
elif isinstance(d, datetime.date):
return calendar.timegm(d.timetuple()) * 1000
else:
raise d

json_obj = orjson.loads(
orjson.dumps(
Expand Down
9 changes: 9 additions & 0 deletions ibis-server/tests/routers/v2/connector/test_bigquery.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@
"expression": "cast('2024-01-01T23:59:59' as timestamp with time zone)",
"type": "timestamp",
},
{
"name": "test_null_time",
"expression": "cast(NULL as timestamp)",
"type": "timestamp",
},
],
"primaryKey": "orderkey",
},
Expand Down Expand Up @@ -95,6 +100,7 @@ def test_query():
"1_370",
1704153599000,
1704153599000,
None,
]
assert result["dtypes"] == {
"orderkey": "int64",
Expand All @@ -105,6 +111,7 @@ def test_query():
"order_cust_key": "object",
"timestamp": "datetime64[ns]",
"timestamptz": "datetime64[ns, UTC]",
"test_null_time": "datetime64[ns]",
}


Expand Down Expand Up @@ -136,6 +143,7 @@ def test_query_with_column_dtypes():
"1_370",
"2024-01-01 23:59:59.000000",
"2024-01-01 23:59:59.000000 UTC",
None,
]
assert result["dtypes"] == {
"orderkey": "int64",
Expand All @@ -146,6 +154,7 @@ def test_query_with_column_dtypes():
"order_cust_key": "object",
"timestamp": "object",
"timestamptz": "object",
"test_null_time": "datetime64[ns]",
}


Expand Down
15 changes: 12 additions & 3 deletions ibis-server/tests/routers/v2/connector/test_clickhouse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@
"expression": "toDateTime64('2024-01-01T23:59:59', 9, 'UTC')",
"type": "timestamp",
},
{
"name": "test_null_time",
"expression": "toDateTime64(NULL, 9)",
"type": "timestamp",
},
{
"name": "customer",
"type": "Customer",
Expand Down Expand Up @@ -162,7 +167,7 @@ def test_query(clickhouse: ClickHouseContainer):
)
assert response.status_code == 200
result = response.json()
assert len(result["columns"]) == 9
assert len(result["columns"]) == 10
assert len(result["data"]) == 1
assert result["data"][0] == [
1,
Expand All @@ -173,6 +178,7 @@ def test_query(clickhouse: ClickHouseContainer):
"1_370",
1704153599000,
1704153599000,
None,
"Customer#000000370",
]
assert result["dtypes"] == {
Expand All @@ -184,6 +190,7 @@ def test_query(clickhouse: ClickHouseContainer):
"order_cust_key": "object",
"timestamp": "datetime64[ns]",
"timestamptz": "datetime64[ns, UTC]",
"test_null_time": "object",
"customer_name": "object",
}

Expand All @@ -200,7 +207,7 @@ def test_query_with_connection_url(clickhouse: ClickHouseContainer):
)
assert response.status_code == 200
result = response.json()
assert len(result["columns"]) == 9
assert len(result["columns"]) == 10
assert len(result["data"]) == 1
assert result["data"][0][0] == 1
assert result["dtypes"] is not None
Expand All @@ -224,7 +231,7 @@ def test_query_with_column_dtypes(clickhouse: ClickHouseContainer):
)
assert response.status_code == 200
result = response.json()
assert len(result["columns"]) == 9
assert len(result["columns"]) == 10
assert len(result["data"]) == 1
assert result["data"][0] == [
1,
Expand All @@ -235,6 +242,7 @@ def test_query_with_column_dtypes(clickhouse: ClickHouseContainer):
"1_370",
"2024-01-01 23:59:59.000000",
"2024-01-01 23:59:59.000000 UTC",
None,
"Customer#000000370",
]
assert result["dtypes"] == {
Expand All @@ -246,6 +254,7 @@ def test_query_with_column_dtypes(clickhouse: ClickHouseContainer):
"order_cust_key": "object",
"timestamp": "object",
"timestamptz": "object",
"test_null_time": "object",
"customer_name": "object",
}

Expand Down
9 changes: 9 additions & 0 deletions ibis-server/tests/routers/v2/connector/test_mssql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@
"expression": "cast('2024-01-01T23:59:59' as timestamp with time zone)",
"type": "timestamp",
},
{
"name": "test_null_time",
"expression": "cast(NULL as timestamp)",
"type": "timestamp",
},
],
"primaryKey": "orderkey",
},
Expand Down Expand Up @@ -111,6 +116,7 @@ def test_query(mssql: SqlServerContainer):
"1_370",
1704153599000,
1704153599000,
None,
]
assert result["dtypes"] == {
"orderkey": "int32",
Expand All @@ -121,6 +127,7 @@ def test_query(mssql: SqlServerContainer):
"order_cust_key": "object",
"timestamp": "datetime64[ns]",
"timestamptz": "datetime64[ns, UTC]",
"test_null_time": "datetime64[ns]",
}


Expand Down Expand Up @@ -174,6 +181,7 @@ def test_query_with_column_dtypes(mssql: SqlServerContainer):
"1_370",
"2024-01-01 23:59:59.000000",
"2024-01-01 23:59:59.000000 UTC",
None,
]
assert result["dtypes"] == {
"orderkey": "int32",
Expand All @@ -184,6 +192,7 @@ def test_query_with_column_dtypes(mssql: SqlServerContainer):
"order_cust_key": "object",
"timestamp": "object",
"timestamptz": "object",
"test_null_time": "datetime64[ns]",
}


Expand Down
9 changes: 9 additions & 0 deletions ibis-server/tests/routers/v2/connector/test_mysql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@
"expression": "cast('2024-01-01T23:59:59' as timestamp with time zone)",
"type": "timestamp",
},
{
"name": "test_null_time",
"expression": "cast(NULL as timestamp)",
"type": "timestamp",
},
],
"primaryKey": "orderkey",
},
Expand Down Expand Up @@ -117,6 +122,7 @@ def test_query(mysql: MySqlContainer):
"1_370",
1704153599000,
1704153599000,
None,
]
assert result["dtypes"] == {
"orderkey": "int32",
Expand All @@ -127,6 +133,7 @@ def test_query(mysql: MySqlContainer):
"order_cust_key": "object",
"timestamp": "datetime64[ns]",
"timestamptz": "datetime64[ns]",
"test_null_time": "datetime64[ns]",
}


Expand Down Expand Up @@ -177,6 +184,7 @@ def test_query_with_column_dtypes(mysql: MySqlContainer):
"1_370",
"2024-01-01 23:59:59.000000",
"2024-01-01 23:59:59.000000",
None,
]
assert result["dtypes"] == {
"orderkey": "int32",
Expand All @@ -187,6 +195,7 @@ def test_query_with_column_dtypes(mysql: MySqlContainer):
"order_cust_key": "object",
"timestamp": "object",
"timestamptz": "object",
"test_null_time": "datetime64[ns]",
}


Expand Down
11 changes: 10 additions & 1 deletion ibis-server/tests/routers/v2/connector/test_postgres.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@
"expression": "cast('2024-01-01T23:59:59' as timestamp with time zone)",
"type": "timestamp",
},
{
"name": "test_null_time",
"expression": "cast(NULL as timestamp)",
"type": "timestamp",
},
],
"primaryKey": "orderkey",
},
Expand Down Expand Up @@ -109,6 +114,7 @@ def test_query(postgres: PostgresContainer):
"1_370",
1704153599000,
1704153599000,
None,
]
assert result["dtypes"] == {
"orderkey": "int32",
Expand All @@ -119,6 +125,7 @@ def test_query(postgres: PostgresContainer):
"order_cust_key": "object",
"timestamp": "datetime64[ns]",
"timestamptz": "datetime64[ns, UTC]",
"test_null_time": "datetime64[ns]",
}


Expand Down Expand Up @@ -194,6 +201,7 @@ def test_query_with_column_dtypes(postgres: PostgresContainer):
"1_370",
"2024-01-01 23:59:59.000000",
"2024-01-01 23:59:59.000000 UTC",
None,
]
assert result["dtypes"] == {
"orderkey": "int32",
Expand All @@ -204,6 +212,7 @@ def test_query_with_column_dtypes(postgres: PostgresContainer):
"order_cust_key": "object",
"timestamp": "object",
"timestamptz": "object",
"test_null_time": "datetime64[ns]",
}


Expand Down Expand Up @@ -437,7 +446,7 @@ def test_dry_plan():
assert response.status_code == 200
assert (
response.text
== '''"WITH \\"Orders\\" AS (SELECT \\"Orders\\".\\"orderkey\\" AS \\"orderkey\\", \\"Orders\\".\\"custkey\\" AS \\"custkey\\", \\"Orders\\".\\"orderstatus\\" AS \\"orderstatus\\", \\"Orders\\".\\"totalprice\\" AS \\"totalprice\\", \\"Orders\\".\\"orderdate\\" AS \\"orderdate\\", \\"Orders\\".\\"order_cust_key\\" AS \\"order_cust_key\\", \\"Orders\\".\\"timestamp\\" AS \\"timestamp\\", \\"Orders\\".\\"timestamptz\\" AS \\"timestamptz\\" FROM (SELECT \\"Orders\\".\\"orderkey\\" AS \\"orderkey\\", \\"Orders\\".\\"custkey\\" AS \\"custkey\\", \\"Orders\\".\\"orderstatus\\" AS \\"orderstatus\\", \\"Orders\\".\\"totalprice\\" AS \\"totalprice\\", \\"Orders\\".\\"orderdate\\" AS \\"orderdate\\", \\"Orders\\".\\"order_cust_key\\" AS \\"order_cust_key\\", \\"Orders\\".\\"timestamp\\" AS \\"timestamp\\", \\"Orders\\".\\"timestamptz\\" AS \\"timestamptz\\" FROM (SELECT o_orderkey AS \\"orderkey\\", o_custkey AS \\"custkey\\", o_orderstatus AS \\"orderstatus\\", o_totalprice AS \\"totalprice\\", o_orderdate AS \\"orderdate\\", CONCAT(o_orderkey, '_', o_custkey) AS \\"order_cust_key\\", CAST('2024-01-01T23:59:59' AS TIMESTAMP) AS \\"timestamp\\", CAST('2024-01-01T23:59:59' AS TIMESTAMPTZ) AS \\"timestamptz\\" FROM (SELECT * FROM public.orders) AS \\"Orders\\") AS \\"Orders\\") AS \\"Orders\\") SELECT orderkey, order_cust_key FROM \\"Orders\\" LIMIT 1"'''
== '''"WITH \\"Orders\\" AS (SELECT \\"Orders\\".\\"orderkey\\" AS \\"orderkey\\", \\"Orders\\".\\"custkey\\" AS \\"custkey\\", \\"Orders\\".\\"orderstatus\\" AS \\"orderstatus\\", \\"Orders\\".\\"totalprice\\" AS \\"totalprice\\", \\"Orders\\".\\"orderdate\\" AS \\"orderdate\\", \\"Orders\\".\\"order_cust_key\\" AS \\"order_cust_key\\", \\"Orders\\".\\"timestamp\\" AS \\"timestamp\\", \\"Orders\\".\\"timestamptz\\" AS \\"timestamptz\\", \\"Orders\\".\\"test_null_time\\" AS \\"test_null_time\\" FROM (SELECT \\"Orders\\".\\"orderkey\\" AS \\"orderkey\\", \\"Orders\\".\\"custkey\\" AS \\"custkey\\", \\"Orders\\".\\"orderstatus\\" AS \\"orderstatus\\", \\"Orders\\".\\"totalprice\\" AS \\"totalprice\\", \\"Orders\\".\\"orderdate\\" AS \\"orderdate\\", \\"Orders\\".\\"order_cust_key\\" AS \\"order_cust_key\\", \\"Orders\\".\\"timestamp\\" AS \\"timestamp\\", \\"Orders\\".\\"timestamptz\\" AS \\"timestamptz\\", \\"Orders\\".\\"test_null_time\\" AS \\"test_null_time\\" FROM (SELECT o_orderkey AS \\"orderkey\\", o_custkey AS \\"custkey\\", o_orderstatus AS \\"orderstatus\\", o_totalprice AS \\"totalprice\\", o_orderdate AS \\"orderdate\\", CONCAT(o_orderkey, '_', o_custkey) AS \\"order_cust_key\\", CAST('2024-01-01T23:59:59' AS TIMESTAMP) AS \\"timestamp\\", CAST('2024-01-01T23:59:59' AS TIMESTAMPTZ) AS \\"timestamptz\\", CAST(NULL AS TIMESTAMP) AS \\"test_null_time\\" FROM (SELECT * FROM public.orders) AS \\"Orders\\") AS \\"Orders\\") AS \\"Orders\\") SELECT orderkey, order_cust_key FROM \\"Orders\\" LIMIT 1"'''
)


Expand Down
9 changes: 9 additions & 0 deletions ibis-server/tests/routers/v2/connector/test_snowflake.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@
"expression": "cast('2024-01-01T23:59:59' as timestamp with time zone)",
"type": "timestamp",
},
{
"name": "test_null_time",
"expression": "cast(NULL as timestamp)",
"type": "timestamp",
},
],
"primaryKey": "orderkey",
},
Expand Down Expand Up @@ -98,6 +103,7 @@ def test_query():
"1_36901",
1704153599000,
1704153599000,
None,
]
assert result["dtypes"] == {
"orderkey": "int64",
Expand All @@ -108,6 +114,7 @@ def test_query():
"order_cust_key": "object",
"timestamp": "datetime64[ns]",
"timestamptz": "datetime64[ns, UTC]",
"test_null_time": "datetime64[ns]",
}


Expand Down Expand Up @@ -139,6 +146,7 @@ def test_query_with_column_dtypes():
"1_36901",
"2024-01-01 23:59:59.000000",
"2024-01-01 23:59:59.000000 UTC",
None,
]
assert result["dtypes"] == {
"orderkey": "int64",
Expand All @@ -149,6 +157,7 @@ def test_query_with_column_dtypes():
"order_cust_key": "object",
"timestamp": "object",
"timestamptz": "object",
"test_null_time": "datetime64[ns]",
}


Expand Down
9 changes: 9 additions & 0 deletions ibis-server/tests/routers/v2/connector/test_trino.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@
"expression": "with_timezone(TIMESTAMP '2024-01-01 23:59:59', 'UTC')",
"type": "timestamp",
},
{
"name": "test_null_time",
"expression": "cast(NULL as timestamp)",
"type": "timestamp",
},
],
"primaryKey": "orderkey",
},
Expand Down Expand Up @@ -88,6 +93,7 @@ def test_query(trino: TrinoContainer):
"1_370",
1704153599000,
1704153599000,
None,
]
assert result["dtypes"] == {
"orderkey": "int64",
Expand All @@ -98,6 +104,7 @@ def test_query(trino: TrinoContainer):
"order_cust_key": "object",
"timestamp": "datetime64[ns]",
"timestamptz": "datetime64[ns, UTC]",
"test_null_time": "datetime64[ns]",
}


Expand Down Expand Up @@ -148,6 +155,7 @@ def test_query_with_column_dtypes(trino: TrinoContainer):
"1_370",
"2024-01-01 23:59:59.000000",
"2024-01-01 23:59:59.000000 UTC",
None,
]
assert result["dtypes"] == {
"orderkey": "int64",
Expand All @@ -158,6 +166,7 @@ def test_query_with_column_dtypes(trino: TrinoContainer):
"order_cust_key": "object",
"timestamp": "object",
"timestamptz": "object",
"test_null_time": "datetime64[ns]",
}


Expand Down

0 comments on commit a109f91

Please sign in to comment.