Skip to content

Commit

Permalink
fix: move oli data to its own source (#2820)
Browse files Browse the repository at this point in the history
fix: move oli datato its own source
  • Loading branch information
ccerv1 authored Jan 19, 2025
1 parent a788e47 commit 62b0d53
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 20 deletions.
20 changes: 0 additions & 20 deletions warehouse/oso_dagster/assets/growthepie.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
from dlt.sources.rest_api.typing import RESTAPIConfig

from ..factories import IntervalGCSAsset, interval_gcs_import_asset
from ..factories.rest import create_rest_factory_asset
from ..utils.common import SourceMode, TimeInterval

config: RESTAPIConfig = {
"client": {
Expand All @@ -28,22 +26,4 @@

growthepie_assets = dlt_assets(
key_prefix="growthepie",
)

growthepie_oli_data = interval_gcs_import_asset(
IntervalGCSAsset(
key_prefix="growthepie",
name="oli",
project_id="opensource-observer",
bucket_name="oso-dataset-transfer-bucket",
path_base="growthepie/oli",
file_match=r"oli_tag_mapping_(?P<interval_timestamp>\d\d\d\d-\d\d-\d\d).parquet",
destination_table="oli_tag_mapping",
raw_dataset_name="oso_raw_sources",
clean_dataset_name="growthepie",
interval=TimeInterval.Daily,
mode=SourceMode.Overwrite,
retention_days=30,
format="PARQUET",
),
)
20 changes: 20 additions & 0 deletions warehouse/oso_dagster/assets/openlabelsinitiative.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
from ..factories import IntervalGCSAsset, interval_gcs_import_asset
from ..utils.common import SourceMode, TimeInterval

openlabelsinitiative_data = interval_gcs_import_asset(
IntervalGCSAsset(
key_prefix="openlabelsinitiative",
name="oli",
project_id="opensource-observer",
bucket_name="oso-dataset-transfer-bucket",
path_base="openlabelsinitiative/oli",
file_match=r"oli_tag_mapping_(?P<interval_timestamp>\d\d\d\d-\d\d-\d\d).parquet",
destination_table="oli_tag_mapping",
raw_dataset_name="oso_raw_sources",
clean_dataset_name="openlabelsinitiative",
interval=TimeInterval.Daily,
mode=SourceMode.Overwrite,
retention_days=30,
format="PARQUET",
),
)

0 comments on commit 62b0d53

Please sign in to comment.