diff --git a/warehouse/oso_dagster/assets/growthepie.py b/warehouse/oso_dagster/assets/growthepie.py index e647c7c4b..428f10aee 100644 --- a/warehouse/oso_dagster/assets/growthepie.py +++ b/warehouse/oso_dagster/assets/growthepie.py @@ -1,8 +1,6 @@ from dlt.sources.rest_api.typing import RESTAPIConfig -from ..factories import IntervalGCSAsset, interval_gcs_import_asset from ..factories.rest import create_rest_factory_asset -from ..utils.common import SourceMode, TimeInterval config: RESTAPIConfig = { "client": { @@ -28,22 +26,4 @@ growthepie_assets = dlt_assets( key_prefix="growthepie", -) - -growthepie_oli_data = interval_gcs_import_asset( - IntervalGCSAsset( - key_prefix="growthepie", - name="oli", - project_id="opensource-observer", - bucket_name="oso-dataset-transfer-bucket", - path_base="growthepie/oli", - file_match=r"oli_tag_mapping_(?P\d\d\d\d-\d\d-\d\d).parquet", - destination_table="oli_tag_mapping", - raw_dataset_name="oso_raw_sources", - clean_dataset_name="growthepie", - interval=TimeInterval.Daily, - mode=SourceMode.Overwrite, - retention_days=30, - format="PARQUET", - ), ) \ No newline at end of file diff --git a/warehouse/oso_dagster/assets/openlabelsinitiative.py b/warehouse/oso_dagster/assets/openlabelsinitiative.py new file mode 100644 index 000000000..0a91b2b05 --- /dev/null +++ b/warehouse/oso_dagster/assets/openlabelsinitiative.py @@ -0,0 +1,20 @@ +from ..factories import IntervalGCSAsset, interval_gcs_import_asset +from ..utils.common import SourceMode, TimeInterval + +openlabelsinitiative_data = interval_gcs_import_asset( + IntervalGCSAsset( + key_prefix="openlabelsinitiative", + name="oli", + project_id="opensource-observer", + bucket_name="oso-dataset-transfer-bucket", + path_base="openlabelsinitiative/oli", + file_match=r"oli_tag_mapping_(?P\d\d\d\d-\d\d-\d\d).parquet", + destination_table="oli_tag_mapping", + raw_dataset_name="oso_raw_sources", + clean_dataset_name="openlabelsinitiative", + interval=TimeInterval.Daily, + mode=SourceMode.Overwrite, + retention_days=30, + format="PARQUET", + ), +) \ No newline at end of file