dagster-gcp-pandas

import pandas as pd from dagster import asset, Definitions from dagster_gcp_pandas import GCSPandasIOManager import os @asset def my_pandas_dataframe_asset() -> pd.DataFrame: """Produces a Pandas DataFrame.""" return pd.DataFrame({"value": [1, 2, 3], "label": ["A", "B", "C"]}) # Configure the GCSPandasIOManager to store DataFrames in a specified GCS bucket. # Ensure the GCS_BUCKET_NAME environment variable is set or replace "your-gcs-bucket-name". # You also need appropriate GCP credentials configured (e.g., GOOGLE_APPLICATION_CREDENTIALS). gcs_io_manager = GCSPandasIOManager( gcs_bucket=os.environ.get("GCS_BUCKET_NAME", "your-gcs-bucket-name"), gcs_prefix="dagster_assets/pandas" ) defs = Definitions( assets=[my_pandas_dataframe_asset], resources={ "io_manager": gcs_io_manager } ) # To run: # 1. Save this code as a Python file (e.g., my_project/repo.py) # 2. Set the GCS_BUCKET_NAME environment variable: # export GCS_BUCKET_NAME="your-actual-bucket-name" # 3. Ensure GCP credentials are set up (e.g., using `gcloud auth application-default login` # or `GOOGLE_APPLICATION_CREDENTIALS` environment variable pointing to a service account key). # 4. Execute: `dagster dev -f my_project/repo.py` # 5. Navigate to the Dagster UI (usually http://localhost:3000) and materialize the asset.