taxi-rides-job.yaml

apiVersion: batch/v1
kind: Job
metadata:
  name: taxi-rides-job
spec:
  template:
    spec:
      initContainers:
      - name: taxi-rides
        workingDir: /app
        env:
        - name: DATA_DIR
          value: "/app/data"
        volumeMounts:
        - name: config-volume
          mountPath: /app/config.yaml
          subPath: config.yaml
        - name: data-volume
          mountPath: /app/data
        image: ghcr.io/serkosi/taxi-rides:latest
      containers:
      - name: data-handler
        image: python:3.9-slim
        command: ["sh", "-c"]
        args:
        - |
          pip install pandas pyarrow
          echo "Processing complete, Parquet data is ready"
          python3 -c '
          import pandas as pd
          import os
          import sys
          
          # Verify Parquet files are readable
          parquet_path = "/app/data/taxi_trips.parquet"
          if not os.path.exists(parquet_path):
            print(f"Error: Parquet file not found at {parquet_path}")
            sys.exit(1)

          df = pd.read_parquet(parquet_path)
          print(f"Parquet file verified: {df.shape[0]} rows")
          '
          echo "Data verification complete"
        volumeMounts:
        - name: data-volume
          mountPath: /app/data          
      restartPolicy: Never
      imagePullSecrets:
      - name: ghcr-secret
      volumes:
      - name: config-volume
        configMap:
          name: taxi-rides-config
      - name: data-volume
        persistentVolumeClaim:
          claimName: taxi-rides-pvc