run_twitter.yml

name: Run Twitter Pipeline
"on":
  schedule:
    - cron: "0 */12 * * *" # Run every 12 hours
  workflow_dispatch: # Allow manual triggers
  push:
    branches:
      - main
    paths:
      - "pipeline/**"
      - ".github/workflows/run_twitter.yml"

env:
  # DLT specific environment variables
  TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }}
  motherduck_token: ${{ secrets.MOTHERDUCK_TOKEN }}

  # Optional: Add any other environment variables needed
  PYTHONUNBUFFERED: "1"

jobs:
  maybe_skip:
    runs-on: ubuntu-latest
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
      - id: skip_check
        uses: fkirc/skip-duplicate-actions@v5
        with:
          concurrent_skipping: always
          skip_after_successful_duplicate: false

  run_pipeline:
    needs: maybe_skip
    if: needs.maybe_skip.outputs.should_skip != 'true'
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: pipeline

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up uv
        uses: astral-sh/setup-uv@v4
        with:
          python-version: "3.10"
          enable-cache: true
          cache-dependency-glob: |
            pipeline/pyproject.toml
            pipeline/uv.lock

      - name: Run Twitter pipeline
        run: uv run python twitter_pipeline.py