main.yml

name: Get a list of domains from m.kuku.lu
on:
  workflow_dispatch:
  schedule:
    - cron: "0 * * * *"

jobs:
  crawler:
    runs-on: ubuntu-latest
    permissions:
      contents: write
      packages: write
    strategy:
      max-parallel: 2
      fail-fast: false
      matrix:
        loop: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
    steps:
      # chekout this repo
      - name: Checkout repository
        uses: actions/checkout@v2

      # setup python3.9
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.9"

      # pip install -r requirements.txt
      - name: Install dependencies
        run: |
          pip install -r ./src/requirements.txt

      # run crawler
      - name: Run scraping script
        run: |
          python ./src/robot.py

      # push files
      - name: commit files
        run: |
          git config --global user.name "github-actions[bot]"
          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add domains.txt
          if git diff --cached --quiet; then
            echo "No changes detected."
          else
            git commit -m 'update: add new domain to domains.txt'
            git push origin main
          fi
  
  sort:
    runs-on: ubuntu-latest
    needs: crawler
    permissions:
      contents: write
      packages: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.9"

      #- name: Install dependencies
      #  run: |
      #    pip install -r ./src/requirements.txt

      - name: Sort domains
        run: |
          python ./src/sort.py

      - name: commit files
        run: |
          git config --global user.name "github-actions[bot]"
          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add domains.txt
          if git diff --cached --quiet; then
            echo "No changes detected."
          else
            git commit -m 'update: sorted domains.txt'
            git push origin main
          fi