crawl.yaml

name: Crawl Workflow

on: [push]

jobs:
  build:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2

    - name: Set up Python 3.8
      uses: actions/setup-python@v2
      with:
        python-version: 3.8

    - name: Install dependencies
      run: |
        python -m venv .env
        source .env/bin/activate
        pip install -r requirements.txt

    - name: Test crawler and indexer
      run: |
        source .env/bin/activate
        python3 -m src.phantom --num_threads 2 --urls "https://github.com/AnsahMohammad" "https://github.com/AnsahMohammad" --sleep 6

        python3 -m nltk.downloader stopwords
        python3 -m nltk.downloader punkt

        python3 -m src.phantom_indexing

        echo "Crawling completed"
      
    - name: test search Engine server
      run: |
        source .env/bin/activate
        python3 phantom.py &
        sleep 10
        curl -f http://localhost:5000/
        echo "Flask test completed"