onyx/.github/workflows/pr-python-model-tests.yml

name: Model Server Tests

on:
  schedule:
    # This cron expression runs the job daily at 16:00 UTC (9am PT)
    - cron: "0 16 * * *"
  workflow_dispatch:
    inputs:
      branch:
        description: 'Branch to run the workflow on'
        required: false
        default: 'main'
        
env:
  # Bedrock
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }}

  # OpenAI
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

jobs:
  model-check:
    # See https://runs-on.com/runners/linux/
    runs-on: [runs-on,runner=8cpu-linux-x64,"run-id=${{ github.run_id }}"]

    env:
      PYTHONPATH: ./backend

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_TOKEN }}

      # tag every docker image with "test" so that we can spin up the correct set
      # of images during testing

      # We don't need to build the Web Docker image since it's not yet used
      # in the integration tests. We have a separate action to verify that it builds
      # successfully.
      - name: Pull Model Server Docker image
        run: |
          docker pull onyxdotapp/onyx-model-server:latest
          docker tag onyxdotapp/onyx-model-server:latest onyxdotapp/onyx-model-server:test
          
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: "pip"
          cache-dependency-path: |
            backend/requirements/default.txt
            backend/requirements/dev.txt

      - name: Install Dependencies
        run: |
          python -m pip install --upgrade pip
          pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
          pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt

      - name: Start Docker containers
        run: |
          cd deployment/docker_compose
          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
          AUTH_TYPE=basic \
          REQUIRE_EMAIL_VERIFICATION=false \
          DISABLE_TELEMETRY=true \
          IMAGE_TAG=test \
          docker compose -f docker-compose.dev.yml -p onyx-stack up -d indexing_model_server
        id: start_docker

      - name: Wait for service to be ready
        run: |
          echo "Starting wait-for-service script..."

          start_time=$(date +%s)
          timeout=300  # 5 minutes in seconds

          while true; do
            current_time=$(date +%s)
            elapsed_time=$((current_time - start_time))
            
            if [ $elapsed_time -ge $timeout ]; then
              echo "Timeout reached. Service did not become ready in 5 minutes."
              exit 1
            fi
            
            # Use curl with error handling to ignore specific exit code 56
            response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:9000/api/health || echo "curl_error")
            
            if [ "$response" = "200" ]; then
              echo "Service is ready!"
              break
            elif [ "$response" = "curl_error" ]; then
              echo "Curl encountered an error, possibly exit code 56. Continuing to retry..."
            else
              echo "Service not ready yet (HTTP status $response). Retrying in 5 seconds..."
            fi
            
            sleep 5
          done
          echo "Finished waiting for service."
          
      - name: Run Tests
        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
        run: |
          py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/llm
          py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/embedding

      - name: Alert on Failure
        if: failure() && github.event_name == 'schedule'
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
        run: |
          curl -X POST \
            -H 'Content-type: application/json' \
            --data '{"text":"Scheduled Model Tests failed! Check the run at: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' \
            $SLACK_WEBHOOK
            
      - name: Stop Docker containers
        if: always()
        run: |
          cd deployment/docker_compose
          docker compose -f docker-compose.dev.yml -p onyx-stack down -v