onyx/.github/workflows/pr-python-model-tests.yml

name: Model Server Tests

on:
  schedule:
    # This cron expression runs the job daily at 16:00 UTC (9am PT)
    - cron: "0 16 * * *"
  workflow_dispatch:

permissions:
  contents: read

env:
  # Bedrock
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_REGION_NAME: ${{ vars.AWS_REGION_NAME }}

  # API keys for testing
  COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
  AZURE_API_URL: ${{ vars.AZURE_API_URL }}

jobs:
  model-check:
    # See https://runs-on.com/runners/linux/
    runs-on:
      - runs-on
      - runner=4cpu-linux-arm64
      - "run-id=${{ github.run_id }}-model-check"
      - "extras=ecr-cache"
    environment: ci-protected
    timeout-minutes: 20

    env:
      PYTHONPATH: ./backend

    steps:
      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Setup Python and Install Dependencies
        uses: ./.github/actions/setup-python-and-install-dependencies
        with:
          requirements: |
            backend/requirements/default.txt
            backend/requirements/dev.txt

      - name: Format branch name for cache
        id: format-branch
        env:
          PR_NUMBER: ${{ github.event.pull_request.number }}
          REF_NAME: ${{ github.ref_name }}
        run: |
          if [ -n "${PR_NUMBER}" ]; then
            CACHE_SUFFIX="${PR_NUMBER}"
          else
            # shellcheck disable=SC2001
            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
          fi
          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT

      - name: Log in to ECR pull-through cache
        uses: ./.github/actions/login-ecr-pullthrough-cache
        with:
          ecr-registry: ${{ vars.ECR_REGISTRY }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # ratchet:docker/setup-buildx-action@v4

      - name: Build and load
        uses: docker/bake-action@6614cfa25eff9a0b2b2697efb0b6159e7680d584 # ratchet:docker/bake-action@v7.2.0
        env:
          TAG: model-server-${{ github.run_id }}
        with:
          load: true
          targets: model-server
          set: |
            model-server.cache-from=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }}
            model-server.cache-from=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }}
            model-server.cache-from=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache
            model-server.cache-from=type=registry,ref=${{ env.BASE_IMAGE_REGISTRY }}/onyxdotapp/onyx-model-server:latest
            model-server.cache-to=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
            model-server.cache-to=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
            model-server.cache-to=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache,mode=max

      - name: Start Docker containers
        id: start_docker
        env:
          IMAGE_TAG: model-server-${{ github.run_id }}
        run: |
          cd deployment/docker_compose
          docker compose \
            -f docker-compose.yml \
            -f docker-compose.dev.yml \
            up -d --wait \
            inference_model_server

      - name: Run Tests
        run: |
          py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/embedding

      - name: Alert on Failure
        if: failure() && github.event_name == 'schedule'
        uses: ./.github/actions/slack-notify
        with:
          webhook-url: ${{ secrets.SLACK_WEBHOOK }}
          failed-jobs: model-check
          title: "🚨 Scheduled Model Tests failed!"
          ref-name: ${{ github.ref_name }}

      - name: Dump all-container logs (optional)
        if: always()
        run: |
          cd deployment/docker_compose
          docker compose logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true

      - name: Upload logs
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: docker-all-logs
          path: ${{ github.workspace }}/docker-compose.log