onyx/.github/workflows/pr-python-connector-tests.yml

name: Connector Tests
concurrency:
  group: Connector-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
  cancel-in-progress: true

on:
  merge_group:
  pull_request:
    branches: [main]
    paths:
      - "backend/**"
      - "pyproject.toml"
      - "uv.lock"
      - ".github/workflows/pr-python-connector-tests.yml"
      - ".github/actions/setup-python-and-install-dependencies/**"
      - ".github/actions/setup-playwright/**"
  push:
    tags:
      - "v*.*.*"
  schedule:
    # This cron expression runs the job daily at 16:00 UTC (9am PT)
    - cron: "0 16 * * *"

permissions:
  id-token: write # Required for OIDC-based AWS credential exchange
  contents: read

env:
  PYTHONPATH: ./backend
  DISABLE_TELEMETRY: "true"
  R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS: ${{ vars.R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS }}
  CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }}
  CONFLUENCE_TEST_SPACE: ${{ vars.CONFLUENCE_TEST_SPACE }}
  CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }}
  SF_USERNAME: ${{ vars.SF_USERNAME }}
  IMAP_HOST: ${{ vars.IMAP_HOST }}
  IMAP_USERNAME: ${{ vars.IMAP_USERNAME }}
  IMAP_MAILBOXES: ${{ vars.IMAP_MAILBOXES }}
  AIRTABLE_TEST_BASE_ID: ${{ vars.AIRTABLE_TEST_BASE_ID }}
  AIRTABLE_TEST_TABLE_ID: ${{ vars.AIRTABLE_TEST_TABLE_ID }}
  AIRTABLE_TEST_TABLE_NAME: ${{ vars.AIRTABLE_TEST_TABLE_NAME }}
  SHAREPOINT_CLIENT_ID: ${{ vars.SHAREPOINT_CLIENT_ID }}
  SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ vars.SHAREPOINT_CLIENT_DIRECTORY_ID }}
  SHAREPOINT_SITE: ${{ vars.SHAREPOINT_SITE }}
  BITBUCKET_EMAIL: ${{ vars.BITBUCKET_EMAIL }}

jobs:
  connectors-check:
    # See https://runs-on.com/runners/linux/
    runs-on:
      [
        runs-on,
        runner=8cpu-linux-x64,
        "run-id=${{ github.run_id }}-connectors-check",
        "extras=s3-cache",
      ]
    timeout-minutes: 45
    environment: ci-protected

    steps:
      - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60

      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Setup Python and Install Dependencies
        uses: ./.github/actions/setup-python-and-install-dependencies
        with:
          requirements: |
            backend/requirements/default.txt
            backend/requirements/dev.txt

      - name: Setup Playwright
        uses: ./.github/actions/setup-playwright

      - name: Detect Connector changes
        id: changes
        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # ratchet:dorny/paths-filter@v3
        with:
          filters: |
            hubspot:
              - 'backend/onyx/connectors/hubspot/**'
              - 'backend/tests/daily/connectors/hubspot/**'
              - 'uv.lock'
            salesforce:
              - 'backend/onyx/connectors/salesforce/**'
              - 'backend/tests/daily/connectors/salesforce/**'
              - 'uv.lock'
            github:
              - 'backend/onyx/connectors/github/**'
              - 'backend/tests/daily/connectors/github/**'
              - 'uv.lock'
            file_processing:
              - 'backend/onyx/file_processing/**'
              - 'uv.lock'

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # ratchet:aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: us-east-2

      - name: Run Tests (excluding HubSpot, Salesforce, GitHub, and Coda)
        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
        run: |
          py.test \
            -n 8 \
            --dist loadfile \
            --durations=8 \
            -o junit_family=xunit2 \
            -xv \
            --ff \
            backend/tests/daily/connectors \
            --ignore backend/tests/daily/connectors/hubspot \
            --ignore backend/tests/daily/connectors/salesforce \
            --ignore backend/tests/daily/connectors/github \
            --ignore backend/tests/daily/connectors/coda

      - name: Run HubSpot Connector Tests
        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.hubspot == 'true' || steps.changes.outputs.file_processing == 'true' }}
        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
        run: |
          py.test \
            -n 8 \
            --dist loadfile \
            --durations=8 \
            -o junit_family=xunit2 \
            -xv \
            --ff \
            backend/tests/daily/connectors/hubspot

      - name: Run Salesforce Connector Tests
        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.salesforce == 'true' || steps.changes.outputs.file_processing == 'true' }}
        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
        run: |
          py.test \
            -n 8 \
            --dist loadfile \
            --durations=8 \
            -o junit_family=xunit2 \
            -xv \
            --ff \
            backend/tests/daily/connectors/salesforce

      - name: Run GitHub Connector Tests
        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.github == 'true' || steps.changes.outputs.file_processing == 'true' }}
        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
        run: |
          py.test \
            -n 8 \
            --dist loadfile \
            --durations=8 \
            -o junit_family=xunit2 \
            -xv \
            --ff \
            backend/tests/daily/connectors/github

      - name: Alert on Failure
        if: failure() && github.event_name == 'schedule'
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
        run: |
          curl -X POST \
            -H 'Content-type: application/json' \
            --data "{\"text\":\"Scheduled Connector Tests failed! Check the run at: https://github.com/${REPO}/actions/runs/${RUN_ID}\"}" \
            $SLACK_WEBHOOK