diff --git a/.github/actions/prepare-build/action.yml b/.github/actions/prepare-build/action.yml deleted file mode 100644 index a2788e2bb4..0000000000 --- a/.github/actions/prepare-build/action.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: "Prepare Build (OpenAPI generation)" -description: "Sets up Python with uv, installs deps, generates OpenAPI schema and Python client, uploads artifact" -inputs: - docker-username: - required: true - docker-password: - required: true -runs: - using: "composite" - steps: - - name: Setup Python and Install Dependencies - uses: ./.github/actions/setup-python-and-install-dependencies - - - name: Generate OpenAPI schema - shell: bash - working-directory: backend - env: - PYTHONPATH: "." - run: | - python scripts/onyx_openapi_schema.py --filename generated/openapi.json - - # needed for pulling openapitools/openapi-generator-cli - # otherwise, we hit the "Unauthenticated users" limit - # https://docs.docker.com/docker-hub/usage/ - - name: Login to Docker Hub - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3 - with: - username: ${{ inputs['docker-username'] }} - password: ${{ inputs['docker-password'] }} - - - name: Generate OpenAPI Python client - shell: bash - run: | - docker run --rm \ - -v "${{ github.workspace }}/backend/generated:/local" \ - openapitools/openapi-generator-cli generate \ - -i /local/openapi.json \ - -g python \ - -o /local/onyx_openapi_client \ - --package-name onyx_openapi_client \ - --skip-validate-spec \ - --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=true,SET_OAS3_NULLABLE=true" diff --git a/.github/actions/setup-python-and-install-dependencies/action.yml b/.github/actions/setup-python-and-install-dependencies/action.yml index b3c622e917..dcbc168ca5 100644 --- a/.github/actions/setup-python-and-install-dependencies/action.yml +++ b/.github/actions/setup-python-and-install-dependencies/action.yml @@ -1,5 +1,9 @@ name: "Setup Python and Install Dependencies" description: "Sets up Python with uv and installs deps" +inputs: + requirements: + description: "Newline-separated list of requirement files to install (relative to repo root)" + required: true runs: using: "composite" steps: @@ -9,11 +13,26 @@ runs: # with: # enable-cache: true + - name: Compute requirements hash + id: req-hash + shell: bash + env: + REQUIREMENTS: ${{ inputs.requirements }} + run: | + # Hash the contents of the specified requirement files + hash="" + while IFS= read -r req; do + if [ -n "$req" ] && [ -f "$req" ]; then + hash="$hash$(sha256sum "$req")" + fi + done <<< "$REQUIREMENTS" + echo "hash=$(echo "$hash" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + - name: Cache uv cache directory uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4 with: path: ~/.cache/uv - key: ${{ runner.os }}-uv-${{ hashFiles('backend/requirements/*.txt', 'backend/pyproject.toml') }} + key: ${{ runner.os }}-uv-${{ steps.req-hash.outputs.hash }} restore-keys: | ${{ runner.os }}-uv- @@ -38,9 +57,16 @@ runs: - name: Install Python dependencies with uv shell: bash + env: + REQUIREMENTS: ${{ inputs.requirements }} run: | - uv pip install \ - -r backend/requirements/default.txt \ - -r backend/requirements/dev.txt \ - -r backend/requirements/model_server.txt \ - -r backend/requirements/ee.txt + # Build the uv pip install command with each requirement file as array elements + cmd=("uv" "pip" "install") + while IFS= read -r req; do + # Skip empty lines + if [ -n "$req" ]; then + cmd+=("-r" "$req") + fi + done <<< "$REQUIREMENTS" + echo "Running: ${cmd[*]}" + "${cmd[@]}" diff --git a/.github/workflows/pr-external-dependency-unit-tests.yml b/.github/workflows/pr-external-dependency-unit-tests.yml index 085f70c0b3..4665771a84 100644 --- a/.github/workflows/pr-external-dependency-unit-tests.yml +++ b/.github/workflows/pr-external-dependency-unit-tests.yml @@ -83,6 +83,11 @@ jobs: - name: Setup Python and Install Dependencies uses: ./.github/actions/setup-python-and-install-dependencies + with: + requirements: | + backend/requirements/default.txt + backend/requirements/dev.txt + backend/requirements/ee.txt - name: Setup Playwright uses: ./.github/actions/setup-playwright diff --git a/.github/workflows/pr-python-checks.yml b/.github/workflows/pr-python-checks.yml index de576cbf6e..5378a063c9 100644 --- a/.github/workflows/pr-python-checks.yml +++ b/.github/workflows/pr-python-checks.yml @@ -46,6 +46,23 @@ jobs: with: persist-credentials: false + - name: Setup Python and Install Dependencies + uses: ./.github/actions/setup-python-and-install-dependencies + with: + requirements: | + backend/requirements/default.txt + backend/requirements/dev.txt + backend/requirements/model_server.txt + backend/requirements/ee.txt + + - name: Generate OpenAPI schema + shell: bash + working-directory: backend + env: + PYTHONPATH: "." + run: | + python scripts/onyx_openapi_schema.py --filename generated/openapi.json + # needed for pulling openapitools/openapi-generator-cli # otherwise, we hit the "Unauthenticated users" limit # https://docs.docker.com/docker-hub/usage/ @@ -55,11 +72,18 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - name: Prepare build - uses: ./.github/actions/prepare-build - with: - docker-username: ${{ secrets.DOCKER_USERNAME }} - docker-password: ${{ secrets.DOCKER_TOKEN }} + - name: Generate OpenAPI Python client + shell: bash + run: | + docker run --rm \ + -v "${{ github.workspace }}/backend/generated:/local" \ + openapitools/openapi-generator-cli generate \ + -i /local/openapi.json \ + -g python \ + -o /local/onyx_openapi_client \ + --package-name onyx_openapi_client \ + --skip-validate-spec \ + --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=true,SET_OAS3_NULLABLE=true" - name: Cache mypy cache if: ${{ vars.DISABLE_MYPY_CACHE != 'true' }} diff --git a/.github/workflows/pr-python-connector-tests.yml b/.github/workflows/pr-python-connector-tests.yml index d1c96e0da7..1b1f43c27b 100644 --- a/.github/workflows/pr-python-connector-tests.yml +++ b/.github/workflows/pr-python-connector-tests.yml @@ -141,6 +141,10 @@ jobs: - name: Setup Python and Install Dependencies uses: ./.github/actions/setup-python-and-install-dependencies + with: + requirements: | + backend/requirements/default.txt + backend/requirements/dev.txt - name: Setup Playwright uses: ./.github/actions/setup-playwright diff --git a/.github/workflows/pr-python-tests.yml b/.github/workflows/pr-python-tests.yml index 7e5e05bce6..13af8557a1 100644 --- a/.github/workflows/pr-python-tests.yml +++ b/.github/workflows/pr-python-tests.yml @@ -37,6 +37,12 @@ jobs: - name: Setup Python and Install Dependencies uses: ./.github/actions/setup-python-and-install-dependencies + with: + requirements: | + backend/requirements/default.txt + backend/requirements/dev.txt + backend/requirements/model_server.txt + backend/requirements/ee.txt - name: Run Tests shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"