From c90ff701dcbfdfee2d043ece6d8eb3c81cd0f624 Mon Sep 17 00:00:00 2001 From: Wenxi Date: Thu, 16 Oct 2025 16:42:24 -0700 Subject: [PATCH] chore: move gh non-secrets to vars (#5744) --- .github/workflows/hotfix-release-branches.yml | 171 ------------------ .../pr-external-dependency-unit-tests.yml | 7 +- .github/workflows/pr-integration-tests.yml | 4 +- .../workflows/pr-mit-integration-tests.yml | 4 +- .../workflows/pr-python-connector-tests.yml | 30 +-- .github/workflows/pr-python-model-tests.yml | 4 +- .github/workflows/tag-nightly.yml | 3 + .../fireflies/test_fireflies_connector.py | 2 +- .../fireflies/test_fireflies_data.json | 2 +- .../connectors/confluence/conftest.py | 2 +- 10 files changed, 31 insertions(+), 198 deletions(-) delete mode 100644 .github/workflows/hotfix-release-branches.yml diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml deleted file mode 100644 index 6e14fa8269..0000000000 --- a/.github/workflows/hotfix-release-branches.yml +++ /dev/null @@ -1,171 +0,0 @@ -# This workflow is intended to be manually triggered via the GitHub Action tab. -# Given a hotfix branch, it will attempt to open a PR to all release branches and -# by default auto merge them - -name: Hotfix release branches - -on: - workflow_dispatch: - inputs: - hotfix_commit: - description: "Hotfix commit hash" - required: true - hotfix_suffix: - description: "Hotfix branch suffix (e.g. hotfix/v0.8-{suffix})" - required: true - release_branch_pattern: - description: "Release branch pattern (regex)" - required: true - default: "release/.*" - auto_merge: - description: "Automatically merge the hotfix PRs" - required: true - type: choice - default: "true" - options: - - true - - false - -jobs: - hotfix_release_branches: - permissions: write-all - # See https://runs-on.com/runners/linux/ - # use a lower powered instance since this just does i/o to docker hub - runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"] - steps: - # needs RKUO_DEPLOY_KEY for write access to merge PR's - - name: Checkout Repository - uses: actions/checkout@v4 - with: - ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}" - fetch-depth: 0 - - - name: Set up Git user - run: | - git config user.name "Richard Kuo [bot]" - git config user.email "rkuo[bot]@onyx.app" - - - name: Fetch All Branches - run: | - git fetch --all --prune - - - name: Verify Hotfix Commit Exists - run: | - git rev-parse --verify "${{ github.event.inputs.hotfix_commit }}" || { echo "Commit not found: ${{ github.event.inputs.hotfix_commit }}"; exit 1; } - - - name: Get Release Branches - id: get_release_branches - run: | - BRANCHES=$(git branch -r | grep -E "${{ github.event.inputs.release_branch_pattern }}" | sed 's|origin/||' | tr -d ' ') - if [ -z "$BRANCHES" ]; then - echo "No release branches found matching pattern '${{ github.event.inputs.release_branch_pattern }}'." - exit 1 - fi - - echo "Found release branches:" - echo "$BRANCHES" - - # Join the branches into a single line separated by commas - BRANCHES_JOINED=$(echo "$BRANCHES" | tr '\n' ',' | sed 's/,$//') - - # Set the branches as an output - echo "branches=$BRANCHES_JOINED" >> $GITHUB_OUTPUT - - # notes on all the vagaries of wiring up automated PR's - # https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs - # we must use a custom token for GH_TOKEN to trigger the subsequent PR checks - - name: Create and Merge Pull Requests to Matching Release Branches - env: - HOTFIX_COMMIT: ${{ github.event.inputs.hotfix_commit }} - HOTFIX_SUFFIX: ${{ github.event.inputs.hotfix_suffix }} - AUTO_MERGE: ${{ github.event.inputs.auto_merge }} - GH_TOKEN: ${{ secrets.RKUO_PERSONAL_ACCESS_TOKEN }} - run: | - # Get the branches from the previous step - BRANCHES="${{ steps.get_release_branches.outputs.branches }}" - - # Convert BRANCHES to an array - IFS=$',' read -ra BRANCH_ARRAY <<< "$BRANCHES" - - # Loop through each release branch and create and merge a PR - for RELEASE_BRANCH in "${BRANCH_ARRAY[@]}"; do - echo "Processing $RELEASE_BRANCH..." - - # Parse out the release version by removing "release/" from the branch name - RELEASE_VERSION=${RELEASE_BRANCH#release/} - echo "Release version parsed: $RELEASE_VERSION" - - HOTFIX_BRANCH="hotfix/${RELEASE_VERSION}-${HOTFIX_SUFFIX}" - echo "Creating PR from $HOTFIX_BRANCH to $RELEASE_BRANCH" - - # Checkout the release branch - echo "Checking out $RELEASE_BRANCH" - git checkout "$RELEASE_BRANCH" - - # Create the new hotfix branch - if git rev-parse --verify "$HOTFIX_BRANCH" >/dev/null 2>&1; then - echo "Hotfix branch $HOTFIX_BRANCH already exists. Skipping branch creation." - else - echo "Branching $RELEASE_BRANCH to $HOTFIX_BRANCH" - git checkout -b "$HOTFIX_BRANCH" - fi - - # Check if the hotfix commit is a merge commit - if git rev-list --merges -n 1 "$HOTFIX_COMMIT" >/dev/null 2>&1; then - # -m 1 uses the target branch as the base (which is what we want) - echo "Hotfix commit $HOTFIX_COMMIT is a merge commit, using -m 1 for cherry-pick" - CHERRY_PICK_CMD="git cherry-pick -m 1 $HOTFIX_COMMIT" - else - CHERRY_PICK_CMD="git cherry-pick $HOTFIX_COMMIT" - fi - - # Perform the cherry-pick - echo "Executing: $CHERRY_PICK_CMD" - eval "$CHERRY_PICK_CMD" - - if [ $? -ne 0 ]; then - echo "Cherry-pick failed for $HOTFIX_COMMIT on $HOTFIX_BRANCH. Aborting..." - git cherry-pick --abort - continue - fi - - # Push the hotfix branch to the remote - echo "Pushing $HOTFIX_BRANCH..." - git push origin "$HOTFIX_BRANCH" - echo "Hotfix branch $HOTFIX_BRANCH created and pushed." - - # Check if PR already exists - EXISTING_PR=$(gh pr list --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH" --state open --json number --jq '.[0].number') - - if [ -n "$EXISTING_PR" ]; then - echo "An open PR already exists: #$EXISTING_PR. Skipping..." - continue - fi - - # Create a new PR and capture the output - PR_OUTPUT=$(gh pr create --title "Merge $HOTFIX_BRANCH into $RELEASE_BRANCH" \ - --body "Automated PR to merge \`$HOTFIX_BRANCH\` into \`$RELEASE_BRANCH\`." \ - --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH") - - # Extract the URL from the output - PR_URL=$(echo "$PR_OUTPUT" | grep -Eo 'https://github.com/[^ ]+') - echo "Pull request created: $PR_URL" - - # Extract PR number from URL - PR_NUMBER=$(basename "$PR_URL") - echo "Pull request created: $PR_NUMBER" - - if [ "$AUTO_MERGE" == "true" ]; then - echo "Attempting to merge pull request #$PR_NUMBER" - - # Attempt to merge the PR - gh pr merge "$PR_NUMBER" --merge --auto --delete-branch - - if [ $? -eq 0 ]; then - echo "Pull request #$PR_NUMBER merged successfully." - else - # Optionally, handle the error or continue - echo "Failed to merge pull request #$PR_NUMBER." - fi - fi - done diff --git a/.github/workflows/pr-external-dependency-unit-tests.yml b/.github/workflows/pr-external-dependency-unit-tests.yml index c282c44e15..88040074d6 100644 --- a/.github/workflows/pr-external-dependency-unit-tests.yml +++ b/.github/workflows/pr-external-dependency-unit-tests.yml @@ -14,11 +14,10 @@ env: S3_ENDPOINT_URL: "http://localhost:9004" # Confluence - CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }} - CONFLUENCE_TEST_SPACE: ${{ secrets.CONFLUENCE_TEST_SPACE }} + CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }} + CONFLUENCE_TEST_SPACE: ${{ vars.CONFLUENCE_TEST_SPACE }} CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }} - CONFLUENCE_IS_CLOUD: ${{ secrets.CONFLUENCE_IS_CLOUD }} - CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }} + CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }} CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }} diff --git a/.github/workflows/pr-integration-tests.yml b/.github/workflows/pr-integration-tests.yml index a4d5d82025..7fa6eeeaaa 100644 --- a/.github/workflows/pr-integration-tests.yml +++ b/.github/workflows/pr-integration-tests.yml @@ -19,8 +19,8 @@ env: # Test Environment Variables OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }} - CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }} + CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }} + CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }} CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }} JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} diff --git a/.github/workflows/pr-mit-integration-tests.yml b/.github/workflows/pr-mit-integration-tests.yml index 0bfeea1b39..c43efc2c1e 100644 --- a/.github/workflows/pr-mit-integration-tests.yml +++ b/.github/workflows/pr-mit-integration-tests.yml @@ -16,8 +16,8 @@ env: # Test Environment Variables OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }} - CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }} + CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }} + CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }} CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }} JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} diff --git a/.github/workflows/pr-python-connector-tests.yml b/.github/workflows/pr-python-connector-tests.yml index 741db09964..a3fe490cb4 100644 --- a/.github/workflows/pr-python-connector-tests.yml +++ b/.github/workflows/pr-python-connector-tests.yml @@ -14,11 +14,10 @@ env: AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }} # Confluence - CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }} - CONFLUENCE_TEST_SPACE: ${{ secrets.CONFLUENCE_TEST_SPACE }} + CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }} + CONFLUENCE_TEST_SPACE: ${{ vars.CONFLUENCE_TEST_SPACE }} CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }} - CONFLUENCE_IS_CLOUD: ${{ secrets.CONFLUENCE_IS_CLOUD }} - CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }} + CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }} CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }} @@ -56,22 +55,22 @@ env: HUBSPOT_ACCESS_TOKEN: ${{ secrets.HUBSPOT_ACCESS_TOKEN }} # IMAP - IMAP_HOST: ${{ secrets.IMAP_HOST }} - IMAP_USERNAME: ${{ secrets.IMAP_USERNAME }} + IMAP_HOST: ${{ vars.IMAP_HOST }} + IMAP_USERNAME: ${{ vars.IMAP_USERNAME }} IMAP_PASSWORD: ${{ secrets.IMAP_PASSWORD }} - IMAP_MAILBOXES: ${{ secrets.IMAP_MAILBOXES }} + IMAP_MAILBOXES: ${{ vars.IMAP_MAILBOXES }} # Airtable - AIRTABLE_TEST_BASE_ID: ${{ secrets.AIRTABLE_TEST_BASE_ID }} - AIRTABLE_TEST_TABLE_ID: ${{ secrets.AIRTABLE_TEST_TABLE_ID }} - AIRTABLE_TEST_TABLE_NAME: ${{ secrets.AIRTABLE_TEST_TABLE_NAME }} + AIRTABLE_TEST_BASE_ID: ${{ vars.AIRTABLE_TEST_BASE_ID }} + AIRTABLE_TEST_TABLE_ID: ${{ vars.AIRTABLE_TEST_TABLE_ID }} + AIRTABLE_TEST_TABLE_NAME: ${{ vars.AIRTABLE_TEST_TABLE_NAME }} AIRTABLE_ACCESS_TOKEN: ${{ secrets.AIRTABLE_ACCESS_TOKEN }} # Sharepoint - SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }} + SHAREPOINT_CLIENT_ID: ${{ vars.SHAREPOINT_CLIENT_ID }} SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }} - SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ secrets.SHAREPOINT_CLIENT_DIRECTORY_ID }} - SHAREPOINT_SITE: ${{ secrets.SHAREPOINT_SITE }} + SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ vars.SHAREPOINT_CLIENT_DIRECTORY_ID }} + SHAREPOINT_SITE: ${{ vars.SHAREPOINT_SITE }} # Github ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }} @@ -102,9 +101,12 @@ env: BITBUCKET_WORKSPACE: ${{ secrets.BITBUCKET_WORKSPACE }} BITBUCKET_REPOSITORIES: ${{ secrets.BITBUCKET_REPOSITORIES }} BITBUCKET_PROJECTS: ${{ secrets.BITBUCKET_PROJECTS }} - BITBUCKET_EMAIL: ${{ secrets.BITBUCKET_EMAIL }} + BITBUCKET_EMAIL: ${{ vars.BITBUCKET_EMAIL }} BITBUCKET_API_TOKEN: ${{ secrets.BITBUCKET_API_TOKEN }} + # Fireflies + FIREFLIES_API_KEY: ${{ secrets.FIREFLIES_API_KEY }} + jobs: connectors-check: # See https://runs-on.com/runners/linux/ diff --git a/.github/workflows/pr-python-model-tests.yml b/.github/workflows/pr-python-model-tests.yml index cd00d1ba49..a0f172f4c9 100644 --- a/.github/workflows/pr-python-model-tests.yml +++ b/.github/workflows/pr-python-model-tests.yml @@ -15,7 +15,7 @@ env: # Bedrock AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }} + AWS_REGION_NAME: ${{ vars.AWS_REGION_NAME }} # API keys for testing COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} @@ -23,7 +23,7 @@ env: LITELLM_API_URL: ${{ secrets.LITELLM_API_URL }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} - AZURE_API_URL: ${{ secrets.AZURE_API_URL }} + AZURE_API_URL: ${{ vars.AZURE_API_URL }} jobs: model-check: diff --git a/.github/workflows/tag-nightly.yml b/.github/workflows/tag-nightly.yml index 61d1818f4d..3a99023332 100644 --- a/.github/workflows/tag-nightly.yml +++ b/.github/workflows/tag-nightly.yml @@ -15,6 +15,9 @@ jobs: # actions using GITHUB_TOKEN cannot trigger another workflow, but we do want this to trigger docker pushes # see https://github.com/orgs/community/discussions/27028#discussioncomment-3254367 for the workaround we # implement here which needs an actual user's deploy key + + # Additional NOTE: even though this is named "rkuo", the actual key is tied to the onyx repo + # and not rkuo's personal account. It is fine to leave this key as is! - name: Checkout code uses: actions/checkout@v4 with: diff --git a/backend/tests/daily/connectors/fireflies/test_fireflies_connector.py b/backend/tests/daily/connectors/fireflies/test_fireflies_connector.py index 3e1dd2bd61..260adae452 100644 --- a/backend/tests/daily/connectors/fireflies/test_fireflies_connector.py +++ b/backend/tests/daily/connectors/fireflies/test_fireflies_connector.py @@ -27,7 +27,7 @@ def fireflies_connector() -> FirefliesConnector: @pytest.mark.xfail( - reason="Environment variable not set for some reason", + reason="We don't have the key that is stored in GitHub Secrets and the returned data is different than expected", ) def test_fireflies_connector_basic(fireflies_connector: FirefliesConnector) -> None: test_data = load_test_data() diff --git a/backend/tests/daily/connectors/fireflies/test_fireflies_data.json b/backend/tests/daily/connectors/fireflies/test_fireflies_data.json index ce69712686..adf6c62431 100644 --- a/backend/tests/daily/connectors/fireflies/test_fireflies_data.json +++ b/backend/tests/daily/connectors/fireflies/test_fireflies_data.json @@ -11,7 +11,7 @@ "sections": [ { "link": "https://app.fireflies.ai/view/VcBdZpuV82rImQCA?t=153.1", - "text": "test_user_1 1: Hey, David, thanks for taking the time today." + "text": "test_user_1 1: Hey, David, thanks for taking the time today." }, { "link": "https://app.fireflies.ai/view/VcBdZpuV82rImQCA?t=158.14", diff --git a/backend/tests/external_dependency_unit/connectors/confluence/conftest.py b/backend/tests/external_dependency_unit/connectors/confluence/conftest.py index 1fe381837f..175adfa798 100644 --- a/backend/tests/external_dependency_unit/connectors/confluence/conftest.py +++ b/backend/tests/external_dependency_unit/connectors/confluence/conftest.py @@ -9,7 +9,7 @@ def confluence_connector_config() -> dict[str, Any]: url_base = os.environ.get("CONFLUENCE_TEST_SPACE_URL") space_key = os.environ.get("CONFLUENCE_SPACE_KEY") page_id = os.environ.get("CONFLUENCE_PAGE_ID") - is_cloud = os.environ.get("CONFLUENCE_IS_CLOUD", "").lower() == "true" + is_cloud = os.environ.get("CONFLUENCE_IS_CLOUD", "true").lower() == "true" assert url_base, "CONFLUENCE_URL environment variable is required"