mirror of
https://github.com/onyx-dot-app/onyx.git
synced 2026-10-19 01:50:51 -05:00
feat(ci): alert on deploy-blocking findings in a Slack thread (#15345)
This commit is contained in:
Nikolas Garza authored and GitHub committed
1 parent
c0049da2ef
commit
3fe68e62ef
3 files changed
+123
-63
No files matched your search
@@ -1,18 +1,18 @@
|
||||
### Turns each package with a blocking vulnerability into a tracking issue, a fix
|
||||
### PR, and a Slack post to #cve-reviews, and closes the issue once the finding
|
||||
### is fixed or suppressed.
|
||||
### Turns each package with a deploy-blocking vulnerability into a tracking
|
||||
### issue, a fix PR when a pin applies, and a Slack post to #cve-reviews, and
|
||||
### closes the issue once the finding is fixed or suppressed.
|
||||
###
|
||||
### Every run scans every tracked lockfile, the open Dependabot alerts, the
|
||||
### pinned Actions, and the images (edge on main, the line's newest release on
|
||||
### a release branch), then `ods audit alert` diffs the blocking findings
|
||||
### against the open cve-alert issues of the branch it runs on.
|
||||
### Every run scans what the deploy gate scans: the root and web lockfiles, the
|
||||
### open Dependabot alerts, the pinned Actions, and the audited images (edge on
|
||||
### main, the line's newest release on a release branch). `ods audit alert`
|
||||
### then diffs the blocking findings against the branch's open cve-alert issues.
|
||||
###
|
||||
### Per alert to announce, triage has Claude pin the fix (npm and PyPI only) and
|
||||
### judge whether Onyx is affected, and publish commits the checked result onto
|
||||
### Dependabot's PR or a fixed branch, then posts to the issue and Slack. A
|
||||
### release run cherry-picks a fix main already merged, or else fixes on main
|
||||
### and backports to every release line from its own up. A reviewer drawn
|
||||
### from .github/actions/slack-notify/cve-reviewers.json owns it.
|
||||
### Per alert, triage has Claude pin the fix (npm and PyPI only) and judge
|
||||
### whether Onyx is affected, and publish commits the checked result onto
|
||||
### Dependabot's PR or a fixed branch. A release run cherry-picks a fix main
|
||||
### already merged, or else fixes on main and backports to every release line
|
||||
### from its own up. Every alert posts to Slack tagging a reviewer drawn from
|
||||
### cve-reviewers.json, with the fix PR or the no-fix line, details threaded.
|
||||
|
||||
name: CVE Alerts
|
||||
|
||||
@@ -41,6 +41,9 @@ defaults:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
# #cve-reviews, posted to with the Internal Tools bot so the details can go
|
||||
# in a thread.
|
||||
CVE_REVIEWS_CHANNEL: C0C5SMAM4TT
|
||||
# jq sort key for a package version: a pre-release ranks below its release,
|
||||
# which `sort -V` gets wrong.
|
||||
VERSION_KEY: 'split("-") | [(.[0] | split(".") | map(tonumber? // .)), (if length == 1 then 1 else 0 end), (.[1:] | join("-") | split(".") | map(tonumber? // .))]'
|
||||
@@ -91,7 +94,7 @@ jobs:
|
||||
if [ "${GITHUB_REF_NAME}" != "main" ]; then
|
||||
sources=(--web --python --actions)
|
||||
fi
|
||||
"${RUNNER_TEMP}/ods-audit" --all-lockfiles --strict "${sources[@]}" --ignore-url "" --format=json > deps.json || true
|
||||
"${RUNNER_TEMP}/ods-audit" --strict "${sources[@]}" --ignore-url "" --format=json > deps.json || true
|
||||
jq -e 'has("blocking")' deps.json > /dev/null
|
||||
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # ratchet:actions/upload-artifact@v4
|
||||
@@ -122,8 +125,6 @@ jobs:
|
||||
repo: onyxdotapp/onyx-backend
|
||||
- component: model-server
|
||||
repo: onyxdotapp/onyx-model-server
|
||||
- component: sandbox
|
||||
repo: onyxdotapp/sandbox
|
||||
steps:
|
||||
- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc
|
||||
|
||||
@@ -268,8 +269,8 @@ jobs:
|
||||
args+=(--results "$f")
|
||||
done
|
||||
skipped=(results/*.skipped)
|
||||
if [ "$(( ${#args[@]} / 2 + ${#skipped[@]} ))" -ne 5 ]; then
|
||||
echo "::error::Expected 5 scan results, found: ${args[*]} ${skipped[*]}"
|
||||
if [ "$(( ${#args[@]} / 2 + ${#skipped[@]} ))" -ne 4 ]; then
|
||||
echo "::error::Expected 4 scan results, found: ${args[*]} ${skipped[*]}"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${#skipped[@]}" -gt 0 ]; then
|
||||
@@ -566,18 +567,19 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# A retry keeps the reviewer an earlier run assigned, so one person owns
|
||||
# the alert.
|
||||
- name: Write the alert and draw a reviewer
|
||||
id: alert
|
||||
run: |
|
||||
printf '%s' "${ALERT}" > "${RUNNER_TEMP}/alert.json"
|
||||
echo "reviewer=$(jq -r '.[]' .github/actions/slack-notify/cve-reviewers.json | shuf -n 1)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Assign the reviewer
|
||||
id: assign
|
||||
env:
|
||||
ISSUE: ${{ matrix.alert.issue }}
|
||||
REVIEWER: ${{ steps.alert.outputs.reviewer }}
|
||||
run: gh issue edit "${ISSUE}" --add-assignee "${REVIEWER}"
|
||||
run: |
|
||||
printf '%s' "${ALERT}" > "${RUNNER_TEMP}/alert.json"
|
||||
reviewer="$(gh issue view "${ISSUE}" --json assignees --jq '.assignees[0].login // ""')"
|
||||
if [ -z "${reviewer}" ]; then
|
||||
reviewer="$(jq -r '.[]' .github/actions/slack-notify/cve-reviewers.json | shuf -n 1)"
|
||||
fi
|
||||
echo "reviewer=${reviewer}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# A missing artifact means triage failed, which the note below reports.
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # ratchet:actions/download-artifact@v4
|
||||
@@ -886,8 +888,8 @@ jobs:
|
||||
echo "::error::The backports onto release lines ${FAILED} failed, likely on a conflict. The alert stays pending."
|
||||
exit 1
|
||||
|
||||
# Post even after a failed step. The pending label stays until the Slack
|
||||
# post succeeds, so a later run retries a lost one.
|
||||
# Post even after a failed step. The pending label stays until a fix PR
|
||||
# is posted to Slack, so a later run retries.
|
||||
- name: Fill in a failed triage
|
||||
if: ${{ !cancelled() && steps.alert.outcome == 'success' }}
|
||||
id: triage
|
||||
@@ -903,16 +905,35 @@ jobs:
|
||||
echo "ok=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# A pending alert with no fix PR is triaged again every run, so a retry
|
||||
# comments only when it found a fix or the issue still lacks a finished
|
||||
# verdict.
|
||||
- name: Post the note on the issue
|
||||
if: ${{ !cancelled() && steps.alert.outcome == 'success' }}
|
||||
id: note
|
||||
env:
|
||||
ISSUE: ${{ matrix.alert.issue }}
|
||||
ALERT_REASON: ${{ matrix.alert.reason }}
|
||||
TRIAGE_OK: ${{ steps.triage.outputs.ok }}
|
||||
PR_URL: ${{ steps.pr.outputs.url || steps.cp.outputs.url }}
|
||||
BACKPORTS: ${{ steps.backport.outputs.urls }}
|
||||
BACKPORT_FAILED: ${{ steps.backport.outputs.failed }}
|
||||
BACKPORT_OUTCOME: ${{ steps.backport.outcome }}
|
||||
CP_OUTCOME: ${{ steps.cp.outcome }}
|
||||
run: |
|
||||
if [ "${ALERT_REASON}" = "retry" ] && [ -z "${PR_URL}" ]; then
|
||||
last="$(gh issue view "${ISSUE}" --json comments --jq '[.comments[] | select(.author.login == "github-actions")] | last.body // ""')"
|
||||
if [ "${TRIAGE_OK}" != "true" ] || { [ -n "${last}" ] && ! grep -q "triage did not finish" <<< "${last}"; }; then
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
result="${RUNNER_TEMP}/triage/result.json"
|
||||
# Keep the drafted --reason a plain double-quoted shell string.
|
||||
reason="$(jq -r .ignore_reason "${result}" | tr '"\t\r' "' " | tr -d '`$' | tr -d "\\\\" | head -c 120)"
|
||||
# `ods audit ignore add` rejects an empty --reason.
|
||||
reason="${reason:-<why this is not exploitable in Onyx>}"
|
||||
expires="$(date -u -d '+30 days' +%Y-%m-%d)"
|
||||
ids="$(jq -r '.advisories[].id' "${RUNNER_TEMP}/alert.json")"
|
||||
{
|
||||
cat "${RUNNER_TEMP}/triage/note.md"
|
||||
echo
|
||||
@@ -935,21 +956,43 @@ jobs:
|
||||
echo "No fix PR: nothing was changed."
|
||||
fi
|
||||
fi
|
||||
echo
|
||||
echo "If Onyx is not affected, suppress it:"
|
||||
echo '```'
|
||||
for id in ${ids}; do echo "ods audit ignore add ${id} --reason \"${reason}\""; done
|
||||
echo '```'
|
||||
echo "If Onyx is affected and no fix exists yet, suppress it for 30 days:"
|
||||
echo '```'
|
||||
for id in ${ids}; do echo "ods audit ignore add ${id} --reason \"${reason}\" --expires ${expires}"; done
|
||||
echo '```'
|
||||
} > "${RUNNER_TEMP}/issue-comment.md"
|
||||
gh issue comment "${ISSUE}" --body-file "${RUNNER_TEMP}/issue-comment.md"
|
||||
|
||||
- name: Build the Slack message
|
||||
# The reviewer is tagged in the Slack post and owns the alert, fix PR or
|
||||
# not. The issue holds Claude's note and the ignore commands.
|
||||
- name: Assign the reviewer
|
||||
if: ${{ !cancelled() && steps.alert.outcome == 'success' }}
|
||||
id: assign
|
||||
env:
|
||||
ISSUE: ${{ matrix.alert.issue }}
|
||||
REVIEWER: ${{ steps.alert.outputs.reviewer }}
|
||||
run: gh issue edit "${ISSUE}" --add-assignee "${REVIEWER}"
|
||||
|
||||
- name: Build the Slack post
|
||||
if: ${{ !cancelled() && steps.alert.outcome == 'success' }}
|
||||
id: slack
|
||||
env:
|
||||
REVIEWER: ${{ steps.alert.outputs.reviewer }}
|
||||
PACKAGE: ${{ matrix.alert.package }}
|
||||
ECOSYSTEM: ${{ matrix.alert.ecosystem }}
|
||||
ISSUE_URL: ${{ matrix.alert.issue_url }}
|
||||
PR_URL: ${{ steps.pr.outputs.url || steps.cp.outputs.url }}
|
||||
BACKPORTS: ${{ steps.backport.outputs.urls }}
|
||||
BACKPORT_FAILED: ${{ steps.backport.outputs.failed }}
|
||||
BACKPORT_OUTCOME: ${{ steps.backport.outcome }}
|
||||
CHANGED: ${{ steps.changes.outputs.changed }}
|
||||
CP_OUTCOME: ${{ steps.cp.outcome }}
|
||||
ISSUE_URL: ${{ matrix.alert.issue_url }}
|
||||
ECOSYSTEM: ${{ matrix.alert.ecosystem }}
|
||||
TRIAGE_OK: ${{ steps.triage.outputs.ok }}
|
||||
run: |
|
||||
alert="${RUNNER_TEMP}/alert.json"
|
||||
result="${RUNNER_TEMP}/triage/result.json"
|
||||
@@ -962,22 +1005,22 @@ jobs:
|
||||
# `ods audit ignore add` rejects an empty --reason.
|
||||
reason="${reason:-<why this is not exploitable in Onyx>}"
|
||||
expires="$(date -u -d '+30 days' +%Y-%m-%d)"
|
||||
# Slack rejects a section over 3000 characters, so the message lists at
|
||||
# most five advisories and drafts commands for two. The issue has all.
|
||||
# The thread lists at most five advisories and drafts commands for
|
||||
# two, so it stays readable. The issue has all of them.
|
||||
count="$(jq '.advisories | length' "${alert}")"
|
||||
ids="$(jq -r '.advisories[:2][].id' "${alert}")"
|
||||
mention="$(jq -r --arg gh "${REVIEWER}" 'to_entries[] | select(.value | ascii_downcase == ($gh | ascii_downcase)) | "<@\(.key)>"' .github/actions/slack-notify/user-mappings.json | head -1)"
|
||||
label="fix PR"
|
||||
if [ "${CHANGED}" = "true" ] && [ -n "${version}" ]; then
|
||||
label="pins ${version}"
|
||||
fi
|
||||
# A fixed delimiter could appear in the note and end the output early.
|
||||
eof="SLACK_DETAILS_$(openssl rand -hex 16)"
|
||||
eof="SLACK_$(openssl rand -hex 16)"
|
||||
{
|
||||
echo "details<<${eof}"
|
||||
echo "{mention} this one is yours."
|
||||
echo "*Advisories:* $(jq -r '[.advisories[:5][] | "<\(.url)|\(.id)>"] | join(", ")' "${alert}")$( [ "${count}" -gt 5 ] && echo ", and $((count - 5)) more in the issue")"
|
||||
echo "*Installed:* $(jq -r '(.versions // []) | if length == 0 then "unknown" else join(", ") end' "${alert}" | head -c 200)"
|
||||
echo "text<<${eof}"
|
||||
echo "*🚨 Vulnerable dependency: ${PACKAGE} (${ECOSYSTEM})* ${mention:-@${REVIEWER}}"
|
||||
echo "*Branch:* ${GITHUB_REF_NAME}"
|
||||
if [ -n "${PR_URL}" ]; then
|
||||
label="fix PR"
|
||||
if [ "${CHANGED}" = "true" ] && [ -n "${version}" ]; then
|
||||
label="pins ${version}"
|
||||
fi
|
||||
echo "*Fix PR:* <${PR_URL}|${label}>. Approve and merge it."
|
||||
if [ -n "${BACKPORTS}" ]; then
|
||||
echo "*Backports:* $(for b in ${BACKPORTS}; do printf '<%s|%s> ' "${b}" "${b##*/pull/}"; done)"
|
||||
@@ -989,17 +1032,19 @@ jobs:
|
||||
if [ "${BACKPORT_OUTCOME}" = "failure" ]; then
|
||||
echo "*Backports:* broke before opening a PR, see the run."
|
||||
fi
|
||||
elif [[ "${ECOSYSTEM}" == *:* ]] && [ "${GITHUB_REF_NAME}" = "main" ]; then
|
||||
echo "*Fix PR:* started the base image digest refresh, which opens a PR if an upstream image has the fix."
|
||||
elif [ "${CP_OUTCOME}" = "failure" ]; then
|
||||
echo "*Fix PR:* the cherry-pick of main's fix failed, likely on a conflict. Pick it by hand, see the run."
|
||||
elif [ "${TRIAGE_OK}" != "true" ]; then
|
||||
echo "*Fix PR:* triage did not finish, so nothing was checked. The next run retries, see the run."
|
||||
else
|
||||
echo "*Fix PR:* none. Nothing was changed, see the note."
|
||||
echo "*Fix PR:* no fix available, see ods ignore commands in thread or bump the version manually."
|
||||
fi
|
||||
echo "*Issue:* <${ISSUE_URL}|tracking issue>"
|
||||
echo "*Claude's verdict:* $(jq -r .verdict "${result}")"
|
||||
slack_escape < "${RUNNER_TEMP}/triage/note.md" | head -c 700
|
||||
echo
|
||||
echo "${eof}"
|
||||
echo "thread<<${eof}"
|
||||
echo "*Description:* Verdict $(jq -r .verdict "${result}"). $(slack_escape < "${RUNNER_TEMP}/triage/note.md" | head -c 2500)"
|
||||
echo "*Advisories:* $(jq -r '[.advisories[:5][] | "<\(.url)|\(.id)>"] | join(", ")' "${alert}")$( [ "${count}" -gt 5 ] && echo ", and $((count - 5)) more in the issue")"
|
||||
echo "*Installed:* $(jq -r '(.versions // []) | if length == 0 then "unknown" else join(", ") end' "${alert}" | head -c 200)"
|
||||
echo "If Onyx is not affected, suppress it:"
|
||||
echo '```'
|
||||
for id in ${ids}; do echo "ods audit ignore add ${id} --reason \"${reason}\""; done
|
||||
@@ -1011,25 +1056,40 @@ jobs:
|
||||
if [ "${count}" -gt 2 ]; then
|
||||
echo "Repeat for the other $((count - 2)) advisories in the issue."
|
||||
fi
|
||||
echo "*Run:* <${RUN_URL}|CVE Alerts run>"
|
||||
echo "${eof}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
# The top post carries the reviewer, the fix PR, and the issue, and the
|
||||
# thread the rest. Both must land before the pending label clears, and a
|
||||
# top post whose thread failed is deleted so the retry posts no second.
|
||||
- name: Post to #cve-reviews
|
||||
if: ${{ !cancelled() && steps.slack.outcome == 'success' }}
|
||||
id: post
|
||||
uses: ./.github/actions/slack-notify
|
||||
with:
|
||||
webhook-url: ${{ secrets.CVE_REVIEWS_WEBHOOK }}
|
||||
title: "🚨 Vulnerable dependency: ${{ matrix.alert.package }} (${{ matrix.alert.ecosystem }})${{ github.ref_name != 'main' && format(' on {0}', github.ref_name) || '' }}"
|
||||
mention: ${{ steps.alert.outputs.reviewer }}
|
||||
details: ${{ steps.slack.outputs.details }}
|
||||
# The pending label clears only after Slack accepts the post.
|
||||
fail-on-error: "true"
|
||||
env:
|
||||
SLACK_BOT_TOKEN: ${{ secrets.CVE_REVIEWS_BOT_TOKEN }}
|
||||
TEXT: ${{ steps.slack.outputs.text }}
|
||||
THREAD: ${{ steps.slack.outputs.thread }}
|
||||
run: |
|
||||
post() {
|
||||
jq -cn --arg channel "${CVE_REVIEWS_CHANNEL}" --arg text "$1" --arg ts "${2:-}" \
|
||||
'{channel: $channel, text: $text, unfurl_links: false, unfurl_media: false} + (if $ts == "" then {} else {thread_ts: $ts} end)' \
|
||||
| curl -sS --fail-with-body -X POST https://slack.com/api/chat.postMessage \
|
||||
-H "Authorization: Bearer ${SLACK_BOT_TOKEN}" -H 'Content-type: application/json; charset=utf-8' --data-binary @- \
|
||||
| jq -re 'if .ok then .ts else error(.error) end'
|
||||
}
|
||||
ts="$(post "${TEXT}")"
|
||||
if ! post "${THREAD}" "${ts}" > /dev/null; then
|
||||
jq -cn --arg channel "${CVE_REVIEWS_CHANNEL}" --arg ts "${ts}" '{channel: $channel, ts: $ts}' \
|
||||
| curl -sS -X POST https://slack.com/api/chat.delete \
|
||||
-H "Authorization: Bearer ${SLACK_BOT_TOKEN}" -H 'Content-type: application/json; charset=utf-8' --data-binary @- > /dev/null
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A failed triage or assignment keeps the label, so the next run retries
|
||||
# the whole alert.
|
||||
# A failed triage, note, assignment, or post keeps the label, so the next
|
||||
# run retries the whole alert.
|
||||
- name: Clear the pending label
|
||||
if: ${{ !cancelled() && steps.post.outcome == 'success' && steps.assign.outcome == 'success' && steps.triage.outputs.ok == 'true' && steps.cp.outcome != 'failure' && steps.pr.outcome != 'failure' && steps.backport.outcome != 'failure' && steps.backport.outputs.failed == '' }}
|
||||
if: ${{ !cancelled() && steps.note.outcome == 'success' && steps.assign.outcome == 'success' && steps.post.outcome == 'success' && steps.triage.outputs.ok == 'true' && steps.cp.outcome != 'failure' && steps.pr.outcome != 'failure' && steps.backport.outcome != 'failure' && steps.backport.outputs.failed == '' }}
|
||||
env:
|
||||
ISSUE: ${{ matrix.alert.issue }}
|
||||
run: gh issue edit "${ISSUE}" --remove-label cve-alert-pending
|
||||
|
||||
@@ -2049,7 +2049,7 @@ jobs:
|
||||
# run once its images are published, since a release branch has no schedule
|
||||
# and that run is what closes the branch's issues after a fix ships.
|
||||
dispatch-cve-alerts:
|
||||
needs: [determine-builds, audit-gate, image-audit, merge-web, merge-backend, merge-model-server, merge-sandbox]
|
||||
needs: [determine-builds, audit-gate, image-audit, merge-web, merge-backend, merge-model-server]
|
||||
if: >-
|
||||
always() && !cancelled() && needs.determine-builds.outputs.is-test-run != 'true' &&
|
||||
(needs.audit-gate.result == 'failure' || needs.image-audit.result == 'failure' || startsWith(github.ref_name, 'v'))
|
||||
|
||||
+3
-3
@@ -748,11 +748,11 @@ when the package gains an advisory, and closes the issues of packages that no
|
||||
longer block. It prints the alerts that need announcing as JSON, including any
|
||||
still labelled `cve-alert-pending` from a run that never announced them. The
|
||||
[CVE Alerts workflow](../../.github/workflows/cve-alerts.yml) runs it daily and
|
||||
after a failed deploy audit, then posts to Slack per alert and opens a fix PR
|
||||
when a pin applies.
|
||||
after a failed deploy audit, opens a fix PR when a pin applies, and posts each
|
||||
alert to Slack with the details in a thread.
|
||||
|
||||
```shell
|
||||
ods audit --all-lockfiles --web --python --ignore-url "" --format=json > deps.json
|
||||
ods audit --web --python --ignore-url "" --format=json > deps.json
|
||||
ods audit image docker.io/onyxdotapp/onyx-backend:edge --ignore-url "" --format=json > backend.json
|
||||
ods audit alert --results deps.json --results backend.json --dry-run
|
||||
```
|
||||
|
||||
Reference in new issue
Block a user