feat(connectors): checks report timeouts with their own kind and row text

Add TimedOutValidationError. The runner gives it the timeout kind, with the
sign-in policy and the Stack Trace of an unreachable source. Jira, HubSpot,
Notion and Linear raise it for a request timeout, and GitHub and Confluence
raise it for a sign-in timeout.

Add CapabilityCheck.timeout_text. The hang guard shows it instead of the
shared "Timed out" text. Each named-check source sets the design's text:
sign-in rows get "Sign-in timed out", other rows get the source's shared
timeout text.

Stop a cycle in an error's cause chain from making _http_detail loop.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KFMazQbuG8ui3jxBxm47sC
This commit is contained in:
Claude committed 2026-10-17 03:36:46 +00:00
1 parent 85aadc22f4
commit b29dbe67c6
22 files changed
+516 -62

No files matched your search

+5 -1
View File
@@ -478,7 +478,11 @@ A source gateway's API error subclasses `exceptions.py:SourceApiError`.
A result's row text is `summary` (a short line) and `description` (up to three
lines). A check's `ValidationError` can carry both, and the runner falls back to
a default for a timeout, a rate limit, an unreachable source or an unexpected
error, and to the "Not run: …" line for a stopped check. `message` stays the
error, and to the "Not run: …" line for a stopped check. A request that the
source does not answer in time raises `exceptions.py:TimedOutValidationError`
(kind `timeout`). When the hang guard stops a check, the row shows the check's
`timeout_text` (`models.py:CheckRowText`; sign-in checks use
`sign_in_timeout_text`), else the default. `message` stays the
full text. A passing check gives its line in `CheckOutput.summary`. Checks that
share a `row_id` are one row on the form: the variants of a probe (token types,
deployments) and a probe's discovery step.
@@ -33,6 +33,7 @@ from onyx.connectors.exceptions import (
InsufficientPermissionsError,
RateLimitedValidationError,
SourceApiError,
TimedOutValidationError,
UnreachableValidationError,
)
from onyx.connectors.interfaces import BaseConnector
@@ -301,8 +302,10 @@ def _scrubbed(text: str, secrets: Iterable[str]) -> str:
def _http_detail(error: BaseException) -> CheckErrorDetail | None:
"""The HTTP response of the first error in the chain that has one, with
the whole body: ``check_error_detail`` redacts it, then cuts it."""
seen: set[int] = set()
current: BaseException | None = error
while current is not None:
while current is not None and id(current) not in seen:
seen.add(id(current))
if isinstance(current, SourceApiError):
return CheckErrorDetail(
status_code=current.status_code,
@@ -343,6 +346,31 @@ def check_error_detail(
return detail if detail != CheckErrorDetail() else None
class CheckRowText(BaseModel):
"""A row's line and description. In a hang-guard text,
``TIME_LIMIT_PLACEHOLDER`` in the description gives the guard's time
limit, e.g. "10 minutes"."""
model_config = ConfigDict(frozen=True)
summary: str
description: str
TIME_LIMIT_PLACEHOLDER = "{time_limit}"
def sign_in_timeout_text(source_name: str) -> CheckRowText:
"""The row text of a sign-in that the source did not answer in time."""
return CheckRowText(
summary="Sign-in timed out",
description=(
f"{source_name} did not answer the sign-in in time.\nRe-run, or "
f"check that {source_name} is reachable from Onyx."
),
)
class CheckNotRunReason(str, Enum):
"""Why a SKIPPED or STOPPED check did not run: a gate of the run, or the
admin."""
@@ -391,7 +419,7 @@ def check_error_kind(error: Exception) -> CheckErrorKind:
return CheckErrorKind.RATE_LIMITED
if isinstance(error, InconclusiveValidationError):
return CheckErrorKind.INCONCLUSIVE
if isinstance(error, TimeoutError):
if isinstance(error, (TimeoutError, TimedOutValidationError)):
return CheckErrorKind.TIMEOUT
if isinstance(error, UnreachableValidationError):
return CheckErrorKind.UNREACHABLE
@@ -525,6 +553,7 @@ class CapabilityCheck(ABC, Generic[ConfigT]):
depends_on: tuple[str, ...] = (),
row_id: str | None = None,
step: CheckStep | None = None,
timeout_text: CheckRowText | None = None,
) -> None:
self.capability = capability
self.check_id = check_id
@@ -578,6 +607,9 @@ class CapabilityCheck(ABC, Generic[ConfigT]):
self.row_id = row_id or (step.step_of if step is not None else check_id)
# Set on the second step of a two-step row.
self.step = step
# The row text when the hang guard stops the check. None gives the
# runner's default.
self.timeout_text = timeout_text
def outcome_status(
self, error_kind: CheckErrorKind, form_run: bool
@@ -10,6 +10,7 @@ from onyx.configs.constants import DocumentSource
from onyx.connectors.capability_checks.findings_setting import check_findings_enabled
from onyx.connectors.capability_checks.models import (
INCOMPLETE_STATUSES,
TIME_LIMIT_PLACEHOLDER,
CapabilityCheck,
CapabilityCheckContext,
CapabilityCheckResult,
@@ -21,6 +22,7 @@ from onyx.connectors.capability_checks.models import (
CheckNotApplicableError,
CheckNotRunReason,
CheckOutput,
CheckRowText,
CredentialCapability,
CredentialCapabilityReport,
check_error_detail,
@@ -203,12 +205,17 @@ def _error_outcome(
form_run: bool,
message: str | None = None,
duration_ms: int | None = None,
default_row: CheckRowText | None = None,
) -> _CheckOutcome:
"""Maps a check's exception to an outcome, per the check exception
contract and the check's ``outcome_status``. The row text is the
error's own, else the default of its kind."""
error's own, else ``default_row``, else the default of its kind."""
error_kind = check_error_kind(error)
summary, description = _DEFAULT_ROW_TEXT_BY_KIND.get(error_kind, (None, None))
summary, description = (
(default_row.summary, default_row.description)
if default_row is not None
else _DEFAULT_ROW_TEXT_BY_KIND.get(error_kind, (None, None))
)
if isinstance(error, ValidationError):
summary = error.summary or summary
description = error.description or description
@@ -249,6 +256,28 @@ def effective_check_timeout_seconds(check: CapabilityCheck[Any]) -> float:
return check.timeout_seconds or CAPABILITY_CHECK_TIMEOUT_SECONDS
def _time_limit_words(timeout_seconds: float) -> str:
seconds = round(timeout_seconds)
if seconds >= 60 and seconds % 60 == 0:
minutes = seconds // 60
return f"{minutes} minute{'' if minutes == 1 else 's'}"
return f"{seconds} second{'' if seconds == 1 else 's'}"
def _hang_guard_row(
check: CapabilityCheck, timeout_seconds: float
) -> CheckRowText | None:
"""The check's own text for its hang guard, with the guard's time limit."""
if check.timeout_text is None:
return None
return CheckRowText(
summary=check.timeout_text.summary,
description=check.timeout_text.description.replace(
TIME_LIMIT_PLACEHOLDER, _time_limit_words(timeout_seconds)
),
)
def _capped(timeout_seconds: float, cap_seconds: float | None) -> float:
return timeout_seconds if cap_seconds is None else min(timeout_seconds, cap_seconds)
@@ -315,6 +344,7 @@ def _execute_check(
form_run,
message=_TIMEOUT_MESSAGE,
duration_ms=elapsed_ms(),
default_row=_hang_guard_row(check, timeout_seconds),
)
except Exception as e:
if check_error_kind(e) == CheckErrorKind.UNEXPECTED:
@@ -39,9 +39,11 @@ from onyx.connectors.capability_checks.models import (
CheckAdvisory,
CheckFindingKey,
CheckOutput,
CheckRowText,
CheckStep,
CredentialCapability,
OptionsFinding,
sign_in_timeout_text,
)
from onyx.connectors.confluence.config import (
ConfluenceConnectorConfig,
@@ -85,6 +87,7 @@ from onyx.connectors.exceptions import (
InsufficientPermissionsError,
NothingToCheckError,
RateLimitedValidationError,
TimedOutValidationError,
UnexpectedValidationError,
UnreachableValidationError,
)
@@ -93,6 +96,14 @@ from onyx.utils.logger import setup_logger
logger = setup_logger()
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/confluence"
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description=(
"Request timed out after {time_limit}. Unable to confirm access.\n"
"Re-run, or check the site's status."
),
)
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("Confluence")
_SITE_FIELDS = frozenset({"wiki_base", "is_cloud"})
_SPACE_FIELD = "space"
@@ -378,6 +389,7 @@ class _ConfluenceCheck(CapabilityCheck[ConfluenceConnectorConfig]):
fails_when_unreachable: bool = False,
row_id: str | None = None,
step: CheckStep | None = None,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=capability,
@@ -394,6 +406,7 @@ class _ConfluenceCheck(CapabilityCheck[ConfluenceConnectorConfig]):
fails_when_unreachable=fails_when_unreachable,
depends_on=_DEPENDS_ON.get(check_id, ()),
step=step,
timeout_text=timeout_text,
)
@@ -413,6 +426,7 @@ class _SiteAuthCheck(_ConfluenceCheck):
row_id="confluence_auth",
validates_binding=True,
fails_when_unreachable=True,
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
display_name="Sign in to Confluence",
remediation=(
"Scoped API tokens work only with Confluence Cloud. Check the "
@@ -424,6 +438,7 @@ class _SiteAuthCheck(_ConfluenceCheck):
check_id="confluence_auth",
validates_binding=True,
fails_when_unreachable=True,
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
display_name="Sign in to Confluence",
remediation=(
"Check the site URL and the Cloud setting. For an API token, "
@@ -508,7 +523,13 @@ class _SiteAuthCheck(_ConfluenceCheck):
),
),
)
except (requests.ConnectionError, requests.Timeout) as e:
except requests.Timeout as e:
raise TimedOutValidationError(
f"{config.wiki_base} did not answer the sign-in in time: {e}",
summary=_SIGN_IN_TIMEOUT_TEXT.summary,
description=_SIGN_IN_TIMEOUT_TEXT.description,
) from e
except requests.ConnectionError as e:
raise UnreachableValidationError(
f"Onyx cannot connect to {config.wiki_base}. Check the site URL.",
summary=f"Cannot reach {urlparse(config.wiki_base).hostname}",
+15
View File
@@ -94,6 +94,21 @@ class UnreachableValidationError(UnexpectedValidationError):
super().__init__(message, summary=summary, description=description)
class TimedOutValidationError(UnreachableValidationError):
"""Raised when the source does not answer a request of the validation in
time. Like an unreachable source, a slow one says nothing about the
credential."""
def __init__(
self,
message: str = "The source did not answer in time",
*,
summary: str | None = None,
description: str | None = None,
):
super().__init__(message, summary=summary, description=description)
class InconclusiveValidationError(UnexpectedValidationError):
"""Raised when a probe ran, but what it sampled cannot decide: the sample
held nothing to test, or the source refused every try. Not proof of a
@@ -27,7 +27,9 @@ from onyx.connectors.capability_checks.models import (
CapabilityCheckContext,
CheckAdvisory,
CheckOutput,
CheckRowText,
CredentialCapability,
sign_in_timeout_text,
)
from onyx.connectors.exceptions import (
ConnectorValidationError,
@@ -37,6 +39,7 @@ from onyx.connectors.exceptions import (
InsufficientPermissionsError,
NothingToCheckError,
RateLimitedValidationError,
TimedOutValidationError,
UnexpectedValidationError,
UnreachableValidationError,
)
@@ -63,6 +66,11 @@ from onyx.connectors.models import ConnectorMissingCredentialError
from onyx.connectors.source_operations import SourceOperations
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/github"
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description="The check ran past its time limit.\nRe-run the checks later.",
)
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("GitHub")
_SSO_GUIDE_LINK = (
"https://docs.github.com/en/enterprise-cloud@latest/authentication/"
"authenticating-with-saml-single-sign-on/"
@@ -278,6 +286,7 @@ class _GitHubCheck(CapabilityCheck[GithubConnectorConfig]):
requires_fields: frozenset[str] = _OWNER_FIELDS,
requires_connector_config: bool = False,
fails_when_unreachable: bool = False,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=capability,
@@ -291,6 +300,7 @@ class _GitHubCheck(CapabilityCheck[GithubConnectorConfig]):
remediation=remediation,
docs_link=_DOCS_LINK,
fails_when_unreachable=fails_when_unreachable,
timeout_text=timeout_text,
)
@@ -316,6 +326,7 @@ class _AuthCheck(_GitHubCheck):
),
requires_fields=frozenset(),
fails_when_unreachable=True,
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
)
def run(self, context: CapabilityCheckContext) -> CheckOutput:
@@ -330,7 +341,13 @@ class _AuthCheck(_GitHubCheck):
"token in API Token."
),
) from e
except (requests.ConnectionError, requests.Timeout) as e:
except requests.Timeout as e:
raise TimedOutValidationError(
f"GitHub did not answer the sign-in in time: {e}",
summary=_SIGN_IN_TIMEOUT_TEXT.summary,
description=_SIGN_IN_TIMEOUT_TEXT.description,
) from e
except requests.ConnectionError as e:
raise UnreachableValidationError(
"Onyx cannot connect to GitHub. Check the Enterprise Server URL, "
"if one is set.",
@@ -37,10 +37,12 @@ from onyx.connectors.capability_checks.models import (
CheckAdvisory,
CheckFindingKey,
CheckOutput,
CheckRowText,
CheckStep,
CountFinding,
CredentialCapability,
OptionsFinding,
sign_in_timeout_text,
)
from onyx.connectors.exceptions import (
ConnectorValidationError,
@@ -97,6 +99,10 @@ from onyx.connectors.models import ConnectorMissingCredentialError
from onyx.connectors.source_operations import SourceOperations
_DOCS_LINK = SCOPE_DOC_URL
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description="The check ran past its time limit.\nRe-run the checks.",
)
_DRIVE_API = "Google Drive API"
_ADMIN_API = "Admin SDK API"
@@ -496,6 +502,7 @@ class _GoogleDriveCheck(CapabilityCheck[GoogleDriveConnectorConfig]):
credential_kind: GoogleCredentialKind | None = None,
discovery: bool = False,
step: CheckStep | None = None,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=capability,
@@ -515,6 +522,7 @@ class _GoogleDriveCheck(CapabilityCheck[GoogleDriveConnectorConfig]):
docs_link=_DOCS_LINK,
discovery=discovery,
step=step,
timeout_text=timeout_text,
)
self._credential_kind = credential_kind
@@ -547,6 +555,7 @@ class _AuthCheck(_GoogleDriveCheck):
"OAuth: connect Google Drive again. Turn on the Google Drive API "
"in the Google Cloud project of the credential."
),
timeout_text=sign_in_timeout_text("Google Drive"),
)
def run(self, context: CapabilityCheckContext) -> CheckOutput:
@@ -23,8 +23,10 @@ from onyx.connectors.capability_checks.models import (
CapabilityCheck,
CapabilityCheckContext,
CheckOutput,
CheckRowText,
CredentialCapability,
form_config,
sign_in_timeout_text,
)
from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
RateLimitTriedTooManyTimesError,
@@ -36,6 +38,7 @@ from onyx.connectors.exceptions import (
InsufficientPermissionsError,
NothingToCheckError,
RateLimitedValidationError,
TimedOutValidationError,
UnexpectedValidationError,
UnreachableValidationError,
)
@@ -60,6 +63,14 @@ from onyx.utils.logger import setup_logger
logger = setup_logger()
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/hubspot"
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description=(
"The check stopped after {time_limit} without an answer from HubSpot.\n"
"Re-run the checks."
),
)
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("HubSpot")
_PROBE_LIMIT = 1
_NOTE_BODY_PROPERTY = "hs_note_body"
_EPOCH = datetime.fromtimestamp(0, tz=timezone.utc)
@@ -154,6 +165,10 @@ _UNREACHABLE_ROW = _RowText(
"api.hubapi.com."
),
)
_REQUEST_TIMEOUT_ROW = _RowText(
summary="Timed out",
description="HubSpot did not answer the request in time.\nRe-run the checks.",
)
_UNEXPECTED_ROW = _RowText(
summary="Unexpected response from HubSpot",
description=(
@@ -181,6 +196,12 @@ def _raise_for_error(
"a minute, then re-run."
),
) from error
if isinstance(error, HubSpotConnectionError) and error.timed_out:
raise TimedOutValidationError(
f"HubSpot did not answer in time: {error}",
summary=_REQUEST_TIMEOUT_ROW.summary,
description=_REQUEST_TIMEOUT_ROW.description,
) from error
if isinstance(error, HubSpotConnectionError):
raise UnreachableValidationError(
f"Onyx cannot connect to HubSpot: {error}",
@@ -287,6 +308,7 @@ class _HubSpotCheck(CapabilityCheck[HubSpotConnectorConfig]):
required: bool = True,
remediation: str | None = None,
fails_when_unreachable: bool = False,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=capability,
@@ -298,6 +320,7 @@ class _HubSpotCheck(CapabilityCheck[HubSpotConnectorConfig]):
remediation=remediation,
docs_link=_DOCS_LINK,
fails_when_unreachable=fails_when_unreachable,
timeout_text=timeout_text,
)
@@ -308,6 +331,7 @@ class _TokenCheck(_HubSpotCheck):
display_name="Sign in to HubSpot",
remediation="Create a private app in HubSpot and paste its access token.",
fails_when_unreachable=True,
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
)
def run(self, context: CapabilityCheckContext) -> CheckOutput:
@@ -325,13 +349,10 @@ class _TokenCheck(_HubSpotCheck):
except HubSpotConnectionError as error:
if not error.timed_out:
_raise_for_error(error, denied="read the portal")
raise UnreachableValidationError(
raise TimedOutValidationError(
f"HubSpot did not answer the sign-in in time: {error}",
summary="Sign-in timed out",
description=(
"HubSpot did not answer the sign-in in time.\nRe-run, or check "
"that HubSpot is reachable from Onyx."
),
summary=_SIGN_IN_TIMEOUT_TEXT.summary,
description=_SIGN_IN_TIMEOUT_TEXT.description,
) from error
except ValueError as error:
# A body that is not HubSpot's portal info, e.g. a proxy page.
@@ -32,8 +32,10 @@ from onyx.connectors.capability_checks.models import (
CapabilityCheckContext,
CheckAdvisory,
CheckOutput,
CheckRowText,
CheckStep,
CredentialCapability,
sign_in_timeout_text,
)
from onyx.connectors.exceptions import (
ConnectorValidationError,
@@ -42,6 +44,7 @@ from onyx.connectors.exceptions import (
InsufficientPermissionsError,
NothingToCheckError,
RateLimitedValidationError,
TimedOutValidationError,
UnexpectedValidationError,
UnreachableValidationError,
)
@@ -71,6 +74,13 @@ from onyx.connectors.jira.utils import (
from onyx.connectors.source_operations import SourceOperations
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/jira"
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description=(
"The check stopped because Jira did not answer in time.\nRe-run the checks."
),
)
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("Jira")
_SITE_FIELDS = frozenset({"jira_base_url"})
_ERROR_DETAIL_CHARS = 300
_CLOUD_HOST_SUFFIX = ".atlassian.net"
@@ -148,12 +158,10 @@ def _error_detail(error: JiraApiError) -> str:
def _raise_for_timeout(error: JiraTimeoutError) -> NoReturn:
"""INCOMPLETE: a slow Jira says nothing about the credential."""
raise UnreachableValidationError(
raise TimedOutValidationError(
f"{error} Run the checks again, or check the status of Jira.",
summary="Timed out",
description=(
"The check stopped because Jira did not answer in time.\nRe-run the checks."
),
summary=_TIMEOUT_TEXT.summary,
description=_TIMEOUT_TEXT.description,
) from error
@@ -278,6 +286,7 @@ class _JiraCheck(CapabilityCheck[JiraConnectorConfig]):
fails_when_unreachable: bool = False,
row_id: str | None = None,
step: CheckStep | None = None,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=capability,
@@ -293,6 +302,7 @@ class _JiraCheck(CapabilityCheck[JiraConnectorConfig]):
fails_when_unreachable=fails_when_unreachable,
depends_on=_DEPENDS_ON.get(check_id, ()),
step=step,
timeout_text=timeout_text,
)
@@ -318,6 +328,7 @@ class _SiteAuthCheck(_JiraCheck):
row_id="jira_auth",
validates_binding=True,
fails_when_unreachable=True,
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
display_name="Sign in to Jira",
remediation=(
"Scoped API tokens work only with Jira Cloud. Enter the email "
@@ -330,6 +341,7 @@ class _SiteAuthCheck(_JiraCheck):
check_id="jira_auth",
validates_binding=True,
fails_when_unreachable=True,
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
display_name="Sign in to Jira",
remediation=(
"Jira Cloud: enter the email of the token's Atlassian account "
@@ -382,14 +394,11 @@ class _SiteAuthCheck(_JiraCheck):
) from e
except (JiraTimeoutError, requests.Timeout) as e:
# A raw Timeout comes from the scoped token's tenant_info call.
raise UnreachableValidationError(
raise TimedOutValidationError(
f"{self.config(context).jira_base_url} did not answer the sign-in "
f"in time: {e}",
summary="Sign-in timed out",
description=(
"Jira did not answer the sign-in in time.\n"
"Re-run, or check that Jira is reachable from Onyx."
),
summary=_SIGN_IN_TIMEOUT_TEXT.summary,
description=_SIGN_IN_TIMEOUT_TEXT.description,
) from e
except requests.ConnectionError as e:
raise UnreachableValidationError(
@@ -12,15 +12,15 @@ from collections.abc import Callable
from enum import Enum
from typing import NoReturn, TypeVar
from pydantic import BaseModel
from onyx.connectors.capability_checks.form_state import FormState
from onyx.connectors.capability_checks.models import (
CapabilityCheck,
CapabilityCheckContext,
CheckOutput,
CheckRowText,
CredentialCapability,
form_config,
sign_in_timeout_text,
)
from onyx.connectors.exceptions import (
ConnectorValidationError,
@@ -28,6 +28,7 @@ from onyx.connectors.exceptions import (
CredentialInvalidError,
InconclusiveValidationError,
RateLimitedValidationError,
TimedOutValidationError,
UnexpectedValidationError,
UnreachableValidationError,
)
@@ -77,13 +78,6 @@ _MAX_SUMMARY_CHARS: int = 40
_REFRESH_REFUSED_STATUSES: frozenset[int] = frozenset({400, 401})
class _RowText(BaseModel):
"""The row text of one outcome of a probe."""
summary: str
description: str
class _CredentialKind(str, Enum):
API_KEY = "api_key"
OAUTH = "oauth"
@@ -126,19 +120,11 @@ def config_from_context(context: CapabilityCheckContext) -> LinearConnectorConfi
return form_config(context, LinearConnectorConfig)
def _timeout_row(sign_in: bool) -> _RowText:
if sign_in:
return _RowText(
summary="Sign-in timed out",
description=(
"Linear did not answer the sign-in in time.\nRe-run, or check "
"that Linear is reachable from Onyx."
),
)
return _RowText(
summary="Timed out",
description="The check ran past its time limit.\nRe-run the checks later.",
)
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description="The check ran past its time limit.\nRe-run the checks later.",
)
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("Linear")
def _raise_for_linear_error(
@@ -195,8 +181,8 @@ def _raise_for_linear_error(
),
) from error
if isinstance(error, LinearUnreachableError) and error.timed_out:
row: _RowText = _timeout_row(sign_in)
raise UnreachableValidationError(
row: CheckRowText = _SIGN_IN_TIMEOUT_TEXT if sign_in else _TIMEOUT_TEXT
raise TimedOutValidationError(
f"Linear did not answer in time: {error}",
summary=row.summary,
description=row.description,
@@ -307,6 +293,7 @@ class _LinearCheck(CapabilityCheck[LinearConnectorConfig]):
remediation: str,
requires_connector_config: bool = False,
fails_when_unreachable: bool = False,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=CredentialCapability.INDEXING,
@@ -318,6 +305,7 @@ class _LinearCheck(CapabilityCheck[LinearConnectorConfig]):
remediation=remediation,
docs_link=DOCS_LINK,
fails_when_unreachable=fails_when_unreachable,
timeout_text=timeout_text,
)
@@ -328,6 +316,7 @@ class _TokenCheck(_LinearCheck):
display_name="Sign in to Linear",
remediation="Check the API key, or reconnect through OAuth.",
fails_when_unreachable=True,
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
)
def run(self, context: CapabilityCheckContext) -> CheckOutput:
@@ -26,7 +26,9 @@ from onyx.connectors.capability_checks.models import (
CheckAdvisory,
CheckNotApplicableError,
CheckOutput,
CheckRowText,
CredentialCapability,
sign_in_timeout_text,
)
from onyx.connectors.exceptions import (
ConnectorValidationError,
@@ -36,6 +38,7 @@ from onyx.connectors.exceptions import (
InsufficientPermissionsError,
NothingToCheckError,
RateLimitedValidationError,
TimedOutValidationError,
UnexpectedValidationError,
UnreachableValidationError,
)
@@ -49,6 +52,14 @@ from onyx.connectors.notion.source_operations import (
from onyx.connectors.source_operations import SourceOperations
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/notion"
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description=(
"The check stopped after {time_limit} without an answer from Notion.\n"
"Re-run the checks."
),
)
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("Notion")
_ERROR_DETAIL_CHARS = 300
# Keeps a row's line short when it names a page or database.
_TITLE_CHARS = 24
@@ -162,7 +173,7 @@ def _raise_for_api_error(
),
) from error
if status is None and error.timed_out:
raise UnreachableValidationError(
raise TimedOutValidationError(
f"Notion did not answer in time: {detail}",
summary="Timed out",
description="Notion did not answer the request in time.\nRe-run the checks.",
@@ -260,6 +271,7 @@ class _NotionCheck(CapabilityCheck[NotionConnectorConfig]):
remediation: str,
required: bool = True,
fails_when_unreachable: bool = False,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=CredentialCapability.INDEXING,
@@ -271,6 +283,7 @@ class _NotionCheck(CapabilityCheck[NotionConnectorConfig]):
remediation=remediation,
docs_link=_DOCS_LINK,
fails_when_unreachable=fails_when_unreachable,
timeout_text=timeout_text,
)
@@ -296,6 +309,7 @@ class _AuthCheck(_NotionCheck):
"Configuration) and update the credential."
),
fails_when_unreachable=True,
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
)
def run(self, context: CapabilityCheckContext) -> CheckOutput:
@@ -313,13 +327,10 @@ class _AuthCheck(_NotionCheck):
) from e
except NotionApiError as e:
if e.status_code is None and e.timed_out:
raise UnreachableValidationError(
raise TimedOutValidationError(
f"Notion did not answer the sign-in in time: {e}",
summary="Sign-in timed out",
description=(
"Notion did not answer the sign-in in time.\nRe-run, or "
"check that Notion is reachable from Onyx."
),
summary=_SIGN_IN_TIMEOUT_TEXT.summary,
description=_SIGN_IN_TIMEOUT_TEXT.description,
) from e
_raise_for_api_error(
e,
@@ -23,10 +23,12 @@ from onyx.connectors.capability_checks.models import (
CheckAdvisory,
CheckFindingKey,
CheckOutput,
CheckRowText,
CheckStep,
CountFinding,
CredentialCapability,
form_config,
sign_in_timeout_text,
)
from onyx.connectors.exceptions import (
ConnectorValidationError,
@@ -64,6 +66,10 @@ from onyx.connectors.onedrive.source_operations import (
)
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/onedrive"
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description="The check did not finish within its time limit.\nRe-run the checks.",
)
_PROBE_PAGE_SIZE = 1
_CANDIDATE_PAGES = 20
# A bound on the pages the drive count reads. Pages hold up to 999 users, so
@@ -370,6 +376,7 @@ class _OneDriveCheck(CapabilityCheck[OneDriveConnectorConfig]):
discovery: bool = False,
row_id: str | None = None,
step: CheckStep | None = None,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=capability,
@@ -385,6 +392,7 @@ class _OneDriveCheck(CapabilityCheck[OneDriveConnectorConfig]):
discovery=discovery,
row_id=row_id,
step=step,
timeout_text=timeout_text,
)
@@ -400,6 +408,7 @@ class _TokenCheck(_OneDriveCheck):
remediation="Check the app id, tenant id, and client credential.",
validates_binding=True,
fails_when_unreachable=True,
timeout_text=sign_in_timeout_text("OneDrive"),
)
def run(self, context: CapabilityCheckContext) -> CheckOutput:
@@ -41,9 +41,11 @@ from onyx.connectors.capability_checks.models import (
CheckAdvisory,
CheckFindingKey,
CheckOutput,
CheckRowText,
CheckStep,
CountFinding,
CredentialCapability,
sign_in_timeout_text,
)
from onyx.connectors.exceptions import (
ConnectorValidationError,
@@ -96,6 +98,10 @@ from onyx.connectors.outlook.source_operations import OutlookSourceOperations
T = TypeVar("T")
_OUTLOOK_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/outlook"
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description="The check did not finish within its time limit.\nRe-run the checks.",
)
# A well-known folder every mailbox has, used to prove name resolution works.
_PROBE_WELL_KNOWN_FOLDER = "junkemail"
@@ -475,6 +481,7 @@ class _OutlookCheck(CapabilityCheck[OutlookConnectorConfig]):
discovery: bool = False,
row_id: str | None = None,
step: CheckStep | None = None,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=capability,
@@ -490,6 +497,7 @@ class _OutlookCheck(CapabilityCheck[OutlookConnectorConfig]):
discovery=discovery,
row_id=row_id,
step=step,
timeout_text=timeout_text,
)
@@ -512,6 +520,7 @@ class _TokenAuthCheck(_OutlookCheck):
),
validates_binding=True,
fails_when_unreachable=True,
timeout_text=sign_in_timeout_text("Outlook"),
)
def run(self, context: CapabilityCheckContext) -> CheckOutput:
@@ -25,8 +25,10 @@ from onyx.connectors.capability_checks.models import (
CapabilityCheckContext,
CheckAdvisory,
CheckOutput,
CheckRowText,
CredentialCapability,
form_config,
sign_in_timeout_text,
)
from onyx.connectors.exceptions import (
ConnectorValidationError,
@@ -115,6 +117,10 @@ from onyx.utils.logger import setup_logger
logger = setup_logger()
_DOCS_LINK: str = "https://docs.onyx.app/admins/connectors/official/sharepoint"
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description="The check did not finish within its time limit.\nRe-run the checks.",
)
_PROBE_SITES: int = 5
# Libraries whose first delta page holds no indexable file before the probe
# stops, so a tenant of empty libraries cannot spend the check's budget.
@@ -849,6 +855,7 @@ class _SharepointCheck(CapabilityCheck[SharepointConnectorConfig]):
requires_connector_config: bool = True,
validates_binding: bool = False,
fails_when_unreachable: bool = False,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=capability,
@@ -861,6 +868,7 @@ class _SharepointCheck(CapabilityCheck[SharepointConnectorConfig]):
validates_binding=validates_binding,
fails_when_unreachable=fails_when_unreachable,
depends_on=_DEPENDS_ON.get(check_id, ()),
timeout_text=timeout_text,
)
@@ -874,6 +882,7 @@ class _TokenCheck(_SharepointCheck):
display_name="Sign in to Microsoft",
validates_binding=True,
fails_when_unreachable=True,
timeout_text=sign_in_timeout_text("SharePoint"),
remediation=(
"Check the client id, directory id and client secret or "
"certificate, and that the Graph API and authority hosts match "
@@ -46,9 +46,11 @@ from onyx.connectors.capability_checks.models import (
CheckFindingKey,
CheckNotApplicableError,
CheckOutput,
CheckRowText,
CheckStep,
CredentialCapability,
OptionsFinding,
sign_in_timeout_text,
)
from onyx.connectors.exceptions import (
ConnectorValidationError,
@@ -83,6 +85,13 @@ from onyx.connectors.source_operations import SourceOperations
_SLACK_DOCS_LINK = (
"https://docs.onyx.app/admins/connectors/official/slack/slack_indexed"
)
_TIMEOUT_TEXT = CheckRowText(
summary="Timed out",
description=(
"The check stopped after {time_limit}.\nRe-run the checks when no Slack "
"indexing job is running."
),
)
_ADD_SCOPE_REMEDIATION = (
"Add the `{scope}` bot scope under OAuth & Permissions in the Slack app "
@@ -490,6 +499,7 @@ class _SlackCheck(CapabilityCheck[SlackConnectorConfig]):
discovery: bool = False,
fails_when_unreachable: bool = False,
step: CheckStep | None = None,
timeout_text: CheckRowText = _TIMEOUT_TEXT,
) -> None:
super().__init__(
capability=capability,
@@ -507,6 +517,7 @@ class _SlackCheck(CapabilityCheck[SlackConnectorConfig]):
discovery=discovery,
fails_when_unreachable=fails_when_unreachable,
step=step,
timeout_text=timeout_text,
)
@@ -523,6 +534,7 @@ class _TokenAuthCheck(_SlackCheck):
),
validates_binding=True,
fails_when_unreachable=True,
timeout_text=sign_in_timeout_text("Slack"),
)
def run(self, context: CapabilityCheckContext) -> CheckOutput:
@@ -0,0 +1,207 @@
"""Timeouts: a ``TimedOutValidationError`` has the timeout kind and the
sign-in policy of an unreachable source, the hang guard uses a check's own
timeout text, and an error detail survives a cycle in the cause chain."""
import time
import pytest
from onyx.configs.constants import DocumentSource
from onyx.connectors.capability_checks.models import (
CapabilityCheck,
CapabilityCheckContext,
CapabilityCheckResult,
CapabilityCheckStatus,
CheckErrorKind,
CheckRowText,
CredentialCapability,
check_error_detail,
sign_in_timeout_text,
)
from onyx.connectors.capability_checks.registry import get_capability_checks
from onyx.connectors.capability_checks.runner import (
_hang_guard_row,
run_capability_checks,
)
from onyx.connectors.exceptions import (
SourceApiError,
TimedOutValidationError,
UnreachableValidationError,
)
from onyx.utils.threadpool_concurrency import run_with_timeout
_TIMEOUT_SECONDS = 0.2
_OWN_TEXT = CheckRowText(summary="Sign-in timed out", description="Slow.\nRe-run.")
class _SlowCheck(CapabilityCheck):
"""Raises ``error``, or sleeps past its hang guard when it is None."""
def __init__(
self,
error: Exception | None = None,
*,
fails_when_unreachable: bool = False,
timeout_text: CheckRowText | None = None,
) -> None:
super().__init__(
capability=CredentialCapability.INDEXING,
check_id="slow_check",
display_name="Slow check",
requires_connector_instance=False,
timeout_seconds=_TIMEOUT_SECONDS,
fails_when_unreachable=fails_when_unreachable,
timeout_text=timeout_text,
)
self._error = error
def run(self, context: CapabilityCheckContext) -> None: # noqa: ARG002
if self._error is None:
time.sleep(_TIMEOUT_SECONDS * 5)
return
raise self._error
def _run(check: CapabilityCheck, *, form_run: bool = False) -> CapabilityCheckResult:
context = CapabilityCheckContext(source=DocumentSource.SLACK, credential_json={})
(result,) = run_capability_checks([check], context, form_run=form_run)
return result
@pytest.mark.parametrize(
("fails_when_unreachable", "form_run", "status"),
[
(True, True, CapabilityCheckStatus.FAILED),
(True, False, CapabilityCheckStatus.INCOMPLETE),
(False, True, CapabilityCheckStatus.INCOMPLETE),
],
)
def test_a_timed_out_error_follows_the_unreachable_policy(
fails_when_unreachable: bool, form_run: bool, status: CapabilityCheckStatus
) -> None:
check = _SlowCheck(
TimedOutValidationError("Slow."), fails_when_unreachable=fails_when_unreachable
)
result = _run(check, form_run=form_run)
assert (result.status, result.error_kind) == (status, CheckErrorKind.TIMEOUT)
assert result.summary == "Timed out"
assert result.error_detail is not None
assert result.error_detail.stack_trace is not None
def test_a_timed_out_error_is_still_an_unreachable_error() -> None:
assert isinstance(TimedOutValidationError(), UnreachableValidationError)
def test_the_hang_guard_uses_the_check_s_timeout_text() -> None:
result = _run(
_SlowCheck(fails_when_unreachable=True, timeout_text=_OWN_TEXT), form_run=True
)
assert result.status == CapabilityCheckStatus.FAILED
assert result.error_kind == CheckErrorKind.TIMEOUT
assert (result.summary, result.description) == (
_OWN_TEXT.summary,
_OWN_TEXT.description,
)
def test_the_hang_guard_keeps_the_default_text_without_one() -> None:
result = _run(_SlowCheck())
assert result.summary == "Timed out"
assert result.description is not None
assert result.description.startswith("The check did not finish in time.")
def test_an_error_s_own_text_wins_over_the_timeout_text() -> None:
error = TimedOutValidationError("Slow.", summary="Own line", description="Own.")
result = _run(_SlowCheck(error, timeout_text=_OWN_TEXT))
assert (result.summary, result.description) == ("Own line", "Own.")
@pytest.mark.parametrize(
("seconds", "words"),
[(600, "10 minutes"), (60, "1 minute"), (90, "90 seconds"), (1, "1 second")],
)
def test_the_timeout_text_names_the_guard_s_time_limit(
seconds: float, words: str
) -> None:
check = _SlowCheck(
timeout_text=CheckRowText(
summary="Timed out", description="Stopped after {time_limit}."
)
)
row = _hang_guard_row(check, seconds)
assert row is not None
assert row.description == f"Stopped after {words}."
_SIGN_IN_CHECKS: dict[DocumentSource, tuple[str, set[str]]] = {
DocumentSource.CONFLUENCE: (
"Confluence",
{"confluence_auth", "confluence_scoped_token_auth"},
),
DocumentSource.JIRA: ("Jira", {"jira_auth", "jira_scoped_token_auth"}),
DocumentSource.GOOGLE_DRIVE: ("Google Drive", {"google_drive_auth"}),
DocumentSource.GITHUB: ("GitHub", {"github_auth"}),
DocumentSource.LINEAR: ("Linear", {"linear_token"}),
DocumentSource.SLACK: ("Slack", {"slack_token_auth"}),
DocumentSource.NOTION: ("Notion", {"notion_auth"}),
DocumentSource.HUBSPOT: ("HubSpot", {"hubspot_token"}),
DocumentSource.SHAREPOINT: ("SharePoint", {"sharepoint_token_auth"}),
DocumentSource.ONEDRIVE: ("OneDrive", {"onedrive_token_auth"}),
DocumentSource.OUTLOOK: ("Outlook", {"outlook_token_auth"}),
}
@pytest.mark.parametrize("source", list(_SIGN_IN_CHECKS))
def test_each_check_has_the_design_s_timeout_text(source: DocumentSource) -> None:
"""Sign-in rows have their own line; the other rows share the source's."""
name, sign_in_ids = _SIGN_IN_CHECKS[source]
checks = get_capability_checks(source)
for check in checks:
if check.check_id in sign_in_ids:
assert check.timeout_text == sign_in_timeout_text(name), check.check_id
else:
assert check.timeout_text is not None, check.check_id
assert check.timeout_text.summary == "Timed out", check.check_id
assert sign_in_ids <= {check.check_id for check in checks}
def test_an_error_detail_survives_a_cycle_in_the_cause_chain() -> None:
first = RuntimeError("first")
second = RuntimeError("second")
first.__cause__ = second
second.__cause__ = first
detail = run_with_timeout(
5, check_error_detail, first, CheckErrorKind.UNEXPECTED, []
)
assert detail is not None
assert detail.status_code is None
assert detail.stack_trace is not None
def test_an_error_detail_finds_the_response_before_a_cycle() -> None:
api_error = SourceApiError(
"HTTP 500", status_code=500, method="GET", url="https://api.example.com/x"
)
wrapper = RuntimeError("wrapper")
wrapper.__cause__ = api_error
api_error.__cause__ = wrapper
detail = run_with_timeout(
5, check_error_detail, wrapper, CheckErrorKind.UNEXPECTED, []
)
assert detail is not None
assert (detail.status_code, detail.request) == (500, "GET api.example.com/x")
@@ -48,6 +48,7 @@ from onyx.connectors.exceptions import (
CredentialExpiredError,
InconclusiveValidationError,
InsufficientPermissionsError,
TimedOutValidationError,
UnexpectedValidationError,
)
@@ -310,6 +311,16 @@ def test_auth_connection_error_is_incomplete() -> None:
_run("confluence_auth", _context(gateway, _form()))
def test_auth_timeout_is_a_sign_in_timeout() -> None:
gateway = _gateway()
gateway.probe_site.side_effect = requests.ConnectTimeout("slow")
with pytest.raises(TimedOutValidationError) as raised:
_run("confluence_auth", _context(gateway, _form()))
assert raised.value.summary == "Sign-in timed out"
# confluence_spaces_visible
@@ -5,6 +5,7 @@ tested nothing says."""
from typing import Any
from unittest.mock import MagicMock
import pytest
import requests
from onyx.configs.constants import DocumentSource
@@ -115,16 +116,31 @@ def test_a_missing_token_says_so() -> None:
assert result.summary == "No token saved in the account"
def test_an_unreachable_server_fails_only_a_form_run() -> None:
@pytest.mark.parametrize(
"error,summary,kind",
[
(
requests.ConnectionError("refused"),
"Cannot reach GitHub",
CheckErrorKind.UNREACHABLE,
),
(requests.ReadTimeout("slow"), "Sign-in timed out", CheckErrorKind.TIMEOUT),
],
ids=["unreachable", "timeout"],
)
def test_an_unreachable_server_fails_only_a_form_run(
error: Exception, summary: str, kind: CheckErrorKind
) -> None:
gateway = _gateway()
gateway.get_token_info.side_effect = requests.ConnectionError("refused")
gateway.get_token_info.side_effect = error
form = _run("github_auth", gateway, form_run=True)
stored = _run("github_auth", gateway)
assert form.status == CapabilityCheckStatus.FAILED
assert stored.status == CapabilityCheckStatus.INCOMPLETE
assert form.summary == stored.summary == "Cannot reach GitHub"
assert form.summary == stored.summary == summary
assert form.error_kind == stored.error_kind == kind
def test_a_rate_limit_is_incomplete_and_stops_the_run() -> None:
@@ -159,6 +159,20 @@ def test_a_sign_in_timeout_fails_only_a_form_run() -> None:
assert form.status == CapabilityCheckStatus.FAILED
assert stored.status == CapabilityCheckStatus.INCOMPLETE
assert form.summary == "Sign-in timed out"
assert form.error_kind == stored.error_kind == CheckErrorKind.TIMEOUT
def test_a_request_timeout_after_sign_in_is_incomplete_on_a_form_run() -> None:
gateway = _gateway()
gateway.list_records.side_effect = HubSpotConnectionError(
"tickets", timed_out=True, detail="read timed out"
)
result = _run(_READ_TICKETS, gateway, form_run=True)
assert result.status == CapabilityCheckStatus.INCOMPLETE
assert result.error_kind == CheckErrorKind.TIMEOUT
assert result.summary == "Timed out"
def test_an_api_error_carries_the_request_and_body() -> None:
@@ -172,7 +172,7 @@ def test_a_sign_in_timeout_fails_only_a_form_run(
assert result.status == status
assert result.summary == "Sign-in timed out"
assert result.description == _SIGN_IN_DESCRIPTION
assert result.error_kind == CheckErrorKind.UNREACHABLE
assert result.error_kind == CheckErrorKind.TIMEOUT
assert result.error_detail is not None
assert result.error_detail.request == "GET example.atlassian.net/rest/api/3/myself"
@@ -197,6 +197,7 @@ def test_a_refused_connection_still_cannot_reach_jira() -> None:
assert result.status == CapabilityCheckStatus.FAILED
assert result.summary == "Cannot reach Jira"
assert result.error_kind == CheckErrorKind.UNREACHABLE
def test_a_timeout_after_sign_in_is_incomplete_on_a_form_run() -> None:
@@ -212,3 +213,4 @@ def test_a_timeout_after_sign_in_is_incomplete_on_a_form_run() -> None:
assert result.description == (
"The check stopped because Jira did not answer in time.\nRe-run the checks."
)
assert result.error_kind == CheckErrorKind.TIMEOUT
@@ -202,6 +202,7 @@ def test_a_sign_in_timeout_says_so() -> None:
assert result.status == CapabilityCheckStatus.FAILED
assert result.summary == "Sign-in timed out"
assert result.error_kind == CheckErrorKind.TIMEOUT
def test_a_busy_refresh_lock_is_incomplete() -> None:
@@ -151,18 +151,23 @@ def test_a_missing_token_fails_sign_in() -> None:
@pytest.mark.parametrize(
"error,summary",
"error,summary,kind",
[
(NotionApiError("reset", status_code=None, code=None), "Cannot reach Notion"),
(
NotionApiError("reset", status_code=None, code=None),
"Cannot reach Notion",
CheckErrorKind.UNREACHABLE,
),
(
NotionApiError("slow", status_code=None, code=None, timed_out=True),
"Sign-in timed out",
CheckErrorKind.TIMEOUT,
),
],
ids=["unreachable", "timeout"],
)
def test_an_unreachable_notion_fails_only_a_form_run(
error: NotionApiError, summary: str
error: NotionApiError, summary: str, kind: CheckErrorKind
) -> None:
gateway = _gateway()
gateway.get_bot_user.side_effect = error
@@ -173,6 +178,7 @@ def test_an_unreachable_notion_fails_only_a_form_run(
assert form.status == CapabilityCheckStatus.FAILED
assert stored.status == CapabilityCheckStatus.INCOMPLETE
assert form.summary == stored.summary == summary
assert form.error_kind == stored.error_kind == kind
def test_a_rate_limit_is_incomplete_and_stops_the_run() -> None: