mirror of
https://github.com/onyx-dot-app/onyx.git
synced 2026-10-19 01:50:51 -05:00
feat(connectors): checks report timeouts with their own kind and row text
Add TimedOutValidationError. The runner gives it the timeout kind, with the sign-in policy and the Stack Trace of an unreachable source. Jira, HubSpot, Notion and Linear raise it for a request timeout, and GitHub and Confluence raise it for a sign-in timeout. Add CapabilityCheck.timeout_text. The hang guard shows it instead of the shared "Timed out" text. Each named-check source sets the design's text: sign-in rows get "Sign-in timed out", other rows get the source's shared timeout text. Stop a cycle in an error's cause chain from making _http_detail loop. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01KFMazQbuG8ui3jxBxm47sC
This commit is contained in:
22 files changed
+516
-62
No files matched your search
@@ -478,7 +478,11 @@ A source gateway's API error subclasses `exceptions.py:SourceApiError`.
|
||||
A result's row text is `summary` (a short line) and `description` (up to three
|
||||
lines). A check's `ValidationError` can carry both, and the runner falls back to
|
||||
a default for a timeout, a rate limit, an unreachable source or an unexpected
|
||||
error, and to the "Not run: …" line for a stopped check. `message` stays the
|
||||
error, and to the "Not run: …" line for a stopped check. A request that the
|
||||
source does not answer in time raises `exceptions.py:TimedOutValidationError`
|
||||
(kind `timeout`). When the hang guard stops a check, the row shows the check's
|
||||
`timeout_text` (`models.py:CheckRowText`; sign-in checks use
|
||||
`sign_in_timeout_text`), else the default. `message` stays the
|
||||
full text. A passing check gives its line in `CheckOutput.summary`. Checks that
|
||||
share a `row_id` are one row on the form: the variants of a probe (token types,
|
||||
deployments) and a probe's discovery step.
|
||||
|
||||
@@ -33,6 +33,7 @@ from onyx.connectors.exceptions import (
|
||||
InsufficientPermissionsError,
|
||||
RateLimitedValidationError,
|
||||
SourceApiError,
|
||||
TimedOutValidationError,
|
||||
UnreachableValidationError,
|
||||
)
|
||||
from onyx.connectors.interfaces import BaseConnector
|
||||
@@ -301,8 +302,10 @@ def _scrubbed(text: str, secrets: Iterable[str]) -> str:
|
||||
def _http_detail(error: BaseException) -> CheckErrorDetail | None:
|
||||
"""The HTTP response of the first error in the chain that has one, with
|
||||
the whole body: ``check_error_detail`` redacts it, then cuts it."""
|
||||
seen: set[int] = set()
|
||||
current: BaseException | None = error
|
||||
while current is not None:
|
||||
while current is not None and id(current) not in seen:
|
||||
seen.add(id(current))
|
||||
if isinstance(current, SourceApiError):
|
||||
return CheckErrorDetail(
|
||||
status_code=current.status_code,
|
||||
@@ -343,6 +346,31 @@ def check_error_detail(
|
||||
return detail if detail != CheckErrorDetail() else None
|
||||
|
||||
|
||||
class CheckRowText(BaseModel):
|
||||
"""A row's line and description. In a hang-guard text,
|
||||
``TIME_LIMIT_PLACEHOLDER`` in the description gives the guard's time
|
||||
limit, e.g. "10 minutes"."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
summary: str
|
||||
description: str
|
||||
|
||||
|
||||
TIME_LIMIT_PLACEHOLDER = "{time_limit}"
|
||||
|
||||
|
||||
def sign_in_timeout_text(source_name: str) -> CheckRowText:
|
||||
"""The row text of a sign-in that the source did not answer in time."""
|
||||
return CheckRowText(
|
||||
summary="Sign-in timed out",
|
||||
description=(
|
||||
f"{source_name} did not answer the sign-in in time.\nRe-run, or "
|
||||
f"check that {source_name} is reachable from Onyx."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class CheckNotRunReason(str, Enum):
|
||||
"""Why a SKIPPED or STOPPED check did not run: a gate of the run, or the
|
||||
admin."""
|
||||
@@ -391,7 +419,7 @@ def check_error_kind(error: Exception) -> CheckErrorKind:
|
||||
return CheckErrorKind.RATE_LIMITED
|
||||
if isinstance(error, InconclusiveValidationError):
|
||||
return CheckErrorKind.INCONCLUSIVE
|
||||
if isinstance(error, TimeoutError):
|
||||
if isinstance(error, (TimeoutError, TimedOutValidationError)):
|
||||
return CheckErrorKind.TIMEOUT
|
||||
if isinstance(error, UnreachableValidationError):
|
||||
return CheckErrorKind.UNREACHABLE
|
||||
@@ -525,6 +553,7 @@ class CapabilityCheck(ABC, Generic[ConfigT]):
|
||||
depends_on: tuple[str, ...] = (),
|
||||
row_id: str | None = None,
|
||||
step: CheckStep | None = None,
|
||||
timeout_text: CheckRowText | None = None,
|
||||
) -> None:
|
||||
self.capability = capability
|
||||
self.check_id = check_id
|
||||
@@ -578,6 +607,9 @@ class CapabilityCheck(ABC, Generic[ConfigT]):
|
||||
self.row_id = row_id or (step.step_of if step is not None else check_id)
|
||||
# Set on the second step of a two-step row.
|
||||
self.step = step
|
||||
# The row text when the hang guard stops the check. None gives the
|
||||
# runner's default.
|
||||
self.timeout_text = timeout_text
|
||||
|
||||
def outcome_status(
|
||||
self, error_kind: CheckErrorKind, form_run: bool
|
||||
|
||||
@@ -10,6 +10,7 @@ from onyx.configs.constants import DocumentSource
|
||||
from onyx.connectors.capability_checks.findings_setting import check_findings_enabled
|
||||
from onyx.connectors.capability_checks.models import (
|
||||
INCOMPLETE_STATUSES,
|
||||
TIME_LIMIT_PLACEHOLDER,
|
||||
CapabilityCheck,
|
||||
CapabilityCheckContext,
|
||||
CapabilityCheckResult,
|
||||
@@ -21,6 +22,7 @@ from onyx.connectors.capability_checks.models import (
|
||||
CheckNotApplicableError,
|
||||
CheckNotRunReason,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CredentialCapability,
|
||||
CredentialCapabilityReport,
|
||||
check_error_detail,
|
||||
@@ -203,12 +205,17 @@ def _error_outcome(
|
||||
form_run: bool,
|
||||
message: str | None = None,
|
||||
duration_ms: int | None = None,
|
||||
default_row: CheckRowText | None = None,
|
||||
) -> _CheckOutcome:
|
||||
"""Maps a check's exception to an outcome, per the check exception
|
||||
contract and the check's ``outcome_status``. The row text is the
|
||||
error's own, else the default of its kind."""
|
||||
error's own, else ``default_row``, else the default of its kind."""
|
||||
error_kind = check_error_kind(error)
|
||||
summary, description = _DEFAULT_ROW_TEXT_BY_KIND.get(error_kind, (None, None))
|
||||
summary, description = (
|
||||
(default_row.summary, default_row.description)
|
||||
if default_row is not None
|
||||
else _DEFAULT_ROW_TEXT_BY_KIND.get(error_kind, (None, None))
|
||||
)
|
||||
if isinstance(error, ValidationError):
|
||||
summary = error.summary or summary
|
||||
description = error.description or description
|
||||
@@ -249,6 +256,28 @@ def effective_check_timeout_seconds(check: CapabilityCheck[Any]) -> float:
|
||||
return check.timeout_seconds or CAPABILITY_CHECK_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def _time_limit_words(timeout_seconds: float) -> str:
|
||||
seconds = round(timeout_seconds)
|
||||
if seconds >= 60 and seconds % 60 == 0:
|
||||
minutes = seconds // 60
|
||||
return f"{minutes} minute{'' if minutes == 1 else 's'}"
|
||||
return f"{seconds} second{'' if seconds == 1 else 's'}"
|
||||
|
||||
|
||||
def _hang_guard_row(
|
||||
check: CapabilityCheck, timeout_seconds: float
|
||||
) -> CheckRowText | None:
|
||||
"""The check's own text for its hang guard, with the guard's time limit."""
|
||||
if check.timeout_text is None:
|
||||
return None
|
||||
return CheckRowText(
|
||||
summary=check.timeout_text.summary,
|
||||
description=check.timeout_text.description.replace(
|
||||
TIME_LIMIT_PLACEHOLDER, _time_limit_words(timeout_seconds)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _capped(timeout_seconds: float, cap_seconds: float | None) -> float:
|
||||
return timeout_seconds if cap_seconds is None else min(timeout_seconds, cap_seconds)
|
||||
|
||||
@@ -315,6 +344,7 @@ def _execute_check(
|
||||
form_run,
|
||||
message=_TIMEOUT_MESSAGE,
|
||||
duration_ms=elapsed_ms(),
|
||||
default_row=_hang_guard_row(check, timeout_seconds),
|
||||
)
|
||||
except Exception as e:
|
||||
if check_error_kind(e) == CheckErrorKind.UNEXPECTED:
|
||||
|
||||
@@ -39,9 +39,11 @@ from onyx.connectors.capability_checks.models import (
|
||||
CheckAdvisory,
|
||||
CheckFindingKey,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CheckStep,
|
||||
CredentialCapability,
|
||||
OptionsFinding,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.confluence.config import (
|
||||
ConfluenceConnectorConfig,
|
||||
@@ -85,6 +87,7 @@ from onyx.connectors.exceptions import (
|
||||
InsufficientPermissionsError,
|
||||
NothingToCheckError,
|
||||
RateLimitedValidationError,
|
||||
TimedOutValidationError,
|
||||
UnexpectedValidationError,
|
||||
UnreachableValidationError,
|
||||
)
|
||||
@@ -93,6 +96,14 @@ from onyx.utils.logger import setup_logger
|
||||
logger = setup_logger()
|
||||
|
||||
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/confluence"
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description=(
|
||||
"Request timed out after {time_limit}. Unable to confirm access.\n"
|
||||
"Re-run, or check the site's status."
|
||||
),
|
||||
)
|
||||
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("Confluence")
|
||||
_SITE_FIELDS = frozenset({"wiki_base", "is_cloud"})
|
||||
_SPACE_FIELD = "space"
|
||||
|
||||
@@ -378,6 +389,7 @@ class _ConfluenceCheck(CapabilityCheck[ConfluenceConnectorConfig]):
|
||||
fails_when_unreachable: bool = False,
|
||||
row_id: str | None = None,
|
||||
step: CheckStep | None = None,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=capability,
|
||||
@@ -394,6 +406,7 @@ class _ConfluenceCheck(CapabilityCheck[ConfluenceConnectorConfig]):
|
||||
fails_when_unreachable=fails_when_unreachable,
|
||||
depends_on=_DEPENDS_ON.get(check_id, ()),
|
||||
step=step,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -413,6 +426,7 @@ class _SiteAuthCheck(_ConfluenceCheck):
|
||||
row_id="confluence_auth",
|
||||
validates_binding=True,
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
|
||||
display_name="Sign in to Confluence",
|
||||
remediation=(
|
||||
"Scoped API tokens work only with Confluence Cloud. Check the "
|
||||
@@ -424,6 +438,7 @@ class _SiteAuthCheck(_ConfluenceCheck):
|
||||
check_id="confluence_auth",
|
||||
validates_binding=True,
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
|
||||
display_name="Sign in to Confluence",
|
||||
remediation=(
|
||||
"Check the site URL and the Cloud setting. For an API token, "
|
||||
@@ -508,7 +523,13 @@ class _SiteAuthCheck(_ConfluenceCheck):
|
||||
),
|
||||
),
|
||||
)
|
||||
except (requests.ConnectionError, requests.Timeout) as e:
|
||||
except requests.Timeout as e:
|
||||
raise TimedOutValidationError(
|
||||
f"{config.wiki_base} did not answer the sign-in in time: {e}",
|
||||
summary=_SIGN_IN_TIMEOUT_TEXT.summary,
|
||||
description=_SIGN_IN_TIMEOUT_TEXT.description,
|
||||
) from e
|
||||
except requests.ConnectionError as e:
|
||||
raise UnreachableValidationError(
|
||||
f"Onyx cannot connect to {config.wiki_base}. Check the site URL.",
|
||||
summary=f"Cannot reach {urlparse(config.wiki_base).hostname}",
|
||||
|
||||
@@ -94,6 +94,21 @@ class UnreachableValidationError(UnexpectedValidationError):
|
||||
super().__init__(message, summary=summary, description=description)
|
||||
|
||||
|
||||
class TimedOutValidationError(UnreachableValidationError):
|
||||
"""Raised when the source does not answer a request of the validation in
|
||||
time. Like an unreachable source, a slow one says nothing about the
|
||||
credential."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str = "The source did not answer in time",
|
||||
*,
|
||||
summary: str | None = None,
|
||||
description: str | None = None,
|
||||
):
|
||||
super().__init__(message, summary=summary, description=description)
|
||||
|
||||
|
||||
class InconclusiveValidationError(UnexpectedValidationError):
|
||||
"""Raised when a probe ran, but what it sampled cannot decide: the sample
|
||||
held nothing to test, or the source refused every try. Not proof of a
|
||||
|
||||
@@ -27,7 +27,9 @@ from onyx.connectors.capability_checks.models import (
|
||||
CapabilityCheckContext,
|
||||
CheckAdvisory,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CredentialCapability,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.exceptions import (
|
||||
ConnectorValidationError,
|
||||
@@ -37,6 +39,7 @@ from onyx.connectors.exceptions import (
|
||||
InsufficientPermissionsError,
|
||||
NothingToCheckError,
|
||||
RateLimitedValidationError,
|
||||
TimedOutValidationError,
|
||||
UnexpectedValidationError,
|
||||
UnreachableValidationError,
|
||||
)
|
||||
@@ -63,6 +66,11 @@ from onyx.connectors.models import ConnectorMissingCredentialError
|
||||
from onyx.connectors.source_operations import SourceOperations
|
||||
|
||||
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/github"
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description="The check ran past its time limit.\nRe-run the checks later.",
|
||||
)
|
||||
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("GitHub")
|
||||
_SSO_GUIDE_LINK = (
|
||||
"https://docs.github.com/en/enterprise-cloud@latest/authentication/"
|
||||
"authenticating-with-saml-single-sign-on/"
|
||||
@@ -278,6 +286,7 @@ class _GitHubCheck(CapabilityCheck[GithubConnectorConfig]):
|
||||
requires_fields: frozenset[str] = _OWNER_FIELDS,
|
||||
requires_connector_config: bool = False,
|
||||
fails_when_unreachable: bool = False,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=capability,
|
||||
@@ -291,6 +300,7 @@ class _GitHubCheck(CapabilityCheck[GithubConnectorConfig]):
|
||||
remediation=remediation,
|
||||
docs_link=_DOCS_LINK,
|
||||
fails_when_unreachable=fails_when_unreachable,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -316,6 +326,7 @@ class _AuthCheck(_GitHubCheck):
|
||||
),
|
||||
requires_fields=frozenset(),
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
|
||||
)
|
||||
|
||||
def run(self, context: CapabilityCheckContext) -> CheckOutput:
|
||||
@@ -330,7 +341,13 @@ class _AuthCheck(_GitHubCheck):
|
||||
"token in API Token."
|
||||
),
|
||||
) from e
|
||||
except (requests.ConnectionError, requests.Timeout) as e:
|
||||
except requests.Timeout as e:
|
||||
raise TimedOutValidationError(
|
||||
f"GitHub did not answer the sign-in in time: {e}",
|
||||
summary=_SIGN_IN_TIMEOUT_TEXT.summary,
|
||||
description=_SIGN_IN_TIMEOUT_TEXT.description,
|
||||
) from e
|
||||
except requests.ConnectionError as e:
|
||||
raise UnreachableValidationError(
|
||||
"Onyx cannot connect to GitHub. Check the Enterprise Server URL, "
|
||||
"if one is set.",
|
||||
|
||||
@@ -37,10 +37,12 @@ from onyx.connectors.capability_checks.models import (
|
||||
CheckAdvisory,
|
||||
CheckFindingKey,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CheckStep,
|
||||
CountFinding,
|
||||
CredentialCapability,
|
||||
OptionsFinding,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.exceptions import (
|
||||
ConnectorValidationError,
|
||||
@@ -97,6 +99,10 @@ from onyx.connectors.models import ConnectorMissingCredentialError
|
||||
from onyx.connectors.source_operations import SourceOperations
|
||||
|
||||
_DOCS_LINK = SCOPE_DOC_URL
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description="The check ran past its time limit.\nRe-run the checks.",
|
||||
)
|
||||
|
||||
_DRIVE_API = "Google Drive API"
|
||||
_ADMIN_API = "Admin SDK API"
|
||||
@@ -496,6 +502,7 @@ class _GoogleDriveCheck(CapabilityCheck[GoogleDriveConnectorConfig]):
|
||||
credential_kind: GoogleCredentialKind | None = None,
|
||||
discovery: bool = False,
|
||||
step: CheckStep | None = None,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=capability,
|
||||
@@ -515,6 +522,7 @@ class _GoogleDriveCheck(CapabilityCheck[GoogleDriveConnectorConfig]):
|
||||
docs_link=_DOCS_LINK,
|
||||
discovery=discovery,
|
||||
step=step,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
self._credential_kind = credential_kind
|
||||
|
||||
@@ -547,6 +555,7 @@ class _AuthCheck(_GoogleDriveCheck):
|
||||
"OAuth: connect Google Drive again. Turn on the Google Drive API "
|
||||
"in the Google Cloud project of the credential."
|
||||
),
|
||||
timeout_text=sign_in_timeout_text("Google Drive"),
|
||||
)
|
||||
|
||||
def run(self, context: CapabilityCheckContext) -> CheckOutput:
|
||||
|
||||
@@ -23,8 +23,10 @@ from onyx.connectors.capability_checks.models import (
|
||||
CapabilityCheck,
|
||||
CapabilityCheckContext,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CredentialCapability,
|
||||
form_config,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
|
||||
RateLimitTriedTooManyTimesError,
|
||||
@@ -36,6 +38,7 @@ from onyx.connectors.exceptions import (
|
||||
InsufficientPermissionsError,
|
||||
NothingToCheckError,
|
||||
RateLimitedValidationError,
|
||||
TimedOutValidationError,
|
||||
UnexpectedValidationError,
|
||||
UnreachableValidationError,
|
||||
)
|
||||
@@ -60,6 +63,14 @@ from onyx.utils.logger import setup_logger
|
||||
logger = setup_logger()
|
||||
|
||||
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/hubspot"
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description=(
|
||||
"The check stopped after {time_limit} without an answer from HubSpot.\n"
|
||||
"Re-run the checks."
|
||||
),
|
||||
)
|
||||
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("HubSpot")
|
||||
_PROBE_LIMIT = 1
|
||||
_NOTE_BODY_PROPERTY = "hs_note_body"
|
||||
_EPOCH = datetime.fromtimestamp(0, tz=timezone.utc)
|
||||
@@ -154,6 +165,10 @@ _UNREACHABLE_ROW = _RowText(
|
||||
"api.hubapi.com."
|
||||
),
|
||||
)
|
||||
_REQUEST_TIMEOUT_ROW = _RowText(
|
||||
summary="Timed out",
|
||||
description="HubSpot did not answer the request in time.\nRe-run the checks.",
|
||||
)
|
||||
_UNEXPECTED_ROW = _RowText(
|
||||
summary="Unexpected response from HubSpot",
|
||||
description=(
|
||||
@@ -181,6 +196,12 @@ def _raise_for_error(
|
||||
"a minute, then re-run."
|
||||
),
|
||||
) from error
|
||||
if isinstance(error, HubSpotConnectionError) and error.timed_out:
|
||||
raise TimedOutValidationError(
|
||||
f"HubSpot did not answer in time: {error}",
|
||||
summary=_REQUEST_TIMEOUT_ROW.summary,
|
||||
description=_REQUEST_TIMEOUT_ROW.description,
|
||||
) from error
|
||||
if isinstance(error, HubSpotConnectionError):
|
||||
raise UnreachableValidationError(
|
||||
f"Onyx cannot connect to HubSpot: {error}",
|
||||
@@ -287,6 +308,7 @@ class _HubSpotCheck(CapabilityCheck[HubSpotConnectorConfig]):
|
||||
required: bool = True,
|
||||
remediation: str | None = None,
|
||||
fails_when_unreachable: bool = False,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=capability,
|
||||
@@ -298,6 +320,7 @@ class _HubSpotCheck(CapabilityCheck[HubSpotConnectorConfig]):
|
||||
remediation=remediation,
|
||||
docs_link=_DOCS_LINK,
|
||||
fails_when_unreachable=fails_when_unreachable,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -308,6 +331,7 @@ class _TokenCheck(_HubSpotCheck):
|
||||
display_name="Sign in to HubSpot",
|
||||
remediation="Create a private app in HubSpot and paste its access token.",
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
|
||||
)
|
||||
|
||||
def run(self, context: CapabilityCheckContext) -> CheckOutput:
|
||||
@@ -325,13 +349,10 @@ class _TokenCheck(_HubSpotCheck):
|
||||
except HubSpotConnectionError as error:
|
||||
if not error.timed_out:
|
||||
_raise_for_error(error, denied="read the portal")
|
||||
raise UnreachableValidationError(
|
||||
raise TimedOutValidationError(
|
||||
f"HubSpot did not answer the sign-in in time: {error}",
|
||||
summary="Sign-in timed out",
|
||||
description=(
|
||||
"HubSpot did not answer the sign-in in time.\nRe-run, or check "
|
||||
"that HubSpot is reachable from Onyx."
|
||||
),
|
||||
summary=_SIGN_IN_TIMEOUT_TEXT.summary,
|
||||
description=_SIGN_IN_TIMEOUT_TEXT.description,
|
||||
) from error
|
||||
except ValueError as error:
|
||||
# A body that is not HubSpot's portal info, e.g. a proxy page.
|
||||
|
||||
@@ -32,8 +32,10 @@ from onyx.connectors.capability_checks.models import (
|
||||
CapabilityCheckContext,
|
||||
CheckAdvisory,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CheckStep,
|
||||
CredentialCapability,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.exceptions import (
|
||||
ConnectorValidationError,
|
||||
@@ -42,6 +44,7 @@ from onyx.connectors.exceptions import (
|
||||
InsufficientPermissionsError,
|
||||
NothingToCheckError,
|
||||
RateLimitedValidationError,
|
||||
TimedOutValidationError,
|
||||
UnexpectedValidationError,
|
||||
UnreachableValidationError,
|
||||
)
|
||||
@@ -71,6 +74,13 @@ from onyx.connectors.jira.utils import (
|
||||
from onyx.connectors.source_operations import SourceOperations
|
||||
|
||||
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/jira"
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description=(
|
||||
"The check stopped because Jira did not answer in time.\nRe-run the checks."
|
||||
),
|
||||
)
|
||||
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("Jira")
|
||||
_SITE_FIELDS = frozenset({"jira_base_url"})
|
||||
_ERROR_DETAIL_CHARS = 300
|
||||
_CLOUD_HOST_SUFFIX = ".atlassian.net"
|
||||
@@ -148,12 +158,10 @@ def _error_detail(error: JiraApiError) -> str:
|
||||
|
||||
def _raise_for_timeout(error: JiraTimeoutError) -> NoReturn:
|
||||
"""INCOMPLETE: a slow Jira says nothing about the credential."""
|
||||
raise UnreachableValidationError(
|
||||
raise TimedOutValidationError(
|
||||
f"{error} Run the checks again, or check the status of Jira.",
|
||||
summary="Timed out",
|
||||
description=(
|
||||
"The check stopped because Jira did not answer in time.\nRe-run the checks."
|
||||
),
|
||||
summary=_TIMEOUT_TEXT.summary,
|
||||
description=_TIMEOUT_TEXT.description,
|
||||
) from error
|
||||
|
||||
|
||||
@@ -278,6 +286,7 @@ class _JiraCheck(CapabilityCheck[JiraConnectorConfig]):
|
||||
fails_when_unreachable: bool = False,
|
||||
row_id: str | None = None,
|
||||
step: CheckStep | None = None,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=capability,
|
||||
@@ -293,6 +302,7 @@ class _JiraCheck(CapabilityCheck[JiraConnectorConfig]):
|
||||
fails_when_unreachable=fails_when_unreachable,
|
||||
depends_on=_DEPENDS_ON.get(check_id, ()),
|
||||
step=step,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -318,6 +328,7 @@ class _SiteAuthCheck(_JiraCheck):
|
||||
row_id="jira_auth",
|
||||
validates_binding=True,
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
|
||||
display_name="Sign in to Jira",
|
||||
remediation=(
|
||||
"Scoped API tokens work only with Jira Cloud. Enter the email "
|
||||
@@ -330,6 +341,7 @@ class _SiteAuthCheck(_JiraCheck):
|
||||
check_id="jira_auth",
|
||||
validates_binding=True,
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
|
||||
display_name="Sign in to Jira",
|
||||
remediation=(
|
||||
"Jira Cloud: enter the email of the token's Atlassian account "
|
||||
@@ -382,14 +394,11 @@ class _SiteAuthCheck(_JiraCheck):
|
||||
) from e
|
||||
except (JiraTimeoutError, requests.Timeout) as e:
|
||||
# A raw Timeout comes from the scoped token's tenant_info call.
|
||||
raise UnreachableValidationError(
|
||||
raise TimedOutValidationError(
|
||||
f"{self.config(context).jira_base_url} did not answer the sign-in "
|
||||
f"in time: {e}",
|
||||
summary="Sign-in timed out",
|
||||
description=(
|
||||
"Jira did not answer the sign-in in time.\n"
|
||||
"Re-run, or check that Jira is reachable from Onyx."
|
||||
),
|
||||
summary=_SIGN_IN_TIMEOUT_TEXT.summary,
|
||||
description=_SIGN_IN_TIMEOUT_TEXT.description,
|
||||
) from e
|
||||
except requests.ConnectionError as e:
|
||||
raise UnreachableValidationError(
|
||||
|
||||
@@ -12,15 +12,15 @@ from collections.abc import Callable
|
||||
from enum import Enum
|
||||
from typing import NoReturn, TypeVar
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from onyx.connectors.capability_checks.form_state import FormState
|
||||
from onyx.connectors.capability_checks.models import (
|
||||
CapabilityCheck,
|
||||
CapabilityCheckContext,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CredentialCapability,
|
||||
form_config,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.exceptions import (
|
||||
ConnectorValidationError,
|
||||
@@ -28,6 +28,7 @@ from onyx.connectors.exceptions import (
|
||||
CredentialInvalidError,
|
||||
InconclusiveValidationError,
|
||||
RateLimitedValidationError,
|
||||
TimedOutValidationError,
|
||||
UnexpectedValidationError,
|
||||
UnreachableValidationError,
|
||||
)
|
||||
@@ -77,13 +78,6 @@ _MAX_SUMMARY_CHARS: int = 40
|
||||
_REFRESH_REFUSED_STATUSES: frozenset[int] = frozenset({400, 401})
|
||||
|
||||
|
||||
class _RowText(BaseModel):
|
||||
"""The row text of one outcome of a probe."""
|
||||
|
||||
summary: str
|
||||
description: str
|
||||
|
||||
|
||||
class _CredentialKind(str, Enum):
|
||||
API_KEY = "api_key"
|
||||
OAUTH = "oauth"
|
||||
@@ -126,19 +120,11 @@ def config_from_context(context: CapabilityCheckContext) -> LinearConnectorConfi
|
||||
return form_config(context, LinearConnectorConfig)
|
||||
|
||||
|
||||
def _timeout_row(sign_in: bool) -> _RowText:
|
||||
if sign_in:
|
||||
return _RowText(
|
||||
summary="Sign-in timed out",
|
||||
description=(
|
||||
"Linear did not answer the sign-in in time.\nRe-run, or check "
|
||||
"that Linear is reachable from Onyx."
|
||||
),
|
||||
)
|
||||
return _RowText(
|
||||
summary="Timed out",
|
||||
description="The check ran past its time limit.\nRe-run the checks later.",
|
||||
)
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description="The check ran past its time limit.\nRe-run the checks later.",
|
||||
)
|
||||
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("Linear")
|
||||
|
||||
|
||||
def _raise_for_linear_error(
|
||||
@@ -195,8 +181,8 @@ def _raise_for_linear_error(
|
||||
),
|
||||
) from error
|
||||
if isinstance(error, LinearUnreachableError) and error.timed_out:
|
||||
row: _RowText = _timeout_row(sign_in)
|
||||
raise UnreachableValidationError(
|
||||
row: CheckRowText = _SIGN_IN_TIMEOUT_TEXT if sign_in else _TIMEOUT_TEXT
|
||||
raise TimedOutValidationError(
|
||||
f"Linear did not answer in time: {error}",
|
||||
summary=row.summary,
|
||||
description=row.description,
|
||||
@@ -307,6 +293,7 @@ class _LinearCheck(CapabilityCheck[LinearConnectorConfig]):
|
||||
remediation: str,
|
||||
requires_connector_config: bool = False,
|
||||
fails_when_unreachable: bool = False,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=CredentialCapability.INDEXING,
|
||||
@@ -318,6 +305,7 @@ class _LinearCheck(CapabilityCheck[LinearConnectorConfig]):
|
||||
remediation=remediation,
|
||||
docs_link=DOCS_LINK,
|
||||
fails_when_unreachable=fails_when_unreachable,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -328,6 +316,7 @@ class _TokenCheck(_LinearCheck):
|
||||
display_name="Sign in to Linear",
|
||||
remediation="Check the API key, or reconnect through OAuth.",
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
|
||||
)
|
||||
|
||||
def run(self, context: CapabilityCheckContext) -> CheckOutput:
|
||||
|
||||
@@ -26,7 +26,9 @@ from onyx.connectors.capability_checks.models import (
|
||||
CheckAdvisory,
|
||||
CheckNotApplicableError,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CredentialCapability,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.exceptions import (
|
||||
ConnectorValidationError,
|
||||
@@ -36,6 +38,7 @@ from onyx.connectors.exceptions import (
|
||||
InsufficientPermissionsError,
|
||||
NothingToCheckError,
|
||||
RateLimitedValidationError,
|
||||
TimedOutValidationError,
|
||||
UnexpectedValidationError,
|
||||
UnreachableValidationError,
|
||||
)
|
||||
@@ -49,6 +52,14 @@ from onyx.connectors.notion.source_operations import (
|
||||
from onyx.connectors.source_operations import SourceOperations
|
||||
|
||||
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/notion"
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description=(
|
||||
"The check stopped after {time_limit} without an answer from Notion.\n"
|
||||
"Re-run the checks."
|
||||
),
|
||||
)
|
||||
_SIGN_IN_TIMEOUT_TEXT = sign_in_timeout_text("Notion")
|
||||
_ERROR_DETAIL_CHARS = 300
|
||||
# Keeps a row's line short when it names a page or database.
|
||||
_TITLE_CHARS = 24
|
||||
@@ -162,7 +173,7 @@ def _raise_for_api_error(
|
||||
),
|
||||
) from error
|
||||
if status is None and error.timed_out:
|
||||
raise UnreachableValidationError(
|
||||
raise TimedOutValidationError(
|
||||
f"Notion did not answer in time: {detail}",
|
||||
summary="Timed out",
|
||||
description="Notion did not answer the request in time.\nRe-run the checks.",
|
||||
@@ -260,6 +271,7 @@ class _NotionCheck(CapabilityCheck[NotionConnectorConfig]):
|
||||
remediation: str,
|
||||
required: bool = True,
|
||||
fails_when_unreachable: bool = False,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=CredentialCapability.INDEXING,
|
||||
@@ -271,6 +283,7 @@ class _NotionCheck(CapabilityCheck[NotionConnectorConfig]):
|
||||
remediation=remediation,
|
||||
docs_link=_DOCS_LINK,
|
||||
fails_when_unreachable=fails_when_unreachable,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -296,6 +309,7 @@ class _AuthCheck(_NotionCheck):
|
||||
"Configuration) and update the credential."
|
||||
),
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=_SIGN_IN_TIMEOUT_TEXT,
|
||||
)
|
||||
|
||||
def run(self, context: CapabilityCheckContext) -> CheckOutput:
|
||||
@@ -313,13 +327,10 @@ class _AuthCheck(_NotionCheck):
|
||||
) from e
|
||||
except NotionApiError as e:
|
||||
if e.status_code is None and e.timed_out:
|
||||
raise UnreachableValidationError(
|
||||
raise TimedOutValidationError(
|
||||
f"Notion did not answer the sign-in in time: {e}",
|
||||
summary="Sign-in timed out",
|
||||
description=(
|
||||
"Notion did not answer the sign-in in time.\nRe-run, or "
|
||||
"check that Notion is reachable from Onyx."
|
||||
),
|
||||
summary=_SIGN_IN_TIMEOUT_TEXT.summary,
|
||||
description=_SIGN_IN_TIMEOUT_TEXT.description,
|
||||
) from e
|
||||
_raise_for_api_error(
|
||||
e,
|
||||
|
||||
@@ -23,10 +23,12 @@ from onyx.connectors.capability_checks.models import (
|
||||
CheckAdvisory,
|
||||
CheckFindingKey,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CheckStep,
|
||||
CountFinding,
|
||||
CredentialCapability,
|
||||
form_config,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.exceptions import (
|
||||
ConnectorValidationError,
|
||||
@@ -64,6 +66,10 @@ from onyx.connectors.onedrive.source_operations import (
|
||||
)
|
||||
|
||||
_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/onedrive"
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description="The check did not finish within its time limit.\nRe-run the checks.",
|
||||
)
|
||||
_PROBE_PAGE_SIZE = 1
|
||||
_CANDIDATE_PAGES = 20
|
||||
# A bound on the pages the drive count reads. Pages hold up to 999 users, so
|
||||
@@ -370,6 +376,7 @@ class _OneDriveCheck(CapabilityCheck[OneDriveConnectorConfig]):
|
||||
discovery: bool = False,
|
||||
row_id: str | None = None,
|
||||
step: CheckStep | None = None,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=capability,
|
||||
@@ -385,6 +392,7 @@ class _OneDriveCheck(CapabilityCheck[OneDriveConnectorConfig]):
|
||||
discovery=discovery,
|
||||
row_id=row_id,
|
||||
step=step,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -400,6 +408,7 @@ class _TokenCheck(_OneDriveCheck):
|
||||
remediation="Check the app id, tenant id, and client credential.",
|
||||
validates_binding=True,
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=sign_in_timeout_text("OneDrive"),
|
||||
)
|
||||
|
||||
def run(self, context: CapabilityCheckContext) -> CheckOutput:
|
||||
|
||||
@@ -41,9 +41,11 @@ from onyx.connectors.capability_checks.models import (
|
||||
CheckAdvisory,
|
||||
CheckFindingKey,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CheckStep,
|
||||
CountFinding,
|
||||
CredentialCapability,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.exceptions import (
|
||||
ConnectorValidationError,
|
||||
@@ -96,6 +98,10 @@ from onyx.connectors.outlook.source_operations import OutlookSourceOperations
|
||||
T = TypeVar("T")
|
||||
|
||||
_OUTLOOK_DOCS_LINK = "https://docs.onyx.app/admins/connectors/official/outlook"
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description="The check did not finish within its time limit.\nRe-run the checks.",
|
||||
)
|
||||
|
||||
# A well-known folder every mailbox has, used to prove name resolution works.
|
||||
_PROBE_WELL_KNOWN_FOLDER = "junkemail"
|
||||
@@ -475,6 +481,7 @@ class _OutlookCheck(CapabilityCheck[OutlookConnectorConfig]):
|
||||
discovery: bool = False,
|
||||
row_id: str | None = None,
|
||||
step: CheckStep | None = None,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=capability,
|
||||
@@ -490,6 +497,7 @@ class _OutlookCheck(CapabilityCheck[OutlookConnectorConfig]):
|
||||
discovery=discovery,
|
||||
row_id=row_id,
|
||||
step=step,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -512,6 +520,7 @@ class _TokenAuthCheck(_OutlookCheck):
|
||||
),
|
||||
validates_binding=True,
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=sign_in_timeout_text("Outlook"),
|
||||
)
|
||||
|
||||
def run(self, context: CapabilityCheckContext) -> CheckOutput:
|
||||
|
||||
@@ -25,8 +25,10 @@ from onyx.connectors.capability_checks.models import (
|
||||
CapabilityCheckContext,
|
||||
CheckAdvisory,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CredentialCapability,
|
||||
form_config,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.exceptions import (
|
||||
ConnectorValidationError,
|
||||
@@ -115,6 +117,10 @@ from onyx.utils.logger import setup_logger
|
||||
logger = setup_logger()
|
||||
|
||||
_DOCS_LINK: str = "https://docs.onyx.app/admins/connectors/official/sharepoint"
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description="The check did not finish within its time limit.\nRe-run the checks.",
|
||||
)
|
||||
_PROBE_SITES: int = 5
|
||||
# Libraries whose first delta page holds no indexable file before the probe
|
||||
# stops, so a tenant of empty libraries cannot spend the check's budget.
|
||||
@@ -849,6 +855,7 @@ class _SharepointCheck(CapabilityCheck[SharepointConnectorConfig]):
|
||||
requires_connector_config: bool = True,
|
||||
validates_binding: bool = False,
|
||||
fails_when_unreachable: bool = False,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=capability,
|
||||
@@ -861,6 +868,7 @@ class _SharepointCheck(CapabilityCheck[SharepointConnectorConfig]):
|
||||
validates_binding=validates_binding,
|
||||
fails_when_unreachable=fails_when_unreachable,
|
||||
depends_on=_DEPENDS_ON.get(check_id, ()),
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -874,6 +882,7 @@ class _TokenCheck(_SharepointCheck):
|
||||
display_name="Sign in to Microsoft",
|
||||
validates_binding=True,
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=sign_in_timeout_text("SharePoint"),
|
||||
remediation=(
|
||||
"Check the client id, directory id and client secret or "
|
||||
"certificate, and that the Graph API and authority hosts match "
|
||||
|
||||
@@ -46,9 +46,11 @@ from onyx.connectors.capability_checks.models import (
|
||||
CheckFindingKey,
|
||||
CheckNotApplicableError,
|
||||
CheckOutput,
|
||||
CheckRowText,
|
||||
CheckStep,
|
||||
CredentialCapability,
|
||||
OptionsFinding,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.exceptions import (
|
||||
ConnectorValidationError,
|
||||
@@ -83,6 +85,13 @@ from onyx.connectors.source_operations import SourceOperations
|
||||
_SLACK_DOCS_LINK = (
|
||||
"https://docs.onyx.app/admins/connectors/official/slack/slack_indexed"
|
||||
)
|
||||
_TIMEOUT_TEXT = CheckRowText(
|
||||
summary="Timed out",
|
||||
description=(
|
||||
"The check stopped after {time_limit}.\nRe-run the checks when no Slack "
|
||||
"indexing job is running."
|
||||
),
|
||||
)
|
||||
|
||||
_ADD_SCOPE_REMEDIATION = (
|
||||
"Add the `{scope}` bot scope under OAuth & Permissions in the Slack app "
|
||||
@@ -490,6 +499,7 @@ class _SlackCheck(CapabilityCheck[SlackConnectorConfig]):
|
||||
discovery: bool = False,
|
||||
fails_when_unreachable: bool = False,
|
||||
step: CheckStep | None = None,
|
||||
timeout_text: CheckRowText = _TIMEOUT_TEXT,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=capability,
|
||||
@@ -507,6 +517,7 @@ class _SlackCheck(CapabilityCheck[SlackConnectorConfig]):
|
||||
discovery=discovery,
|
||||
fails_when_unreachable=fails_when_unreachable,
|
||||
step=step,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -523,6 +534,7 @@ class _TokenAuthCheck(_SlackCheck):
|
||||
),
|
||||
validates_binding=True,
|
||||
fails_when_unreachable=True,
|
||||
timeout_text=sign_in_timeout_text("Slack"),
|
||||
)
|
||||
|
||||
def run(self, context: CapabilityCheckContext) -> CheckOutput:
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Timeouts: a ``TimedOutValidationError`` has the timeout kind and the
|
||||
sign-in policy of an unreachable source, the hang guard uses a check's own
|
||||
timeout text, and an error detail survives a cycle in the cause chain."""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from onyx.configs.constants import DocumentSource
|
||||
from onyx.connectors.capability_checks.models import (
|
||||
CapabilityCheck,
|
||||
CapabilityCheckContext,
|
||||
CapabilityCheckResult,
|
||||
CapabilityCheckStatus,
|
||||
CheckErrorKind,
|
||||
CheckRowText,
|
||||
CredentialCapability,
|
||||
check_error_detail,
|
||||
sign_in_timeout_text,
|
||||
)
|
||||
from onyx.connectors.capability_checks.registry import get_capability_checks
|
||||
from onyx.connectors.capability_checks.runner import (
|
||||
_hang_guard_row,
|
||||
run_capability_checks,
|
||||
)
|
||||
from onyx.connectors.exceptions import (
|
||||
SourceApiError,
|
||||
TimedOutValidationError,
|
||||
UnreachableValidationError,
|
||||
)
|
||||
from onyx.utils.threadpool_concurrency import run_with_timeout
|
||||
|
||||
_TIMEOUT_SECONDS = 0.2
|
||||
_OWN_TEXT = CheckRowText(summary="Sign-in timed out", description="Slow.\nRe-run.")
|
||||
|
||||
|
||||
class _SlowCheck(CapabilityCheck):
|
||||
"""Raises ``error``, or sleeps past its hang guard when it is None."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
error: Exception | None = None,
|
||||
*,
|
||||
fails_when_unreachable: bool = False,
|
||||
timeout_text: CheckRowText | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
capability=CredentialCapability.INDEXING,
|
||||
check_id="slow_check",
|
||||
display_name="Slow check",
|
||||
requires_connector_instance=False,
|
||||
timeout_seconds=_TIMEOUT_SECONDS,
|
||||
fails_when_unreachable=fails_when_unreachable,
|
||||
timeout_text=timeout_text,
|
||||
)
|
||||
self._error = error
|
||||
|
||||
def run(self, context: CapabilityCheckContext) -> None: # noqa: ARG002
|
||||
if self._error is None:
|
||||
time.sleep(_TIMEOUT_SECONDS * 5)
|
||||
return
|
||||
raise self._error
|
||||
|
||||
|
||||
def _run(check: CapabilityCheck, *, form_run: bool = False) -> CapabilityCheckResult:
|
||||
context = CapabilityCheckContext(source=DocumentSource.SLACK, credential_json={})
|
||||
(result,) = run_capability_checks([check], context, form_run=form_run)
|
||||
return result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("fails_when_unreachable", "form_run", "status"),
|
||||
[
|
||||
(True, True, CapabilityCheckStatus.FAILED),
|
||||
(True, False, CapabilityCheckStatus.INCOMPLETE),
|
||||
(False, True, CapabilityCheckStatus.INCOMPLETE),
|
||||
],
|
||||
)
|
||||
def test_a_timed_out_error_follows_the_unreachable_policy(
|
||||
fails_when_unreachable: bool, form_run: bool, status: CapabilityCheckStatus
|
||||
) -> None:
|
||||
check = _SlowCheck(
|
||||
TimedOutValidationError("Slow."), fails_when_unreachable=fails_when_unreachable
|
||||
)
|
||||
|
||||
result = _run(check, form_run=form_run)
|
||||
|
||||
assert (result.status, result.error_kind) == (status, CheckErrorKind.TIMEOUT)
|
||||
assert result.summary == "Timed out"
|
||||
assert result.error_detail is not None
|
||||
assert result.error_detail.stack_trace is not None
|
||||
|
||||
|
||||
def test_a_timed_out_error_is_still_an_unreachable_error() -> None:
|
||||
assert isinstance(TimedOutValidationError(), UnreachableValidationError)
|
||||
|
||||
|
||||
def test_the_hang_guard_uses_the_check_s_timeout_text() -> None:
|
||||
result = _run(
|
||||
_SlowCheck(fails_when_unreachable=True, timeout_text=_OWN_TEXT), form_run=True
|
||||
)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.FAILED
|
||||
assert result.error_kind == CheckErrorKind.TIMEOUT
|
||||
assert (result.summary, result.description) == (
|
||||
_OWN_TEXT.summary,
|
||||
_OWN_TEXT.description,
|
||||
)
|
||||
|
||||
|
||||
def test_the_hang_guard_keeps_the_default_text_without_one() -> None:
|
||||
result = _run(_SlowCheck())
|
||||
|
||||
assert result.summary == "Timed out"
|
||||
assert result.description is not None
|
||||
assert result.description.startswith("The check did not finish in time.")
|
||||
|
||||
|
||||
def test_an_error_s_own_text_wins_over_the_timeout_text() -> None:
|
||||
error = TimedOutValidationError("Slow.", summary="Own line", description="Own.")
|
||||
|
||||
result = _run(_SlowCheck(error, timeout_text=_OWN_TEXT))
|
||||
|
||||
assert (result.summary, result.description) == ("Own line", "Own.")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("seconds", "words"),
|
||||
[(600, "10 minutes"), (60, "1 minute"), (90, "90 seconds"), (1, "1 second")],
|
||||
)
|
||||
def test_the_timeout_text_names_the_guard_s_time_limit(
|
||||
seconds: float, words: str
|
||||
) -> None:
|
||||
check = _SlowCheck(
|
||||
timeout_text=CheckRowText(
|
||||
summary="Timed out", description="Stopped after {time_limit}."
|
||||
)
|
||||
)
|
||||
|
||||
row = _hang_guard_row(check, seconds)
|
||||
|
||||
assert row is not None
|
||||
assert row.description == f"Stopped after {words}."
|
||||
|
||||
|
||||
_SIGN_IN_CHECKS: dict[DocumentSource, tuple[str, set[str]]] = {
|
||||
DocumentSource.CONFLUENCE: (
|
||||
"Confluence",
|
||||
{"confluence_auth", "confluence_scoped_token_auth"},
|
||||
),
|
||||
DocumentSource.JIRA: ("Jira", {"jira_auth", "jira_scoped_token_auth"}),
|
||||
DocumentSource.GOOGLE_DRIVE: ("Google Drive", {"google_drive_auth"}),
|
||||
DocumentSource.GITHUB: ("GitHub", {"github_auth"}),
|
||||
DocumentSource.LINEAR: ("Linear", {"linear_token"}),
|
||||
DocumentSource.SLACK: ("Slack", {"slack_token_auth"}),
|
||||
DocumentSource.NOTION: ("Notion", {"notion_auth"}),
|
||||
DocumentSource.HUBSPOT: ("HubSpot", {"hubspot_token"}),
|
||||
DocumentSource.SHAREPOINT: ("SharePoint", {"sharepoint_token_auth"}),
|
||||
DocumentSource.ONEDRIVE: ("OneDrive", {"onedrive_token_auth"}),
|
||||
DocumentSource.OUTLOOK: ("Outlook", {"outlook_token_auth"}),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", list(_SIGN_IN_CHECKS))
|
||||
def test_each_check_has_the_design_s_timeout_text(source: DocumentSource) -> None:
|
||||
"""Sign-in rows have their own line; the other rows share the source's."""
|
||||
name, sign_in_ids = _SIGN_IN_CHECKS[source]
|
||||
checks = get_capability_checks(source)
|
||||
|
||||
for check in checks:
|
||||
if check.check_id in sign_in_ids:
|
||||
assert check.timeout_text == sign_in_timeout_text(name), check.check_id
|
||||
else:
|
||||
assert check.timeout_text is not None, check.check_id
|
||||
assert check.timeout_text.summary == "Timed out", check.check_id
|
||||
assert sign_in_ids <= {check.check_id for check in checks}
|
||||
|
||||
|
||||
def test_an_error_detail_survives_a_cycle_in_the_cause_chain() -> None:
|
||||
first = RuntimeError("first")
|
||||
second = RuntimeError("second")
|
||||
first.__cause__ = second
|
||||
second.__cause__ = first
|
||||
|
||||
detail = run_with_timeout(
|
||||
5, check_error_detail, first, CheckErrorKind.UNEXPECTED, []
|
||||
)
|
||||
|
||||
assert detail is not None
|
||||
assert detail.status_code is None
|
||||
assert detail.stack_trace is not None
|
||||
|
||||
|
||||
def test_an_error_detail_finds_the_response_before_a_cycle() -> None:
|
||||
api_error = SourceApiError(
|
||||
"HTTP 500", status_code=500, method="GET", url="https://api.example.com/x"
|
||||
)
|
||||
wrapper = RuntimeError("wrapper")
|
||||
wrapper.__cause__ = api_error
|
||||
api_error.__cause__ = wrapper
|
||||
|
||||
detail = run_with_timeout(
|
||||
5, check_error_detail, wrapper, CheckErrorKind.UNEXPECTED, []
|
||||
)
|
||||
|
||||
assert detail is not None
|
||||
assert (detail.status_code, detail.request) == (500, "GET api.example.com/x")
|
||||
@@ -48,6 +48,7 @@ from onyx.connectors.exceptions import (
|
||||
CredentialExpiredError,
|
||||
InconclusiveValidationError,
|
||||
InsufficientPermissionsError,
|
||||
TimedOutValidationError,
|
||||
UnexpectedValidationError,
|
||||
)
|
||||
|
||||
@@ -310,6 +311,16 @@ def test_auth_connection_error_is_incomplete() -> None:
|
||||
_run("confluence_auth", _context(gateway, _form()))
|
||||
|
||||
|
||||
def test_auth_timeout_is_a_sign_in_timeout() -> None:
|
||||
gateway = _gateway()
|
||||
gateway.probe_site.side_effect = requests.ConnectTimeout("slow")
|
||||
|
||||
with pytest.raises(TimedOutValidationError) as raised:
|
||||
_run("confluence_auth", _context(gateway, _form()))
|
||||
|
||||
assert raised.value.summary == "Sign-in timed out"
|
||||
|
||||
|
||||
# confluence_spaces_visible
|
||||
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ tested nothing says."""
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from onyx.configs.constants import DocumentSource
|
||||
@@ -115,16 +116,31 @@ def test_a_missing_token_says_so() -> None:
|
||||
assert result.summary == "No token saved in the account"
|
||||
|
||||
|
||||
def test_an_unreachable_server_fails_only_a_form_run() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"error,summary,kind",
|
||||
[
|
||||
(
|
||||
requests.ConnectionError("refused"),
|
||||
"Cannot reach GitHub",
|
||||
CheckErrorKind.UNREACHABLE,
|
||||
),
|
||||
(requests.ReadTimeout("slow"), "Sign-in timed out", CheckErrorKind.TIMEOUT),
|
||||
],
|
||||
ids=["unreachable", "timeout"],
|
||||
)
|
||||
def test_an_unreachable_server_fails_only_a_form_run(
|
||||
error: Exception, summary: str, kind: CheckErrorKind
|
||||
) -> None:
|
||||
gateway = _gateway()
|
||||
gateway.get_token_info.side_effect = requests.ConnectionError("refused")
|
||||
gateway.get_token_info.side_effect = error
|
||||
|
||||
form = _run("github_auth", gateway, form_run=True)
|
||||
stored = _run("github_auth", gateway)
|
||||
|
||||
assert form.status == CapabilityCheckStatus.FAILED
|
||||
assert stored.status == CapabilityCheckStatus.INCOMPLETE
|
||||
assert form.summary == stored.summary == "Cannot reach GitHub"
|
||||
assert form.summary == stored.summary == summary
|
||||
assert form.error_kind == stored.error_kind == kind
|
||||
|
||||
|
||||
def test_a_rate_limit_is_incomplete_and_stops_the_run() -> None:
|
||||
|
||||
@@ -159,6 +159,20 @@ def test_a_sign_in_timeout_fails_only_a_form_run() -> None:
|
||||
assert form.status == CapabilityCheckStatus.FAILED
|
||||
assert stored.status == CapabilityCheckStatus.INCOMPLETE
|
||||
assert form.summary == "Sign-in timed out"
|
||||
assert form.error_kind == stored.error_kind == CheckErrorKind.TIMEOUT
|
||||
|
||||
|
||||
def test_a_request_timeout_after_sign_in_is_incomplete_on_a_form_run() -> None:
|
||||
gateway = _gateway()
|
||||
gateway.list_records.side_effect = HubSpotConnectionError(
|
||||
"tickets", timed_out=True, detail="read timed out"
|
||||
)
|
||||
|
||||
result = _run(_READ_TICKETS, gateway, form_run=True)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.INCOMPLETE
|
||||
assert result.error_kind == CheckErrorKind.TIMEOUT
|
||||
assert result.summary == "Timed out"
|
||||
|
||||
|
||||
def test_an_api_error_carries_the_request_and_body() -> None:
|
||||
|
||||
@@ -172,7 +172,7 @@ def test_a_sign_in_timeout_fails_only_a_form_run(
|
||||
assert result.status == status
|
||||
assert result.summary == "Sign-in timed out"
|
||||
assert result.description == _SIGN_IN_DESCRIPTION
|
||||
assert result.error_kind == CheckErrorKind.UNREACHABLE
|
||||
assert result.error_kind == CheckErrorKind.TIMEOUT
|
||||
assert result.error_detail is not None
|
||||
assert result.error_detail.request == "GET example.atlassian.net/rest/api/3/myself"
|
||||
|
||||
@@ -197,6 +197,7 @@ def test_a_refused_connection_still_cannot_reach_jira() -> None:
|
||||
|
||||
assert result.status == CapabilityCheckStatus.FAILED
|
||||
assert result.summary == "Cannot reach Jira"
|
||||
assert result.error_kind == CheckErrorKind.UNREACHABLE
|
||||
|
||||
|
||||
def test_a_timeout_after_sign_in_is_incomplete_on_a_form_run() -> None:
|
||||
@@ -212,3 +213,4 @@ def test_a_timeout_after_sign_in_is_incomplete_on_a_form_run() -> None:
|
||||
assert result.description == (
|
||||
"The check stopped because Jira did not answer in time.\nRe-run the checks."
|
||||
)
|
||||
assert result.error_kind == CheckErrorKind.TIMEOUT
|
||||
@@ -202,6 +202,7 @@ def test_a_sign_in_timeout_says_so() -> None:
|
||||
|
||||
assert result.status == CapabilityCheckStatus.FAILED
|
||||
assert result.summary == "Sign-in timed out"
|
||||
assert result.error_kind == CheckErrorKind.TIMEOUT
|
||||
|
||||
|
||||
def test_a_busy_refresh_lock_is_incomplete() -> None:
|
||||
|
||||
@@ -151,18 +151,23 @@ def test_a_missing_token_fails_sign_in() -> None:
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error,summary",
|
||||
"error,summary,kind",
|
||||
[
|
||||
(NotionApiError("reset", status_code=None, code=None), "Cannot reach Notion"),
|
||||
(
|
||||
NotionApiError("reset", status_code=None, code=None),
|
||||
"Cannot reach Notion",
|
||||
CheckErrorKind.UNREACHABLE,
|
||||
),
|
||||
(
|
||||
NotionApiError("slow", status_code=None, code=None, timed_out=True),
|
||||
"Sign-in timed out",
|
||||
CheckErrorKind.TIMEOUT,
|
||||
),
|
||||
],
|
||||
ids=["unreachable", "timeout"],
|
||||
)
|
||||
def test_an_unreachable_notion_fails_only_a_form_run(
|
||||
error: NotionApiError, summary: str
|
||||
error: NotionApiError, summary: str, kind: CheckErrorKind
|
||||
) -> None:
|
||||
gateway = _gateway()
|
||||
gateway.get_bot_user.side_effect = error
|
||||
@@ -173,6 +178,7 @@ def test_an_unreachable_notion_fails_only_a_form_run(
|
||||
assert form.status == CapabilityCheckStatus.FAILED
|
||||
assert stored.status == CapabilityCheckStatus.INCOMPLETE
|
||||
assert form.summary == stored.summary == summary
|
||||
assert form.error_kind == stored.error_kind == kind
|
||||
|
||||
|
||||
def test_a_rate_limit_is_incomplete_and_stops_the_run() -> None:
|
||||
|
||||
Reference in new issue
Block a user