feat(connectors): Microsoft checks share their sign-in row and Graph error rows

The SharePoint, OneDrive and Outlook checks share:
- One sign-in probe, "Sign in to Microsoft". Every check after it depends
  on it, so a bad secret is one failed row and the rest are "Not run".
- The design's sign-in lines. Descriptions name the connector's account
  fields for the account's sign-in method. An expired secret or
  certificate says so.
- The certificate's end date, read from the .pfx: "Certificate valid
  until ...", with a caveat in the last 30 days.
- "Signed in as {app name}", from the token's claims.
- Form runs fail an unreachable sign-in. OneDrive and Outlook sign-in
  validate the credential binding, as SharePoint does.
- Graph errors: a 429 is RateLimitedValidationError and stops the run, no
  answer is UnreachableValidationError, and a 401 and an unexpected answer
  have their rows. Callers pass the rows of a 403 and a 404.
- MicrosoftGraphError is a SourceApiError with the request and the
  response body, so Error Details show them.
- One MicrosoftTokenInfo for the three gateways' check_token.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KFMazQbuG8ui3jxBxm47sC
This commit is contained in:
Claude committed 2026-10-17 03:12:02 +00:00
1 parent 218d5ef709
commit bdcc56c746
22 files changed
+1283 -143

No files matched your search

@@ -440,6 +440,11 @@ a gateway today (`<source>/source_operations.py`); most connectors still call th
SharePoint's gateway is also the only importer of `msal`, `requests` and `office365`
under `connectors/sharepoint/` and `ee/.../external_permissions/sharepoint/`. Its
permission reads are shared with Teams through `microsoft_utils/sharepoint_rest.py`.
The SharePoint, OneDrive and Outlook checks share their sign-in probe
(`microsoft_utils/sign_in.py`) and row text (`microsoft_utils/check_text.py`),
and every probe after sign-in depends on it. `MicrosoftGraphError` is a
`SourceApiError` that carries the request and the response body, so a check's
Error Details show them.
The GitHub gateway calls the REST API through PyGithub's requester and parses the
JSON into the models in `github/models.py`, so no PyGithub object leaves it.
Its `GitHubApiError` is a `SourceApiError` that carries the request and the
@@ -0,0 +1,270 @@
"""The row text the Microsoft connectors' capability checks share: the
sign-in rows, the throttled and unreachable rows, and the lines that say
which grant an app registration needs.
A check's row has a short line (``summary``, 40 characters or fewer) and up
to three lines of description. The connectors wrap these in their check
outputs and errors."""
from datetime import datetime, timedelta, timezone
from pydantic import BaseModel, ConfigDict
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
GRAPH_API = "Microsoft Graph"
SHAREPOINT_API = "SharePoint API"
# A certificate this close to its end date gets a caveat on the sign-in row.
CERTIFICATE_EXPIRY_WARNING = timedelta(days=30)
class MicrosoftRowText(BaseModel):
"""The row text of one outcome of a probe."""
model_config = ConfigDict(frozen=True)
summary: str
description: str
class MicrosoftAccountLabels(BaseModel):
"""The names the connector form gives the account's fields, for lines
that tell the admin which field to fix."""
model_config = ConfigDict(frozen=True)
client_id: str
directory_id: str
client_secret: str
private_key: str
certificate_password: str
@classmethod
def with_prefix(
cls, prefix: str, *, private_key: str | None = None
) -> "MicrosoftAccountLabels":
return cls(
client_id=f"{prefix} Client ID",
directory_id=f"{prefix} Directory ID",
client_secret=f"{prefix} Client Secret",
private_key=private_key or f"{prefix} Private Key",
certificate_password=f"{prefix} Certificate Password",
)
DEFAULT_ACCOUNT_LABELS = MicrosoftAccountLabels(
client_id="Client ID",
directory_id="Directory ID",
client_secret="Client Secret",
private_key="Private Key",
certificate_password="Certificate Password",
)
def grant_line(permission: str, api: str = GRAPH_API) -> str:
"""The line that tells the admin which application permission to grant."""
return (
f"Grant `{permission}` ({api}, application) to the app registration and "
"admin-consent it."
)
def format_date(moment: datetime) -> str:
return moment.astimezone(timezone.utc).strftime("%Y/%m/%d")
RATE_LIMITED_ROW = MicrosoftRowText(
summary="Microsoft is throttling requests",
description="Microsoft limited the request rate.\nWait a few minutes, then re-run.",
)
UNREACHABLE_ROW = MicrosoftRowText(
summary="Cannot reach Microsoft",
description=(
"The request did not reach Microsoft, after retries.\nCheck that Onyx can "
"reach Microsoft Graph, then re-run."
),
)
TOKEN_REJECTED_ROW = MicrosoftRowText(
summary="Microsoft rejected the app's token",
description=(
"Microsoft Graph refused the token the sign-in gave. (HTTP 401)\nCheck that "
"the Graph API host matches the app's cloud, then re-run."
),
)
def unexpected_answer_row(request: str) -> MicrosoftRowText:
"""``request`` names the call, such as "the site listing"."""
return MicrosoftRowText(
summary="Unexpected answer from Microsoft",
description=(
f"Microsoft refused {request} in a way the check does not expect.\n"
"Re-run the checks. If it repeats, send the error details to support."
),
)
# Sign in to Microsoft: the same row for every Microsoft connector.
UNKNOWN_SIGN_IN_METHOD_ROW = MicrosoftRowText(
summary="Unknown sign-in method on the account",
description=(
"The account names a sign-in method other than client secret or "
"certificate.\nCreate the account again with one of the two."
),
)
SIGN_IN_UNAVAILABLE_ROW = MicrosoftRowText(
summary="Microsoft sign-in is unavailable",
description="Microsoft's sign-in service did not answer.\nRe-run the checks later.",
)
SIGN_IN_UNEXPECTED_ROW = MicrosoftRowText(
summary="Unexpected answer from Microsoft sign-in",
description=(
"Microsoft's sign-in service answered in a way the check does not "
"expect.\nRe-run the checks. If it repeats, send the error details to "
"support."
),
)
SIGN_IN_UNREACHABLE_ROW = MicrosoftRowText(
summary="Cannot reach Microsoft sign-in",
description=(
"The sign-in request did not reach Microsoft, after retries.\nCheck that "
"Onyx can reach the authority host, then re-run."
),
)
SIGN_IN_REFUSED_ROW = MicrosoftRowText(
summary="Microsoft refused the sign-in request",
description=(
"Microsoft's sign-in service refused the request.\nCheck the error "
"details for the AADSTS code and fix what it names."
),
)
DIRECTORY_NOT_FOUND_ROW = MicrosoftRowText(
summary="Directory ID not found",
description=(
"Microsoft has no directory with this ID at the selected authority "
"host.\nCheck the Directory ID, and that the authority host matches the "
"app's cloud."
),
)
APP_NOT_FOUND_ROW = MicrosoftRowText(
summary="App registration not found",
description=(
"Microsoft did not find this client ID in this directory.\nCopy the "
"Application (client) ID from the app registration's overview."
),
)
def no_token_row(code: str) -> MicrosoftRowText:
return MicrosoftRowText(
summary="Microsoft did not issue a token",
description=(
"Microsoft refused the token request for a reason the check does not "
f"map ({code}).\nCheck the error details for the AADSTS code."
),
)
def missing_field_row(
labels: MicrosoftAccountLabels, *, certificate: bool
) -> MicrosoftRowText:
fields: str = (
f"{labels.client_id}, {labels.directory_id}, {labels.private_key} and "
f"{labels.certificate_password}"
if certificate
else f"{labels.client_id}, {labels.directory_id} and {labels.client_secret}"
)
return MicrosoftRowText(
summary="A required account field is empty",
description=(
f"The account is missing a field the sign-in needs.\nFill in {fields}."
),
)
def certificate_unreadable_row(labels: MicrosoftAccountLabels) -> MicrosoftRowText:
return MicrosoftRowText(
summary="Certificate file does not open",
description=(
"The .pfx file or its password on this account is wrong.\nUpload the "
f".pfx again as {labels.private_key} and re-enter "
f"{labels.certificate_password}."
),
)
def credential_rejected_row(
labels: MicrosoftAccountLabels,
*,
certificate: bool,
certificate_not_after: datetime | None = None,
secret_expired: bool = False,
) -> MicrosoftRowText:
"""``certificate_not_after`` in the past, or ``secret_expired``, names the
expiry as the cause."""
if certificate:
expired: bool = certificate_not_after is not None and (
certificate_not_after < datetime.now(timezone.utc)
)
cause: str = (
f"The certificate on this account expired on "
f"{format_date(certificate_not_after)}."
if expired and certificate_not_after is not None
else "Microsoft refused the certificate on this account."
)
return MicrosoftRowText(
summary="Secret or certificate was rejected",
description=(
f"{cause}\nUpload a current certificate under Certificates & "
"secrets in the app registration, then upload its .pfx as "
f"{labels.private_key}."
),
)
cause = (
"The client secret on this account has expired."
if secret_expired
else "Microsoft refused the secret on this account."
)
return MicrosoftRowText(
summary="Secret or certificate was rejected",
description=(
f"{cause}\nCreate a new secret under Certificates & secrets in the app "
f"registration and paste its Value into {labels.client_secret}."
),
)
class MicrosoftSignIn(BaseModel):
"""What a passing sign-in check shows: its line, and caveats."""
summary: str
description: str | None = None
caveats: list[str] = []
def sign_in_text(
token: MicrosoftTokenInfo, labels: MicrosoftAccountLabels
) -> MicrosoftSignIn:
"""The app's name from the token, and the certificate's end date. A
certificate near its end date gets a caveat."""
summary: str = (
f"Signed in as {token.app_name}"
if token.app_name
else f"Signed in as app {token.app_id}"
if token.app_id
else "Signed in to Microsoft"
)
not_after: datetime | None = token.certificate_not_after
if not_after is None:
return MicrosoftSignIn(summary=summary)
caveats: list[str] = []
if not_after - datetime.now(timezone.utc) <= CERTIFICATE_EXPIRY_WARNING:
caveats.append(
f"The certificate expires on {format_date(not_after)}. Upload a new "
f".pfx as {labels.private_key} before then, or sign-in stops working."
)
return MicrosoftSignIn(
summary=summary,
description=f"Certificate valid until {format_date(not_after)}.",
caveats=caveats,
)
@@ -12,7 +12,10 @@ threshold that marks a credential invalid, so it is not interchangeable with
"""
import base64
import binascii
import json
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Any, assert_never
@@ -22,7 +25,8 @@ from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.serialization import pkcs12
from msal.exceptions import MsalServiceError
from office365.runtime.auth.token_response import TokenResponse
from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict
from pydantic import ValidationError as PydanticValidationError
from onyx.connectors.exceptions import ConnectorValidationError
from onyx.connectors.microsoft_utils.graph_errors import (
@@ -35,6 +39,7 @@ from onyx.connectors.microsoft_utils.graph_errors import (
microsoft_error_from_exception,
msal_http_status,
)
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
from onyx.utils.logger import setup_logger
logger = setup_logger()
@@ -78,6 +83,8 @@ class MicrosoftAuthContext:
app: msal.ConfidentialClientApplication
method: MicrosoftAuthMethod
# The certificate's end date, for a certificate account.
certificate_not_after: datetime | None = None
class CertificateData(BaseModel):
@@ -115,6 +122,61 @@ def load_certificate_from_pfx(pfx_data: bytes, password: str) -> CertificateData
return None
def certificate_not_after(pfx_data: bytes, password: str) -> datetime | None:
"""The end date of the certificate in a .pfx bundle, read with no call to
Microsoft. None when the bundle does not open."""
try:
_, certificate, _ = pkcs12.load_key_and_certificates(
pfx_data, password.encode("utf-8")
)
except ValueError:
logger.debug("The PFX bundle did not open to read its end date")
return None
return certificate.not_valid_after_utc if certificate is not None else None
class _AccessTokenClaims(BaseModel):
"""The claims of an app-only access token that name the app."""
model_config = ConfigDict(extra="ignore")
app_displayname: str | None = None
appid: str | None = None
azp: str | None = None
def _access_token_claims(access_token: str) -> _AccessTokenClaims:
"""The token's claims, read without verifying its signature: they only
name the app on the sign-in row. Empty when the token is not a JWT."""
parts: list[str] = access_token.split(".")
if len(parts) != 3:
return _AccessTokenClaims()
payload: str = parts[1] + "=" * (-len(parts[1]) % 4)
try:
return _AccessTokenClaims.model_validate(
json.loads(base64.urlsafe_b64decode(payload))
)
except (binascii.Error, ValueError, PydanticValidationError):
logger.debug("The access token's claims did not decode")
return _AccessTokenClaims()
def token_info(
auth_context: MicrosoftAuthContext, token_response: dict[str, Any]
) -> MicrosoftTokenInfo:
"""What a token response tells about the app that signed in."""
claims: _AccessTokenClaims = _access_token_claims(
str(token_response.get("access_token") or "")
)
expires_in: int | str | None = token_response.get("expires_in")
return MicrosoftTokenInfo(
expires_in=int(expires_in) if expires_in is not None else None,
app_name=claims.app_displayname,
app_id=claims.appid or claims.azp,
certificate_not_after=auth_context.certificate_not_after,
)
def build_msal_app(
*,
client_id: str,
@@ -144,9 +206,8 @@ def build_msal_app(
"Private key and certificate password are required for certificate authentication"
)
certificate_data = load_certificate_from_pfx(
base64.b64decode(private_key_b64), certificate_password
)
pfx_data: bytes = base64.b64decode(private_key_b64)
certificate_data = load_certificate_from_pfx(pfx_data, certificate_password)
if certificate_data is None:
raise RuntimeError("Failed to load certificate")
@@ -158,6 +219,7 @@ def build_msal_app(
client_credential=certificate_data.model_dump(),
),
method=MicrosoftAuthMethod.CERTIFICATE,
certificate_not_after=certificate_not_after(pfx_data, certificate_password),
)
if auth_method is MicrosoftAuthMethod.CLIENT_SECRET:
@@ -197,21 +259,26 @@ def build_graph_auth_context(
except ConnectorValidationError as error:
raise MicrosoftAuthError(INVALID_AUTH_METHOD_CODE, str(error)) from error
certificate: bool = auth_method is MicrosoftAuthMethod.CERTIFICATE
required = [client_id, directory_id]
if auth_method is MicrosoftAuthMethod.CLIENT_SECRET:
required.append(client_secret)
else:
if certificate:
required.extend((private_key_b64, certificate_password))
else:
required.append(client_secret)
if any(not str(value or "").strip() for value in required):
raise MicrosoftAuthError(
MISSING_CREDENTIAL_CODE, "one or more required fields are missing"
MISSING_CREDENTIAL_CODE,
"one or more required fields are missing",
certificate=certificate,
)
if auth_method is MicrosoftAuthMethod.CERTIFICATE:
if certificate:
try:
base64.b64decode(private_key_b64 or "", validate=True)
except ValueError as error:
raise MicrosoftAuthError(INVALID_CERTIFICATE_CODE, str(error)) from error
raise MicrosoftAuthError(
INVALID_CERTIFICATE_CODE, str(error), certificate=True
) from error
try:
return build_msal_app(
@@ -232,9 +299,13 @@ def build_graph_auth_context(
or is_msal_decode_error(error)
):
raise microsoft_error_from_exception(error) from error
raise MicrosoftAuthError(INVALID_AUTHORITY_CODE, str(error)) from error
raise MicrosoftAuthError(
INVALID_AUTHORITY_CODE, str(error), certificate=certificate
) from error
except RuntimeError as error:
raise MicrosoftAuthError(INVALID_CERTIFICATE_CODE, str(error)) from error
raise MicrosoftAuthError(
INVALID_CERTIFICATE_CODE, str(error), certificate=certificate
) from error
except (MsalServiceError, requests.RequestException) as error:
raise microsoft_error_from_exception(error) from error
@@ -251,6 +322,8 @@ def acquire_graph_token_response(
raise MicrosoftAuthError(
str(response.get("error") or "unknown_error"),
str(response.get("error_description") or ""),
certificate=auth_context.method is MicrosoftAuthMethod.CERTIFICATE,
certificate_not_after=auth_context.certificate_not_after,
)
return response
@@ -4,6 +4,7 @@ import json
import re
from collections.abc import Generator, Iterator
from contextlib import contextmanager
from datetime import datetime
from typing import Any, NoReturn
import requests
@@ -14,9 +15,34 @@ from onyx.connectors.exceptions import (
CredentialExpiredError,
CredentialInvalidError,
InsufficientPermissionsError,
RateLimitedValidationError,
SourceApiError,
UnexpectedValidationError,
UnreachableValidationError,
ValidationError,
)
from onyx.connectors.microsoft_utils.check_text import (
APP_NOT_FOUND_ROW,
DEFAULT_ACCOUNT_LABELS,
DIRECTORY_NOT_FOUND_ROW,
RATE_LIMITED_ROW,
SIGN_IN_REFUSED_ROW,
SIGN_IN_UNAVAILABLE_ROW,
SIGN_IN_UNEXPECTED_ROW,
SIGN_IN_UNREACHABLE_ROW,
TOKEN_REJECTED_ROW,
UNKNOWN_SIGN_IN_METHOD_ROW,
UNREACHABLE_ROW,
MicrosoftAccountLabels,
MicrosoftRowText,
certificate_unreadable_row,
credential_rejected_row,
missing_field_row,
no_token_row,
unexpected_answer_row,
)
from onyx.connectors.microsoft_utils.graph_client import (
TRANSIENT_TRANSPORT_EXCEPTIONS,
is_permanent_refusal_status,
)
@@ -38,11 +64,29 @@ class MicrosoftGraphErrorDetails(BaseModel):
message: str
class MicrosoftGraphError(Exception):
def __init__(self, status: int | None, code: str, message: str) -> None:
class MicrosoftGraphError(SourceApiError):
"""A Graph, SharePoint REST or token endpoint refusal. ``status`` is None
for a failure with no answer, such as a dropped connection."""
def __init__(
self,
status: int | None,
code: str,
message: str,
*,
method: str | None = None,
url: str | None = None,
body: str | None = None,
) -> None:
super().__init__(
f"Graph {status} {code}: {message}",
status_code=status,
method=method,
url=url,
body=body,
)
self.status = status
self.code = code
super().__init__(f"Graph {status} {code}: {message}")
@property
def is_permanent_refusal(self) -> bool:
@@ -54,8 +98,21 @@ class MicrosoftGraphError(Exception):
class MicrosoftAuthError(Exception):
def __init__(self, code: str, description: str) -> None:
"""Entra refused to sign the app in. ``certificate`` marks a certificate
account; ``certificate_not_after`` is its certificate's end date, when
the certificate opened."""
def __init__(
self,
code: str,
description: str,
*,
certificate: bool = False,
certificate_not_after: datetime | None = None,
) -> None:
self.code = code
self.certificate = certificate
self.certificate_not_after = certificate_not_after
super().__init__(f"{code}: {description}")
@@ -122,12 +179,25 @@ def parse_graph_error(error: Exception) -> MicrosoftGraphErrorDetails:
def microsoft_error_from_exception(error: Exception) -> MicrosoftGraphError:
"""The error with its status and code, and, when it carries an answer,
the request and response body for Error Details."""
details = (
parse_graph_error(error)
if isinstance(error, requests.RequestException)
else parse_msal_error(error)
)
return MicrosoftGraphError(details.status, details.code, details.message)
response = error.response if isinstance(error, requests.RequestException) else None
if response is None:
return MicrosoftGraphError(details.status, details.code, details.message)
request = response.request
return MicrosoftGraphError(
details.status,
details.code,
details.message,
method=request.method if request is not None else None,
url=response.url or (request.url if request is not None else None),
body=response.text or None,
)
@contextmanager
@@ -142,42 +212,144 @@ def raise_microsoft_errors() -> Iterator[None]:
raise microsoft_error_from_exception(error) from error
def raise_for_auth_error(error: MicrosoftAuthError) -> NoReturn:
# Entra's code for an expired client secret.
_EXPIRED_SECRET_CODE = "AADSTS7000222"
def _raise_with_row(
error_class: type[ValidationError],
message: str,
row: MicrosoftRowText | None,
cause: Exception,
) -> NoReturn:
raise error_class(
message,
summary=row.summary if row else None,
description=row.description if row else None,
) from cause
def raise_for_auth_error(
error: MicrosoftAuthError,
labels: MicrosoftAccountLabels = DEFAULT_ACCOUNT_LABELS,
) -> NoReturn:
"""Maps an Entra sign-in refusal onto the validation family, with the
sign-in row's text. ``labels`` name the account's fields."""
if error.code in TRANSIENT_OAUTH_CODES:
raise UnexpectedValidationError(
"Microsoft's token endpoint is unavailable. Re-run the checks later."
) from error
_raise_with_row(
UnexpectedValidationError,
"Microsoft's token endpoint is unavailable. Re-run the checks later.",
SIGN_IN_UNAVAILABLE_ROW,
error,
)
if error.code == MISSING_CREDENTIAL_CODE:
raise CredentialInvalidError(
"Microsoft credentials are incomplete; check all required fields."
) from error
_raise_with_row(
CredentialInvalidError,
"Microsoft credentials are incomplete; check all required fields.",
missing_field_row(labels, certificate=error.certificate),
error,
)
if error.code == INVALID_AUTHORITY_CODE:
raise CredentialInvalidError(
_raise_with_row(
CredentialInvalidError,
"Microsoft does not know this directory. Check the directory id and "
"authority host."
) from error
"authority host.",
DIRECTORY_NOT_FOUND_ROW,
error,
)
if error.code == INVALID_CERTIFICATE_CODE:
raise CredentialInvalidError(
_raise_with_row(
CredentialInvalidError,
"The PFX bundle could not be opened. Check the file and certificate "
"password."
) from error
"password.",
certificate_unreadable_row(labels),
error,
)
if error.code == INVALID_AUTH_METHOD_CODE:
raise CredentialInvalidError(
"The Microsoft authentication method is invalid."
) from error
_raise_with_row(
CredentialInvalidError,
"The Microsoft authentication method is invalid.",
UNKNOWN_SIGN_IN_METHOD_ROW,
error,
)
if error.code == "invalid_client":
raise CredentialInvalidError(
_raise_with_row(
CredentialInvalidError,
"Microsoft rejected the client secret or certificate. It is wrong, "
"expired, or belongs to a different app registration."
) from error
"expired, or belongs to a different app registration.",
credential_rejected_row(
labels,
certificate=error.certificate,
certificate_not_after=error.certificate_not_after,
secret_expired=_EXPIRED_SECRET_CODE in str(error),
),
error,
)
if error.code in INVALID_REGISTRATION_CODES:
raise CredentialInvalidError(
_raise_with_row(
CredentialInvalidError,
"Microsoft rejected the app registration. Check the client id and "
"directory id."
) from error
raise CredentialInvalidError(
f"Microsoft did not issue a token ({error.code})."
) from error
"directory id.",
APP_NOT_FOUND_ROW,
error,
)
_raise_with_row(
CredentialInvalidError,
f"Microsoft did not issue a token ({error.code}).",
no_token_row(error.code),
error,
)
def _is_transport_failure(error: MicrosoftGraphError) -> bool:
"""No answer came back: the connection failed or timed out, after
retries."""
return error.status is None and any(
isinstance(wrapped, TRANSIENT_TRANSPORT_EXCEPTIONS)
for wrapped in _exception_chain(error)
)
def raise_for_sign_in_error(error: MicrosoftGraphError) -> NoReturn:
"""Maps a failed token request that Entra did not answer with an OAuth
error: throttled, unreachable, down, or refused outright."""
if error.status == 429:
_raise_with_row(
RateLimitedValidationError,
f"Microsoft's token endpoint is throttling ({error.code}). Re-run the "
"checks later.",
RATE_LIMITED_ROW,
error,
)
if _is_transport_failure(error):
_raise_with_row(
UnreachableValidationError,
f"Onyx cannot reach Microsoft's token endpoint ({error.code}).",
SIGN_IN_UNREACHABLE_ROW,
error,
)
if error.status is not None and error.status >= 500:
_raise_with_row(
UnexpectedValidationError,
f"Microsoft's token endpoint is unavailable ({error.status} "
f"{error.code}). Re-run the checks later.",
SIGN_IN_UNAVAILABLE_ROW,
error,
)
if error.status is not None and error.status >= 400:
_raise_with_row(
CredentialInvalidError,
f"Microsoft refused the token request. Graph reported `{error.code}`.",
SIGN_IN_REFUSED_ROW,
error,
)
_raise_with_row(
UnexpectedValidationError,
f"Microsoft's token endpoint answered unexpectedly ({error.status} "
f"{error.code}).",
SIGN_IN_UNEXPECTED_ROW,
error,
)
def raise_for_graph_error(
@@ -186,23 +358,58 @@ def raise_for_graph_error(
*,
remediation: str | None = None,
permanent_refusal_message: str | None = None,
denied_row: MicrosoftRowText | None = None,
not_found_row: MicrosoftRowText | None = None,
request: str = "the request",
) -> NoReturn:
"""Maps a Graph or SharePoint REST refusal onto the validation family.
``denied_row`` is the row of a 403, ``not_found_row`` of a 404 or 423;
``request`` names the call in the unexpected-answer row."""
if error.status == 401:
raise CredentialExpiredError(
f"Graph rejected the access token ({error.code})."
) from error
_raise_with_row(
CredentialExpiredError,
f"Graph rejected the access token ({error.code}).",
TOKEN_REJECTED_ROW,
error,
)
if error.status == 403:
detail = f"{denied_message} Graph reported `{error.code}`."
if remediation:
detail = f"{detail} {remediation}"
raise InsufficientPermissionsError(detail) from error
_raise_with_row(InsufficientPermissionsError, detail, denied_row, error)
if error.is_permanent_refusal:
raise ConnectorValidationError(
permanent_refusal_message or denied_message
) from error
_raise_with_row(
ConnectorValidationError,
permanent_refusal_message or denied_message,
not_found_row,
error,
)
if error.status == 429:
_raise_with_row(
RateLimitedValidationError,
f"Graph is throttling ({error.status} {error.code}). Re-run the checks "
"later.",
RATE_LIMITED_ROW,
error,
)
if _is_transport_failure(error):
_raise_with_row(
UnreachableValidationError,
f"Graph is unreachable ({error.code}). Re-run the checks later.",
UNREACHABLE_ROW,
error,
)
if error.fails_the_attempt:
raise UnexpectedValidationError(
f"Graph is throttling or unreachable ({error.status} {error.code}). "
"Re-run the checks later."
) from error
raise UnexpectedValidationError(f"Unexpected Graph error: {error}") from error
_raise_with_row(
UnexpectedValidationError,
f"Graph answered with an error ({error.status} {error.code}). Re-run "
"the checks later.",
unexpected_answer_row(request),
error,
)
_raise_with_row(
UnexpectedValidationError,
f"Unexpected Graph error: {error}",
unexpected_answer_row(request),
error,
)
@@ -8,11 +8,13 @@ from onyx.connectors.microsoft_utils.graph_auth import (
MicrosoftAuthContext,
acquire_graph_token_response,
build_graph_auth_context,
token_info,
)
from onyx.connectors.microsoft_utils.graph_client import GraphApiClient
from onyx.connectors.microsoft_utils.graph_errors import (
microsoft_error_from_exception,
)
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
class MicrosoftGraphAuthConfig(BaseModel):
@@ -66,6 +68,10 @@ class MicrosoftGraphGateway:
def token_response(self) -> dict[str, Any]:
return acquire_graph_token_response(self.auth_context, self.graph_api_host)
def token_info(self) -> MicrosoftTokenInfo:
"""Signs in, and reads what the token tells about the app."""
return token_info(self.auth_context, self.token_response())
def access_token(self) -> str:
return str(self.token_response()["access_token"])
@@ -1,7 +1,9 @@
"""The plain data SharePoint permission sync works on: principals and role
assignments from SharePoint REST, and the members of an Entra group from
Graph. The office365 objects never leave ``sharepoint_rest``."""
"""Plain data the Microsoft connectors share: what an app-only sign-in tells,
and what SharePoint permission sync works on (principals and role
assignments from SharePoint REST, the members of an Entra group from Graph).
The office365 objects never leave ``sharepoint_rest``."""
from datetime import datetime
from enum import Enum
from pydantic import BaseModel, ConfigDict
@@ -61,3 +63,16 @@ class EntraMember(BaseModel):
display_name: str | None = None
user_principal_name: str | None = None
mail: str | None = None
class MicrosoftTokenInfo(BaseModel):
"""What an app-only sign-in tells about the app: the token's lifetime,
the app's name and id from the token's claims, and, for a certificate
account, the certificate's end date."""
model_config = ConfigDict(frozen=True)
expires_in: int | None = None
app_name: str | None = None
app_id: str | None = None
certificate_not_after: datetime | None = None
@@ -0,0 +1,31 @@
"""The sign-in probe every Microsoft connector's first check runs: one
app-only token request, mapped onto the sign-in row."""
from collections.abc import Callable
from onyx.connectors.microsoft_utils.check_text import (
MicrosoftAccountLabels,
MicrosoftSignIn,
sign_in_text,
)
from onyx.connectors.microsoft_utils.graph_errors import (
MicrosoftAuthError,
MicrosoftGraphError,
raise_for_auth_error,
raise_for_sign_in_error,
)
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
def sign_in(
check_token: Callable[[], MicrosoftTokenInfo], labels: MicrosoftAccountLabels
) -> MicrosoftSignIn:
"""Signs in with ``check_token`` and returns the passing row's text.
Raises the validation family with the sign-in row's text."""
try:
token: MicrosoftTokenInfo = check_token()
except MicrosoftAuthError as error:
raise_for_auth_error(error, labels)
except MicrosoftGraphError as error:
raise_for_sign_in_error(error)
return sign_in_text(token, labels)
@@ -18,22 +18,21 @@ from onyx.connectors.exceptions import (
ConnectorValidationError,
UnexpectedValidationError,
)
from onyx.connectors.microsoft_utils.check_text import (
MicrosoftAccountLabels,
MicrosoftSignIn,
)
from onyx.connectors.microsoft_utils.drive_delta import (
DRIVE_DELTA_SELECT_FIELDS,
DriveDeltaItem,
build_delta_start_url,
)
from onyx.connectors.microsoft_utils.entra import EntraGroup
from onyx.connectors.microsoft_utils.graph_errors import (
MicrosoftAuthError as OneDriveAuthError,
)
from onyx.connectors.microsoft_utils.graph_errors import (
MicrosoftGraphError as OneDriveGraphError,
)
from onyx.connectors.microsoft_utils.graph_errors import (
raise_for_auth_error,
raise_for_graph_error,
)
from onyx.connectors.microsoft_utils.graph_errors import raise_for_graph_error
from onyx.connectors.microsoft_utils.sign_in import sign_in
from onyx.connectors.onedrive.config import OneDriveConnectorConfig
from onyx.connectors.onedrive.models import (
OneDriveDrive,
@@ -55,6 +54,12 @@ _COUNT_PAGES = 20
T = TypeVar("T")
_HIDDEN_MEMBERSHIP_VISIBILITY = "HiddenMembership"
_ACCOUNT_LABELS = MicrosoftAccountLabels.with_prefix(
"OneDrive", private_key="OneDrive Certificate"
)
_SIGN_IN_CHECK_ID = "onedrive_token_auth"
# Every probe signs in first, so a failed sign-in is one row.
_AFTER_SIGN_IN: tuple[str, ...] = (_SIGN_IN_CHECK_ID,)
_EMPTY_DRIVE_ADVISORY = CheckAdvisory(
message=(
@@ -242,20 +247,24 @@ class _TokenCheck(CapabilityCheck):
def __init__(self) -> None:
super().__init__(
capability=CredentialCapability.INDEXING,
check_id="onedrive_token_auth",
display_name="App registration can sign in",
check_id=_SIGN_IN_CHECK_ID,
display_name="Sign in to Microsoft",
requires_connector_instance=False,
remediation="Check the app id, tenant id, and client credential.",
docs_link=_DOCS_LINK,
validates_binding=True,
fails_when_unreachable=True,
)
def run(self, context: CapabilityCheckContext) -> None:
try:
_gateway(context).check_token()
except OneDriveAuthError as error:
raise_for_auth_error(error)
except OneDriveGraphError as error:
raise_for_graph_error(error, "Microsoft refused the token request.")
def run(self, context: CapabilityCheckContext) -> CheckOutput:
signed_in: MicrosoftSignIn = sign_in(
_gateway(context).check_token, _ACCOUNT_LABELS
)
return CheckOutput(
summary=signed_in.summary,
description=signed_in.description,
advisories=[CheckAdvisory(message=caveat) for caveat in signed_in.caveats],
)
class _UsersCheck(CapabilityCheck):
@@ -267,6 +276,7 @@ class _UsersCheck(CapabilityCheck):
requires_connector_instance=False,
remediation="Grant and admin-consent `User.Read.All`.",
docs_link=_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> None:
@@ -309,6 +319,7 @@ class _DriveCountCheck(CapabilityCheck[OneDriveConnectorConfig]):
requires_connector_instance=False,
remediation="Grant and admin-consent `User.Read.All`.",
docs_link=_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
discovery=True,
)
@@ -339,6 +350,7 @@ class _ConfiguredUsersCheck(CapabilityCheck[OneDriveConnectorConfig]):
requires_connector_config=True,
remediation="Use enabled member user principal names.",
docs_link=_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> None:
@@ -364,6 +376,7 @@ class _DriveCheck(CapabilityCheck[OneDriveConnectorConfig]):
requires_connector_config=True,
remediation="Grant `Sites.Read.All` or a selected personal-site read grant.",
docs_link=_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> None:
@@ -390,6 +403,7 @@ class _DeltaCheck(CapabilityCheck[OneDriveConnectorConfig]):
requires_connector_config=True,
remediation="Grant `Sites.Read.All` or a selected personal-site read grant.",
docs_link=_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> None:
@@ -428,6 +442,7 @@ class _PermissionCheck(CapabilityCheck[OneDriveConnectorConfig]):
requires_connector_config=True,
remediation="Grant `Sites.Read.All` or a selected personal-site read grant.",
docs_link=_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> CheckOutput | None:
@@ -470,6 +485,7 @@ class _GroupListCheck(CapabilityCheck):
requires_connector_instance=False,
remediation="Grant and admin-consent `GroupMember.ReadBasic.All`.",
docs_link=_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> None:
@@ -488,6 +504,7 @@ class _GroupMembershipCheck(CapabilityCheck):
requires_connector_instance=False,
remediation="Grant `GroupMember.ReadBasic.All`.",
docs_link=_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> CheckOutput | None:
@@ -66,10 +66,6 @@ class OneDriveDrive(BaseModel):
web_url: str | None = None
class OneDriveTokenInfo(BaseModel):
expires_in: int | None = None
class OneDriveDiscoveredFile(BaseModel):
model_config = ConfigDict(frozen=True)
@@ -49,12 +49,12 @@ from onyx.connectors.microsoft_utils.graph_gateway import (
MicrosoftGraphGateway,
build_graph_user_url,
)
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
from onyx.connectors.onedrive.models import (
OneDriveCredentials,
OneDriveDrive,
OneDrivePermission,
OneDrivePermissionPage,
OneDriveTokenInfo,
OneDriveUser,
OneDriveUserPage,
)
@@ -133,9 +133,6 @@ class OneDriveSourceOperations(SourceOperations):
)
return self._graph_gateway
def _token_response(self) -> dict[str, Any]:
return self._gateway().token_response()
def _access_token(self) -> str:
return self._gateway().access_token()
@@ -149,12 +146,8 @@ class OneDriveSourceOperations(SourceOperations):
capabilities={CredentialCapability.INDEXING},
consumes=OperationConsumes.CREDENTIAL,
)
def check_token(self) -> OneDriveTokenInfo:
response = self._token_response()
expires = response.get("expires_in")
return OneDriveTokenInfo(
expires_in=int(expires) if expires is not None else None
)
def check_token(self) -> MicrosoftTokenInfo:
return self._gateway().token_info()
@source_operation(
capabilities={CredentialCapability.INDEXING},
@@ -42,14 +42,15 @@ from onyx.connectors.exceptions import (
InsufficientPermissionsError,
UnexpectedValidationError,
)
from onyx.connectors.microsoft_utils.entra import EntraGroup
from onyx.connectors.microsoft_utils.graph_errors import (
MicrosoftAuthError as OutlookAuthError,
from onyx.connectors.microsoft_utils.check_text import (
MicrosoftAccountLabels,
MicrosoftSignIn,
)
from onyx.connectors.microsoft_utils.entra import EntraGroup
from onyx.connectors.microsoft_utils.graph_errors import (
MicrosoftGraphError as OutlookGraphError,
)
from onyx.connectors.microsoft_utils.graph_errors import raise_for_auth_error
from onyx.connectors.microsoft_utils.sign_in import sign_in
from onyx.connectors.outlook.config import OutlookConnectorConfig
from onyx.connectors.outlook.errors import (
CALENDAR_READ_REMEDIATION,
@@ -99,7 +100,10 @@ _CANDIDATE_PAGES = 20
# the count reaches its cap long before this unless Graph sends empty pages.
_COUNT_PAGES = 20
_TOKEN_ENDPOINT_DENIED = "Microsoft's token endpoint refused the request."
_ACCOUNT_LABELS = MicrosoftAccountLabels.with_prefix("Microsoft Outlook")
_SIGN_IN_CHECK_ID = "outlook_token_auth"
# Every probe signs in first, so a failed sign-in is one row.
_AFTER_SIGN_IN: tuple[str, ...] = (_SIGN_IN_CHECK_ID,)
# Mail.ReadBasic.All answers every metadata call but refuses bodies, so the
# read probe must fetch one body to tell the two grants apart.
@@ -273,23 +277,27 @@ class _TokenAuthCheck(CapabilityCheck):
def __init__(self) -> None:
super().__init__(
capability=CredentialCapability.INDEXING,
check_id="outlook_token_auth",
display_name="App registration can sign in",
check_id=_SIGN_IN_CHECK_ID,
display_name="Sign in to Microsoft",
requires_connector_instance=False,
remediation=(
"Enter the client id, directory (tenant) id and a current "
"client secret or certificate of the Entra app registration."
),
docs_link=_OUTLOOK_DOCS_LINK,
validates_binding=True,
fails_when_unreachable=True,
)
def run(self, context: CapabilityCheckContext) -> None:
try:
_gateway(context).check_token()
except OutlookAuthError as e:
raise_for_auth_error(e)
except OutlookGraphError as e:
raise_for_graph_error(e, _TOKEN_ENDPOINT_DENIED)
def run(self, context: CapabilityCheckContext) -> CheckOutput:
signed_in: MicrosoftSignIn = sign_in(
_gateway(context).check_token, _ACCOUNT_LABELS
)
return CheckOutput(
summary=signed_in.summary,
description=signed_in.description,
advisories=[CheckAdvisory(message=caveat) for caveat in signed_in.caveats],
)
class _MailboxListingCheck(CapabilityCheck):
@@ -308,6 +316,7 @@ class _MailboxListingCheck(CapabilityCheck):
"registration and admin-consent it."
),
docs_link=_OUTLOOK_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> None:
@@ -354,6 +363,7 @@ class _MailboxCountCheck(CapabilityCheck[OutlookConnectorConfig]):
"registration and admin-consent it."
),
docs_link=_OUTLOOK_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
discovery=True,
)
@@ -404,6 +414,7 @@ class _MailReadCheck(CapabilityCheck[OutlookConnectorConfig]):
requires_connector_instance=False,
remediation=EXCHANGE_SCOPE_REMEDIATION,
docs_link=_OUTLOOK_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> CheckOutput:
@@ -500,6 +511,7 @@ class _CalendarReadCheck(CapabilityCheck[OutlookConnectorConfig]):
requires_connector_config=True,
remediation=CALENDAR_READ_REMEDIATION,
docs_link=_OUTLOOK_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> CheckOutput | None:
@@ -569,6 +581,7 @@ class _ConfiguredMailboxesCheck(CapabilityCheck[OutlookConnectorConfig]):
requires_connector_config=True,
remediation=f"{MAILBOX_UNAVAILABLE_REMEDIATION} {EXCHANGE_SCOPE_REMEDIATION}",
docs_link=_OUTLOOK_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> None:
@@ -594,6 +607,7 @@ class _ConfiguredGroupsCheck(CapabilityCheck):
requires_connector_config=True,
remediation=f"{GROUP_UNAVAILABLE_REMEDIATION} {GROUP_LISTING_REMEDIATION}",
docs_link=_OUTLOOK_DOCS_LINK,
depends_on=_AFTER_SIGN_IN,
)
def run(self, context: CapabilityCheckContext) -> None:
@@ -12,10 +12,6 @@ from pydantic import BaseModel, ConfigDict
from onyx.connectors.models import ConnectorFailure, Document, HierarchyNode
class OutlookTokenInfo(BaseModel):
expires_in: int | None = None
class OutlookMailbox(BaseModel):
model_config = ConfigDict(frozen=True)
@@ -52,6 +52,7 @@ from onyx.connectors.microsoft_utils.graph_gateway import (
MicrosoftGraphGateway,
build_graph_user_url,
)
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
from onyx.connectors.outlook.models import (
OutlookAttachment,
OutlookDeltaPage,
@@ -65,7 +66,6 @@ from onyx.connectors.outlook.models import (
OutlookMessageChange,
OutlookMessagePage,
OutlookRecipient,
OutlookTokenInfo,
)
from onyx.connectors.source_operations import (
OperationConsumes,
@@ -400,9 +400,6 @@ class OutlookSourceOperations(SourceOperations):
)
return self._graph_gateway
def _token_response(self) -> dict[str, Any]:
return self._gateway().token_response()
def _access_token(self) -> str:
return self._gateway().access_token()
@@ -451,13 +448,9 @@ class OutlookSourceOperations(SourceOperations):
capabilities={CredentialCapability.INDEXING},
consumes=OperationConsumes.CREDENTIAL,
)
def check_token(self) -> OutlookTokenInfo:
def check_token(self) -> MicrosoftTokenInfo:
"""Acquire an app-only token: proves client id, directory id and credential agree."""
response = self._token_response()
expires_in = response.get("expires_in")
return OutlookTokenInfo(
expires_in=int(expires_in) if expires_in is not None else None
)
return self._gateway().token_info()
@source_operation(
capabilities={CredentialCapability.INDEXING},
@@ -16,6 +16,8 @@ from onyx.connectors.capability_checks.form_state import FormState
from onyx.connectors.capability_checks.models import (
CapabilityCheck,
CapabilityCheckContext,
CheckAdvisory,
CheckOutput,
CredentialCapability,
form_config,
)
@@ -23,6 +25,10 @@ from onyx.connectors.exceptions import (
ConnectorValidationError,
UnexpectedValidationError,
)
from onyx.connectors.microsoft_utils.check_text import (
MicrosoftAccountLabels,
MicrosoftSignIn,
)
from onyx.connectors.microsoft_utils.drive_delta import (
DriveDeltaFetchResult,
DriveDeltaItem,
@@ -57,6 +63,7 @@ from onyx.connectors.microsoft_utils.sharepoint_principals import (
is_public_login_name,
resolve_group_id,
)
from onyx.connectors.microsoft_utils.sign_in import sign_in
from onyx.connectors.sharepoint.config import SharepointConnectorConfig
from onyx.connectors.sharepoint.connector import (
ONEDRIVE_HOST_MARKER,
@@ -124,6 +131,10 @@ _CERTIFICATE_REMEDIATION: str = (
_SITE_SECURABLE: SharepointSecurable = SharepointSecurable(
kind=SharepointSecurableKind.SITE
)
_ACCOUNT_LABELS: MicrosoftAccountLabels = MicrosoftAccountLabels.with_prefix(
"SharePoint"
)
_SIGN_IN_CHECK_ID: str = "sharepoint_token_auth"
def _gateway(context: CapabilityCheckContext) -> SharepointSourceOperations:
@@ -460,6 +471,9 @@ class _SharepointCheck(CapabilityCheck[SharepointConnectorConfig]):
capability: CredentialCapability = CredentialCapability.INDEXING,
requires_connector_config: bool = True,
validates_binding: bool = False,
fails_when_unreachable: bool = False,
# Every probe signs in first, so a failed sign-in is one row.
depends_on: tuple[str, ...] = (_SIGN_IN_CHECK_ID,),
) -> None:
super().__init__(
capability=capability,
@@ -470,6 +484,8 @@ class _SharepointCheck(CapabilityCheck[SharepointConnectorConfig]):
remediation=remediation,
docs_link=_DOCS_LINK,
validates_binding=validates_binding,
fails_when_unreachable=fails_when_unreachable,
depends_on=depends_on,
)
@@ -479,9 +495,11 @@ class _TokenCheck(_SharepointCheck):
def __init__(self) -> None:
super().__init__(
check_id="sharepoint_token_auth",
display_name="App registration can sign in",
check_id=_SIGN_IN_CHECK_ID,
display_name="Sign in to Microsoft",
validates_binding=True,
fails_when_unreachable=True,
depends_on=(),
remediation=(
"Check the client id, directory id and client secret or "
"certificate, and that the Graph API and authority hosts match "
@@ -489,13 +507,15 @@ class _TokenCheck(_SharepointCheck):
),
)
def run(self, context: CapabilityCheckContext) -> None:
try:
_gateway(context).check_token()
except MicrosoftAuthError as error:
raise_for_auth_error(error)
except MicrosoftGraphError as error:
raise_for_graph_error(error, "Microsoft refused the token request.")
def run(self, context: CapabilityCheckContext) -> CheckOutput:
signed_in: MicrosoftSignIn = sign_in(
_gateway(context).check_token, _ACCOUNT_LABELS
)
return CheckOutput(
summary=signed_in.summary,
description=signed_in.description,
advisories=[CheckAdvisory(message=caveat) for caveat in signed_in.caveats],
)
class _ContentTypesCheck(_SharepointCheck):
@@ -504,6 +524,7 @@ class _ContentTypesCheck(_SharepointCheck):
check_id="sharepoint_content_types",
display_name="A content type is selected",
remediation="Turn on site documents, site pages, or both.",
depends_on=(),
)
def run(self, context: CapabilityCheckContext) -> None:
@@ -781,7 +802,7 @@ class _CertificateAuthCheck(_SharepointCheck):
try:
method: MicrosoftAuthMethod = _gateway(context).get_auth_method()
except MicrosoftAuthError as error:
raise_for_auth_error(error)
raise_for_auth_error(error, _ACCOUNT_LABELS)
if not method.supports_sharepoint_rest:
raise ConnectorValidationError(
"Permission sync needs the SharePoint REST API, which only accepts "
@@ -14,12 +14,6 @@ class SharepointCredentials(BaseModel):
sp_certificate_password: str | None = None
class SharepointTokenInfo(BaseModel):
model_config = ConfigDict(frozen=True)
expires_in: int | None
class SharepointDrive(BaseModel):
"""A document library as Graph lists it. ``list_id`` is the SharePoint
list behind it, which the permission reads address. Graph can list a
@@ -78,6 +78,7 @@ from onyx.connectors.microsoft_utils.graph_gateway import (
)
from onyx.connectors.microsoft_utils.models import (
EntraMember,
MicrosoftTokenInfo,
SharepointPrincipal,
SharepointRoleAssignment,
SharepointSecurable,
@@ -90,7 +91,6 @@ from onyx.connectors.sharepoint.connector_utils import (
from onyx.connectors.sharepoint.models import (
SharepointCredentials,
SharepointDrive,
SharepointTokenInfo,
SitePagesPage,
)
from onyx.connectors.source_operations import (
@@ -297,12 +297,8 @@ class SharepointSourceOperations(SourceOperations):
capabilities={CredentialCapability.INDEXING},
consumes=OperationConsumes.CREDENTIAL,
)
def check_token(self) -> SharepointTokenInfo:
response: dict[str, Any] = self._gateway().token_response()
expires: int | str | None = response.get("expires_in")
return SharepointTokenInfo(
expires_in=int(expires) if expires is not None else None
)
def check_token(self) -> MicrosoftTokenInfo:
return self._gateway().token_info()
@source_operation(
capabilities={
@@ -105,7 +105,15 @@ def test_a_bad_secret_fails_the_token_check(
_credentials(test_secrets, secret="not-the-secret"), _config([_site()])
)
assert _statuses(results)["sharepoint_token_auth"] is CapabilityCheckStatus.FAILED
by_id: dict[str, CapabilityCheckResult] = {
result.check_id: result for result in results
}
assert by_id["sharepoint_token_auth"].status is CapabilityCheckStatus.FAILED
assert by_id["sharepoint_token_auth"].summary == (
"Secret or certificate was rejected"
)
# The probes after sign-in do not run.
assert by_id["sharepoint_documents_read"].blocked_by == "sharepoint_token_auth"
assert verdict is CapabilityVerdict.FAILED
@@ -0,0 +1,497 @@
"""The row text and states every Microsoft connector's checks share: the
sign-in row, the throttled and unreachable rows, Error Details from a Graph
refusal, the certificate's end date and the app's name from the token, and
the checks after a failed sign-in, which do not run."""
import base64
import json
from datetime import datetime, timedelta, timezone
from typing import Any
from unittest.mock import MagicMock, create_autospec
import pytest
import requests
from cryptography import x509
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.hazmat.primitives.serialization import BestAvailableEncryption, pkcs12
from cryptography.x509.oid import NameOID
from onyx.configs.constants import DocumentSource
from onyx.connectors.capability_checks.models import (
CapabilityCheck,
CapabilityCheckContext,
CapabilityCheckResult,
CapabilityCheckStatus,
CheckErrorKind,
CheckNotRunReason,
check_error_detail,
)
from onyx.connectors.capability_checks.runner import run_capability_checks
from onyx.connectors.exceptions import (
CredentialExpiredError,
CredentialInvalidError,
InsufficientPermissionsError,
RateLimitedValidationError,
SourceApiError,
UnexpectedValidationError,
UnreachableValidationError,
ValidationError,
)
from onyx.connectors.microsoft_utils import check_text
from onyx.connectors.microsoft_utils.check_text import (
MicrosoftAccountLabels,
MicrosoftRowText,
sign_in_text,
)
from onyx.connectors.microsoft_utils.graph_auth import (
MicrosoftAuthContext,
MicrosoftAuthMethod,
certificate_not_after,
token_info,
)
from onyx.connectors.microsoft_utils.graph_errors import (
INVALID_AUTH_METHOD_CODE,
INVALID_AUTHORITY_CODE,
INVALID_CERTIFICATE_CODE,
MISSING_CREDENTIAL_CODE,
MicrosoftAuthError,
MicrosoftGraphError,
microsoft_error_from_exception,
raise_for_auth_error,
raise_for_graph_error,
raise_for_sign_in_error,
)
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
from onyx.connectors.onedrive.capability_checks import build_onedrive_indexing_checks
from onyx.connectors.onedrive.source_operations import OneDriveSourceOperations
from onyx.connectors.outlook.capability_checks import build_outlook_indexing_checks
from onyx.connectors.outlook.source_operations import OutlookSourceOperations
from onyx.connectors.sharepoint.capability_checks import (
build_sharepoint_indexing_checks,
)
from onyx.connectors.sharepoint.source_operations import SharepointSourceOperations
_LABELS = MicrosoftAccountLabels.with_prefix("SharePoint")
_GRAPH_URL = "https://graph.microsoft.com/v1.0/sites?search=*"
def _raised(raise_it: Any, *args: Any, **kwargs: Any) -> ValidationError:
with pytest.raises(ValidationError) as raised:
raise_it(*args, **kwargs)
return raised.value
def _transport_error() -> MicrosoftGraphError:
try:
raise microsoft_error_from_exception(
requests.ConnectionError("refused")
) from requests.ConnectionError("refused")
except MicrosoftGraphError as error:
return error
# --- Every row line fits the card ---
def test_every_shared_line_is_short_enough() -> None:
rows: list[MicrosoftRowText] = [
value
for value in vars(check_text).values()
if isinstance(value, MicrosoftRowText)
] + [
check_text.missing_field_row(_LABELS, certificate=True),
check_text.certificate_unreadable_row(_LABELS),
check_text.credential_rejected_row(_LABELS, certificate=False),
check_text.no_token_row("x"),
check_text.unexpected_answer_row("the site listing"),
]
for row in rows:
assert len(row.summary) <= 40, row.summary
assert len(row.description.splitlines()) <= 3, row.description
# --- Sign-in refusals ---
def test_a_missing_field_names_the_fields_of_the_sign_in_method() -> None:
secret = _raised(
raise_for_auth_error, MicrosoftAuthError(MISSING_CREDENTIAL_CODE, ""), _LABELS
)
certificate = _raised(
raise_for_auth_error,
MicrosoftAuthError(MISSING_CREDENTIAL_CODE, "", certificate=True),
_LABELS,
)
assert isinstance(secret, CredentialInvalidError)
assert secret.summary == "A required account field is empty"
assert secret.description is not None
assert secret.description.endswith(
"SharePoint Client ID, SharePoint Directory ID and SharePoint Client Secret."
)
assert certificate.description is not None
assert "SharePoint Private Key" in certificate.description
@pytest.mark.parametrize(
("code", "summary"),
[
(INVALID_AUTH_METHOD_CODE, "Unknown sign-in method on the account"),
(INVALID_CERTIFICATE_CODE, "Certificate file does not open"),
(INVALID_AUTHORITY_CODE, "Directory ID not found"),
("invalid_client", "Secret or certificate was rejected"),
("unauthorized_client", "App registration not found"),
("access_denied", "Microsoft did not issue a token"),
("temporarily_unavailable", "Microsoft sign-in is unavailable"),
],
)
def test_each_sign_in_refusal_has_its_line(code: str, summary: str) -> None:
error = _raised(raise_for_auth_error, MicrosoftAuthError(code, "x"), _LABELS)
assert error.summary == summary
def test_an_expired_secret_says_so() -> None:
error = _raised(
raise_for_auth_error,
MicrosoftAuthError("invalid_client", "AADSTS7000222: The secret expired."),
_LABELS,
)
assert error.description is not None
assert error.description.startswith("The client secret on this account has")
assert "SharePoint Client Secret" in error.description
def test_an_expired_certificate_names_its_end_date() -> None:
error = _raised(
raise_for_auth_error,
MicrosoftAuthError(
"invalid_client",
"AADSTS700027",
certificate=True,
certificate_not_after=datetime(2026, 1, 31, tzinfo=timezone.utc),
),
_LABELS,
)
assert error.description is not None
assert error.description.startswith(
"The certificate on this account expired on 2026/01/31."
)
@pytest.mark.parametrize(
("error", "error_class", "summary"),
[
(
MicrosoftGraphError(429, "tooManyRequests", "slow down"),
RateLimitedValidationError,
"Microsoft is throttling requests",
),
(
_transport_error(),
UnreachableValidationError,
"Cannot reach Microsoft sign-in",
),
(
MicrosoftGraphError(503, "serviceUnavailable", "down"),
UnexpectedValidationError,
"Microsoft sign-in is unavailable",
),
(
MicrosoftGraphError(400, "invalid_request", "AADSTS900023"),
CredentialInvalidError,
"Microsoft refused the sign-in request",
),
(
MicrosoftGraphError(None, "JSONDecodeError", "not json"),
UnexpectedValidationError,
"Unexpected answer from Microsoft sign-in",
),
],
)
def test_a_token_request_without_an_oauth_error_maps_by_status(
error: MicrosoftGraphError, error_class: type[ValidationError], summary: str
) -> None:
raised = _raised(raise_for_sign_in_error, error)
assert type(raised) is error_class
assert raised.summary == summary
# --- Graph refusals after sign-in ---
def test_a_throttled_graph_call_stops_the_run() -> None:
raised = _raised(
raise_for_graph_error,
MicrosoftGraphError(429, "tooManyRequests", "slow down"),
"denied",
)
assert isinstance(raised, RateLimitedValidationError)
assert raised.summary == "Microsoft is throttling requests"
def test_a_dropped_connection_is_unreachable() -> None:
raised = _raised(raise_for_graph_error, _transport_error(), "denied")
assert isinstance(raised, UnreachableValidationError)
assert raised.summary == "Cannot reach Microsoft"
def test_a_rejected_token_is_rejected() -> None:
raised = _raised(
raise_for_graph_error,
MicrosoftGraphError(401, "InvalidAuthenticationToken", ""),
"x",
)
assert isinstance(raised, CredentialExpiredError)
assert raised.summary == "Microsoft rejected the app's token"
def test_a_denial_takes_the_caller_row() -> None:
row = MicrosoftRowText(summary="No permission to list sites", description="d")
raised = _raised(
raise_for_graph_error,
MicrosoftGraphError(403, "accessDenied", "no"),
"The app cannot list sites.",
denied_row=row,
)
assert isinstance(raised, InsufficientPermissionsError)
assert raised.summary == row.summary
assert "accessDenied" in raised.message
def test_a_server_error_names_the_request() -> None:
raised = _raised(
raise_for_graph_error,
MicrosoftGraphError(502, "badGateway", ""),
"x",
request="the site listing",
)
assert raised.summary == "Unexpected answer from Microsoft"
assert raised.description is not None
assert "the site listing" in raised.description
# --- Error Details ---
def test_a_graph_refusal_carries_the_request_and_body() -> None:
response = requests.Response()
response.status_code = 403
response.url = _GRAPH_URL
response._content = b'{"error": {"code": "accessDenied", "message": "no"}}'
response.request = requests.Request("GET", _GRAPH_URL).prepare()
error = microsoft_error_from_exception(requests.HTTPError(response=response))
assert isinstance(error, SourceApiError)
assert error.status == 403
assert error.code == "accessDenied"
detail = check_error_detail(error, CheckErrorKind.PERMISSION, [])
assert detail is not None
assert detail.request == "GET graph.microsoft.com/v1.0/sites"
assert detail.response_excerpt is not None
assert "accessDenied" in detail.response_excerpt
# --- The certificate's end date and the app's name ---
def _pfx(not_after: datetime) -> bytes:
key = ec.generate_private_key(ec.SECP256R1())
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "onyx")])
certificate = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(1)
.not_valid_before(not_after - timedelta(days=365))
.not_valid_after(not_after)
.sign(key, hashes.SHA256())
)
return pkcs12.serialize_key_and_certificates(
b"onyx", key, certificate, None, BestAvailableEncryption(b"secret")
)
def test_the_certificate_end_date_reads_from_the_pfx() -> None:
end = datetime(2027, 3, 1, tzinfo=timezone.utc)
assert certificate_not_after(_pfx(end), "secret") == end
assert certificate_not_after(_pfx(end), "wrong") is None
assert certificate_not_after(b"not a pfx", "secret") is None
def _jwt(claims: dict[str, str]) -> str:
payload = base64.urlsafe_b64encode(json.dumps(claims).encode()).decode()
return f"header.{payload.rstrip('=')}.signature"
@pytest.mark.parametrize(
("access_token", "summary"),
[
(
_jwt({"app_displayname": "Onyx Indexer", "appid": "a1"}),
"Signed in as Onyx Indexer",
),
(_jwt({"appid": "a1"}), "Signed in as app a1"),
("opaque-token", "Signed in to Microsoft"),
],
)
def test_sign_in_names_the_app_from_the_token(access_token: str, summary: str) -> None:
context = MicrosoftAuthContext(
app=MagicMock(), method=MicrosoftAuthMethod.CLIENT_SECRET
)
info = token_info(context, {"access_token": access_token, "expires_in": 3599})
assert info.expires_in == 3599
assert sign_in_text(info, _LABELS).summary == summary
def test_a_certificate_near_its_end_date_gets_a_caveat() -> None:
soon = datetime.now(timezone.utc) + timedelta(days=10)
later = datetime.now(timezone.utc) + timedelta(days=200)
near = sign_in_text(MicrosoftTokenInfo(certificate_not_after=soon), _LABELS)
far = sign_in_text(MicrosoftTokenInfo(certificate_not_after=later), _LABELS)
assert near.caveats and "SharePoint Private Key" in near.caveats[0]
assert far.caveats == []
assert far.description is not None
assert far.description.startswith("Certificate valid until ")
# --- The checks after sign-in ---
class _Connector:
def __init__(
self,
source: DocumentSource,
gateway_class: type,
checks: list[CapabilityCheck],
sign_in_id: str,
config: dict[str, Any],
) -> None:
self.source = source
self.gateway_class = gateway_class
self.checks = checks
self.sign_in_id = sign_in_id
self.config = config
_CONNECTORS = [
_Connector(
DocumentSource.SHAREPOINT,
SharepointSourceOperations,
build_sharepoint_indexing_checks(),
"sharepoint_token_auth",
{"sites": ["https://contoso.sharepoint.com/sites/eng"]},
),
_Connector(
DocumentSource.ONEDRIVE,
OneDriveSourceOperations,
build_onedrive_indexing_checks(),
"onedrive_token_auth",
{},
),
_Connector(
DocumentSource.OUTLOOK,
OutlookSourceOperations,
build_outlook_indexing_checks(),
"outlook_token_auth",
{},
),
]
def _run(
connector: _Connector, gateway: MagicMock, *, form_run: bool = False
) -> dict[str, CapabilityCheckResult]:
context = CapabilityCheckContext(
source=connector.source,
credential_json={},
connector_specific_config=connector.config,
source_operations=gateway,
)
return {
result.check_id: result
for result in run_capability_checks(
connector.checks, context, form_run=form_run
)
}
@pytest.mark.parametrize("connector", _CONNECTORS, ids=lambda c: c.source.value)
def test_a_failed_sign_in_is_one_row(connector: _Connector) -> None:
gateway: MagicMock = create_autospec(connector.gateway_class, instance=True)
gateway.check_token.side_effect = MicrosoftAuthError("invalid_client", "")
results = _run(connector, gateway)
sign_in = results.pop(connector.sign_in_id)
assert sign_in.display_name == "Sign in to Microsoft"
assert sign_in.status == CapabilityCheckStatus.FAILED
assert sign_in.summary == "Secret or certificate was rejected"
for result in results.values():
# A settings check makes no call, so it runs.
if not result.applicable or result.status == CapabilityCheckStatus.PASSED:
continue
assert result.status == CapabilityCheckStatus.SKIPPED, result.check_id
assert result.blocked_by == connector.sign_in_id
@pytest.mark.parametrize("connector", _CONNECTORS, ids=lambda c: c.source.value)
def test_an_unreachable_sign_in_fails_only_a_form_run(connector: _Connector) -> None:
gateway: MagicMock = create_autospec(connector.gateway_class, instance=True)
gateway.check_token.side_effect = _transport_error()
form = _run(connector, gateway, form_run=True)[connector.sign_in_id]
stored = _run(connector, gateway)[connector.sign_in_id]
assert form.status == CapabilityCheckStatus.FAILED
assert stored.status == CapabilityCheckStatus.INCOMPLETE
assert form.summary == stored.summary == "Cannot reach Microsoft sign-in"
@pytest.mark.parametrize("connector", _CONNECTORS, ids=lambda c: c.source.value)
def test_a_throttled_sign_in_stops_the_run(connector: _Connector) -> None:
gateway: MagicMock = create_autospec(connector.gateway_class, instance=True)
gateway.check_token.side_effect = MicrosoftGraphError(429, "tooManyRequests", "")
results = _run(connector, gateway)
sign_in = results.pop(connector.sign_in_id)
assert sign_in.status == CapabilityCheckStatus.INCOMPLETE
assert sign_in.error_kind == CheckErrorKind.RATE_LIMITED
assert sign_in.summary == "Microsoft is throttling requests"
for result in results.values():
if result.applicable and result.status != CapabilityCheckStatus.PASSED:
assert result.not_run_reason in (
CheckNotRunReason.DEPENDENCY_INCOMPLETE,
CheckNotRunReason.RATE_LIMITED,
), result.check_id
@pytest.mark.parametrize("connector", _CONNECTORS, ids=lambda c: c.source.value)
def test_sign_in_names_the_app(connector: _Connector) -> None:
gateway: MagicMock = create_autospec(connector.gateway_class, instance=True)
gateway.check_token.return_value = MicrosoftTokenInfo(app_name="Onyx Indexer")
result = _run(connector, gateway)[connector.sign_in_id]
assert result.status == CapabilityCheckStatus.PASSED
assert result.summary == "Signed in as Onyx Indexer"
assert result.validates_binding
@@ -49,6 +49,7 @@ from onyx.connectors.microsoft_utils.graph_errors import (
from onyx.connectors.microsoft_utils.graph_errors import (
MicrosoftGraphError as OneDriveGraphError,
)
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
from onyx.connectors.models import (
ConnectorFailure,
Document,
@@ -1158,6 +1159,7 @@ def test_onedrive_drive_count_maps_listing_errors(
def test_onedrive_named_capability_checks_pass_through_gateway() -> None:
gateway: Any = create_autospec(OneDriveSourceOperations, instance=True)
gateway.check_token.return_value = MicrosoftTokenInfo(expires_in=3599)
gateway.list_users.return_value = OneDriveUserPage(users=[_user()])
gateway.get_user.return_value = _user()
gateway.get_default_drive.return_value = _drive()
@@ -43,6 +43,7 @@ from onyx.connectors.microsoft_utils.graph_errors import (
from onyx.connectors.microsoft_utils.graph_errors import (
MicrosoftGraphError as OutlookGraphError,
)
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
from onyx.connectors.outlook.capability_checks import (
build_outlook_doc_permission_sync_checks,
build_outlook_indexing_checks,
@@ -52,7 +53,6 @@ from onyx.connectors.outlook.models import (
OutlookEventPage,
OutlookFolderPage,
OutlookMailboxPage,
OutlookTokenInfo,
)
from onyx.connectors.outlook.source_operations import OutlookSourceOperations
from tests.unit.onyx.connectors.outlook.outlook_api_shapes import (
@@ -83,7 +83,9 @@ def _advisories(output: CheckOutput | None) -> list[str]:
def _gateway() -> MagicMock:
"""A healthy tenant: one user with a readable mailbox holding one message."""
gateway = create_autospec(OutlookSourceOperations, instance=True)
gateway.check_token.return_value = OutlookTokenInfo(expires_in=3599)
gateway.check_token.return_value = MicrosoftTokenInfo(
expires_in=3599, app_name="Onyx"
)
gateway.list_mailbox_users.return_value = OutlookMailboxPage(mailboxes=[mailbox()])
gateway.resolve_mailbox.return_value = mailbox()
gateway.probe_mailbox.return_value = folder()
@@ -12,6 +12,7 @@ from unittest.mock import MagicMock
from onyx.connectors.credentials_provider import OnyxStaticCredentialsProvider
from onyx.connectors.microsoft_utils.graph_client import GraphApiClient
from onyx.connectors.microsoft_utils.graph_gateway import MicrosoftGraphGateway
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
from onyx.connectors.sharepoint.connector import SharepointConnector
from onyx.connectors.sharepoint.source_operations import (
CONFIG_SITES,
@@ -74,6 +75,7 @@ def fake_gateway(
"access_token": FAKE_TOKEN,
"expires_in": 3600,
}
graph.token_info.return_value = MicrosoftTokenInfo(expires_in=3600)
gateway._graph_gateway = graph
if sdk_client is not None:
gateway._sdk_client = sdk_client
@@ -36,13 +36,13 @@ from onyx.connectors.microsoft_utils.graph_errors import (
MicrosoftAuthError,
MicrosoftGraphError,
)
from onyx.connectors.microsoft_utils.models import MicrosoftTokenInfo
from onyx.connectors.sharepoint import connector_utils
from onyx.connectors.sharepoint.capability_checks import (
build_sharepoint_indexing_checks,
)
from onyx.connectors.sharepoint.models import (
SharepointDrive,
SharepointTokenInfo,
SitePagesPage,
)
from onyx.connectors.sharepoint.source_operations import SharepointSourceOperations
@@ -89,7 +89,9 @@ def _gateway() -> MagicMock:
"""A healthy tenant: one site with one library holding one small file and
one page."""
gateway: MagicMock = create_autospec(SharepointSourceOperations, instance=True)
gateway.check_token.return_value = SharepointTokenInfo(expires_in=3599)
gateway.check_token.return_value = MicrosoftTokenInfo(
expires_in=3599, app_name="Onyx"
)
gateway.list_site_urls.return_value = [SITE_URL, OTHER_SITE_URL]
gateway.get_site_id.return_value = "site-id"
gateway.list_drives.return_value = [DRIVE]