mirror of
https://github.com/onyx-dot-app/onyx.git
synced 2026-10-19 01:50:51 -05:00
feat(connectors): config models check the design's field formats (Drive, Microsoft, Slack, Notion)
The Google Drive, SharePoint, OneDrive and Notion config models get field validators for the design's format rules: My Drive Emails holds at least one email, each SharePoint site is a full site URL, each OneDrive user is an email, and the Notion root page is a page URL or its 32-character ID. A Notion page URL becomes the page ID that it holds. The plan endpoint returns the rules as form_errors. The checks no longer test these formats. Slack and Outlook get no rule: the design gives Slack a server-side rule only, and gives Outlook no rule. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01KFMazQbuG8ui3jxBxm47sC
This commit is contained in:
19 files changed
+298
-136
No files matched your search
@@ -1128,15 +1128,6 @@ class _ConfiguredMyDrivesCheck(_GoogleDriveCheck):
|
||||
config: GoogleDriveConnectorConfig = self.config(context)
|
||||
ops: GoogleDriveSourceOperations = _gateway(context)
|
||||
requested: list[str] = extract_str_list_from_comma_str(config.my_drive_emails)
|
||||
if not requested:
|
||||
raise ConnectorValidationError(
|
||||
"My Drive emails holds no email.",
|
||||
summary="My Drive Emails has no email",
|
||||
description=(
|
||||
"My Drive Emails holds only commas or spaces.\nEnter at least "
|
||||
"one email in My Drive Emails, or clear the field."
|
||||
),
|
||||
)
|
||||
with _google_errors(
|
||||
context,
|
||||
api=_ADMIN_API,
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
from typing import Annotated, Any, ClassVar
|
||||
|
||||
from pydantic import field_validator
|
||||
|
||||
from onyx.configs.app_configs import INDEX_BATCH_SIZE
|
||||
from onyx.connectors.connector_config import ConnectorConfig, CredentialBinding
|
||||
from onyx.connectors.connector_config import (
|
||||
ConnectorConfig,
|
||||
CredentialBinding,
|
||||
field_format_error,
|
||||
)
|
||||
from onyx.connectors.exceptions import ConnectorValidationError
|
||||
from onyx.connectors.field_policy import (
|
||||
FieldClass,
|
||||
@@ -16,6 +22,8 @@ from onyx.connectors.google_utils.shared_constants import (
|
||||
)
|
||||
from onyx.connectors.planning_rule import ConnectorChangeOverride
|
||||
|
||||
MY_DRIVE_EMAILS_EMPTY_ERROR = "Enter at least one email."
|
||||
|
||||
_COSMETIC = FieldPolicy(FieldClass.COSMETIC)
|
||||
_SPECIFIC_REQUEST_FIELDS = (
|
||||
"shared_drive_urls",
|
||||
@@ -85,6 +93,15 @@ class GoogleDriveConnectorConfig(GoogleDriveCredentialBinding, ConnectorConfig):
|
||||
only_org_public: Annotated[bool | None, _COSMETIC] = None
|
||||
continue_on_failure: Annotated[bool | None, _COSMETIC] = None
|
||||
|
||||
# A value of only commas or spaces turns on the specific mode with no My
|
||||
# Drive to index.
|
||||
@field_validator("my_drive_emails")
|
||||
@classmethod
|
||||
def _my_drive_emails_name_an_email(cls, value: str | None) -> str | None:
|
||||
if value and not value.replace(",", "").strip():
|
||||
raise field_format_error(MY_DRIVE_EMAILS_EMPTY_ERROR)
|
||||
return value
|
||||
|
||||
|
||||
def has_specific_requests(config: GoogleDriveConnectorConfig) -> bool:
|
||||
return bool(
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
import re
|
||||
from typing import Annotated
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from pydantic import field_validator
|
||||
|
||||
from onyx.configs.app_configs import (
|
||||
INDEX_BATCH_SIZE,
|
||||
NOTION_CONNECTOR_DISABLE_RECURSIVE_PAGE_LOOKUP,
|
||||
)
|
||||
from onyx.connectors.connector_config import ConnectorConfig
|
||||
from onyx.connectors.connector_config import ConnectorConfig, field_format_error
|
||||
from onyx.connectors.field_policy import (
|
||||
FieldClass,
|
||||
FieldPolicy,
|
||||
@@ -16,6 +20,44 @@ from onyx.connectors.planning_rule import ConnectorChangeOverride
|
||||
|
||||
_RECURSIVE_INDEX_ENABLED = "recursive_index_enabled"
|
||||
|
||||
ROOT_PAGE_ID_FORMAT_ERROR = "Paste the page URL or its 32-character ID."
|
||||
|
||||
_HEX_ID = re.compile(r"[0-9a-fA-F]{32}")
|
||||
_DASHED_ID = re.compile(
|
||||
r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
|
||||
)
|
||||
|
||||
|
||||
def _dashed(hex_id: str) -> str:
|
||||
return (
|
||||
f"{hex_id[0:8]}-{hex_id[8:12]}-{hex_id[12:16]}-{hex_id[16:20]}-{hex_id[20:]}"
|
||||
).lower()
|
||||
|
||||
|
||||
def page_id_from_url(url: str) -> str | None:
|
||||
"""The page ID (a dashed UUID) in a Notion page URL, or None."""
|
||||
parsed = urlparse(url)
|
||||
netloc = parsed.netloc.lower()
|
||||
if not (
|
||||
"notion.so" in netloc or "notion.site" in netloc or "app.notion.com" in netloc
|
||||
):
|
||||
return None
|
||||
|
||||
# The path ends in "Title-PageID".
|
||||
path_last = parsed.path.split("/")[-1]
|
||||
candidate = path_last.split("-")[-1] if "-" in path_last else path_last
|
||||
cleaned = re.sub(r"[^0-9a-fA-F-]", "", candidate).replace("-", "")
|
||||
if _HEX_ID.fullmatch(cleaned):
|
||||
return _dashed(cleaned)
|
||||
|
||||
params = parse_qs(parsed.query)
|
||||
for key in ("p", "page_id"):
|
||||
if key in params and params[key]:
|
||||
candidate = params[key][0].replace("-", "")
|
||||
if _HEX_ID.fullmatch(candidate):
|
||||
return _dashed(candidate)
|
||||
return None
|
||||
|
||||
|
||||
class NotionConnectorConfig(ConnectorConfig):
|
||||
batch_size: Annotated[int, FieldPolicy(FieldClass.COSMETIC)] = INDEX_BATCH_SIZE
|
||||
@@ -29,6 +71,19 @@ class NotionConnectorConfig(ConnectorConfig):
|
||||
FieldPolicy(FieldClass.SCOPE, scope=ScopeInclude(empty_means_all=True)),
|
||||
] = None
|
||||
|
||||
# The connector sends the value to Notion as a page ID, so a URL becomes
|
||||
# the ID it holds.
|
||||
@field_validator("root_page_id")
|
||||
@classmethod
|
||||
def _root_page_is_an_id_or_url(cls, value: str | None) -> str | None:
|
||||
root: str = (value or "").strip()
|
||||
if not root or _HEX_ID.fullmatch(root) or _DASHED_ID.fullmatch(root):
|
||||
return value
|
||||
page_id: str | None = page_id_from_url(root)
|
||||
if page_id is None:
|
||||
raise field_format_error(ROOT_PAGE_ID_FORMAT_ERROR)
|
||||
return page_id
|
||||
|
||||
|
||||
def notion_planning_rule(
|
||||
old: NotionConnectorConfig, new: NotionConnectorConfig
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import re
|
||||
from collections.abc import Generator
|
||||
from datetime import datetime
|
||||
from itertools import islice
|
||||
from typing import Any, cast
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from pydantic import BaseModel
|
||||
from typing_extensions import override
|
||||
@@ -37,6 +35,7 @@ from onyx.connectors.models import (
|
||||
SlimDocument,
|
||||
TextSection,
|
||||
)
|
||||
from onyx.connectors.notion.config import page_id_from_url
|
||||
from onyx.connectors.notion.models import NotionSearchObjectType
|
||||
from onyx.connectors.notion.source_operations import (
|
||||
NotionApiError,
|
||||
@@ -189,46 +188,9 @@ class NotionConnector(LoadConnector, PollConnector, SlimConnector):
|
||||
@override
|
||||
def normalize_url(cls, url: str) -> NormalizationResult:
|
||||
"""Normalize a Notion URL to extract the page ID (UUID format)."""
|
||||
parsed = urlparse(url)
|
||||
netloc = parsed.netloc.lower()
|
||||
|
||||
if not (
|
||||
"notion.so" in netloc
|
||||
or "notion.site" in netloc
|
||||
or "app.notion.com" in netloc
|
||||
):
|
||||
return NormalizationResult(normalized_url=None, use_default=False)
|
||||
|
||||
# Extract page ID from path (format: "Title-PageID")
|
||||
path_last = parsed.path.split("/")[-1]
|
||||
candidate = path_last.split("-")[-1] if "-" in path_last else path_last
|
||||
|
||||
# Clean and format as UUID
|
||||
candidate = re.sub(r"[^0-9a-fA-F-]", "", candidate)
|
||||
cleaned = candidate.replace("-", "")
|
||||
|
||||
if len(cleaned) == 32 and re.fullmatch(r"[0-9a-fA-F]{32}", cleaned):
|
||||
normalized_uuid = (
|
||||
f"{cleaned[0:8]}-{cleaned[8:12]}-{cleaned[12:16]}-{cleaned[16:20]}-{cleaned[20:]}"
|
||||
).lower()
|
||||
return NormalizationResult(
|
||||
normalized_url=normalized_uuid, use_default=False
|
||||
)
|
||||
|
||||
# Try query params
|
||||
params = parse_qs(parsed.query)
|
||||
for key in ("p", "page_id"):
|
||||
if key in params and params[key]:
|
||||
candidate = params[key][0].replace("-", "")
|
||||
if len(candidate) == 32 and re.fullmatch(r"[0-9a-fA-F]{32}", candidate):
|
||||
normalized_uuid = (
|
||||
f"{candidate[0:8]}-{candidate[8:12]}-{candidate[12:16]}-{candidate[16:20]}-{candidate[20:]}"
|
||||
).lower()
|
||||
return NormalizationResult(
|
||||
normalized_url=normalized_uuid, use_default=False
|
||||
)
|
||||
|
||||
return NormalizationResult(normalized_url=None, use_default=False)
|
||||
return NormalizationResult(
|
||||
normalized_url=page_id_from_url(url), use_default=False
|
||||
)
|
||||
|
||||
@retry_builder(tries=3, delay=1, backoff=2)
|
||||
def _fetch_child_blocks(
|
||||
|
||||
@@ -168,36 +168,15 @@ def _raise_row(message: str, row: MicrosoftRowText) -> NoReturn:
|
||||
)
|
||||
|
||||
|
||||
def _is_invalid_entry(user: str) -> bool:
|
||||
if not user.strip():
|
||||
return False
|
||||
try:
|
||||
normalize_configured_users([user])
|
||||
except ConnectorValidationError:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _configured_users(context: CapabilityCheckContext) -> list[str]:
|
||||
"""The Users entries, normalized as indexing reads them. An entry that is
|
||||
not an email, or a list of blanks, fails with its row."""
|
||||
"""The Users entries, normalized as indexing reads them. A list of blanks
|
||||
fails with its row; the config model rejects an entry that is not an
|
||||
email."""
|
||||
users: list[str] = _config(context).users or []
|
||||
try:
|
||||
return normalize_configured_users(users)
|
||||
except ConnectorValidationError as error:
|
||||
if not any(user.strip() for user in users):
|
||||
_raise_row(error.message, _BLANK_USERS_ROW)
|
||||
entry: str = next(
|
||||
(user.strip() for user in users if _is_invalid_entry(user)), ""
|
||||
)
|
||||
_raise_row(
|
||||
error.message,
|
||||
_row(
|
||||
_line("Not an email address: ", entry),
|
||||
"Each entry in Users must be a user principal name or primary "
|
||||
"email address.",
|
||||
),
|
||||
)
|
||||
_raise_row(error.message, _BLANK_USERS_ROW)
|
||||
|
||||
|
||||
def _candidate_users(
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from typing import Annotated
|
||||
|
||||
from onyx.connectors.connector_config import ConnectorConfig
|
||||
from pydantic import field_validator
|
||||
|
||||
from onyx.connectors.connector_config import ConnectorConfig, field_format_error
|
||||
from onyx.connectors.field_policy import (
|
||||
FieldClass,
|
||||
FieldPolicy,
|
||||
@@ -8,6 +10,9 @@ from onyx.connectors.field_policy import (
|
||||
ScopeInclude,
|
||||
)
|
||||
from onyx.connectors.microsoft_utils.config import MicrosoftCloudBinding
|
||||
from onyx.connectors.onedrive.scope import normalized_user_email
|
||||
|
||||
USER_EMAIL_FORMAT_ERROR = "Enter email addresses, e.g. jane@acme.com."
|
||||
|
||||
|
||||
class OneDriveConnectorConfig(MicrosoftCloudBinding, ConnectorConfig):
|
||||
@@ -22,3 +27,15 @@ class OneDriveConnectorConfig(MicrosoftCloudBinding, ConnectorConfig):
|
||||
treat_organization_link_as_public: Annotated[
|
||||
bool, FieldPolicy(FieldClass.BEHAVIOR)
|
||||
] = False
|
||||
|
||||
# Blank entries are skipped. A list of only blanks fails the "Find
|
||||
# selected users" check, as the design's row says.
|
||||
@field_validator("users")
|
||||
@classmethod
|
||||
def _users_are_emails(cls, value: list[str] | None) -> list[str] | None:
|
||||
if any(
|
||||
entry.strip() and normalized_user_email(entry) is None
|
||||
for entry in value or []
|
||||
):
|
||||
raise field_format_error(USER_EMAIL_FORMAT_ERROR)
|
||||
return value
|
||||
@@ -5,6 +5,15 @@ from onyx.connectors.exceptions import ConnectorValidationError
|
||||
_EMAIL_ADAPTER = TypeAdapter(EmailStr)
|
||||
|
||||
|
||||
def normalized_user_email(entry: str) -> str | None:
|
||||
"""A Users entry as the email indexing reads, or None when it is not an
|
||||
email. The caller skips blank entries."""
|
||||
try:
|
||||
return str(_EMAIL_ADAPTER.validate_python(entry.strip())).casefold()
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def normalize_configured_users(users: list[str] | None) -> list[str]:
|
||||
raw_users = users or []
|
||||
normalized: list[str] = []
|
||||
@@ -13,12 +22,11 @@ def normalize_configured_users(users: list[str] | None) -> list[str]:
|
||||
candidate = raw_user.strip()
|
||||
if not candidate:
|
||||
continue
|
||||
try:
|
||||
user = str(_EMAIL_ADAPTER.validate_python(candidate)).casefold()
|
||||
except ValidationError as error:
|
||||
user = normalized_user_email(candidate)
|
||||
if user is None:
|
||||
raise ConnectorValidationError(
|
||||
f"Invalid OneDrive user email: {candidate!r}."
|
||||
) from error
|
||||
)
|
||||
if user in seen:
|
||||
continue
|
||||
seen.add(user)
|
||||
|
||||
@@ -100,7 +100,6 @@ from onyx.connectors.sharepoint.connector import (
|
||||
)
|
||||
from onyx.connectors.sharepoint.connector_utils import (
|
||||
SiteUrlBlockedError,
|
||||
SiteUrlFormatError,
|
||||
SiteUrlHostError,
|
||||
tenant_domain_from_site_urls,
|
||||
validate_content_types,
|
||||
@@ -967,23 +966,14 @@ def _site_problem(error: MicrosoftGraphError, site_url: str) -> _SiteProblem:
|
||||
def _validate_site_urls(
|
||||
site_urls: list[str], environment: MicrosoftGraphEnvironment
|
||||
) -> None:
|
||||
"""Each URL is a full site URL, safe to request, on the first URL's
|
||||
tenant host: the rules indexing applies."""
|
||||
"""Each URL is safe to request and on the first URL's tenant host: the
|
||||
rules indexing applies. The config model checks the URL format."""
|
||||
tenant_domain: str | None = tenant_domain_from_site_urls(site_urls)
|
||||
for site_url in site_urls:
|
||||
try:
|
||||
connector_utils.validate_site_url(
|
||||
site_url, environment.sharepoint_domain_suffix, tenant_domain
|
||||
)
|
||||
except SiteUrlFormatError as error:
|
||||
raise ConnectorValidationError(
|
||||
error.message,
|
||||
summary=_line("Not a site URL: ", site_url or "(blank)"),
|
||||
description=(
|
||||
"A site URL must start with https:// and contain /sites/, "
|
||||
"/teams/ or /personal/ followed by the site name."
|
||||
),
|
||||
) from error
|
||||
except SiteUrlBlockedError as error:
|
||||
raise ConnectorValidationError(
|
||||
error.message,
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
from typing import Annotated
|
||||
|
||||
from pydantic import field_validator
|
||||
|
||||
from onyx.configs.app_configs import (
|
||||
INDEX_BATCH_SIZE,
|
||||
SHAREPOINT_EXHAUSTIVE_AD_ENUMERATION,
|
||||
)
|
||||
from onyx.connectors.connector_config import ConnectorConfig
|
||||
from onyx.connectors.connector_config import ConnectorConfig, field_format_error
|
||||
from onyx.connectors.field_policy import (
|
||||
FieldClass,
|
||||
FieldPolicy,
|
||||
@@ -17,12 +19,23 @@ from onyx.connectors.microsoft_utils.config import (
|
||||
MicrosoftCloudBinding,
|
||||
)
|
||||
|
||||
SITE_URL_FORMAT_ERROR = (
|
||||
"Enter a full site URL, e.g. https://acme.sharepoint.com/sites/hr."
|
||||
)
|
||||
|
||||
_COSMETIC = FieldPolicy(FieldClass.COSMETIC)
|
||||
_CONTENT_TYPE_TOGGLE = FieldPolicy(
|
||||
FieldClass.SCOPE, scope=ScopeToggle(widens_when=True)
|
||||
)
|
||||
|
||||
|
||||
def is_site_url_well_formed(site_url: str) -> bool:
|
||||
"""A full SharePoint or OneDrive site URL: https and a site path."""
|
||||
return site_url.startswith("https://") and (
|
||||
"/sites/" in site_url or "/teams/" in site_url or "/personal/" in site_url
|
||||
)
|
||||
|
||||
|
||||
class SharepointCredentialBinding(MicrosoftCloudBinding):
|
||||
# Ignored: the connector uses the suffix of the cloud graph_api_host selects.
|
||||
sharepoint_domain_suffix: Annotated[str, _COSMETIC] = (
|
||||
@@ -52,3 +65,12 @@ class SharepointConnectorConfig(SharepointCredentialBinding, ConnectorConfig):
|
||||
exhaustive_ad_enumeration: Annotated[bool, _COSMETIC] = (
|
||||
SHAREPOINT_EXHAUSTIVE_AD_ENUMERATION
|
||||
)
|
||||
|
||||
# Indexing validates every entry, blanks included. The host and the
|
||||
# outbound URL guard depend on the cloud and DNS, so the checks keep them.
|
||||
@field_validator("sites")
|
||||
@classmethod
|
||||
def _sites_are_full_site_urls(cls, value: list[str] | None) -> list[str] | None:
|
||||
if any(not is_site_url_well_formed(site_url) for site_url in value or []):
|
||||
raise field_format_error(SITE_URL_FORMAT_ERROR)
|
||||
return value
|
||||
@@ -7,6 +7,7 @@ from onyx.connectors.exceptions import ConnectorValidationError
|
||||
from onyx.connectors.microsoft_utils.drive_items import DriveItemData
|
||||
from onyx.connectors.microsoft_utils.sharepoint_rest import SharepointPermissionReader
|
||||
from onyx.connectors.models import ExternalAccess
|
||||
from onyx.connectors.sharepoint.config import is_site_url_well_formed
|
||||
from onyx.db.enums import HierarchyNodeType
|
||||
from onyx.utils.url import SSRFException, validate_outbound_http_url
|
||||
from onyx.utils.variable_functionality import (
|
||||
@@ -150,13 +151,6 @@ def validate_content_types(
|
||||
)
|
||||
|
||||
|
||||
def is_site_url_well_formed(site_url: str) -> bool:
|
||||
"""A full SharePoint or OneDrive site URL: https and a site path."""
|
||||
return site_url.startswith("https://") and (
|
||||
"/sites/" in site_url or "/teams/" in site_url or "/personal/" in site_url
|
||||
)
|
||||
|
||||
|
||||
def validate_site_url(
|
||||
site_url: str, sharepoint_domain_suffix: str, tenant_domain: str | None
|
||||
) -> None:
|
||||
|
||||
+13
@@ -12,6 +12,7 @@ from onyx.connectors.capability_checks.models import (
|
||||
CapabilityCheckContext,
|
||||
CapabilityCheckResult,
|
||||
CapabilityCheckStatus,
|
||||
CheckErrorKind,
|
||||
CheckFindingKey,
|
||||
CountFinding,
|
||||
FindingOption,
|
||||
@@ -21,6 +22,7 @@ from onyx.connectors.capability_checks.runner import run_capability_checks
|
||||
from onyx.connectors.google_drive.capability_checks import (
|
||||
build_google_drive_indexing_checks,
|
||||
)
|
||||
from onyx.connectors.google_drive.config import MY_DRIVE_EMAILS_EMPTY_ERROR
|
||||
from onyx.connectors.google_drive.connector import (
|
||||
extract_ids_from_urls,
|
||||
extract_str_list_from_comma_str,
|
||||
@@ -451,6 +453,17 @@ def test_my_drive_alias_is_reported_as_skipped() -> None:
|
||||
assert f"alias@{_DOMAIN}" in result.message
|
||||
|
||||
|
||||
def test_my_drive_emails_without_an_email_fail_as_a_setting() -> None:
|
||||
ops = _ops()
|
||||
|
||||
result = _run("google_drive_configured_my_drives", ops, {"my_drive_emails": ","})
|
||||
|
||||
assert result.status == CapabilityCheckStatus.FAILED
|
||||
assert result.error_kind == CheckErrorKind.INVALID
|
||||
assert f"my_drive_emails: {MY_DRIVE_EMAILS_EMPTY_ERROR}" in result.message
|
||||
ops.list_user_emails.assert_not_called()
|
||||
|
||||
|
||||
def test_my_drive_of_a_listed_user_passes() -> None:
|
||||
ops = _ops()
|
||||
|
||||
|
||||
@@ -10,11 +10,13 @@ from onyx.connectors.capability_checks.models import (
|
||||
CapabilityCheckContext,
|
||||
CapabilityCheckResult,
|
||||
CapabilityCheckStatus,
|
||||
CheckErrorKind,
|
||||
CheckNotRunReason,
|
||||
)
|
||||
from onyx.connectors.capability_checks.registry import get_capability_checks
|
||||
from onyx.connectors.capability_checks.runner import run_capability_checks
|
||||
from onyx.connectors.notion.capability_checks import build_notion_indexing_checks
|
||||
from onyx.connectors.notion.config import ROOT_PAGE_ID_FORMAT_ERROR
|
||||
from onyx.connectors.notion.connector import NotionConnector
|
||||
from onyx.connectors.notion.models import NotionSearchObjectType, NotionSearchResponse
|
||||
from onyx.connectors.notion.source_operations import (
|
||||
@@ -22,6 +24,7 @@ from onyx.connectors.notion.source_operations import (
|
||||
NotionSourceOperations,
|
||||
)
|
||||
|
||||
_ROOT_PAGE_ID = "17ab3186873d418fb899c3f6a43f68de"
|
||||
_CREDENTIAL = {"notion_integration_token": "secret"}
|
||||
_PAGE = {"id": "page-1", "object": "page"}
|
||||
_DATA_SOURCE = {
|
||||
@@ -97,7 +100,7 @@ def test_every_check_probes_without_waiting_on_rate_limits() -> None:
|
||||
gateway: MagicMock = _gateway()
|
||||
|
||||
for check in build_notion_indexing_checks():
|
||||
_run(check.check_id, gateway, {"root_page_id": "root"})
|
||||
_run(check.check_id, gateway, {"root_page_id": _ROOT_PAGE_ID})
|
||||
|
||||
calls = [call for call in gateway.mock_calls if call[0]]
|
||||
assert calls
|
||||
@@ -178,7 +181,7 @@ def test_shared_content_fails_without_the_read_content_capability() -> None:
|
||||
|
||||
def test_shared_content_does_not_apply_with_a_root_page() -> None:
|
||||
result: CapabilityCheckResult = _run(
|
||||
"notion_shared_content", _gateway(), {"root_page_id": "root"}
|
||||
"notion_shared_content", _gateway(), {"root_page_id": _ROOT_PAGE_ID}
|
||||
)
|
||||
|
||||
assert not result.applicable
|
||||
@@ -224,11 +227,11 @@ def test_root_page_passes() -> None:
|
||||
gateway: MagicMock = _gateway()
|
||||
|
||||
result: CapabilityCheckResult = _run(
|
||||
"notion_root_page", gateway, {"root_page_id": " root "}
|
||||
"notion_root_page", gateway, {"root_page_id": f" {_ROOT_PAGE_ID} "}
|
||||
)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.PASSED
|
||||
gateway.get_page.assert_called_once_with(page_id="root", fast=True)
|
||||
gateway.get_page.assert_called_once_with(page_id=_ROOT_PAGE_ID, fast=True)
|
||||
gateway.get_database.assert_not_called()
|
||||
|
||||
|
||||
@@ -238,11 +241,11 @@ def test_root_page_passes_for_a_wiki() -> None:
|
||||
gateway.get_page.side_effect = _api_error(404, "object_not_found")
|
||||
|
||||
result: CapabilityCheckResult = _run(
|
||||
"notion_root_page", gateway, {"root_page_id": "root"}
|
||||
"notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID}
|
||||
)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.PASSED
|
||||
gateway.get_database.assert_called_once_with(database_id="root", fast=True)
|
||||
gateway.get_database.assert_called_once_with(database_id=_ROOT_PAGE_ID, fast=True)
|
||||
|
||||
|
||||
def test_root_page_fails_when_not_shared() -> None:
|
||||
@@ -251,7 +254,7 @@ def test_root_page_fails_when_not_shared() -> None:
|
||||
gateway.get_database.side_effect = _api_error(404, "object_not_found")
|
||||
|
||||
result: CapabilityCheckResult = _run(
|
||||
"notion_root_page", gateway, {"root_page_id": "root"}
|
||||
"notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID}
|
||||
)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.FAILED
|
||||
@@ -259,17 +262,45 @@ def test_root_page_fails_when_not_shared() -> None:
|
||||
assert "shared with the integration" in result.message
|
||||
|
||||
|
||||
def test_root_page_fails_for_an_invalid_id() -> None:
|
||||
def test_root_page_fails_when_notion_rejects_the_id() -> None:
|
||||
gateway: MagicMock = _gateway()
|
||||
gateway.get_page.side_effect = _api_error(400, "validation_error")
|
||||
|
||||
result: CapabilityCheckResult = _run(
|
||||
"notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID}
|
||||
)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.FAILED
|
||||
assert "not a valid Notion page ID" in result.message
|
||||
gateway.get_database.assert_not_called()
|
||||
|
||||
|
||||
def test_root_page_that_is_not_an_id_or_url_fails_as_a_setting() -> None:
|
||||
gateway: MagicMock = _gateway()
|
||||
|
||||
result: CapabilityCheckResult = _run(
|
||||
"notion_root_page", gateway, {"root_page_id": "My Page"}
|
||||
)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.FAILED
|
||||
assert "not a valid Notion page ID" in result.message
|
||||
gateway.get_database.assert_not_called()
|
||||
assert result.error_kind == CheckErrorKind.INVALID
|
||||
assert f"root_page_id: {ROOT_PAGE_ID_FORMAT_ERROR}" in result.message
|
||||
gateway.get_page.assert_not_called()
|
||||
|
||||
|
||||
def test_root_page_reads_the_id_in_a_page_url() -> None:
|
||||
gateway: MagicMock = _gateway()
|
||||
|
||||
result: CapabilityCheckResult = _run(
|
||||
"notion_root_page",
|
||||
gateway,
|
||||
{"root_page_id": f"https://www.notion.so/acme/Handbook-{_ROOT_PAGE_ID}"},
|
||||
)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.PASSED
|
||||
gateway.get_page.assert_called_once_with(
|
||||
page_id="17ab3186-873d-418f-b899-c3f6a43f68de", fast=True
|
||||
)
|
||||
|
||||
|
||||
# --- notion_page_content ---
|
||||
@@ -290,12 +321,12 @@ def test_page_content_reads_the_root_page_when_search_is_empty() -> None:
|
||||
gateway: MagicMock = _gateway(pages=[])
|
||||
|
||||
result: CapabilityCheckResult = _run(
|
||||
"notion_page_content", gateway, {"root_page_id": "root"}
|
||||
"notion_page_content", gateway, {"root_page_id": _ROOT_PAGE_ID}
|
||||
)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.PASSED
|
||||
gateway.get_block_children.assert_called_once_with(
|
||||
block_id="root", page_size=1, fast=True
|
||||
block_id=_ROOT_PAGE_ID, page_size=1, fast=True
|
||||
)
|
||||
|
||||
|
||||
@@ -336,7 +367,7 @@ def test_page_content_leaves_an_unreadable_root_page_to_its_check() -> None:
|
||||
gateway.get_block_children.side_effect = _api_error(404, "object_not_found")
|
||||
|
||||
result: CapabilityCheckResult = _run(
|
||||
"notion_page_content", gateway, {"root_page_id": "root"}
|
||||
"notion_page_content", gateway, {"root_page_id": _ROOT_PAGE_ID}
|
||||
)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.INCOMPLETE
|
||||
|
||||
@@ -25,6 +25,7 @@ from tests.unit.onyx.connectors.notion.test_notion_capability_checks import (
|
||||
_gateway,
|
||||
)
|
||||
|
||||
_ROOT_PAGE_ID = "17ab3186873d418fb899c3f6a43f68de"
|
||||
_ALL_CHECK_IDS = [check.check_id for check in build_notion_indexing_checks()]
|
||||
_BOT_USER = {
|
||||
"object": "user",
|
||||
@@ -290,7 +291,7 @@ def test_root_page_names_the_page() -> None:
|
||||
gateway = _gateway()
|
||||
gateway.get_page.return_value = _page("Handbook")
|
||||
|
||||
result = _run("notion_root_page", gateway, {"root_page_id": "root"})
|
||||
result = _run("notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID})
|
||||
|
||||
assert result.summary == "Read root page Handbook"
|
||||
|
||||
@@ -300,7 +301,7 @@ def test_root_page_names_a_wiki() -> None:
|
||||
gateway.get_page.side_effect = _api_error(404, "object_not_found")
|
||||
gateway.get_database.return_value = {"object": "database", **_data_source("Wiki")}
|
||||
|
||||
result = _run("notion_root_page", gateway, {"root_page_id": "root"})
|
||||
result = _run("notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID})
|
||||
|
||||
assert result.summary == "Read root page Wiki"
|
||||
|
||||
@@ -320,7 +321,7 @@ def test_root_page_failures_have_the_design_lines(
|
||||
gateway.get_page.side_effect = _api_error(status)
|
||||
gateway.get_database.side_effect = _api_error(status)
|
||||
|
||||
result = _run("notion_root_page", gateway, {"root_page_id": "root"})
|
||||
result = _run("notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID})
|
||||
|
||||
assert result.status == CapabilityCheckStatus.FAILED
|
||||
assert result.error_kind == error_kind
|
||||
@@ -343,7 +344,7 @@ def test_page_content_runs_after_the_root_page() -> None:
|
||||
gateway = _gateway()
|
||||
gateway.get_page.side_effect = _api_error(400, "validation_error")
|
||||
|
||||
results = _run_all(_ALL_CHECK_IDS, gateway, {"root_page_id": "root"})
|
||||
results = _run_all(_ALL_CHECK_IDS, gateway, {"root_page_id": _ROOT_PAGE_ID})
|
||||
|
||||
assert not results["notion_shared_content"].applicable
|
||||
content = results["notion_page_content"]
|
||||
@@ -380,7 +381,7 @@ def test_page_content_is_incomplete_when_the_root_page_has_no_blocks() -> None:
|
||||
gateway = _gateway(pages=[])
|
||||
gateway.get_block_children.side_effect = _api_error(404, "object_not_found")
|
||||
|
||||
result = _run("notion_page_content", gateway, {"root_page_id": "root"})
|
||||
result = _run("notion_page_content", gateway, {"root_page_id": _ROOT_PAGE_ID})
|
||||
|
||||
assert result.status == CapabilityCheckStatus.INCOMPLETE
|
||||
assert result.error_kind == CheckErrorKind.INCONCLUSIVE
|
||||
|
||||
@@ -32,6 +32,7 @@ from onyx.connectors.onedrive.capability_checks import (
|
||||
build_onedrive_group_sync_checks,
|
||||
build_onedrive_indexing_checks,
|
||||
)
|
||||
from onyx.connectors.onedrive.config import USER_EMAIL_FORMAT_ERROR
|
||||
from onyx.connectors.onedrive.models import (
|
||||
OneDriveDrive,
|
||||
OneDriveUser,
|
||||
@@ -152,11 +153,15 @@ def test_all_selected_users_found() -> None:
|
||||
assert result.summary == "All 2 selected users found"
|
||||
|
||||
|
||||
def test_an_entry_that_is_not_an_email_names_it() -> None:
|
||||
result = _run("onedrive_configured_users", _gateway(), {"users": ["not-an-email"]})
|
||||
def test_an_entry_that_is_not_an_email_fails_as_a_setting() -> None:
|
||||
gateway = _gateway()
|
||||
|
||||
result = _run("onedrive_configured_users", gateway, {"users": ["not-an-email"]})
|
||||
|
||||
assert result.status == CapabilityCheckStatus.FAILED
|
||||
assert result.summary == "Not an email address: not-an-email"
|
||||
assert result.error_kind == CheckErrorKind.INVALID
|
||||
assert f"users: {USER_EMAIL_FORMAT_ERROR}" in result.message
|
||||
gateway.get_user.assert_not_called()
|
||||
|
||||
|
||||
def test_blank_entries_say_so() -> None:
|
||||
|
||||
@@ -15,6 +15,7 @@ from onyx.connectors.capability_checks.models import (
|
||||
CapabilityCheckContext,
|
||||
CapabilityCheckResult,
|
||||
CapabilityCheckStatus,
|
||||
CheckErrorKind,
|
||||
)
|
||||
from onyx.connectors.capability_checks.runner import run_capability_checks
|
||||
from onyx.connectors.exceptions import (
|
||||
@@ -43,6 +44,7 @@ from onyx.connectors.sharepoint import connector_utils
|
||||
from onyx.connectors.sharepoint.capability_checks import (
|
||||
build_sharepoint_indexing_checks,
|
||||
)
|
||||
from onyx.connectors.sharepoint.config import SITE_URL_FORMAT_ERROR
|
||||
from onyx.connectors.sharepoint.models import (
|
||||
SharepointDrive,
|
||||
SitePagesPage,
|
||||
@@ -300,12 +302,14 @@ def test_configured_sites_check_is_skipped_without_sites() -> None:
|
||||
def test_a_blank_site_entry_fails_like_indexing() -> None:
|
||||
gateway: MagicMock = _gateway()
|
||||
|
||||
with pytest.raises(ConnectorValidationError, match="not a full"):
|
||||
_run(
|
||||
"sharepoint_configured_sites",
|
||||
_context(gateway, _config(sites=["", SITE_URL])),
|
||||
)
|
||||
(result,) = run_capability_checks(
|
||||
[_CHECKS_BY_ID["sharepoint_configured_sites"]],
|
||||
_context(gateway, _config(sites=["", SITE_URL])),
|
||||
)
|
||||
|
||||
assert result.status is CapabilityCheckStatus.FAILED
|
||||
assert result.error_kind is CheckErrorKind.INVALID
|
||||
assert f"sites: {SITE_URL_FORMAT_ERROR}" in result.message
|
||||
gateway.get_site_id.assert_not_called()
|
||||
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ from onyx.connectors.sharepoint.capability_checks import (
|
||||
build_sharepoint_group_sync_checks,
|
||||
build_sharepoint_indexing_checks,
|
||||
)
|
||||
from onyx.connectors.sharepoint.config import SITE_URL_FORMAT_ERROR
|
||||
from onyx.connectors.sharepoint.models import SitePagesPage
|
||||
from tests.unit.onyx.connectors.sharepoint.test_sharepoint_perm_sync_checks import (
|
||||
ITEM_JSON,
|
||||
@@ -165,16 +166,19 @@ def test_the_group_listing_names_the_group() -> None:
|
||||
# --- Find selected sites ---
|
||||
|
||||
|
||||
def test_a_site_url_without_a_site_path_names_the_url() -> None:
|
||||
def test_a_site_url_without_a_site_path_fails_as_a_setting() -> None:
|
||||
gateway = _healthy()
|
||||
|
||||
result = _run(
|
||||
"sharepoint_configured_sites",
|
||||
_healthy(),
|
||||
gateway,
|
||||
_config(sites=["https://contoso.sharepoint.com/eng"]),
|
||||
)
|
||||
|
||||
assert result.status == CapabilityCheckStatus.FAILED
|
||||
assert result.summary is not None
|
||||
assert result.summary.startswith("Not a site URL: ")
|
||||
assert result.error_kind == CheckErrorKind.INVALID
|
||||
assert f"sites: {SITE_URL_FORMAT_ERROR}" in result.message
|
||||
gateway.get_site_id.assert_not_called()
|
||||
|
||||
|
||||
def test_a_site_on_another_tenant_says_so() -> None:
|
||||
|
||||
@@ -20,6 +20,7 @@ _BITBUCKET_WORKSPACE = {"workspace": "onyx"}
|
||||
_DRUPAL_SITE = {"base_url": "https://wiki.example.com"}
|
||||
_ASANA_WORKSPACE = {"asana_workspace_id": "w"}
|
||||
_JIRA_SITE = {"jira_base_url": "https://example.atlassian.net"}
|
||||
_NOTION_PAGE_ID = "17ab3186873d418fb899c3f6a43f68de"
|
||||
|
||||
|
||||
def _classify(
|
||||
@@ -282,8 +283,8 @@ def test_every_rule_reads_the_config_class_of_its_source() -> None:
|
||||
# Notion: a root page is always followed recursively.
|
||||
(
|
||||
DocumentSource.NOTION,
|
||||
{"root_page_id": "r", "recursive_index_enabled": False},
|
||||
{"root_page_id": "r", "recursive_index_enabled": True},
|
||||
{"root_page_id": _NOTION_PAGE_ID, "recursive_index_enabled": False},
|
||||
{"root_page_id": _NOTION_PAGE_ID, "recursive_index_enabled": True},
|
||||
{},
|
||||
),
|
||||
(
|
||||
|
||||
@@ -306,7 +306,7 @@ _CONNECTOR_CONFIGS: dict[DocumentSource, list[dict[str, Any]]] = {
|
||||
{"repo_owner": "acme", "repositories": "docs"},
|
||||
],
|
||||
# The root page check reads a page only when a root page is set.
|
||||
DocumentSource.NOTION: [{"root_page_id": "root"}],
|
||||
DocumentSource.NOTION: [{"root_page_id": "17ab3186873d418fb899c3f6a43f68de"}],
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -15,8 +15,12 @@ from onyx.connectors.confluence.config import (
|
||||
)
|
||||
from onyx.connectors.factory import build_connector_kwargs, validate_connector_config
|
||||
from onyx.connectors.github.config import REPOSITORY_OWNER_ERROR
|
||||
from onyx.connectors.google_drive.config import MY_DRIVE_EMAILS_EMPTY_ERROR
|
||||
from onyx.connectors.jira.config import JQL_ORDER_BY_ERROR
|
||||
from onyx.connectors.notion.config import ROOT_PAGE_ID_FORMAT_ERROR
|
||||
from onyx.connectors.onedrive.config import USER_EMAIL_FORMAT_ERROR
|
||||
from onyx.connectors.registry import CONNECTOR_CLASS_MAP
|
||||
from onyx.connectors.sharepoint.config import SITE_URL_FORMAT_ERROR
|
||||
from onyx.server.documents.capability_check_runs import plan_draft_capability_checks
|
||||
|
||||
_CONFLUENCE_FORM: dict[str, Any] = {
|
||||
@@ -25,6 +29,9 @@ _CONFLUENCE_FORM: dict[str, Any] = {
|
||||
}
|
||||
_JIRA_FORM: dict[str, Any] = {"jira_base_url": "https://example.atlassian.net"}
|
||||
_GITHUB_FORM: dict[str, Any] = {"repo_owner": "onyx-dot-app"}
|
||||
_NOTION_PAGE_ID = "17ab3186873d418fb899c3f6a43f68de"
|
||||
_NOTION_DASHED_PAGE_ID = "17ab3186-873d-418f-b899-c3f6a43f68de"
|
||||
_SHAREPOINT_SITE = "https://acme.sharepoint.com/sites/hr"
|
||||
|
||||
|
||||
def _form_errors(source: DocumentSource, form_values: dict[str, Any]) -> dict[str, str]:
|
||||
@@ -72,6 +79,43 @@ def _form_errors(source: DocumentSource, form_values: dict[str, Any]) -> dict[st
|
||||
"repositories",
|
||||
REPOSITORY_OWNER_ERROR,
|
||||
),
|
||||
(
|
||||
DocumentSource.GOOGLE_DRIVE,
|
||||
{"my_drive_emails": " , "},
|
||||
"my_drive_emails",
|
||||
MY_DRIVE_EMAILS_EMPTY_ERROR,
|
||||
),
|
||||
(
|
||||
DocumentSource.SHAREPOINT,
|
||||
{"sites": [_SHAREPOINT_SITE, "acme.sharepoint.com/hr"]},
|
||||
"sites",
|
||||
SITE_URL_FORMAT_ERROR,
|
||||
),
|
||||
# Indexing validates every entry, blanks included.
|
||||
(
|
||||
DocumentSource.SHAREPOINT,
|
||||
{"sites": [""]},
|
||||
"sites",
|
||||
SITE_URL_FORMAT_ERROR,
|
||||
),
|
||||
(
|
||||
DocumentSource.ONEDRIVE,
|
||||
{"users": ["jane@acme.com", "jane"]},
|
||||
"users",
|
||||
USER_EMAIL_FORMAT_ERROR,
|
||||
),
|
||||
(
|
||||
DocumentSource.NOTION,
|
||||
{"root_page_id": "roadmap"},
|
||||
"root_page_id",
|
||||
ROOT_PAGE_ID_FORMAT_ERROR,
|
||||
),
|
||||
(
|
||||
DocumentSource.NOTION,
|
||||
{"root_page_id": "https://example.com/Roadmap-" + _NOTION_PAGE_ID},
|
||||
"root_page_id",
|
||||
ROOT_PAGE_ID_FORMAT_ERROR,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_value_that_breaks_a_format_rule_is_a_form_error(
|
||||
@@ -104,6 +148,22 @@ def test_a_value_that_breaks_a_format_rule_is_a_form_error(
|
||||
(DocumentSource.JIRA, {**_JIRA_FORM, "jql_query": None}),
|
||||
(DocumentSource.GITHUB, {**_GITHUB_FORM, "repositories": "onyx, docs"}),
|
||||
(DocumentSource.GITHUB, {**_GITHUB_FORM, "repositories": ""}),
|
||||
(
|
||||
DocumentSource.GOOGLE_DRIVE,
|
||||
{"my_drive_emails": "a@acme.com, ,b@acme.com"},
|
||||
),
|
||||
(DocumentSource.GOOGLE_DRIVE, {"my_drive_emails": ""}),
|
||||
(DocumentSource.SHAREPOINT, {"sites": [_SHAREPOINT_SITE]}),
|
||||
(DocumentSource.SHAREPOINT, {"sites": []}),
|
||||
(DocumentSource.ONEDRIVE, {"users": [" Jane@acme.com ", " "]}),
|
||||
(DocumentSource.ONEDRIVE, {"users": None}),
|
||||
(DocumentSource.NOTION, {"root_page_id": f" {_NOTION_PAGE_ID} "}),
|
||||
(DocumentSource.NOTION, {"root_page_id": _NOTION_DASHED_PAGE_ID}),
|
||||
(
|
||||
DocumentSource.NOTION,
|
||||
{"root_page_id": f"https://www.notion.so/acme/Roadmap-{_NOTION_PAGE_ID}"},
|
||||
),
|
||||
(DocumentSource.NOTION, {"root_page_id": ""}),
|
||||
],
|
||||
)
|
||||
def test_a_value_that_follows_the_format_rules_is_no_form_error(
|
||||
@@ -116,3 +176,11 @@ def test_a_stored_config_that_breaks_a_format_rule_still_builds_the_connector()
|
||||
stored = {**_GITHUB_FORM, "repositories": "onyx-dot-app/onyx"}
|
||||
|
||||
assert build_connector_kwargs(DocumentSource.GITHUB, stored) == stored
|
||||
|
||||
|
||||
def test_a_notion_page_url_reaches_the_connector_as_its_page_id() -> None:
|
||||
url = f"https://acme.notion.site/Roadmap-{_NOTION_PAGE_ID}?pvs=4"
|
||||
|
||||
assert build_connector_kwargs(DocumentSource.NOTION, {"root_page_id": url}) == {
|
||||
"root_page_id": _NOTION_DASHED_PAGE_ID
|
||||
}
|
||||
Reference in new issue
Block a user