feat(connectors): config models check the design's field formats (Drive, Microsoft, Slack, Notion)

The Google Drive, SharePoint, OneDrive and Notion config models get field
validators for the design's format rules: My Drive Emails holds at least
one email, each SharePoint site is a full site URL, each OneDrive user is
an email, and the Notion root page is a page URL or its 32-character ID.
A Notion page URL becomes the page ID that it holds. The plan endpoint
returns the rules as form_errors. The checks no longer test these formats.
Slack and Outlook get no rule: the design gives Slack a server-side rule
only, and gives Outlook no rule.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KFMazQbuG8ui3jxBxm47sC
This commit is contained in:
Claude committed 2026-10-17 03:38:43 +00:00
1 parent b29dbe67c6
commit 7e9a16b004
19 files changed
+298 -136

No files matched your search

@@ -1128,15 +1128,6 @@ class _ConfiguredMyDrivesCheck(_GoogleDriveCheck):
config: GoogleDriveConnectorConfig = self.config(context)
ops: GoogleDriveSourceOperations = _gateway(context)
requested: list[str] = extract_str_list_from_comma_str(config.my_drive_emails)
if not requested:
raise ConnectorValidationError(
"My Drive emails holds no email.",
summary="My Drive Emails has no email",
description=(
"My Drive Emails holds only commas or spaces.\nEnter at least "
"one email in My Drive Emails, or clear the field."
),
)
with _google_errors(
context,
api=_ADMIN_API,
+18 -1
View File
@@ -1,7 +1,13 @@
from typing import Annotated, Any, ClassVar
from pydantic import field_validator
from onyx.configs.app_configs import INDEX_BATCH_SIZE
from onyx.connectors.connector_config import ConnectorConfig, CredentialBinding
from onyx.connectors.connector_config import (
ConnectorConfig,
CredentialBinding,
field_format_error,
)
from onyx.connectors.exceptions import ConnectorValidationError
from onyx.connectors.field_policy import (
FieldClass,
@@ -16,6 +22,8 @@ from onyx.connectors.google_utils.shared_constants import (
)
from onyx.connectors.planning_rule import ConnectorChangeOverride
MY_DRIVE_EMAILS_EMPTY_ERROR = "Enter at least one email."
_COSMETIC = FieldPolicy(FieldClass.COSMETIC)
_SPECIFIC_REQUEST_FIELDS = (
"shared_drive_urls",
@@ -85,6 +93,15 @@ class GoogleDriveConnectorConfig(GoogleDriveCredentialBinding, ConnectorConfig):
only_org_public: Annotated[bool | None, _COSMETIC] = None
continue_on_failure: Annotated[bool | None, _COSMETIC] = None
# A value of only commas or spaces turns on the specific mode with no My
# Drive to index.
@field_validator("my_drive_emails")
@classmethod
def _my_drive_emails_name_an_email(cls, value: str | None) -> str | None:
if value and not value.replace(",", "").strip():
raise field_format_error(MY_DRIVE_EMAILS_EMPTY_ERROR)
return value
def has_specific_requests(config: GoogleDriveConnectorConfig) -> bool:
return bool(
+56 -1
View File
@@ -1,10 +1,14 @@
import re
from typing import Annotated
from urllib.parse import parse_qs, urlparse
from pydantic import field_validator
from onyx.configs.app_configs import (
INDEX_BATCH_SIZE,
NOTION_CONNECTOR_DISABLE_RECURSIVE_PAGE_LOOKUP,
)
from onyx.connectors.connector_config import ConnectorConfig
from onyx.connectors.connector_config import ConnectorConfig, field_format_error
from onyx.connectors.field_policy import (
FieldClass,
FieldPolicy,
@@ -16,6 +20,44 @@ from onyx.connectors.planning_rule import ConnectorChangeOverride
_RECURSIVE_INDEX_ENABLED = "recursive_index_enabled"
ROOT_PAGE_ID_FORMAT_ERROR = "Paste the page URL or its 32-character ID."
_HEX_ID = re.compile(r"[0-9a-fA-F]{32}")
_DASHED_ID = re.compile(
r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
)
def _dashed(hex_id: str) -> str:
return (
f"{hex_id[0:8]}-{hex_id[8:12]}-{hex_id[12:16]}-{hex_id[16:20]}-{hex_id[20:]}"
).lower()
def page_id_from_url(url: str) -> str | None:
"""The page ID (a dashed UUID) in a Notion page URL, or None."""
parsed = urlparse(url)
netloc = parsed.netloc.lower()
if not (
"notion.so" in netloc or "notion.site" in netloc or "app.notion.com" in netloc
):
return None
# The path ends in "Title-PageID".
path_last = parsed.path.split("/")[-1]
candidate = path_last.split("-")[-1] if "-" in path_last else path_last
cleaned = re.sub(r"[^0-9a-fA-F-]", "", candidate).replace("-", "")
if _HEX_ID.fullmatch(cleaned):
return _dashed(cleaned)
params = parse_qs(parsed.query)
for key in ("p", "page_id"):
if key in params and params[key]:
candidate = params[key][0].replace("-", "")
if _HEX_ID.fullmatch(candidate):
return _dashed(candidate)
return None
class NotionConnectorConfig(ConnectorConfig):
batch_size: Annotated[int, FieldPolicy(FieldClass.COSMETIC)] = INDEX_BATCH_SIZE
@@ -29,6 +71,19 @@ class NotionConnectorConfig(ConnectorConfig):
FieldPolicy(FieldClass.SCOPE, scope=ScopeInclude(empty_means_all=True)),
] = None
# The connector sends the value to Notion as a page ID, so a URL becomes
# the ID it holds.
@field_validator("root_page_id")
@classmethod
def _root_page_is_an_id_or_url(cls, value: str | None) -> str | None:
root: str = (value or "").strip()
if not root or _HEX_ID.fullmatch(root) or _DASHED_ID.fullmatch(root):
return value
page_id: str | None = page_id_from_url(root)
if page_id is None:
raise field_format_error(ROOT_PAGE_ID_FORMAT_ERROR)
return page_id
def notion_planning_rule(
old: NotionConnectorConfig, new: NotionConnectorConfig
+4 -42
View File
@@ -1,9 +1,7 @@
import re
from collections.abc import Generator
from datetime import datetime
from itertools import islice
from typing import Any, cast
from urllib.parse import parse_qs, urlparse
from pydantic import BaseModel
from typing_extensions import override
@@ -37,6 +35,7 @@ from onyx.connectors.models import (
SlimDocument,
TextSection,
)
from onyx.connectors.notion.config import page_id_from_url
from onyx.connectors.notion.models import NotionSearchObjectType
from onyx.connectors.notion.source_operations import (
NotionApiError,
@@ -189,46 +188,9 @@ class NotionConnector(LoadConnector, PollConnector, SlimConnector):
@override
def normalize_url(cls, url: str) -> NormalizationResult:
"""Normalize a Notion URL to extract the page ID (UUID format)."""
parsed = urlparse(url)
netloc = parsed.netloc.lower()
if not (
"notion.so" in netloc
or "notion.site" in netloc
or "app.notion.com" in netloc
):
return NormalizationResult(normalized_url=None, use_default=False)
# Extract page ID from path (format: "Title-PageID")
path_last = parsed.path.split("/")[-1]
candidate = path_last.split("-")[-1] if "-" in path_last else path_last
# Clean and format as UUID
candidate = re.sub(r"[^0-9a-fA-F-]", "", candidate)
cleaned = candidate.replace("-", "")
if len(cleaned) == 32 and re.fullmatch(r"[0-9a-fA-F]{32}", cleaned):
normalized_uuid = (
f"{cleaned[0:8]}-{cleaned[8:12]}-{cleaned[12:16]}-{cleaned[16:20]}-{cleaned[20:]}"
).lower()
return NormalizationResult(
normalized_url=normalized_uuid, use_default=False
)
# Try query params
params = parse_qs(parsed.query)
for key in ("p", "page_id"):
if key in params and params[key]:
candidate = params[key][0].replace("-", "")
if len(candidate) == 32 and re.fullmatch(r"[0-9a-fA-F]{32}", candidate):
normalized_uuid = (
f"{candidate[0:8]}-{candidate[8:12]}-{candidate[12:16]}-{candidate[16:20]}-{candidate[20:]}"
).lower()
return NormalizationResult(
normalized_url=normalized_uuid, use_default=False
)
return NormalizationResult(normalized_url=None, use_default=False)
return NormalizationResult(
normalized_url=page_id_from_url(url), use_default=False
)
@retry_builder(tries=3, delay=1, backoff=2)
def _fetch_child_blocks(
@@ -168,36 +168,15 @@ def _raise_row(message: str, row: MicrosoftRowText) -> NoReturn:
)
def _is_invalid_entry(user: str) -> bool:
if not user.strip():
return False
try:
normalize_configured_users([user])
except ConnectorValidationError:
return True
return False
def _configured_users(context: CapabilityCheckContext) -> list[str]:
"""The Users entries, normalized as indexing reads them. An entry that is
not an email, or a list of blanks, fails with its row."""
"""The Users entries, normalized as indexing reads them. A list of blanks
fails with its row; the config model rejects an entry that is not an
email."""
users: list[str] = _config(context).users or []
try:
return normalize_configured_users(users)
except ConnectorValidationError as error:
if not any(user.strip() for user in users):
_raise_row(error.message, _BLANK_USERS_ROW)
entry: str = next(
(user.strip() for user in users if _is_invalid_entry(user)), ""
)
_raise_row(
error.message,
_row(
_line("Not an email address: ", entry),
"Each entry in Users must be a user principal name or primary "
"email address.",
),
)
_raise_row(error.message, _BLANK_USERS_ROW)
def _candidate_users(
+18 -1
View File
@@ -1,6 +1,8 @@
from typing import Annotated
from onyx.connectors.connector_config import ConnectorConfig
from pydantic import field_validator
from onyx.connectors.connector_config import ConnectorConfig, field_format_error
from onyx.connectors.field_policy import (
FieldClass,
FieldPolicy,
@@ -8,6 +10,9 @@ from onyx.connectors.field_policy import (
ScopeInclude,
)
from onyx.connectors.microsoft_utils.config import MicrosoftCloudBinding
from onyx.connectors.onedrive.scope import normalized_user_email
USER_EMAIL_FORMAT_ERROR = "Enter email addresses, e.g. jane@acme.com."
class OneDriveConnectorConfig(MicrosoftCloudBinding, ConnectorConfig):
@@ -22,3 +27,15 @@ class OneDriveConnectorConfig(MicrosoftCloudBinding, ConnectorConfig):
treat_organization_link_as_public: Annotated[
bool, FieldPolicy(FieldClass.BEHAVIOR)
] = False
# Blank entries are skipped. A list of only blanks fails the "Find
# selected users" check, as the design's row says.
@field_validator("users")
@classmethod
def _users_are_emails(cls, value: list[str] | None) -> list[str] | None:
if any(
entry.strip() and normalized_user_email(entry) is None
for entry in value or []
):
raise field_format_error(USER_EMAIL_FORMAT_ERROR)
return value
+12 -4
View File
@@ -5,6 +5,15 @@ from onyx.connectors.exceptions import ConnectorValidationError
_EMAIL_ADAPTER = TypeAdapter(EmailStr)
def normalized_user_email(entry: str) -> str | None:
"""A Users entry as the email indexing reads, or None when it is not an
email. The caller skips blank entries."""
try:
return str(_EMAIL_ADAPTER.validate_python(entry.strip())).casefold()
except ValidationError:
return None
def normalize_configured_users(users: list[str] | None) -> list[str]:
raw_users = users or []
normalized: list[str] = []
@@ -13,12 +22,11 @@ def normalize_configured_users(users: list[str] | None) -> list[str]:
candidate = raw_user.strip()
if not candidate:
continue
try:
user = str(_EMAIL_ADAPTER.validate_python(candidate)).casefold()
except ValidationError as error:
user = normalized_user_email(candidate)
if user is None:
raise ConnectorValidationError(
f"Invalid OneDrive user email: {candidate!r}."
) from error
)
if user in seen:
continue
seen.add(user)
@@ -100,7 +100,6 @@ from onyx.connectors.sharepoint.connector import (
)
from onyx.connectors.sharepoint.connector_utils import (
SiteUrlBlockedError,
SiteUrlFormatError,
SiteUrlHostError,
tenant_domain_from_site_urls,
validate_content_types,
@@ -967,23 +966,14 @@ def _site_problem(error: MicrosoftGraphError, site_url: str) -> _SiteProblem:
def _validate_site_urls(
site_urls: list[str], environment: MicrosoftGraphEnvironment
) -> None:
"""Each URL is a full site URL, safe to request, on the first URL's
tenant host: the rules indexing applies."""
"""Each URL is safe to request and on the first URL's tenant host: the
rules indexing applies. The config model checks the URL format."""
tenant_domain: str | None = tenant_domain_from_site_urls(site_urls)
for site_url in site_urls:
try:
connector_utils.validate_site_url(
site_url, environment.sharepoint_domain_suffix, tenant_domain
)
except SiteUrlFormatError as error:
raise ConnectorValidationError(
error.message,
summary=_line("Not a site URL: ", site_url or "(blank)"),
description=(
"A site URL must start with https:// and contain /sites/, "
"/teams/ or /personal/ followed by the site name."
),
) from error
except SiteUrlBlockedError as error:
raise ConnectorValidationError(
error.message,
+23 -1
View File
@@ -1,10 +1,12 @@
from typing import Annotated
from pydantic import field_validator
from onyx.configs.app_configs import (
INDEX_BATCH_SIZE,
SHAREPOINT_EXHAUSTIVE_AD_ENUMERATION,
)
from onyx.connectors.connector_config import ConnectorConfig
from onyx.connectors.connector_config import ConnectorConfig, field_format_error
from onyx.connectors.field_policy import (
FieldClass,
FieldPolicy,
@@ -17,12 +19,23 @@ from onyx.connectors.microsoft_utils.config import (
MicrosoftCloudBinding,
)
SITE_URL_FORMAT_ERROR = (
"Enter a full site URL, e.g. https://acme.sharepoint.com/sites/hr."
)
_COSMETIC = FieldPolicy(FieldClass.COSMETIC)
_CONTENT_TYPE_TOGGLE = FieldPolicy(
FieldClass.SCOPE, scope=ScopeToggle(widens_when=True)
)
def is_site_url_well_formed(site_url: str) -> bool:
"""A full SharePoint or OneDrive site URL: https and a site path."""
return site_url.startswith("https://") and (
"/sites/" in site_url or "/teams/" in site_url or "/personal/" in site_url
)
class SharepointCredentialBinding(MicrosoftCloudBinding):
# Ignored: the connector uses the suffix of the cloud graph_api_host selects.
sharepoint_domain_suffix: Annotated[str, _COSMETIC] = (
@@ -52,3 +65,12 @@ class SharepointConnectorConfig(SharepointCredentialBinding, ConnectorConfig):
exhaustive_ad_enumeration: Annotated[bool, _COSMETIC] = (
SHAREPOINT_EXHAUSTIVE_AD_ENUMERATION
)
# Indexing validates every entry, blanks included. The host and the
# outbound URL guard depend on the cloud and DNS, so the checks keep them.
@field_validator("sites")
@classmethod
def _sites_are_full_site_urls(cls, value: list[str] | None) -> list[str] | None:
if any(not is_site_url_well_formed(site_url) for site_url in value or []):
raise field_format_error(SITE_URL_FORMAT_ERROR)
return value
@@ -7,6 +7,7 @@ from onyx.connectors.exceptions import ConnectorValidationError
from onyx.connectors.microsoft_utils.drive_items import DriveItemData
from onyx.connectors.microsoft_utils.sharepoint_rest import SharepointPermissionReader
from onyx.connectors.models import ExternalAccess
from onyx.connectors.sharepoint.config import is_site_url_well_formed
from onyx.db.enums import HierarchyNodeType
from onyx.utils.url import SSRFException, validate_outbound_http_url
from onyx.utils.variable_functionality import (
@@ -150,13 +151,6 @@ def validate_content_types(
)
def is_site_url_well_formed(site_url: str) -> bool:
"""A full SharePoint or OneDrive site URL: https and a site path."""
return site_url.startswith("https://") and (
"/sites/" in site_url or "/teams/" in site_url or "/personal/" in site_url
)
def validate_site_url(
site_url: str, sharepoint_domain_suffix: str, tenant_domain: str | None
) -> None:
@@ -12,6 +12,7 @@ from onyx.connectors.capability_checks.models import (
CapabilityCheckContext,
CapabilityCheckResult,
CapabilityCheckStatus,
CheckErrorKind,
CheckFindingKey,
CountFinding,
FindingOption,
@@ -21,6 +22,7 @@ from onyx.connectors.capability_checks.runner import run_capability_checks
from onyx.connectors.google_drive.capability_checks import (
build_google_drive_indexing_checks,
)
from onyx.connectors.google_drive.config import MY_DRIVE_EMAILS_EMPTY_ERROR
from onyx.connectors.google_drive.connector import (
extract_ids_from_urls,
extract_str_list_from_comma_str,
@@ -451,6 +453,17 @@ def test_my_drive_alias_is_reported_as_skipped() -> None:
assert f"alias@{_DOMAIN}" in result.message
def test_my_drive_emails_without_an_email_fail_as_a_setting() -> None:
ops = _ops()
result = _run("google_drive_configured_my_drives", ops, {"my_drive_emails": ","})
assert result.status == CapabilityCheckStatus.FAILED
assert result.error_kind == CheckErrorKind.INVALID
assert f"my_drive_emails: {MY_DRIVE_EMAILS_EMPTY_ERROR}" in result.message
ops.list_user_emails.assert_not_called()
def test_my_drive_of_a_listed_user_passes() -> None:
ops = _ops()
@@ -10,11 +10,13 @@ from onyx.connectors.capability_checks.models import (
CapabilityCheckContext,
CapabilityCheckResult,
CapabilityCheckStatus,
CheckErrorKind,
CheckNotRunReason,
)
from onyx.connectors.capability_checks.registry import get_capability_checks
from onyx.connectors.capability_checks.runner import run_capability_checks
from onyx.connectors.notion.capability_checks import build_notion_indexing_checks
from onyx.connectors.notion.config import ROOT_PAGE_ID_FORMAT_ERROR
from onyx.connectors.notion.connector import NotionConnector
from onyx.connectors.notion.models import NotionSearchObjectType, NotionSearchResponse
from onyx.connectors.notion.source_operations import (
@@ -22,6 +24,7 @@ from onyx.connectors.notion.source_operations import (
NotionSourceOperations,
)
_ROOT_PAGE_ID = "17ab3186873d418fb899c3f6a43f68de"
_CREDENTIAL = {"notion_integration_token": "secret"}
_PAGE = {"id": "page-1", "object": "page"}
_DATA_SOURCE = {
@@ -97,7 +100,7 @@ def test_every_check_probes_without_waiting_on_rate_limits() -> None:
gateway: MagicMock = _gateway()
for check in build_notion_indexing_checks():
_run(check.check_id, gateway, {"root_page_id": "root"})
_run(check.check_id, gateway, {"root_page_id": _ROOT_PAGE_ID})
calls = [call for call in gateway.mock_calls if call[0]]
assert calls
@@ -178,7 +181,7 @@ def test_shared_content_fails_without_the_read_content_capability() -> None:
def test_shared_content_does_not_apply_with_a_root_page() -> None:
result: CapabilityCheckResult = _run(
"notion_shared_content", _gateway(), {"root_page_id": "root"}
"notion_shared_content", _gateway(), {"root_page_id": _ROOT_PAGE_ID}
)
assert not result.applicable
@@ -224,11 +227,11 @@ def test_root_page_passes() -> None:
gateway: MagicMock = _gateway()
result: CapabilityCheckResult = _run(
"notion_root_page", gateway, {"root_page_id": " root "}
"notion_root_page", gateway, {"root_page_id": f" {_ROOT_PAGE_ID} "}
)
assert result.status == CapabilityCheckStatus.PASSED
gateway.get_page.assert_called_once_with(page_id="root", fast=True)
gateway.get_page.assert_called_once_with(page_id=_ROOT_PAGE_ID, fast=True)
gateway.get_database.assert_not_called()
@@ -238,11 +241,11 @@ def test_root_page_passes_for_a_wiki() -> None:
gateway.get_page.side_effect = _api_error(404, "object_not_found")
result: CapabilityCheckResult = _run(
"notion_root_page", gateway, {"root_page_id": "root"}
"notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID}
)
assert result.status == CapabilityCheckStatus.PASSED
gateway.get_database.assert_called_once_with(database_id="root", fast=True)
gateway.get_database.assert_called_once_with(database_id=_ROOT_PAGE_ID, fast=True)
def test_root_page_fails_when_not_shared() -> None:
@@ -251,7 +254,7 @@ def test_root_page_fails_when_not_shared() -> None:
gateway.get_database.side_effect = _api_error(404, "object_not_found")
result: CapabilityCheckResult = _run(
"notion_root_page", gateway, {"root_page_id": "root"}
"notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID}
)
assert result.status == CapabilityCheckStatus.FAILED
@@ -259,17 +262,45 @@ def test_root_page_fails_when_not_shared() -> None:
assert "shared with the integration" in result.message
def test_root_page_fails_for_an_invalid_id() -> None:
def test_root_page_fails_when_notion_rejects_the_id() -> None:
gateway: MagicMock = _gateway()
gateway.get_page.side_effect = _api_error(400, "validation_error")
result: CapabilityCheckResult = _run(
"notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID}
)
assert result.status == CapabilityCheckStatus.FAILED
assert "not a valid Notion page ID" in result.message
gateway.get_database.assert_not_called()
def test_root_page_that_is_not_an_id_or_url_fails_as_a_setting() -> None:
gateway: MagicMock = _gateway()
result: CapabilityCheckResult = _run(
"notion_root_page", gateway, {"root_page_id": "My Page"}
)
assert result.status == CapabilityCheckStatus.FAILED
assert "not a valid Notion page ID" in result.message
gateway.get_database.assert_not_called()
assert result.error_kind == CheckErrorKind.INVALID
assert f"root_page_id: {ROOT_PAGE_ID_FORMAT_ERROR}" in result.message
gateway.get_page.assert_not_called()
def test_root_page_reads_the_id_in_a_page_url() -> None:
gateway: MagicMock = _gateway()
result: CapabilityCheckResult = _run(
"notion_root_page",
gateway,
{"root_page_id": f"https://www.notion.so/acme/Handbook-{_ROOT_PAGE_ID}"},
)
assert result.status == CapabilityCheckStatus.PASSED
gateway.get_page.assert_called_once_with(
page_id="17ab3186-873d-418f-b899-c3f6a43f68de", fast=True
)
# --- notion_page_content ---
@@ -290,12 +321,12 @@ def test_page_content_reads_the_root_page_when_search_is_empty() -> None:
gateway: MagicMock = _gateway(pages=[])
result: CapabilityCheckResult = _run(
"notion_page_content", gateway, {"root_page_id": "root"}
"notion_page_content", gateway, {"root_page_id": _ROOT_PAGE_ID}
)
assert result.status == CapabilityCheckStatus.PASSED
gateway.get_block_children.assert_called_once_with(
block_id="root", page_size=1, fast=True
block_id=_ROOT_PAGE_ID, page_size=1, fast=True
)
@@ -336,7 +367,7 @@ def test_page_content_leaves_an_unreadable_root_page_to_its_check() -> None:
gateway.get_block_children.side_effect = _api_error(404, "object_not_found")
result: CapabilityCheckResult = _run(
"notion_page_content", gateway, {"root_page_id": "root"}
"notion_page_content", gateway, {"root_page_id": _ROOT_PAGE_ID}
)
assert result.status == CapabilityCheckStatus.INCOMPLETE
@@ -25,6 +25,7 @@ from tests.unit.onyx.connectors.notion.test_notion_capability_checks import (
_gateway,
)
_ROOT_PAGE_ID = "17ab3186873d418fb899c3f6a43f68de"
_ALL_CHECK_IDS = [check.check_id for check in build_notion_indexing_checks()]
_BOT_USER = {
"object": "user",
@@ -290,7 +291,7 @@ def test_root_page_names_the_page() -> None:
gateway = _gateway()
gateway.get_page.return_value = _page("Handbook")
result = _run("notion_root_page", gateway, {"root_page_id": "root"})
result = _run("notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID})
assert result.summary == "Read root page Handbook"
@@ -300,7 +301,7 @@ def test_root_page_names_a_wiki() -> None:
gateway.get_page.side_effect = _api_error(404, "object_not_found")
gateway.get_database.return_value = {"object": "database", **_data_source("Wiki")}
result = _run("notion_root_page", gateway, {"root_page_id": "root"})
result = _run("notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID})
assert result.summary == "Read root page Wiki"
@@ -320,7 +321,7 @@ def test_root_page_failures_have_the_design_lines(
gateway.get_page.side_effect = _api_error(status)
gateway.get_database.side_effect = _api_error(status)
result = _run("notion_root_page", gateway, {"root_page_id": "root"})
result = _run("notion_root_page", gateway, {"root_page_id": _ROOT_PAGE_ID})
assert result.status == CapabilityCheckStatus.FAILED
assert result.error_kind == error_kind
@@ -343,7 +344,7 @@ def test_page_content_runs_after_the_root_page() -> None:
gateway = _gateway()
gateway.get_page.side_effect = _api_error(400, "validation_error")
results = _run_all(_ALL_CHECK_IDS, gateway, {"root_page_id": "root"})
results = _run_all(_ALL_CHECK_IDS, gateway, {"root_page_id": _ROOT_PAGE_ID})
assert not results["notion_shared_content"].applicable
content = results["notion_page_content"]
@@ -380,7 +381,7 @@ def test_page_content_is_incomplete_when_the_root_page_has_no_blocks() -> None:
gateway = _gateway(pages=[])
gateway.get_block_children.side_effect = _api_error(404, "object_not_found")
result = _run("notion_page_content", gateway, {"root_page_id": "root"})
result = _run("notion_page_content", gateway, {"root_page_id": _ROOT_PAGE_ID})
assert result.status == CapabilityCheckStatus.INCOMPLETE
assert result.error_kind == CheckErrorKind.INCONCLUSIVE
@@ -32,6 +32,7 @@ from onyx.connectors.onedrive.capability_checks import (
build_onedrive_group_sync_checks,
build_onedrive_indexing_checks,
)
from onyx.connectors.onedrive.config import USER_EMAIL_FORMAT_ERROR
from onyx.connectors.onedrive.models import (
OneDriveDrive,
OneDriveUser,
@@ -152,11 +153,15 @@ def test_all_selected_users_found() -> None:
assert result.summary == "All 2 selected users found"
def test_an_entry_that_is_not_an_email_names_it() -> None:
result = _run("onedrive_configured_users", _gateway(), {"users": ["not-an-email"]})
def test_an_entry_that_is_not_an_email_fails_as_a_setting() -> None:
gateway = _gateway()
result = _run("onedrive_configured_users", gateway, {"users": ["not-an-email"]})
assert result.status == CapabilityCheckStatus.FAILED
assert result.summary == "Not an email address: not-an-email"
assert result.error_kind == CheckErrorKind.INVALID
assert f"users: {USER_EMAIL_FORMAT_ERROR}" in result.message
gateway.get_user.assert_not_called()
def test_blank_entries_say_so() -> None:
@@ -15,6 +15,7 @@ from onyx.connectors.capability_checks.models import (
CapabilityCheckContext,
CapabilityCheckResult,
CapabilityCheckStatus,
CheckErrorKind,
)
from onyx.connectors.capability_checks.runner import run_capability_checks
from onyx.connectors.exceptions import (
@@ -43,6 +44,7 @@ from onyx.connectors.sharepoint import connector_utils
from onyx.connectors.sharepoint.capability_checks import (
build_sharepoint_indexing_checks,
)
from onyx.connectors.sharepoint.config import SITE_URL_FORMAT_ERROR
from onyx.connectors.sharepoint.models import (
SharepointDrive,
SitePagesPage,
@@ -300,12 +302,14 @@ def test_configured_sites_check_is_skipped_without_sites() -> None:
def test_a_blank_site_entry_fails_like_indexing() -> None:
gateway: MagicMock = _gateway()
with pytest.raises(ConnectorValidationError, match="not a full"):
_run(
"sharepoint_configured_sites",
_context(gateway, _config(sites=["", SITE_URL])),
)
(result,) = run_capability_checks(
[_CHECKS_BY_ID["sharepoint_configured_sites"]],
_context(gateway, _config(sites=["", SITE_URL])),
)
assert result.status is CapabilityCheckStatus.FAILED
assert result.error_kind is CheckErrorKind.INVALID
assert f"sites: {SITE_URL_FORMAT_ERROR}" in result.message
gateway.get_site_id.assert_not_called()
@@ -27,6 +27,7 @@ from onyx.connectors.sharepoint.capability_checks import (
build_sharepoint_group_sync_checks,
build_sharepoint_indexing_checks,
)
from onyx.connectors.sharepoint.config import SITE_URL_FORMAT_ERROR
from onyx.connectors.sharepoint.models import SitePagesPage
from tests.unit.onyx.connectors.sharepoint.test_sharepoint_perm_sync_checks import (
ITEM_JSON,
@@ -165,16 +166,19 @@ def test_the_group_listing_names_the_group() -> None:
# --- Find selected sites ---
def test_a_site_url_without_a_site_path_names_the_url() -> None:
def test_a_site_url_without_a_site_path_fails_as_a_setting() -> None:
gateway = _healthy()
result = _run(
"sharepoint_configured_sites",
_healthy(),
gateway,
_config(sites=["https://contoso.sharepoint.com/eng"]),
)
assert result.status == CapabilityCheckStatus.FAILED
assert result.summary is not None
assert result.summary.startswith("Not a site URL: ")
assert result.error_kind == CheckErrorKind.INVALID
assert f"sites: {SITE_URL_FORMAT_ERROR}" in result.message
gateway.get_site_id.assert_not_called()
def test_a_site_on_another_tenant_says_so() -> None:
@@ -20,6 +20,7 @@ _BITBUCKET_WORKSPACE = {"workspace": "onyx"}
_DRUPAL_SITE = {"base_url": "https://wiki.example.com"}
_ASANA_WORKSPACE = {"asana_workspace_id": "w"}
_JIRA_SITE = {"jira_base_url": "https://example.atlassian.net"}
_NOTION_PAGE_ID = "17ab3186873d418fb899c3f6a43f68de"
def _classify(
@@ -282,8 +283,8 @@ def test_every_rule_reads_the_config_class_of_its_source() -> None:
# Notion: a root page is always followed recursively.
(
DocumentSource.NOTION,
{"root_page_id": "r", "recursive_index_enabled": False},
{"root_page_id": "r", "recursive_index_enabled": True},
{"root_page_id": _NOTION_PAGE_ID, "recursive_index_enabled": False},
{"root_page_id": _NOTION_PAGE_ID, "recursive_index_enabled": True},
{},
),
(
@@ -306,7 +306,7 @@ _CONNECTOR_CONFIGS: dict[DocumentSource, list[dict[str, Any]]] = {
{"repo_owner": "acme", "repositories": "docs"},
],
# The root page check reads a page only when a root page is set.
DocumentSource.NOTION: [{"root_page_id": "root"}],
DocumentSource.NOTION: [{"root_page_id": "17ab3186873d418fb899c3f6a43f68de"}],
}
@@ -15,8 +15,12 @@ from onyx.connectors.confluence.config import (
)
from onyx.connectors.factory import build_connector_kwargs, validate_connector_config
from onyx.connectors.github.config import REPOSITORY_OWNER_ERROR
from onyx.connectors.google_drive.config import MY_DRIVE_EMAILS_EMPTY_ERROR
from onyx.connectors.jira.config import JQL_ORDER_BY_ERROR
from onyx.connectors.notion.config import ROOT_PAGE_ID_FORMAT_ERROR
from onyx.connectors.onedrive.config import USER_EMAIL_FORMAT_ERROR
from onyx.connectors.registry import CONNECTOR_CLASS_MAP
from onyx.connectors.sharepoint.config import SITE_URL_FORMAT_ERROR
from onyx.server.documents.capability_check_runs import plan_draft_capability_checks
_CONFLUENCE_FORM: dict[str, Any] = {
@@ -25,6 +29,9 @@ _CONFLUENCE_FORM: dict[str, Any] = {
}
_JIRA_FORM: dict[str, Any] = {"jira_base_url": "https://example.atlassian.net"}
_GITHUB_FORM: dict[str, Any] = {"repo_owner": "onyx-dot-app"}
_NOTION_PAGE_ID = "17ab3186873d418fb899c3f6a43f68de"
_NOTION_DASHED_PAGE_ID = "17ab3186-873d-418f-b899-c3f6a43f68de"
_SHAREPOINT_SITE = "https://acme.sharepoint.com/sites/hr"
def _form_errors(source: DocumentSource, form_values: dict[str, Any]) -> dict[str, str]:
@@ -72,6 +79,43 @@ def _form_errors(source: DocumentSource, form_values: dict[str, Any]) -> dict[st
"repositories",
REPOSITORY_OWNER_ERROR,
),
(
DocumentSource.GOOGLE_DRIVE,
{"my_drive_emails": " , "},
"my_drive_emails",
MY_DRIVE_EMAILS_EMPTY_ERROR,
),
(
DocumentSource.SHAREPOINT,
{"sites": [_SHAREPOINT_SITE, "acme.sharepoint.com/hr"]},
"sites",
SITE_URL_FORMAT_ERROR,
),
# Indexing validates every entry, blanks included.
(
DocumentSource.SHAREPOINT,
{"sites": [""]},
"sites",
SITE_URL_FORMAT_ERROR,
),
(
DocumentSource.ONEDRIVE,
{"users": ["jane@acme.com", "jane"]},
"users",
USER_EMAIL_FORMAT_ERROR,
),
(
DocumentSource.NOTION,
{"root_page_id": "roadmap"},
"root_page_id",
ROOT_PAGE_ID_FORMAT_ERROR,
),
(
DocumentSource.NOTION,
{"root_page_id": "https://example.com/Roadmap-" + _NOTION_PAGE_ID},
"root_page_id",
ROOT_PAGE_ID_FORMAT_ERROR,
),
],
)
def test_a_value_that_breaks_a_format_rule_is_a_form_error(
@@ -104,6 +148,22 @@ def test_a_value_that_breaks_a_format_rule_is_a_form_error(
(DocumentSource.JIRA, {**_JIRA_FORM, "jql_query": None}),
(DocumentSource.GITHUB, {**_GITHUB_FORM, "repositories": "onyx, docs"}),
(DocumentSource.GITHUB, {**_GITHUB_FORM, "repositories": ""}),
(
DocumentSource.GOOGLE_DRIVE,
{"my_drive_emails": "a@acme.com, ,b@acme.com"},
),
(DocumentSource.GOOGLE_DRIVE, {"my_drive_emails": ""}),
(DocumentSource.SHAREPOINT, {"sites": [_SHAREPOINT_SITE]}),
(DocumentSource.SHAREPOINT, {"sites": []}),
(DocumentSource.ONEDRIVE, {"users": [" Jane@acme.com ", " "]}),
(DocumentSource.ONEDRIVE, {"users": None}),
(DocumentSource.NOTION, {"root_page_id": f" {_NOTION_PAGE_ID} "}),
(DocumentSource.NOTION, {"root_page_id": _NOTION_DASHED_PAGE_ID}),
(
DocumentSource.NOTION,
{"root_page_id": f"https://www.notion.so/acme/Roadmap-{_NOTION_PAGE_ID}"},
),
(DocumentSource.NOTION, {"root_page_id": ""}),
],
)
def test_a_value_that_follows_the_format_rules_is_no_form_error(
@@ -116,3 +176,11 @@ def test_a_stored_config_that_breaks_a_format_rule_still_builds_the_connector()
stored = {**_GITHUB_FORM, "repositories": "onyx-dot-app/onyx"}
assert build_connector_kwargs(DocumentSource.GITHUB, stored) == stored
def test_a_notion_page_url_reaches_the_connector_as_its_page_id() -> None:
url = f"https://acme.notion.site/Roadmap-{_NOTION_PAGE_ID}?pvs=4"
assert build_connector_kwargs(DocumentSource.NOTION, {"root_page_id": url}) == {
"root_page_id": _NOTION_DASHED_PAGE_ID
}