starred/onyx
1
0 Fork 0
mirror of https://github.com/onyx-dot-app/onyx.git synced 2026-10-19 01:50:51 -05:00
Code Issues Packages Projects Releases 10 Wiki Activity

refactor(connectors): Confluence source-operations gateway (#15453)

Browse Source 
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Evan LohnandClaude Opus 5.5 authored and GitHub committed 2026-10-03 04:38:44 +00:00
1 parent b7dca10601
commit 3c8fd80472
24 files changed
+2251 -1849
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/ee/onyx/external_permissions/confluence/group_sync.py
+1 -5
View File
@@ -9,9 +9,6 @@ from onyx.connectors.confluence.onyx_confluence import (
get_user_email_from_username__server,
)
from onyx.connectors.credentials_provider import OnyxDBCredentialsProvider
from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
credential_uses_scoped_token,
)
from onyx.db.engine.sql_engine import get_session_with_current_tenant
from onyx.db.models import ConnectorCredentialPair
from onyx.db.users import get_all_users
@@ -168,10 +165,9 @@ def confluence_group_sync(
tenant_id, cc_pair.connector.source, cc_pair.credential_id
)
is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False)
# The credential carries the flag; older connectors set it in their config.
scoped_token = cc_pair.connector.connector_specific_config.get(
"scoped_token", False
) or credential_uses_scoped_token(provider.get_credentials())
)
wiki_base: str = cc_pair.connector.connector_specific_config["wiki_base"]
url = wiki_base.rstrip("/")
backend/ee/onyx/external_permissions/jira/group_sync.py
+2 -7
View File
@@ -7,9 +7,6 @@ from jira.exceptions import JIRAError
from ee.onyx.db.external_perm import ExternalUserGroup
from ee.onyx.external_permissions.utils import credential_json
from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
credential_uses_scoped_token,
)
from onyx.connectors.jira.utils import build_jira_client
from onyx.db.models import ConnectorCredentialPair
from onyx.utils.logger import setup_logger
@@ -158,17 +155,15 @@ def jira_group_sync(
Streams group-by-group rather than accumulating all groups in memory.
"""
jira_base_url = cc_pair.connector.connector_specific_config.get("jira_base_url", "")
credentials = credential_json(cc_pair)
# The credential carries the flag; older connectors set it in their config.
scoped_token = cc_pair.connector.connector_specific_config.get(
"scoped_token", False
) or credential_uses_scoped_token(credentials)
)
if not jira_base_url:
raise ValueError("No jira_base_url found in connector config")
jira_client = build_jira_client(
credentials=credentials,
credentials=credential_json(cc_pair),
jira_base=jira_base_url,
scoped_token=scoped_token,
)
backend/onyx/connectors/confluence/connector.py
+68 -78
View File
@@ -5,7 +5,6 @@ from datetime import datetime, timedelta, timezone
from typing import Any
from urllib.parse import quote
from atlassian.errors import ApiError
from requests.exceptions import HTTPError
from typing_extensions import override
@@ -22,21 +21,25 @@ from onyx.connectors.confluence.access import (
get_page_restrictions,
get_page_restrictions_with_per_ancestor_fetch,
)
from onyx.connectors.confluence.onyx_confluence import (
from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
from onyx.connectors.confluence.source_operations import (
Confcloud77618Error,
OnyxConfluence,
extract_text_from_confluence_html,
ConfluenceProbeVariant,
ConfluenceSearchVariant,
ConfluenceSourceOperations,
ConfluenceSpaceNotFoundError,
)
from onyx.connectors.confluence.utils import (
build_confluence_document_id,
build_cql_url,
convert_attachment_to_content,
datetime_from_string,
extract_text_from_confluence_html,
update_param_in_path,
validate_attachment_filetype,
)
from onyx.connectors.credentials_provider import OnyxStaticCredentialsProvider
from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
credential_uses_scoped_token,
is_atlassian_date_error,
)
from onyx.connectors.exceptions import (
@@ -200,8 +203,7 @@ class ConfluenceConnector(
self.timezone_offset = timezone_offset
self.scoped_token = scoped_token
self.include_attachments = include_attachments
self._confluence_client: OnyxConfluence | None = None
self._low_timeout_confluence_client: OnyxConfluence | None = None
self._source_operations: ConfluenceSourceOperations | None = None
self._fetched_titles: set[str] = set()
self.allow_images = False
@@ -241,16 +243,6 @@ class ConfluenceConnector(
self.timezone: timezone = timezone(offset=timedelta(hours=timezone_offset))
self.credentials_provider: CredentialsProviderInterface | None = None
self.probe_kwargs = {
"max_backoff_retries": 6,
"max_backoff_seconds": 10,
}
self.final_kwargs = {
"max_backoff_retries": 10,
"max_backoff_seconds": 60,
}
# deprecated
self.continue_on_failure = continue_on_failure
@@ -264,7 +256,7 @@ class ConfluenceConnector(
"""Yield hierarchy nodes for all spaces we're indexing."""
space_keys = [self.space] if self.space else None
for space in self.confluence_client.retrieve_confluence_spaces(
for space in self.source_operations.list_spaces(
space_keys=space_keys,
limit=50,
):
@@ -416,51 +408,38 @@ class ConfluenceConnector(
)
@property
def confluence_client(self) -> OnyxConfluence:
if self._confluence_client is None:
def source_operations(self) -> ConfluenceSourceOperations:
if self._source_operations is None:
raise ConnectorMissingCredentialError("Confluence")
return self._confluence_client
return self._source_operations
@property
def low_timeout_confluence_client(self) -> OnyxConfluence:
if self._low_timeout_confluence_client is None:
raise ConnectorMissingCredentialError("Confluence")
return self._low_timeout_confluence_client
def _perm_sync_client(self) -> OnyxConfluence:
"""Temporary: the EE perm-sync helpers still take the client. #15454
moves them to gateway operations."""
return self.source_operations._legacy_client()
def set_credentials_provider(
self, credentials_provider: CredentialsProviderInterface
) -> None:
self.credentials_provider = credentials_provider
# The credential now carries the flag; older connectors set it in
# their own config.
self.scoped_token = self.scoped_token or credential_uses_scoped_token(
credentials_provider.get_credentials()
source_operations = ConfluenceSourceOperations(
credentials_provider=credentials_provider,
connector_specific_config={
"wiki_base": self.wiki_base,
"is_cloud": self.is_cloud,
"scoped_token": self.scoped_token,
},
)
# raises exception if there's a problem
confluence_client = OnyxConfluence(
is_cloud=self.is_cloud,
url=self.wiki_base,
credentials_provider=credentials_provider,
scoped_token=self.scoped_token,
source_operations.probe_site(
variant=(
ConfluenceProbeVariant.SCOPED
if self.scoped_token
else ConfluenceProbeVariant.UNSCOPED
)
)
confluence_client._probe_connection(**self.probe_kwargs)
confluence_client._initialize_connection(**self.final_kwargs)
self._confluence_client = confluence_client
# create a low timeout confluence client for sync flows
low_timeout_confluence_client = OnyxConfluence(
is_cloud=self.is_cloud,
url=self.wiki_base,
credentials_provider=credentials_provider,
timeout=3,
scoped_token=self.scoped_token,
)
low_timeout_confluence_client._probe_connection(**self.probe_kwargs)
low_timeout_confluence_client._initialize_connection(**self.final_kwargs)
self._low_timeout_confluence_client = low_timeout_confluence_client
self._source_operations = source_operations
def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError("Use set_credentials_provider with this connector.")
@@ -520,13 +499,13 @@ class ConfluenceConnector(
comment_cql += self.cql_label_filter
expand = ",".join(_COMMENT_EXPANSION_FIELDS)
for comment in self.confluence_client.paginated_cql_retrieval(
for comment in self.source_operations.search_comments(
cql=comment_cql,
expand=expand,
):
comment_string += "\nComment:\n"
comment_string += extract_text_from_confluence_html(
confluence_client=self.confluence_client,
source_operations=self.source_operations,
confluence_object=comment,
fetched_titles=set(),
)
@@ -551,7 +530,7 @@ class ConfluenceConnector(
# Get the page content
page_content = extract_text_from_confluence_html(
self.confluence_client, page, self._fetched_titles
self.source_operations, page, self._fetched_titles
)
# Create the main section for the page content
@@ -651,7 +630,8 @@ class ConfluenceConnector(
page_hierarchy_node_yielded = False
try:
for attachment in self.confluence_client.paginated_cql_retrieval(
for attachment in self.source_operations.search_attachments(
variant=ConfluenceSearchVariant.CONTENT,
cql=attachment_query,
expand=",".join(_ATTACHMENT_EXPANSION_FIELDS),
):
@@ -705,11 +685,10 @@ class ConfluenceConnector(
continue
try:
response = convert_attachment_to_content(
confluence_client=self.confluence_client,
source_operations=self.source_operations,
attachment=attachment,
page_id=_get_page_id(page),
allow_images=self.allow_images,
is_cloud=self.is_cloud,
)
if response is None:
continue
@@ -895,8 +874,8 @@ class ConfluenceConnector(
def store_next_page_url(next_page_url: str) -> None:
checkpoint.next_page_url = next_page_url
for page in self.confluence_client.paginated_page_retrieval(
cql_url=page_query_url,
for page in self.source_operations.search_pages_from_url(
url=page_query_url,
limit=self.batch_size,
next_page_callback=store_next_page_url,
):
@@ -940,9 +919,7 @@ class ConfluenceConnector(
or paginated_page_retrieval methods.
"""
page_query = self._construct_page_cql_query(start, end)
cql_url = self.confluence_client.build_cql_url(
page_query, expand=",".join(_PAGE_EXPANSION_FIELDS)
)
cql_url = build_cql_url(page_query, expand=",".join(_PAGE_EXPANSION_FIELDS))
return update_param_in_path(cql_url, "limit", str(limit))
@override
@@ -1005,7 +982,7 @@ class ConfluenceConnector(
space_level_access: dict[str, ExternalAccess] = (
get_all_space_permissions(
self.confluence_client, self.is_cloud, add_prefix=True
self._perm_sync_client, self.is_cloud, add_prefix=True
)
if include_permissions
else {}
@@ -1022,7 +999,12 @@ class ConfluenceConnector(
cql = "type=page and id IN (%s)" % quoted_ids
seen_page_ids: set[str] = set()
for page in self.confluence_client.paginated_cql_retrieval(
for page in self.source_operations.search_pages(
variant=(
ConfluenceSearchVariant.WITH_RESTRICTIONS
if include_permissions
else ConfluenceSearchVariant.CONTENT
),
cql=cql,
expand=",".join(expand_fields),
):
@@ -1055,7 +1037,7 @@ class ConfluenceConnector(
if include_permissions:
space_key = page.get("space", {}).get("key") or ""
doc_or_failure.external_access = get_page_restrictions(
self.confluence_client,
self._perm_sync_client,
doc_or_failure.id,
page.get("restrictions") or {},
page.get("ancestors", []),
@@ -1136,6 +1118,7 @@ class ConfluenceConnector(
def _retrieve_attachments_for_slim_page(
self,
page_id: str,
variant: ConfluenceSearchVariant,
expand: str,
start: SecondsSinceUnixEpoch | None,
end: SecondsSinceUnixEpoch | None,
@@ -1153,10 +1136,12 @@ class ConfluenceConnector(
while True:
try:
return list(
self.confluence_client.cql_paginate_all_expansions(
self.source_operations.search_attachments(
variant=variant,
cql=attachment_query,
expand=expand,
limit=_SLIM_DOC_BATCH_SIZE,
follow_expansion_links=True,
)
)
except HTTPError as e:
@@ -1187,6 +1172,11 @@ class ConfluenceConnector(
# Pruning skips the restrictions expand (the CONFCLOUD-77618
# trigger) but still needs space + ancestors for hierarchy.
search_variant = (
ConfluenceSearchVariant.WITH_RESTRICTIONS
if include_permissions
else ConfluenceSearchVariant.SLIM
)
if not include_permissions:
restrictions_expand = ",".join(_PRUNING_EXPANSION_FIELDS)
elif expand_per_page:
@@ -1197,7 +1187,7 @@ class ConfluenceConnector(
space_level_access_info: dict[str, ExternalAccess] = {}
if include_permissions:
space_level_access_info = get_all_space_permissions(
self.confluence_client, self.is_cloud
self._perm_sync_client, self.is_cloud
)
# Yield space hierarchy nodes first
@@ -1214,7 +1204,7 @@ class ConfluenceConnector(
) -> ExternalAccess | None:
if expand_per_page:
resolved = get_page_restrictions_with_per_ancestor_fetch(
self.confluence_client,
self._perm_sync_client,
doc_id,
restrictions,
ancestors,
@@ -1222,7 +1212,7 @@ class ConfluenceConnector(
)
else:
resolved = get_page_restrictions(
self.confluence_client, doc_id, restrictions, ancestors
self._perm_sync_client, doc_id, restrictions, ancestors
)
return resolved or (
space_level_access_info.get(space_key) if space_key else None
@@ -1230,10 +1220,12 @@ class ConfluenceConnector(
# Query pages (with optional time filtering for indexing_start)
page_query = self._construct_page_cql_query(start, end)
for page in self.confluence_client.cql_paginate_all_expansions(
for page in self.source_operations.search_pages(
variant=search_variant,
cql=page_query,
expand=restrictions_expand,
limit=_SLIM_DOC_BATCH_SIZE,
follow_expansion_links=True,
):
# Yield ancestor hierarchy nodes for this page
doc_metadata_list.extend(self._yield_ancestor_hierarchy_nodes(page))
@@ -1271,7 +1263,7 @@ class ConfluenceConnector(
attachment_results: Iterable[dict[str, Any]] = ()
if self.include_attachments:
attachment_results = self._retrieve_attachments_for_slim_page(
_get_page_id(page), restrictions_expand, start, end
_get_page_id(page), search_variant, restrictions_expand, start, end
)
for attachment in attachment_results:
# admission must mirror the main indexing pass
@@ -1342,9 +1334,7 @@ class ConfluenceConnector(
def validate_connector_settings(self) -> None:
try:
spaces_iter = self.low_timeout_confluence_client.retrieve_confluence_spaces(
limit=1,
)
spaces_iter = self.source_operations.list_spaces(limit=1, fast=True)
first_space = next(spaces_iter, None)
except HTTPError as e:
status_code = _http_status(e)
@@ -1372,8 +1362,8 @@ class ConfluenceConnector(
if self.space:
try:
self.low_timeout_confluence_client.get_space(self.space)
except ApiError as e:
self.source_operations.get_space(space_key=self.space, fast=True)
except ConfluenceSpaceNotFoundError as e:
raise ConnectorValidationError(
"Invalid Confluence space key provided"
) from e
@@ -1396,7 +1386,7 @@ class ConfluenceConnector(
permission model and is validated via its own failure modes
during sync).
"""
client = self.low_timeout_confluence_client
client = self.source_operations._legacy_client(fast=True)
if not client.supports_rest_space_permissions():
return
backend/onyx/connectors/confluence/onyx_confluence.py
+15 -1481
View File
File diff suppressed because it is too large. Load diff
backend/onyx/connectors/confluence/source_operations.py
+1654
View File
File diff suppressed because it is too large. Load diff
backend/onyx/connectors/confluence/utils.py
+152 -117
View File
@@ -1,12 +1,12 @@
import math
import time
from collections.abc import Callable
from datetime import datetime, timedelta, timezone
from io import BytesIO
from pathlib import Path
from typing import TYPE_CHECKING, Any, TypeVar, cast
from typing import TYPE_CHECKING, Any
from urllib.parse import parse_qs, quote, urljoin, urlparse
import bs4
import requests
from pydantic import BaseModel
@@ -18,13 +18,16 @@ from onyx.configs.app_configs import (
from onyx.configs.constants import FileOrigin
from onyx.file_processing.extract_file_text import extract_file_text, get_file_ext
from onyx.file_processing.file_types import OnyxFileExtensions, OnyxMimeTypes
from onyx.file_processing.html_utils import format_document_soup
from onyx.file_processing.image_utils import store_image_and_create_section
from onyx.utils.datetime import datetime_to_utc
from onyx.utils.logger import setup_logger
from onyx.utils.retry_after import parse_retry_after_seconds
if TYPE_CHECKING:
from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
from onyx.connectors.confluence.source_operations import (
ConfluenceSourceOperations,
)
logger = setup_logger()
@@ -71,38 +74,11 @@ class AttachmentProcessingResult(BaseModel):
error: str | None = None
def _make_attachment_link(
confluence_client: "OnyxConfluence",
attachment: dict[str, Any],
parent_content_id: str | None,
is_cloud: bool,
) -> str | None:
download_link = ""
if is_cloud:
# https://developer.atlassian.com/cloud/confluence/rest/v1/api-group-content---attachments/#api-wiki-rest-api-content-id-child-attachment-attachmentid-download-get
if not parent_content_id:
logger.warning(
"parent_content_id is required to download attachments from Confluence Cloud!"
)
return None
download_link = (
confluence_client.url
+ f"/rest/api/content/{parent_content_id}/child/attachment/{attachment['id']}/download"
)
else:
download_link = confluence_client.url + attachment["_links"]["download"]
return download_link
def process_attachment(
confluence_client: "OnyxConfluence",
source_operations: "ConfluenceSourceOperations",
attachment: dict[str, Any],
parent_content_id: str | None,
allow_images: bool,
is_cloud: bool,
) -> AttachmentProcessingResult:
"""
Processes a Confluence attachment. If it's a document, extracts text,
@@ -119,14 +95,6 @@ def process_attachment(
error=f"Unsupported file type: {media_type}",
)
attachment_link = _make_attachment_link(
confluence_client, attachment, parent_content_id, is_cloud
)
if not attachment_link:
return AttachmentProcessingResult(
text=None, file_name=None, error="Failed to make attachment link"
)
attachment_size = attachment["extensions"]["fileSize"]
if media_type.startswith("image/"):
@@ -140,7 +108,7 @@ def process_attachment(
if attachment_size > CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD:
logger.warning(
"Skipping %s due to size. size=%s threshold=%s",
attachment_link,
attachment["title"],
attachment_size,
CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD,
)
@@ -150,28 +118,9 @@ def process_attachment(
error=f"Attachment text too long: {attachment_size} chars",
)
logger.info(
"Downloading attachment: title=%s length=%s link=%s",
attachment["title"],
attachment_size,
attachment_link,
raw_bytes = source_operations.download_attachment(
attachment=attachment, parent_content_id=parent_content_id
)
# Download the attachment
resp: requests.Response = confluence_client._session.get(attachment_link)
if resp.status_code != 200:
logger.warning(
"Failed to fetch %s with status code %s",
attachment_link,
resp.status_code,
)
return AttachmentProcessingResult(
text=None,
file_name=None,
error=f"Attachment download status code is {resp.status_code}",
)
raw_bytes = resp.content
if not raw_bytes:
return AttachmentProcessingResult(
text=None, file_name=None, error="attachment.content is None"
@@ -179,9 +128,7 @@ def process_attachment(
# Process image attachments
if media_type.startswith("image/"):
return _process_image_attachment(
confluence_client, attachment, raw_bytes, media_type
)
return _process_image_attachment(attachment, raw_bytes, media_type)
# Process document attachments
try:
@@ -211,7 +158,6 @@ def process_attachment(
def _process_image_attachment(
confluence_client: "OnyxConfluence", # noqa: ARG001
attachment: dict[str, Any],
raw_bytes: bytes,
media_type: str,
@@ -237,11 +183,10 @@ def _process_image_attachment(
def convert_attachment_to_content(
confluence_client: "OnyxConfluence",
source_operations: "ConfluenceSourceOperations",
attachment: dict[str, Any],
page_id: str,
allow_images: bool,
is_cloud: bool,
) -> tuple[str | None, str | None] | None:
"""
Facade function which:
@@ -259,9 +204,7 @@ def convert_attachment_to_content(
)
return None
result = process_attachment(
confluence_client, attachment, page_id, allow_images, is_cloud
)
result = process_attachment(source_operations, attachment, page_id, allow_images)
if result.error is not None:
logger.warning(
"Attachment %s encountered error: %s", attachment["title"], result.error
@@ -295,6 +238,12 @@ def build_confluence_document_id(
return final_url
def build_cql_url(cql: str, expand: str | None = None) -> str:
"""Returns the content-search path for a CQL query."""
expand_string = f"&expand={expand}" if expand else ""
return f"rest/api/content/search?cql={cql}{expand_string}"
def datetime_from_string(datetime_string: str) -> datetime:
return datetime_to_utc(datetime.fromisoformat(datetime_string))
@@ -337,52 +286,6 @@ def confluence_refresh_tokens(
return new_credentials
F = TypeVar("F", bound=Callable[..., Any])
# https://developer.atlassian.com/cloud/confluence/rate-limiting/
# this uses the native rate limiting option provided by the
# confluence client and otherwise applies a simpler set of error handling
def handle_confluence_rate_limit(confluence_call: F) -> F:
def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
MAX_RETRIES = 5
TIMEOUT = 600
timeout_at = time.monotonic() + TIMEOUT
for attempt in range(MAX_RETRIES):
if time.monotonic() > timeout_at:
raise TimeoutError(
f"Confluence call attempts took longer than {TIMEOUT} seconds."
)
try:
# we're relying more on the client to rate limit itself
# and applying our own retries in a more specific set of circumstances
return confluence_call(*args, **kwargs)
except requests.HTTPError as e:
delay_until = _handle_http_error(e, attempt, MAX_RETRIES)
logger.warning(
"HTTPError in confluence call. Retrying in %s seconds...",
delay_until,
)
while time.monotonic() < delay_until:
# in the future, check a signal here to exit
time.sleep(1)
except AttributeError as e:
# Some error within the Confluence library, unclear why it fails.
# Users reported it to be intermittent, so just retry
if attempt == MAX_RETRIES - 1:
raise e
logger.exception(
"Confluence Client raised an AttributeError. Retrying..."
)
time.sleep(5)
return cast(F, wrapped_call)
def _handle_http_error(e: requests.HTTPError, attempt: int, max_retries: int) -> int:
MIN_DELAY = 2
MAX_DELAY = 60
@@ -403,7 +306,7 @@ def _handle_http_error(e: requests.HTTPError, attempt: int, max_retries: int) ->
"403 error. This sometimes happens when we hit Confluence rate limits. Retrying in %s seconds...",
FORBIDDEN_RETRY_DELAY,
)
return FORBIDDEN_RETRY_DELAY
return math.ceil(time.monotonic() + FORBIDDEN_RETRY_DELAY)
raise e
@@ -478,3 +381,135 @@ def update_param_in_path(path: str, param: str, value: str) -> str:
+ "?"
+ "&".join(f"{k}={quote(v[0])}" for k, v in query_params.items())
)
def sanitize_attachment_title(title: str) -> str:
"""
Sanitize the attachment title to be a valid HTML attribute.
"""
return title.replace("<", "_").replace(">", "_").replace(" ", "_").replace(":", "_")
def extract_text_from_confluence_html(
source_operations: "ConfluenceSourceOperations",
confluence_object: dict[str, Any],
fetched_titles: set[str],
) -> str:
"""Parse a Confluence html page and replace the 'user Id' by the real
User Display Name
Args:
source_operations: The Confluence source-operations gateway
confluence_object (dict): The confluence object as a dict
fetched_titles (set[str]): The titles of the pages that have already been fetched
Returns:
str: loaded and formated Confluence page
"""
body = confluence_object["body"]
object_html = body.get("storage", body.get("view", {})).get("value")
soup = bs4.BeautifulSoup(object_html, "html.parser")
_remove_macro_stylings(soup=soup)
for date_span in soup.findAll("span", {"class": "date-lozenger-container"}):
date_span.replaceWith(date_span.get_text())
for user in soup.findAll("ri:user"):
user_id = (
user.attrs["ri:account-id"]
if "ri:account-id" in user.attrs
else user.get("ri:userkey")
)
if not user_id:
logger.warning(
"ri:userkey not found in ri:user element. Found attrs: %s",
user.attrs,
)
continue
# Include @ sign for tagging, more clear for LLM
user.replaceWith("@" + source_operations.get_user_display_name(user_id=user_id))
for html_page_reference in soup.findAll("ac:structured-macro"):
# Here, we only want to process page within page macros
if html_page_reference.attrs.get("ac:name") != "include":
continue
page_data = html_page_reference.find("ri:page")
if not page_data:
logger.warning(
"Skipping retrieval of %s because because page data is missing",
html_page_reference,
)
continue
page_title = page_data.attrs.get("ri:content-title")
if not page_title:
# only fetch pages that have a title
logger.warning(
"Skipping retrieval of %s because it has no title",
html_page_reference,
)
continue
if page_title in fetched_titles:
# prevent recursive fetching of pages
logger.debug("Skipping %s because it has already been fetched", page_title)
continue
fetched_titles.add(page_title)
# Wrap this in a try-except because there are some pages that might not exist
try:
page_contents = source_operations.find_page_by_title(title=page_title)
except Exception as e:
logger.warning(
"Error getting page contents for object %s: %s",
confluence_object,
e,
)
continue
if not page_contents:
continue
text_from_page = extract_text_from_confluence_html(
source_operations=source_operations,
confluence_object=page_contents,
fetched_titles=fetched_titles,
)
html_page_reference.replaceWith(text_from_page)
for html_link_body in soup.findAll("ac:link-body"):
# This extracts the text from inline links in the page so they can be
# represented in the document text as plain text
try:
text_from_link = html_link_body.text
html_link_body.replaceWith(f"(LINK TEXT: {text_from_link})")
except Exception as e:
logger.warning("Error processing ac:link-body: %s", e)
for html_attachment in soup.findAll("ri:attachment"):
# This extracts the text from inline attachments in the page so they can be
# represented in the document text as plain text
try:
html_attachment.replaceWith(
f"<attachment>{sanitize_attachment_title(html_attachment.attrs['ri:filename'])}</attachment>"
) # to be replaced later
except Exception as e:
logger.warning("Error processing ac:attachment: %s", e)
return format_document_soup(soup)
def _remove_macro_stylings(soup: bs4.BeautifulSoup) -> None:
for macro_root in soup.findAll("ac:structured-macro"):
if not isinstance(macro_root, bs4.Tag):
continue
macro_styling = macro_root.find(name="ac:parameter", attrs={"ac:name": "page"})
if not macro_styling or not isinstance(macro_styling, bs4.Tag):
continue
macro_styling.extract()
backend/onyx/connectors/cross_connector_utils/miscellaneous_utils.py
-12
View File
@@ -206,18 +206,6 @@ def get_cloudId(base_url: str) -> str:
return response.json()["cloudId"]
def credential_uses_scoped_token(credentials: dict[str, Any]) -> bool:
"""Whether an Atlassian credential says its API token has scopes.
The credential form stores a boolean; the edit form can store text, so
"true" counts and "false" does not.
"""
value = credentials.get("scoped_token")
if isinstance(value, str):
return value.strip().lower() == "true"
return value is True
def scoped_url(url: str, product: str) -> str:
parsed = urlparse(url)
base_url = parsed.scheme + "://" + parsed.netloc
backend/onyx/connectors/jira/connector.py
-6
View File
@@ -20,7 +20,6 @@ from onyx.configs.app_configs import (
)
from onyx.configs.constants import DocumentSource
from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
credential_uses_scoped_token,
is_atlassian_date_error,
time_str_to_utc,
)
@@ -705,11 +704,6 @@ class JiraConnector(
return project_key
def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
# The credential now carries the flag; older connectors set it in
# their own config.
self.scoped_token = self.scoped_token or credential_uses_scoped_token(
credentials
)
self._jira_client = build_jira_client(
credentials=credentials,
jira_base=self.jira_base,
backend/tests/daily/connectors/confluence/test_confluence_user_email_overrides.py
+2 -1
View File
@@ -1,7 +1,8 @@
import types
from unittest.mock import patch
from onyx.connectors.confluence.onyx_confluence import ConfluenceUser, OnyxConfluence
from onyx.connectors.confluence.models import ConfluenceUser
from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
from onyx.connectors.interfaces import CredentialsProviderInterface
backend/tests/unit/onyx/connectors/confluence/confluence_gateway_fakes.py
+23
View File
@@ -0,0 +1,23 @@
"""Builds a Confluence gateway around a fake transport client for unit tests."""
from typing import Any
from unittest import mock
from onyx.connectors.confluence.source_operations import ConfluenceSourceOperations
from onyx.connectors.interfaces import CredentialsProviderInterface
def gateway_with_client(
client: Any,
*,
wiki_base: str = "https://example.atlassian.net/wiki",
is_cloud: bool = False,
) -> ConfluenceSourceOperations:
"""Returns a gateway whose main and fast clients are both ``client``."""
gateway = ConfluenceSourceOperations(
credentials_provider=mock.Mock(spec=CredentialsProviderInterface),
connector_specific_config={"wiki_base": wiki_base, "is_cloud": is_cloud},
)
gateway._cached_client = client
gateway._cached_fast_client = client
return gateway
backend/tests/unit/onyx/connectors/confluence/test_attachment_pagination_400.py
+14 -8
View File
@@ -20,8 +20,14 @@ from onyx.connectors.confluence.connector import (
ConfluenceConnector,
_extract_page_id_from_url,
)
from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
from onyx.connectors.confluence.source_operations import ConfluenceSearchVariant
from onyx.connectors.confluence.source_operations import (
_OnyxConfluence as OnyxConfluence,
)
from onyx.connectors.models import ConnectorFailure, Document, DocumentFailure
from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import (
gateway_with_client,
)
_PAGE = {
"id": "111",
@@ -73,9 +79,9 @@ def _fetch_with_pagination_error(
with (
mock.patch.object(
ConfluenceConnector,
"confluence_client",
"source_operations",
new_callable=mock.PropertyMock,
return_value=fake_client,
return_value=gateway_with_client(fake_client),
),
mock.patch.object(
connector, "_maybe_yield_page_hierarchy_node", return_value=None
@@ -155,9 +161,9 @@ def _run_reindex(attachment_error_code: int | None) -> list[Any]:
with (
mock.patch.object(
ConfluenceConnector,
"confluence_client",
"source_operations",
new_callable=mock.PropertyMock,
return_value=fake_client,
return_value=gateway_with_client(fake_client),
),
mock.patch.object(
connector, "_yield_space_hierarchy_nodes", return_value=iter([])
@@ -209,12 +215,12 @@ def _slim_attachments_with(side_effects: list[Any]) -> tuple[list[Any], mock.Moc
with mock.patch.object(
ConfluenceConnector,
"confluence_client",
"source_operations",
new_callable=mock.PropertyMock,
return_value=fake_client,
return_value=gateway_with_client(fake_client),
):
results = connector._retrieve_attachments_for_slim_page(
"111", "space", None, None
"111", ConfluenceSearchVariant.SLIM, "space", None, None
)
return results, fake_client
backend/tests/unit/onyx/connectors/confluence/test_confcloud_77618_fallback.py
+9 -4
View File
@@ -19,12 +19,17 @@ from onyx.connectors.confluence.access import (
get_page_restrictions_with_per_ancestor_fetch as get_page_restrictions_with_per_ancestor_fetch_shim,
)
from onyx.connectors.confluence.connector import ConfluenceConnector
from onyx.connectors.confluence.onyx_confluence import (
from onyx.connectors.confluence.source_operations import (
Confcloud77618Error,
OnyxConfluence,
_is_confcloud_77618_response,
)
from onyx.connectors.confluence.source_operations import (
_OnyxConfluence as OnyxConfluence,
)
from onyx.connectors.interfaces import CredentialsProviderInterface
from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import (
gateway_with_client,
)
_CONFCLOUD_77618_BODY = (
'{"statusCode":404,"data":{"authorized":false,"valid":false,'
@@ -361,9 +366,9 @@ def test_pruning_expand_skips_restrictions_but_keeps_hierarchy(
with mock.patch.object(
ConfluenceConnector,
"confluence_client",
"source_operations",
new_callable=mock.PropertyMock,
return_value=fake_client,
return_value=gateway_with_client(fake_client),
):
with mock.patch.object(
confluence_connector,
backend/tests/unit/onyx/connectors/confluence/test_confluence_attachment_links.py
+10 -5
View File
@@ -5,8 +5,13 @@ from unittest import mock
import pytest
from onyx.connectors.confluence.connector import ConfluenceConnector
from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
from onyx.connectors.confluence.source_operations import (
_OnyxConfluence as OnyxConfluence,
)
from onyx.connectors.models import Document
from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import (
gateway_with_client,
)
_PAGE_ID = "111"
_ATTACHMENT_PAGE_ID = "222"
@@ -70,9 +75,9 @@ def test_attachment_section_link_uses_platform_specific_url(
with (
mock.patch.object(
ConfluenceConnector,
"confluence_client",
"source_operations",
new_callable=mock.PropertyMock,
return_value=confluence_client,
return_value=gateway_with_client(confluence_client),
),
mock.patch(
"onyx.connectors.confluence.connector.convert_attachment_to_content",
@@ -101,9 +106,9 @@ def test_attachment_failure_uses_attachment_document_id() -> None:
with (
mock.patch.object(
ConfluenceConnector,
"confluence_client",
"source_operations",
new_callable=mock.PropertyMock,
return_value=confluence_client,
return_value=gateway_with_client(confluence_client),
),
mock.patch(
"onyx.connectors.confluence.connector.convert_attachment_to_content",
backend/tests/unit/onyx/connectors/confluence/test_confluence_checkpointing.py
+13 -13
View File
@@ -12,7 +12,9 @@ from onyx.connectors.confluence.connector import (
ConfluenceCheckpoint,
ConfluenceConnector,
)
from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
from onyx.connectors.confluence.source_operations import (
_OnyxConfluence as OnyxConfluence,
)
from onyx.connectors.exceptions import (
CredentialExpiredError,
InsufficientPermissionsError,
@@ -25,6 +27,9 @@ from onyx.connectors.models import (
HierarchyNode,
SlimDocument,
)
from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import (
gateway_with_client,
)
from tests.unit.onyx.connectors.utils import (
load_everything_from_checkpoint_connector,
load_everything_from_checkpoint_connector_from_checkpoint,
@@ -69,8 +74,7 @@ def confluence_connector(
batch_size=2,
)
# Initialize the client directly
connector._confluence_client = mock_confluence_client
connector._low_timeout_confluence_client = mock_confluence_client
connector._source_operations = gateway_with_client(mock_confluence_client)
with patch("onyx.connectors.confluence.connector._SLIM_DOC_BATCH_SIZE", 2):
yield connector
@@ -151,8 +155,7 @@ def test_load_from_checkpoint_happy_path(
)
# Mock paginated_cql_retrieval to return our mock pages
confluence_client = confluence_connector._confluence_client
assert confluence_client is not None, "bad test setup"
confluence_client = confluence_connector.source_operations._client()
# Mock space retrieval for hierarchy nodes (called at start of first batch)
confluence_client.retrieve_confluence_spaces = MagicMock(
@@ -222,8 +225,7 @@ def test_load_from_checkpoint_with_page_processing_error(
mock_page2 = create_mock_page(id="2", title="Page 2")
# Mock paginated_cql_retrieval to return our mock pages
confluence_client = confluence_connector._confluence_client
assert confluence_client is not None, "bad test setup"
confluence_client = confluence_connector.source_operations._client()
# Mock space retrieval for hierarchy nodes (called at start of first batch)
confluence_client.retrieve_confluence_spaces = MagicMock(
@@ -317,8 +319,7 @@ def test_retrieve_all_slim_docs_perm_sync(
mock_page2 = create_mock_page(id="2")
# Mock paginated_cql_retrieval to return our mock pages
confluence_client = confluence_connector._confluence_client
assert confluence_client is not None, "bad test setup"
confluence_client = confluence_connector.source_operations._client()
# Mock space retrieval for hierarchy nodes
confluence_client.retrieve_confluence_spaces = MagicMock(
@@ -389,7 +390,7 @@ def test_validate_connector_settings_errors(
error = HTTPError(response=MagicMock(status_code=status_code))
with patch(
"onyx.connectors.confluence.onyx_confluence.OnyxConfluence.retrieve_confluence_spaces"
"onyx.connectors.confluence.source_operations._OnyxConfluence.retrieve_confluence_spaces"
) as mock_retrieve:
mock_retrieve.side_effect = error
@@ -402,7 +403,7 @@ def test_validate_connector_settings_success(
confluence_connector: ConfluenceConnector,
) -> None:
"""Test successful validation"""
low_client = confluence_connector.low_timeout_confluence_client
low_client = confluence_connector.source_operations._client(fast=True)
with (
patch.object(
low_client,
@@ -442,8 +443,7 @@ def test_checkpoint_progress(
)
# Mock paginated_cql_retrieval to return our mock pages
confluence_client = confluence_connector._confluence_client
assert confluence_client is not None, "bad test setup"
confluence_client = confluence_connector.source_operations._client()
# Mock space retrieval for hierarchy nodes (called at start of first batch)
confluence_client.retrieve_confluence_spaces = MagicMock(
backend/tests/unit/onyx/connectors/confluence/test_confluence_failure_document_id.py
+7 -2
View File
@@ -27,8 +27,13 @@ from onyx.connectors.confluence.connector import (
ConfluenceConnector,
_extract_page_id_from_url,
)
from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
from onyx.connectors.confluence.source_operations import (
_OnyxConfluence as OnyxConfluence,
)
from onyx.connectors.models import ConnectorFailure, Document
from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import (
gateway_with_client,
)
_PAGE_ID = "555"
@@ -47,7 +52,7 @@ def _make_connector() -> ConfluenceConnector:
)
confluence_client = mock.Mock(spec=OnyxConfluence)
confluence_client.paginated_cql_retrieval.return_value = iter([])
connector._confluence_client = confluence_client
connector._source_operations = gateway_with_client(confluence_client)
return connector
backend/tests/unit/onyx/connectors/confluence/test_confluence_source_operations.py
+237
View File
@@ -0,0 +1,237 @@
from types import SimpleNamespace
from typing import Any
from unittest import mock
import pytest
import requests
from atlassian.errors import ApiError
from onyx.connectors.confluence import source_operations
from onyx.connectors.confluence import utils as confluence_utils
from onyx.connectors.confluence.connector import ConfluenceConnector
from onyx.connectors.confluence.source_operations import (
ConfluenceProbeVariant,
ConfluenceRetriesExhaustedError,
ConfluenceSourceOperations,
ConfluenceSpaceNotFoundError,
_OnyxConfluence,
)
from onyx.connectors.exceptions import ConnectorValidationError
from onyx.connectors.interfaces import CredentialsProviderInterface
from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import (
gateway_with_client,
)
_WIKI_BASE = "https://confluence.example.com"
_ATTACHMENT: dict[str, Any] = {
"id": "att1",
"title": "spec.pdf",
"_links": {"download": "/download/attachments/1/spec.pdf"},
}
def _provider() -> mock.Mock:
provider = mock.Mock(spec=CredentialsProviderInterface)
provider.is_dynamic.return_value = False
provider.get_credentials.return_value = {
"confluence_username": "user",
"confluence_access_token": "token",
}
provider.get_provider_key.return_value = "key"
provider.__enter__ = mock.Mock(return_value=None)
provider.__exit__ = mock.Mock(return_value=None)
return provider
def _response(status_code: int, content: bytes = b"") -> requests.Response:
response = requests.Response()
response.status_code = status_code
response._content = content
response.url = _WIKI_BASE
return response
def _sdk_client(*responses: requests.Response) -> mock.Mock:
sdk = mock.Mock()
sdk.url = _WIKI_BASE
sdk.session.get.side_effect = list(responses)
return sdk
def _transport(sdk: mock.Mock, is_cloud: bool = False) -> _OnyxConfluence:
client = _OnyxConfluence(
is_cloud=is_cloud, url=_WIKI_BASE, credentials_provider=_provider()
)
client._confluence = sdk
client._kwargs = client.shared_base_kwargs
return client
@pytest.fixture
def fake_clock(monkeypatch: pytest.MonkeyPatch) -> None:
"""Makes retry waits instant: sleep advances a fake monotonic clock."""
now = 0.0
def monotonic() -> float:
return now
def sleep(seconds: float) -> None:
nonlocal now
now += seconds
fake_time = SimpleNamespace(monotonic=monotonic, sleep=sleep)
monkeypatch.setattr(source_operations, "time", fake_time)
monkeypatch.setattr(confluence_utils, "time", fake_time)
@pytest.mark.usefixtures("fake_clock")
def test_download_attachment_retries_server_errors() -> None:
# Precondition.
sdk = _sdk_client(_response(503), _response(200, b"file bytes"))
gateway = gateway_with_client(_transport(sdk), wiki_base=_WIKI_BASE)
# Under test.
content = gateway.download_attachment(attachment=_ATTACHMENT, parent_content_id="1")
# Postcondition.
assert content == b"file bytes"
assert sdk.session.get.call_count == 2
sdk.session.get.assert_called_with(_WIKI_BASE + "/download/attachments/1/spec.pdf")
def test_download_attachment_uses_the_refreshed_client() -> None:
# Precondition.
stale_sdk = _sdk_client(_response(200, b"stale"))
fresh_sdk = _sdk_client(_response(200, b"fresh"))
client = _transport(stale_sdk)
gateway = gateway_with_client(client, wiki_base=_WIKI_BASE)
# Under test.
with (
mock.patch.object(
client, "_renew_credentials", return_value=({"new": "creds"}, True)
),
mock.patch.object(
client, "_initialize_connection_helper", return_value=fresh_sdk
) as rebuild,
):
content = gateway.download_attachment(
attachment=_ATTACHMENT, parent_content_id="1"
)
# Postcondition.
assert content == b"fresh"
rebuild.assert_called_once()
stale_sdk.session.get.assert_not_called()
def test_download_attachment_raises_on_client_error() -> None:
# Precondition.
sdk = _sdk_client(_response(404))
gateway = gateway_with_client(_transport(sdk), wiki_base=_WIKI_BASE)
# Under test / postcondition.
with pytest.raises(requests.HTTPError):
gateway.download_attachment(attachment=_ATTACHMENT, parent_content_id="1")
assert sdk.session.get.call_count == 1
def test_download_attachment_does_not_retry_a_403() -> None:
# Precondition.
sdk = _sdk_client(_response(403), _response(200, b"file bytes"))
gateway = gateway_with_client(_transport(sdk), wiki_base=_WIKI_BASE)
# Under test / postcondition.
with pytest.raises(requests.HTTPError) as raised:
gateway.download_attachment(attachment=_ATTACHMENT, parent_content_id="1")
assert raised.value.response is not None
assert raised.value.response.status_code == 403
assert sdk.session.get.call_count == 1
@pytest.mark.usefixtures("fake_clock")
def test_exhausted_retries_raise_with_the_last_status() -> None:
# Precondition.
client = _transport(_sdk_client())
def always_rate_limited(_confluence: Any) -> bytes:
response = _response(429)
raise requests.HTTPError(response=response)
# Under test / postcondition.
with pytest.raises(ConfluenceRetriesExhaustedError) as raised:
client._call_with_retries(always_rate_limited)
assert raised.value.last_status_code == 429
def test_download_attachment_cloud_needs_parent_id() -> None:
gateway = gateway_with_client(
_transport(_sdk_client(), is_cloud=True), is_cloud=True
)
with pytest.raises(ValueError):
gateway.download_attachment(attachment=_ATTACHMENT, parent_content_id=None)
@pytest.mark.parametrize("scoped_token", [False, True])
def test_connector_probes_the_site_once(scoped_token: bool) -> None:
"""One probe and one tenant_info lookup per connector, even though
validation builds a second, low-timeout client."""
# Precondition.
connector = ConfluenceConnector(
wiki_base=_WIKI_BASE, is_cloud=True, space="KEY", scoped_token=scoped_token
)
with (
mock.patch.object(_OnyxConfluence, "_probe_connection") as probe,
mock.patch.object(
source_operations,
"scoped_url",
return_value="https://api.atlassian.com/ex/confluence/cloud-id",
) as resolve_scoped_url,
mock.patch.object(
ConfluenceSourceOperations,
"list_spaces",
return_value=iter([{"key": "KEY"}]),
),
mock.patch.object(
_OnyxConfluence, "get_space", create=True, return_value={"key": "KEY"}
),
):
# Under test.
connector.set_credentials_provider(_provider())
connector.validate_connector_settings()
# Postcondition.
probe.assert_called_once()
assert resolve_scoped_url.call_count == (1 if scoped_token else 0)
gateway = connector.source_operations
assert gateway._cached_client is not None
assert gateway._cached_fast_client is not None
assert gateway._cached_client is not gateway._cached_fast_client
def test_probe_site_rejects_a_mismatched_variant() -> None:
gateway = ConfluenceSourceOperations(
credentials_provider=_provider(),
connector_specific_config={"wiki_base": _WIKI_BASE, "is_cloud": False},
)
with pytest.raises(ValueError):
gateway.probe_site(variant=ConfluenceProbeVariant.SCOPED)
def test_get_space_translates_the_sdk_error() -> None:
# Precondition.
client = mock.Mock(spec=_OnyxConfluence)
client.get_space = mock.Mock(side_effect=ApiError("no such space"))
gateway = gateway_with_client(client)
connector = ConfluenceConnector(wiki_base=_WIKI_BASE, is_cloud=False, space="X")
connector._source_operations = gateway
# Under test / postcondition.
with pytest.raises(ConfluenceSpaceNotFoundError):
gateway.get_space(space_key="X")
client.retrieve_confluence_spaces.return_value = iter([{"key": "Y"}])
with pytest.raises(ConnectorValidationError, match="Invalid Confluence space key"):
connector.validate_connector_settings()
backend/tests/unit/onyx/connectors/confluence/test_confluence_user_cache_instance_isolation.py
+4 -3
View File
@@ -8,6 +8,7 @@ run in the same multi-tenant worker process."""
from unittest.mock import MagicMock
import onyx.connectors.confluence.onyx_confluence as onyx_confluence
import onyx.connectors.confluence.source_operations as source_operations
def _client(url: str) -> MagicMock:
@@ -64,13 +65,13 @@ def test_userkey_email_cache_is_instance_isolated() -> None:
def test_display_name_cache_is_instance_isolated() -> None:
onyx_confluence._USER_ID_TO_DISPLAY_NAME_CACHE.clear()
source_operations._USER_ID_TO_DISPLAY_NAME_CACHE.clear()
user_id = "user-1"
client_a = _client("https://a.example.com")
client_a.get_user_details_by_userkey.return_value = {"displayName": "Alice A"}
client_b = _client("https://b.example.com")
client_b.get_user_details_by_userkey.return_value = {"displayName": "Bob B"}
assert onyx_confluence._get_user(client_a, user_id) == "Alice A"
assert source_operations._get_user(client_a, user_id) == "Alice A"
# same id on a different instance must not see instance A's display name
assert onyx_confluence._get_user(client_b, user_id) == "Bob B"
assert source_operations._get_user(client_b, user_id) == "Bob B"
backend/tests/unit/onyx/connectors/confluence/test_extract_text.py
+9 -5
View File
@@ -1,16 +1,20 @@
from unittest import mock
from onyx.connectors.confluence.onyx_confluence import extract_text_from_confluence_html
from onyx.connectors.confluence.source_operations import ConfluenceSourceOperations
from onyx.connectors.confluence.utils import extract_text_from_confluence_html
from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import (
gateway_with_client,
)
def _make_confluence_object(html: str) -> dict:
return {"body": {"storage": {"value": html}}}
def _make_mock_client() -> mock.Mock:
def _make_gateway() -> ConfluenceSourceOperations:
client = mock.Mock()
client.paginated_cql_retrieval.return_value = iter([])
return client
return gateway_with_client(client)
def test_date_lozenge_text_is_preserved() -> None:
@@ -21,7 +25,7 @@ def test_date_lozenge_text_is_preserved() -> None:
" at noon.</p>"
)
result = extract_text_from_confluence_html(
confluence_client=_make_mock_client(),
source_operations=_make_gateway(),
confluence_object=_make_confluence_object(html),
fetched_titles=set(),
)
@@ -32,7 +36,7 @@ def test_page_without_date_lozenge_unaffected() -> None:
"""Pages with no date lozenge spans are processed normally."""
html = "<p>No dates here.</p>"
result = extract_text_from_confluence_html(
confluence_client=_make_mock_client(),
source_operations=_make_gateway(),
confluence_object=_make_confluence_object(html),
fetched_titles=set(),
)
backend/tests/unit/onyx/connectors/confluence/test_include_attachments_skip.py
+10 -5
View File
@@ -6,8 +6,13 @@ from typing import Any
from unittest import mock
from onyx.connectors.confluence.connector import ConfluenceConnector
from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
from onyx.connectors.confluence.source_operations import (
_OnyxConfluence as OnyxConfluence,
)
from onyx.connectors.models import SlimDocument
from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import (
gateway_with_client,
)
_PAGE_CQL = "PAGE_CQL"
_ATTACHMENT_CQL = "ATTACHMENT_CQL"
@@ -58,9 +63,9 @@ def _collect_slim_doc_ids(
with (
mock.patch.object(
ConfluenceConnector,
"confluence_client",
"source_operations",
new_callable=mock.PropertyMock,
return_value=fake_client,
return_value=gateway_with_client(fake_client),
),
mock.patch.object(
connector, "_yield_space_hierarchy_nodes", return_value=iter([])
@@ -124,9 +129,9 @@ def test_main_pass_skips_attachment_fetch_when_disabled() -> None:
with mock.patch.object(
ConfluenceConnector,
"confluence_client",
"source_operations",
new_callable=mock.PropertyMock,
return_value=fake_client,
return_value=gateway_with_client(fake_client),
):
docs, failures = connector._fetch_page_attachments(_PAGE)
backend/tests/unit/onyx/connectors/confluence/test_onyx_confluence.py
+11 -6
View File
@@ -7,12 +7,17 @@ import requests
from requests import HTTPError
from onyx.connectors.confluence import onyx_confluence as onyx_confluence_module
from onyx.connectors.confluence import source_operations as source_operations_module
from onyx.connectors.confluence.onyx_confluence import (
get_user_email_from_userkey__server,
)
from onyx.connectors.confluence.source_operations import (
_DEFAULT_PAGINATION_LIMIT,
_MINIMUM_PAGINATION_LIMIT,
ConfluenceRestSpacePermissionsNotAvailableError,
OnyxConfluence,
get_user_email_from_userkey__server,
)
from onyx.connectors.confluence.source_operations import (
_OnyxConfluence as OnyxConfluence,
)
from onyx.connectors.exceptions import (
ConnectorValidationError,
@@ -1493,16 +1498,16 @@ def test_token_refresh_keeps_the_oauth_site(monkeypatch: pytest.MonkeyPatch) ->
redis_client = mock.MagicMock()
redis_client.get.return_value = None
monkeypatch.setattr(
onyx_confluence_module, "get_redis_client", lambda **_: redis_client
source_operations_module, "get_redis_client", lambda **_: redis_client
)
monkeypatch.setattr(
onyx_confluence_module, "OAUTH_CONFLUENCE_CLOUD_CLIENT_ID", "id"
source_operations_module, "OAUTH_CONFLUENCE_CLOUD_CLIENT_ID", "id"
)
monkeypatch.setattr(
onyx_confluence_module, "OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET", "secret"
source_operations_module, "OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET", "secret"
)
monkeypatch.setattr(
onyx_confluence_module,
source_operations_module,
"confluence_refresh_tokens",
lambda *_: {
"confluence_access_token": "new-access",
backend/tests/unit/onyx/connectors/confluence/test_rate_limit_handler.py
-63
View File
@@ -1,63 +0,0 @@
from unittest.mock import Mock
import pytest
from requests import HTTPError
from onyx.connectors.confluence.utils import handle_confluence_rate_limit
@pytest.fixture
def mock_confluence_call() -> Mock:
return Mock()
# ***** Checking call count to sleep() won't correctly reflect test correctness
# especially since we really need to sleep multiple times and check for
# abort signals moving forward. Disabling this test for now until we come up with
# a better way forward.
# @pytest.mark.parametrize(
# "status_code,text,retry_after",
# [
# (429, "Rate limit exceeded", "5"),
# (200, "Rate limit exceeded", None),
# (429, "Some other error", "5"),
# ],
# )
# def test_rate_limit_handling(
# mock_confluence_call: Mock, status_code: int, text: str, retry_after: str | None
# ) -> None:
# with patch("time.sleep") as mock_sleep:
# mock_confluence_call.side_effect = [
# HTTPError(
# response=Mock(
# status_code=status_code,
# text=text,
# headers={"Retry-After": retry_after} if retry_after else {},
# )
# ),
# ] * 2 + ["Success"]
# handled_call = make_confluence_call_handle_rate_limit(mock_confluence_call)
# result = handled_call()
# assert result == "Success"
# assert mock_confluence_call.call_count == 3
# assert mock_sleep.call_count == 2
# if retry_after:
# mock_sleep.assert_called_with(int(retry_after))
# NOTE(rkuo): This tests an older version of rate limiting that is being deprecated
# and probably should go away soon.
def test_non_rate_limit_error(mock_confluence_call: Mock) -> None:
mock_confluence_call.side_effect = HTTPError(
response=Mock(status_code=500, text="Internal Server Error")
)
handled_call = handle_confluence_rate_limit(mock_confluence_call)
with pytest.raises(HTTPError):
handled_call()
assert mock_confluence_call.call_count == 5
backend/tests/unit/onyx/connectors/confluence/test_slim_doc_image_skip.py
+8 -3
View File
@@ -14,8 +14,13 @@ from unittest import mock
import pytest
from onyx.connectors.confluence.connector import ConfluenceConnector
from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
from onyx.connectors.confluence.source_operations import (
_OnyxConfluence as OnyxConfluence,
)
from onyx.connectors.models import SlimDocument
from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import (
gateway_with_client,
)
_PAGE_CQL = "PAGE_CQL"
_ATTACHMENT_CQL = "ATTACHMENT_CQL"
@@ -75,9 +80,9 @@ def _collect_slim_doc_ids(connector: ConfluenceConnector) -> list[str]:
with (
mock.patch.object(
ConfluenceConnector,
"confluence_client",
"source_operations",
new_callable=mock.PropertyMock,
return_value=fake_client,
return_value=gateway_with_client(fake_client),
),
mock.patch.object(
connector, "_yield_space_hierarchy_nodes", return_value=iter([])
backend/tests/unit/onyx/connectors/cross_connector_utils/test_miscellaneous_utils.py
+1 -24
View File
@@ -2,10 +2,7 @@ import datetime
import pytest
from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
credential_uses_scoped_token,
time_str_to_utc,
)
from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
def test_time_str_to_utc() -> None:
@@ -54,23 +51,3 @@ def test_time_str_to_utc_raises_on_impossible_dates() -> None:
):
with pytest.raises(ValueError):
time_str_to_utc(bad)
@pytest.mark.parametrize(
("value", "expected"),
[
(True, True),
(False, False),
("true", True),
("True", True),
("false", False),
("", False),
(None, False),
],
)
def test_credential_uses_scoped_token(value: object, expected: bool) -> None:
assert credential_uses_scoped_token({"scoped_token": value}) is expected
def test_credential_uses_scoped_token_without_the_key() -> None:
assert credential_uses_scoped_token({}) is False
pyproject.toml
+1 -1
View File
@@ -380,7 +380,7 @@ known-first-party = ["onyx", "ee", "tests", "shared_configs", "model_server"]
"backend/onyx/chat/process_message.py" = ["C901"] # 47
"backend/onyx/connectors/blob/connector.py" = ["C901"] # 24
"backend/onyx/connectors/confluence/connector.py" = ["C901"] # 22
"backend/onyx/connectors/confluence/onyx_confluence.py" = ["C901"] # 22
"backend/onyx/connectors/confluence/source_operations.py" = ["C901"] # 22
"backend/onyx/connectors/connector_runner.py" = ["C901"] # 26
"backend/onyx/connectors/drupal_wiki/connector.py" = ["C901"] # 21
"backend/onyx/connectors/github/connector.py" = ["C901"] # 28