From 3c8fd8047288754d7ece424fb8f53887176dbc5c Mon Sep 17 00:00:00 2001 From: Evan Lohn Date: Sat, 3 Oct 2026 04:38:44 +0000 Subject: [PATCH] refactor(connectors): Confluence source-operations gateway (#15453) Co-authored-by: Claude Opus 5.5 --- .../confluence/group_sync.py | 6 +- .../external_permissions/jira/group_sync.py | 9 +- .../onyx/connectors/confluence/connector.py | 146 +- .../connectors/confluence/onyx_confluence.py | 1496 +-------------- .../confluence/source_operations.py | 1654 +++++++++++++++++ backend/onyx/connectors/confluence/utils.py | 269 +-- .../miscellaneous_utils.py | 12 - backend/onyx/connectors/jira/connector.py | 6 - .../test_confluence_user_email_overrides.py | 3 +- .../confluence/confluence_gateway_fakes.py | 23 + .../test_attachment_pagination_400.py | 22 +- .../test_confcloud_77618_fallback.py | 13 +- .../test_confluence_attachment_links.py | 15 +- .../test_confluence_checkpointing.py | 26 +- .../test_confluence_failure_document_id.py | 9 +- .../test_confluence_source_operations.py | 237 +++ ...onfluence_user_cache_instance_isolation.py | 7 +- .../confluence/test_extract_text.py | 14 +- .../test_include_attachments_skip.py | 15 +- .../confluence/test_onyx_confluence.py | 17 +- .../confluence/test_rate_limit_handler.py | 63 - .../confluence/test_slim_doc_image_skip.py | 11 +- .../test_miscellaneous_utils.py | 25 +- pyproject.toml | 2 +- 24 files changed, 2251 insertions(+), 1849 deletions(-) create mode 100644 backend/onyx/connectors/confluence/source_operations.py create mode 100644 backend/tests/unit/onyx/connectors/confluence/confluence_gateway_fakes.py create mode 100644 backend/tests/unit/onyx/connectors/confluence/test_confluence_source_operations.py delete mode 100644 backend/tests/unit/onyx/connectors/confluence/test_rate_limit_handler.py diff --git a/backend/ee/onyx/external_permissions/confluence/group_sync.py b/backend/ee/onyx/external_permissions/confluence/group_sync.py index ccafcce18d..b2e3c7519f 100644 --- a/backend/ee/onyx/external_permissions/confluence/group_sync.py +++ b/backend/ee/onyx/external_permissions/confluence/group_sync.py @@ -9,9 +9,6 @@ from onyx.connectors.confluence.onyx_confluence import ( get_user_email_from_username__server, ) from onyx.connectors.credentials_provider import OnyxDBCredentialsProvider -from onyx.connectors.cross_connector_utils.miscellaneous_utils import ( - credential_uses_scoped_token, -) from onyx.db.engine.sql_engine import get_session_with_current_tenant from onyx.db.models import ConnectorCredentialPair from onyx.db.users import get_all_users @@ -168,10 +165,9 @@ def confluence_group_sync( tenant_id, cc_pair.connector.source, cc_pair.credential_id ) is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False) - # The credential carries the flag; older connectors set it in their config. scoped_token = cc_pair.connector.connector_specific_config.get( "scoped_token", False - ) or credential_uses_scoped_token(provider.get_credentials()) + ) wiki_base: str = cc_pair.connector.connector_specific_config["wiki_base"] url = wiki_base.rstrip("/") diff --git a/backend/ee/onyx/external_permissions/jira/group_sync.py b/backend/ee/onyx/external_permissions/jira/group_sync.py index 400c37dbe6..402e168823 100644 --- a/backend/ee/onyx/external_permissions/jira/group_sync.py +++ b/backend/ee/onyx/external_permissions/jira/group_sync.py @@ -7,9 +7,6 @@ from jira.exceptions import JIRAError from ee.onyx.db.external_perm import ExternalUserGroup from ee.onyx.external_permissions.utils import credential_json -from onyx.connectors.cross_connector_utils.miscellaneous_utils import ( - credential_uses_scoped_token, -) from onyx.connectors.jira.utils import build_jira_client from onyx.db.models import ConnectorCredentialPair from onyx.utils.logger import setup_logger @@ -158,17 +155,15 @@ def jira_group_sync( Streams group-by-group rather than accumulating all groups in memory. """ jira_base_url = cc_pair.connector.connector_specific_config.get("jira_base_url", "") - credentials = credential_json(cc_pair) - # The credential carries the flag; older connectors set it in their config. scoped_token = cc_pair.connector.connector_specific_config.get( "scoped_token", False - ) or credential_uses_scoped_token(credentials) + ) if not jira_base_url: raise ValueError("No jira_base_url found in connector config") jira_client = build_jira_client( - credentials=credentials, + credentials=credential_json(cc_pair), jira_base=jira_base_url, scoped_token=scoped_token, ) diff --git a/backend/onyx/connectors/confluence/connector.py b/backend/onyx/connectors/confluence/connector.py index ad90fb6927..42bce4818b 100644 --- a/backend/onyx/connectors/confluence/connector.py +++ b/backend/onyx/connectors/confluence/connector.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta, timezone from typing import Any from urllib.parse import quote -from atlassian.errors import ApiError from requests.exceptions import HTTPError from typing_extensions import override @@ -22,21 +21,25 @@ from onyx.connectors.confluence.access import ( get_page_restrictions, get_page_restrictions_with_per_ancestor_fetch, ) -from onyx.connectors.confluence.onyx_confluence import ( +from onyx.connectors.confluence.onyx_confluence import OnyxConfluence +from onyx.connectors.confluence.source_operations import ( Confcloud77618Error, - OnyxConfluence, - extract_text_from_confluence_html, + ConfluenceProbeVariant, + ConfluenceSearchVariant, + ConfluenceSourceOperations, + ConfluenceSpaceNotFoundError, ) from onyx.connectors.confluence.utils import ( build_confluence_document_id, + build_cql_url, convert_attachment_to_content, datetime_from_string, + extract_text_from_confluence_html, update_param_in_path, validate_attachment_filetype, ) from onyx.connectors.credentials_provider import OnyxStaticCredentialsProvider from onyx.connectors.cross_connector_utils.miscellaneous_utils import ( - credential_uses_scoped_token, is_atlassian_date_error, ) from onyx.connectors.exceptions import ( @@ -200,8 +203,7 @@ class ConfluenceConnector( self.timezone_offset = timezone_offset self.scoped_token = scoped_token self.include_attachments = include_attachments - self._confluence_client: OnyxConfluence | None = None - self._low_timeout_confluence_client: OnyxConfluence | None = None + self._source_operations: ConfluenceSourceOperations | None = None self._fetched_titles: set[str] = set() self.allow_images = False @@ -241,16 +243,6 @@ class ConfluenceConnector( self.timezone: timezone = timezone(offset=timedelta(hours=timezone_offset)) self.credentials_provider: CredentialsProviderInterface | None = None - self.probe_kwargs = { - "max_backoff_retries": 6, - "max_backoff_seconds": 10, - } - - self.final_kwargs = { - "max_backoff_retries": 10, - "max_backoff_seconds": 60, - } - # deprecated self.continue_on_failure = continue_on_failure @@ -264,7 +256,7 @@ class ConfluenceConnector( """Yield hierarchy nodes for all spaces we're indexing.""" space_keys = [self.space] if self.space else None - for space in self.confluence_client.retrieve_confluence_spaces( + for space in self.source_operations.list_spaces( space_keys=space_keys, limit=50, ): @@ -416,51 +408,38 @@ class ConfluenceConnector( ) @property - def confluence_client(self) -> OnyxConfluence: - if self._confluence_client is None: + def source_operations(self) -> ConfluenceSourceOperations: + if self._source_operations is None: raise ConnectorMissingCredentialError("Confluence") - return self._confluence_client + return self._source_operations @property - def low_timeout_confluence_client(self) -> OnyxConfluence: - if self._low_timeout_confluence_client is None: - raise ConnectorMissingCredentialError("Confluence") - return self._low_timeout_confluence_client + def _perm_sync_client(self) -> OnyxConfluence: + """Temporary: the EE perm-sync helpers still take the client. #15454 + moves them to gateway operations.""" + return self.source_operations._legacy_client() def set_credentials_provider( self, credentials_provider: CredentialsProviderInterface ) -> None: self.credentials_provider = credentials_provider - # The credential now carries the flag; older connectors set it in - # their own config. - self.scoped_token = self.scoped_token or credential_uses_scoped_token( - credentials_provider.get_credentials() + source_operations = ConfluenceSourceOperations( + credentials_provider=credentials_provider, + connector_specific_config={ + "wiki_base": self.wiki_base, + "is_cloud": self.is_cloud, + "scoped_token": self.scoped_token, + }, ) - # raises exception if there's a problem - confluence_client = OnyxConfluence( - is_cloud=self.is_cloud, - url=self.wiki_base, - credentials_provider=credentials_provider, - scoped_token=self.scoped_token, + source_operations.probe_site( + variant=( + ConfluenceProbeVariant.SCOPED + if self.scoped_token + else ConfluenceProbeVariant.UNSCOPED + ) ) - confluence_client._probe_connection(**self.probe_kwargs) - confluence_client._initialize_connection(**self.final_kwargs) - - self._confluence_client = confluence_client - - # create a low timeout confluence client for sync flows - low_timeout_confluence_client = OnyxConfluence( - is_cloud=self.is_cloud, - url=self.wiki_base, - credentials_provider=credentials_provider, - timeout=3, - scoped_token=self.scoped_token, - ) - low_timeout_confluence_client._probe_connection(**self.probe_kwargs) - low_timeout_confluence_client._initialize_connection(**self.final_kwargs) - - self._low_timeout_confluence_client = low_timeout_confluence_client + self._source_operations = source_operations def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: raise NotImplementedError("Use set_credentials_provider with this connector.") @@ -520,13 +499,13 @@ class ConfluenceConnector( comment_cql += self.cql_label_filter expand = ",".join(_COMMENT_EXPANSION_FIELDS) - for comment in self.confluence_client.paginated_cql_retrieval( + for comment in self.source_operations.search_comments( cql=comment_cql, expand=expand, ): comment_string += "\nComment:\n" comment_string += extract_text_from_confluence_html( - confluence_client=self.confluence_client, + source_operations=self.source_operations, confluence_object=comment, fetched_titles=set(), ) @@ -551,7 +530,7 @@ class ConfluenceConnector( # Get the page content page_content = extract_text_from_confluence_html( - self.confluence_client, page, self._fetched_titles + self.source_operations, page, self._fetched_titles ) # Create the main section for the page content @@ -651,7 +630,8 @@ class ConfluenceConnector( page_hierarchy_node_yielded = False try: - for attachment in self.confluence_client.paginated_cql_retrieval( + for attachment in self.source_operations.search_attachments( + variant=ConfluenceSearchVariant.CONTENT, cql=attachment_query, expand=",".join(_ATTACHMENT_EXPANSION_FIELDS), ): @@ -705,11 +685,10 @@ class ConfluenceConnector( continue try: response = convert_attachment_to_content( - confluence_client=self.confluence_client, + source_operations=self.source_operations, attachment=attachment, page_id=_get_page_id(page), allow_images=self.allow_images, - is_cloud=self.is_cloud, ) if response is None: continue @@ -895,8 +874,8 @@ class ConfluenceConnector( def store_next_page_url(next_page_url: str) -> None: checkpoint.next_page_url = next_page_url - for page in self.confluence_client.paginated_page_retrieval( - cql_url=page_query_url, + for page in self.source_operations.search_pages_from_url( + url=page_query_url, limit=self.batch_size, next_page_callback=store_next_page_url, ): @@ -940,9 +919,7 @@ class ConfluenceConnector( or paginated_page_retrieval methods. """ page_query = self._construct_page_cql_query(start, end) - cql_url = self.confluence_client.build_cql_url( - page_query, expand=",".join(_PAGE_EXPANSION_FIELDS) - ) + cql_url = build_cql_url(page_query, expand=",".join(_PAGE_EXPANSION_FIELDS)) return update_param_in_path(cql_url, "limit", str(limit)) @override @@ -1005,7 +982,7 @@ class ConfluenceConnector( space_level_access: dict[str, ExternalAccess] = ( get_all_space_permissions( - self.confluence_client, self.is_cloud, add_prefix=True + self._perm_sync_client, self.is_cloud, add_prefix=True ) if include_permissions else {} @@ -1022,7 +999,12 @@ class ConfluenceConnector( cql = "type=page and id IN (%s)" % quoted_ids seen_page_ids: set[str] = set() - for page in self.confluence_client.paginated_cql_retrieval( + for page in self.source_operations.search_pages( + variant=( + ConfluenceSearchVariant.WITH_RESTRICTIONS + if include_permissions + else ConfluenceSearchVariant.CONTENT + ), cql=cql, expand=",".join(expand_fields), ): @@ -1055,7 +1037,7 @@ class ConfluenceConnector( if include_permissions: space_key = page.get("space", {}).get("key") or "" doc_or_failure.external_access = get_page_restrictions( - self.confluence_client, + self._perm_sync_client, doc_or_failure.id, page.get("restrictions") or {}, page.get("ancestors", []), @@ -1136,6 +1118,7 @@ class ConfluenceConnector( def _retrieve_attachments_for_slim_page( self, page_id: str, + variant: ConfluenceSearchVariant, expand: str, start: SecondsSinceUnixEpoch | None, end: SecondsSinceUnixEpoch | None, @@ -1153,10 +1136,12 @@ class ConfluenceConnector( while True: try: return list( - self.confluence_client.cql_paginate_all_expansions( + self.source_operations.search_attachments( + variant=variant, cql=attachment_query, expand=expand, limit=_SLIM_DOC_BATCH_SIZE, + follow_expansion_links=True, ) ) except HTTPError as e: @@ -1187,6 +1172,11 @@ class ConfluenceConnector( # Pruning skips the restrictions expand (the CONFCLOUD-77618 # trigger) but still needs space + ancestors for hierarchy. + search_variant = ( + ConfluenceSearchVariant.WITH_RESTRICTIONS + if include_permissions + else ConfluenceSearchVariant.SLIM + ) if not include_permissions: restrictions_expand = ",".join(_PRUNING_EXPANSION_FIELDS) elif expand_per_page: @@ -1197,7 +1187,7 @@ class ConfluenceConnector( space_level_access_info: dict[str, ExternalAccess] = {} if include_permissions: space_level_access_info = get_all_space_permissions( - self.confluence_client, self.is_cloud + self._perm_sync_client, self.is_cloud ) # Yield space hierarchy nodes first @@ -1214,7 +1204,7 @@ class ConfluenceConnector( ) -> ExternalAccess | None: if expand_per_page: resolved = get_page_restrictions_with_per_ancestor_fetch( - self.confluence_client, + self._perm_sync_client, doc_id, restrictions, ancestors, @@ -1222,7 +1212,7 @@ class ConfluenceConnector( ) else: resolved = get_page_restrictions( - self.confluence_client, doc_id, restrictions, ancestors + self._perm_sync_client, doc_id, restrictions, ancestors ) return resolved or ( space_level_access_info.get(space_key) if space_key else None @@ -1230,10 +1220,12 @@ class ConfluenceConnector( # Query pages (with optional time filtering for indexing_start) page_query = self._construct_page_cql_query(start, end) - for page in self.confluence_client.cql_paginate_all_expansions( + for page in self.source_operations.search_pages( + variant=search_variant, cql=page_query, expand=restrictions_expand, limit=_SLIM_DOC_BATCH_SIZE, + follow_expansion_links=True, ): # Yield ancestor hierarchy nodes for this page doc_metadata_list.extend(self._yield_ancestor_hierarchy_nodes(page)) @@ -1271,7 +1263,7 @@ class ConfluenceConnector( attachment_results: Iterable[dict[str, Any]] = () if self.include_attachments: attachment_results = self._retrieve_attachments_for_slim_page( - _get_page_id(page), restrictions_expand, start, end + _get_page_id(page), search_variant, restrictions_expand, start, end ) for attachment in attachment_results: # admission must mirror the main indexing pass @@ -1342,9 +1334,7 @@ class ConfluenceConnector( def validate_connector_settings(self) -> None: try: - spaces_iter = self.low_timeout_confluence_client.retrieve_confluence_spaces( - limit=1, - ) + spaces_iter = self.source_operations.list_spaces(limit=1, fast=True) first_space = next(spaces_iter, None) except HTTPError as e: status_code = _http_status(e) @@ -1372,8 +1362,8 @@ class ConfluenceConnector( if self.space: try: - self.low_timeout_confluence_client.get_space(self.space) - except ApiError as e: + self.source_operations.get_space(space_key=self.space, fast=True) + except ConfluenceSpaceNotFoundError as e: raise ConnectorValidationError( "Invalid Confluence space key provided" ) from e @@ -1396,7 +1386,7 @@ class ConfluenceConnector( permission model and is validated via its own failure modes during sync). """ - client = self.low_timeout_confluence_client + client = self.source_operations._legacy_client(fast=True) if not client.supports_rest_space_permissions(): return diff --git a/backend/onyx/connectors/confluence/onyx_confluence.py b/backend/onyx/connectors/confluence/onyx_confluence.py index b314ce95a2..0fdfc59987 100644 --- a/backend/onyx/connectors/confluence/onyx_confluence.py +++ b/backend/onyx/connectors/confluence/onyx_confluence.py @@ -1,1317 +1,37 @@ -""" -# README (notes on Confluence pagination): +"""EE perm-sync helpers that still use the Confluence client directly. -We've noticed that the `search/users` and `users/memberof` endpoints for Confluence Cloud use offset-based pagination as -opposed to cursor-based. We also know that page-retrieval uses cursor-based pagination. - -Our default pagination strategy right now for cloud is to assume cursor-based. -However, if you notice that a cloud API is not being properly paginated (i.e., if the `_links.next` is not appearing in the -returned payload), then you can force offset-based pagination. - -# TODO (@raunakab) -We haven't explored all of the cloud APIs' pagination strategies. @raunakab take time to go through this and figure them out. +Temporary: #15454 moves these calls to ``ConfluenceSourceOperations`` operations +and deletes this module, including the re-exports below. """ -import json -import time -from collections.abc import Callable, Generator, Iterator -from datetime import datetime, timedelta, timezone -from typing import Any, TypeVar, cast -from urllib.parse import quote - -import bs4 -import requests -from atlassian import Confluence from requests import HTTPError -from onyx.configs.app_configs import ( - CONFLUENCE_CONNECTOR_USER_PROFILES_OVERRIDE, - OAUTH_CONFLUENCE_CLOUD_CLIENT_ID, - OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET, +from onyx.connectors.confluence.source_operations import ( + ConfluenceRestSpacePermissionsNotAvailableError, ) -from onyx.connectors.confluence.models import ConfluenceUser -from onyx.connectors.confluence.user_profile_override import ( - process_confluence_user_profiles_override, +from onyx.connectors.confluence.source_operations import ( + _OnyxConfluence as OnyxConfluence, ) -from onyx.connectors.confluence.utils import ( - _handle_http_error, - confluence_refresh_tokens, - get_start_param_from_url, - update_param_in_path, -) -from onyx.connectors.cross_connector_utils.miscellaneous_utils import scoped_url -from onyx.connectors.exceptions import ( - ConnectorValidationError, - InsufficientPermissionsError, -) -from onyx.connectors.interfaces import CredentialsProviderInterface -from onyx.file_processing.html_utils import format_document_soup -from onyx.redis.redis_pool import get_redis_client -from onyx.redis.tenant_redis_client import TenantRedisClient from onyx.utils.logger import setup_logger +__all__ = [ + "ConfluenceRestSpacePermissionsNotAvailableError", + "OnyxConfluence", + "get_user_email_from_userkey__server", + "get_user_email_from_username__server", +] + logger = setup_logger() -# Set by the Confluence Cloud OAuth finalize step. -_OAUTH_SITE_KEYS = ("cloud_name", "wiki_base") - - -F = TypeVar("F", bound=Callable[..., Any]) - - -# https://jira.atlassian.com/browse/CONFCLOUD-76433 -_PROBLEMATIC_EXPANSIONS = "body.storage.value" -_REPLACEMENT_EXPANSIONS = "body.view.value" - -# CONFCLOUD-77618 / CONFCLOUD-76424: ancestor-restrictions expand on -# content/search 404s the whole batch when an ancestor is unreadable -# (draft / outdated / trashed). We detect the body signature and raise. -_ANCESTOR_RESTRICTIONS_EXPAND_PREFIX = "ancestors.restrictions.read.restrictions." -_CONFCLOUD_77618_404_BODY_SIGNATURES = ( - "No content with id", - "Cannot find content. Outdated version/old_draft/trashed", -) - -_USER_NOT_FOUND = "Unknown Confluence User" -# All three caches are keyed by (confluence instance base url, identifier). The +# Both caches are keyed by (confluence instance base url, identifier). The # Confluence Server/DC username and userKey namespaces are per-instance, so a bare # identifier key would let one instance's user resolve to another instance's email # when several Confluence connectors run in the same multi-tenant worker process. -_USER_ID_TO_DISPLAY_NAME_CACHE: dict[tuple[str, str], str | None] = {} _USER_EMAIL_CACHE: dict[tuple[str, str], str | None] = {} # Separate cache from _USER_EMAIL_CACHE: the DC 9.1+ REST space-permissions # response only includes a user's userKey (CONFSERVER-100505), not their # username, so we have to resolve email by a different identifier. _USER_KEY_TO_EMAIL_CACHE: dict[tuple[str, str], str | None] = {} -_DEFAULT_PAGINATION_LIMIT = 1000 -_MINIMUM_PAGINATION_LIMIT = 5 - -_SERVER_ERROR_CODES = {500, 502, 503, 504} - -_CONFLUENCE_SPACES_API_V1 = "rest/api/space" -_CONFLUENCE_SPACES_API_V2 = "wiki/api/v2/spaces" - -# Atlassian KB documenting how Secure Administrator Sessions (WebSudo) breaks -# admin JSON-RPC calls. Surfaced in the validation error so admins can act on -# it without our help. -_WEBSUDO_KB_URL = ( - "https://support.atlassian.com/confluence/kb/" - "json-rpc-api-request-returns-websudorequiredexception-on-confluence/" -) -# Cap how much of an unparseable JSON-RPC response body we put in the error -# message. WebSudo / login HTML pages are well under this; the cap is a -# defense against a runaway response (e.g. a multi-MB error page) ending up -# in our logs and validation surface. -_JSONRPC_ERROR_BODY_SNIPPET_CHARS = 1000 - -# DC 9.1.0 is the first DC release with the REST API for space permissions -# (CONFSERVER-78176). Older DC versions still need the legacy JSON-RPC -# fallback. Server / Data Center only -- Cloud has its own permissions API -# and is branched on `is_cloud` upstream of any version check. -_MIN_DC_VERSION_FOR_REST_SPACE_PERMISSIONS: tuple[int, int] = (9, 1) - -# Atlassian's documented Confluence DC endpoint for build information, -# under the "Server Information" REST API group. Returns a JSON object -# whose top-level `version` field is the upstream Confluence version -# (e.g. "10.2.10"). Confirmed present in the v8.4, v9.3, and v10.x -# DC REST API references; we previously probed the Jira-style -# `/rest/api/serverInfo` slug, which 404s on Confluence DC 10.x. -_DC_SERVER_INFORMATION_PATH = "rest/api/server-information" - - -class ConfluenceRateLimitError(Exception): - pass - - -class Confcloud77618Error(Exception): - """Signal to the perm-sync caller that the ancestor-restrictions - expand 404'd on a draft / outdated / trashed ancestor and the run - must restart with per-page restriction lookups.""" - - def __init__(self, url: str, body: str) -> None: - super().__init__( - f"CONFCLOUD-77618: ancestor-restrictions expand 404 from " - f"{url}: {body[:500]}" - ) - self.url = url - self.body = body - - -class ConfluenceRestSpacePermissionsNotAvailableError(Exception): - """Raised by REST-API space-permissions calls when the endpoint is missing - on the upstream Confluence DC instance (e.g. DC < 9.1.0 returning 404). - - Callers use this as a signal to fall back to the legacy JSON-RPC path. - """ - - -def _is_confcloud_77618_response(response: requests.Response) -> bool: - """Body-signature match for the CONFCLOUD-77618 / CONFCLOUD-76424 404 - so unrelated 404s still propagate.""" - if response.status_code != 404: - return False - body = response.text - return any(sig in body for sig in _CONFCLOUD_77618_404_BODY_SIGNATURES) - - -class OnyxConfluence: - """ - This is a custom Confluence class that: - - A. overrides the default Confluence class to add a custom CQL method. - B. - This is necessary because the default Confluence class does not properly support cql expansions. - All methods are automatically wrapped with handle_confluence_rate_limit. - """ - - CREDENTIAL_PREFIX = "connector:confluence:credential" - CREDENTIAL_TTL = 300 # 5 min - PROBE_TIMEOUT = 5 # 5 seconds - - def __init__( - self, - is_cloud: bool, - url: str, - credentials_provider: CredentialsProviderInterface, - timeout: int | None = None, - scoped_token: bool = False, - # should generally not be passed in, but making it overridable for - # easier testing - confluence_user_profiles_override: list[dict[str, str]] | None = ( - CONFLUENCE_CONNECTOR_USER_PROFILES_OVERRIDE - ), - ) -> None: - self.base_url = url #'/'.join(url.rstrip("/").split("/")[:-1]) - url = scoped_url(url, "confluence") if scoped_token else url - - self._is_cloud = is_cloud - self._url = url.rstrip("/") - self._credentials_provider = credentials_provider - self.scoped_token = scoped_token - self.redis_client: TenantRedisClient | None = None - self.static_credentials: dict[str, Any] | None = None - if self._credentials_provider.is_dynamic(): - self.redis_client = get_redis_client( - tenant_id=credentials_provider.get_tenant_id() - ) - else: - self.static_credentials = self._credentials_provider.get_credentials() - - self._confluence = Confluence(url) - self.credential_key: str = ( - self.CREDENTIAL_PREFIX - + f":credential_{self._credentials_provider.get_provider_key()}" - ) - - self._kwargs: Any = None - - self.shared_base_kwargs: dict[str, str | int | bool] = { - "api_version": "cloud" if is_cloud else "latest", - "backoff_and_retry": False, - "cloud": is_cloud, - } - if timeout: - self.shared_base_kwargs["timeout"] = timeout - - self._confluence_user_profiles_override = ( - process_confluence_user_profiles_override(confluence_user_profiles_override) - if confluence_user_profiles_override - else None - ) - - # Cached result of the server-information probe, populated on - # first `get_server_version()` call. _server_version_probed=True - # with _server_version=None means "we tried and the probe - # failed", so we don't keep retrying every space-permissions sync. - self._server_version: tuple[int, int] | None = None - self._server_version_probed: bool = False - - def _renew_credentials(self) -> tuple[dict[str, Any], bool]: - """credential_json - the current json credentials - Returns a tuple - 1. The up to date credentials - 2. True if the credentials were updated - - This method is intended to be used within a distributed lock. - Lock, call this, update credentials if the tokens were refreshed, then release - """ - # static credentials are preloaded, so no locking/redis required - if self.static_credentials: - return self.static_credentials, False - - if not self.redis_client: - raise RuntimeError("self.redis_client is None") - - # dynamic credentials need locking - # check redis first, then fallback to the DB - credential_bytes = self.redis_client.get(self.credential_key) - if credential_bytes is not None: - credential_str = credential_bytes.decode("utf-8") - credential_json: dict[str, Any] = json.loads(credential_str) - else: - credential_json = self._credentials_provider.get_credentials() - - if "confluence_refresh_token" not in credential_json: - # static credentials ... cache them permanently and return - self.static_credentials = credential_json - return credential_json, False - - if not OAUTH_CONFLUENCE_CLOUD_CLIENT_ID: - raise RuntimeError("OAUTH_CONFLUENCE_CLOUD_CLIENT_ID must be set!") - - if not OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET: - raise RuntimeError("OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET must be set!") - - # check if we should refresh tokens. we're deciding to refresh halfway - # to expiration - now = datetime.now(timezone.utc) - created_at = datetime.fromisoformat(credential_json["created_at"]) - expires_in: int = credential_json["expires_in"] - renew_at = created_at + timedelta(seconds=expires_in // 2) - if now <= renew_at: - # cached/current credentials are reasonably up to date - return credential_json, False - - # we need to refresh - logger.info("Renewing Confluence Cloud credentials...") - new_credentials = { - # The refresh response has no site info; keep what the OAuth - # finalize step stored. - **{ - key: credential_json[key] - for key in _OAUTH_SITE_KEYS - if key in credential_json - }, - **confluence_refresh_tokens( - OAUTH_CONFLUENCE_CLOUD_CLIENT_ID, - OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET, - credential_json["cloud_id"], - credential_json["confluence_refresh_token"], - ), - } - - # store the new credentials to redis and to the db thru the provider - # redis: we use a 5 min TTL because we are given a 10 minute grace period - # when keys are rotated. it's easier to expire the cached credentials - # reasonably frequently rather than trying to handle strong synchronization - # between the db and redis everywhere the credentials might be updated - new_credential_str = json.dumps(new_credentials) - self.redis_client.set( - self.credential_key, new_credential_str, nx=True, ex=self.CREDENTIAL_TTL - ) - self._credentials_provider.set_credentials(new_credentials) - - return new_credentials, True - - @staticmethod - def _make_oauth2_dict(credentials: dict[str, Any]) -> dict[str, Any]: - oauth2_dict: dict[str, Any] = {} - if "confluence_refresh_token" in credentials: - oauth2_dict["client_id"] = OAUTH_CONFLUENCE_CLOUD_CLIENT_ID - oauth2_dict["token"] = {} - oauth2_dict["token"]["access_token"] = credentials[ - "confluence_access_token" - ] - return oauth2_dict - - def _build_spaces_url( - self, - is_v2: bool, - base_url: str, - limit: int, - space_keys: list[str] | None, - start: int | None = None, - ) -> str: - """Build URL for Confluence spaces API with query parameters.""" - key_param = "keys" if is_v2 else "spaceKey" - - params = [f"limit={limit}"] - if space_keys: - params.append(f"{key_param}={','.join(space_keys)}") - if start is not None and not is_v2: - params.append(f"start={start}") - - return f"{base_url}?{'&'.join(params)}" - - def _paginate_spaces_for_endpoint( - self, - is_v2: bool, - base_url: str, - limit: int, - space_keys: list[str] | None, - ) -> Iterator[dict[str, Any]]: - """Paginate spaces. Server stops on missing ``_links.next`` - (and empty ``results``, defensively). Don't stop on - ``len(results) < limit``: ``/rest/api/space`` on DC caps at - ``DefaultRestSpaceManager.MAX_SIZE`` (#4129). ``start`` is - re-derived locally; Confluence under-counts it on capped pages - and CONFSERVER-95272/-95312 returns records past the true end. - """ - start = 0 - url = self._build_spaces_url( - is_v2, base_url, limit, space_keys, start if not is_v2 else None - ) - - while url: - response = self.get(url, advanced_mode=True) - response.raise_for_status() - data = response.json() - - results = data.get("results", []) - if not results: - return - - yield from results - - next_link = data.get("_links", {}).get("next", "") - if not next_link: - return - - if is_v2: - url = next_link - else: - start += len(results) - url = self._build_spaces_url(is_v2, base_url, limit, space_keys, start) - - def retrieve_confluence_spaces( - self, - space_keys: list[str] | None = None, - limit: int = 50, - ) -> Iterator[dict[str, str]]: - """ - Retrieve spaces from Confluence using v2 API (Cloud) or v1 API (Server/fallback). - - Args: - space_keys: Optional list of space keys to filter by - limit: Results per page (default 50) - - Yields: - Space dictionaries with keys: id, key, name, type, status, etc. - - Note: - For Cloud instances, attempts v2 API first. If v2 returns 404, - automatically falls back to v1 API for compatibility with older instances. - """ - # Determine API version once - use_v2 = self._is_cloud and not self.scoped_token - base_url = _CONFLUENCE_SPACES_API_V2 if use_v2 else _CONFLUENCE_SPACES_API_V1 - - try: - yield from self._paginate_spaces_for_endpoint( - use_v2, base_url, limit, space_keys - ) - except HTTPError as e: - if e.response.status_code == 404 and use_v2: - logger.warning( - "v2 spaces API returned 404, falling back to v1 API. This may indicate an older Confluence Cloud instance." - ) - # Fallback to v1 - yield from self._paginate_spaces_for_endpoint( - False, _CONFLUENCE_SPACES_API_V1, limit, space_keys - ) - else: - raise - - def _probe_connection( - self, - **kwargs: Any, - ) -> None: - merged_kwargs = {**self.shared_base_kwargs, **kwargs} - # add special timeout to make sure that we don't hang indefinitely - merged_kwargs["timeout"] = self.PROBE_TIMEOUT - - with self._credentials_provider: - credentials, _ = self._renew_credentials() - if self.scoped_token: - # v2 endpoint doesn't always work with scoped tokens, use v1 - token = credentials["confluence_access_token"] - probe_url = f"{self.base_url}/{_CONFLUENCE_SPACES_API_V1}?limit=1" - import requests - - try: - r = requests.get( - probe_url, - headers={"Authorization": f"Bearer {token}"}, - timeout=10, - ) - r.raise_for_status() - except HTTPError as e: - if e.response.status_code == 403: - logger.warning( - "scoped token authenticated but not valid for probe endpoint (spaces)" - ) - else: - if "WWW-Authenticate" in e.response.headers: - logger.warning( - "WWW-Authenticate: %s", - e.response.headers["WWW-Authenticate"], - ) - logger.warning("Full error: %s", e.response.text) - raise e - return - - # Initialize connection with probe timeout settings - self._confluence = self._initialize_connection_helper( - credentials, **merged_kwargs - ) - - # Retrieve first space to validate connection - spaces_iter = self.retrieve_confluence_spaces(limit=1) - first_space = next(spaces_iter, None) - - if not first_space: - raise RuntimeError( - f"No spaces found at {self._url}! Check your credentials and wiki_base and make sure is_cloud is set correctly." - ) - - logger.info("Confluence probe succeeded.") - - def _initialize_connection( - self, - **kwargs: Any, - ) -> None: - """Called externally to init the connection in a thread safe manner.""" - merged_kwargs = {**self.shared_base_kwargs, **kwargs} - with self._credentials_provider: - credentials, _ = self._renew_credentials() - self._confluence = self._initialize_connection_helper( - credentials, **merged_kwargs - ) - self._kwargs = merged_kwargs - - def _initialize_connection_helper( - self, - credentials: dict[str, Any], - **kwargs: Any, - ) -> Confluence: - """Called internally to init the connection. Distributed locking - to prevent multiple threads from modifying the credentials - must be handled around this function.""" - - confluence = None - - # probe connection with direct client, no retries - if "confluence_refresh_token" in credentials: - logger.info("Connecting to Confluence Cloud with OAuth Access Token.") - - oauth2_dict: dict[str, Any] = OnyxConfluence._make_oauth2_dict(credentials) - url = f"https://api.atlassian.com/ex/confluence/{credentials['cloud_id']}" - confluence = Confluence(url=url, oauth2=oauth2_dict, **kwargs) - else: - logger.info( - "Connecting to Confluence with Personal Access Token as user: %s", - credentials["confluence_username"], - ) - if self._is_cloud: - confluence = Confluence( - url=self._url, - username=credentials["confluence_username"], - password=credentials["confluence_access_token"], - **kwargs, - ) - else: - confluence = Confluence( - url=self._url, - token=credentials["confluence_access_token"], - **kwargs, - ) - - return confluence - - # https://developer.atlassian.com/cloud/confluence/rate-limiting/ - # This uses the native rate limiting option provided by the - # confluence client and otherwise applies a simpler set of error handling. - def _make_rate_limited_confluence_method( - self, name: str, credential_provider: CredentialsProviderInterface | None - ) -> Callable[..., Any]: - def wrapped_call(*args: list[Any], **kwargs: Any) -> Any: - MAX_RETRIES = 5 - - TIMEOUT = 600 - timeout_at = time.monotonic() + TIMEOUT - - for attempt in range(MAX_RETRIES): - if time.monotonic() > timeout_at: - raise TimeoutError( - f"Confluence call attempts took longer than {TIMEOUT} seconds." - ) - - # we're relying more on the client to rate limit itself - # and applying our own retries in a more specific set of circumstances - try: - if credential_provider: - with credential_provider: - credentials, renewed = self._renew_credentials() - if renewed: - self._confluence = self._initialize_connection_helper( - credentials, **self._kwargs - ) - attr = getattr( # ods: ignore[getattr] - self._confluence, name, None - ) - if attr is None: - # The underlying Confluence client doesn't have this attribute - raise AttributeError( - f"'{type(self).__name__}' object has no attribute '{name}'" - ) - - return attr(*args, **kwargs) - else: - attr = getattr( # ods: ignore[getattr] - self._confluence, name, None - ) - if attr is None: - # The underlying Confluence client doesn't have this attribute - raise AttributeError( - f"'{type(self).__name__}' object has no attribute '{name}'" - ) - - return attr(*args, **kwargs) - - except HTTPError as e: - delay_until = _handle_http_error(e, attempt, MAX_RETRIES) - logger.warning( - "HTTPError in confluence call. Retrying in %s seconds...", - delay_until, - ) - while time.monotonic() < delay_until: - # in the future, check a signal here to exit - time.sleep(1) - except AttributeError as e: - # Some error within the Confluence library, unclear why it fails. - # Users reported it to be intermittent, so just retry - if attempt == MAX_RETRIES - 1: - raise e - - logger.exception( - "Confluence Client raised an AttributeError. Retrying..." - ) - time.sleep(5) - - return wrapped_call - - def __getattr__(self, name: str) -> Any: - """Dynamically intercept attribute/method access.""" - attr = getattr(self._confluence, name, None) # ods: ignore[getattr] - if attr is None: - # The underlying Confluence client doesn't have this attribute - raise AttributeError( - f"'{type(self).__name__}' object has no attribute '{name}'" - ) - - # If it's not a method, just return it after ensuring token validity - if not callable(attr): - return attr - - # skip methods that start with "_" - if name.startswith("_"): - return attr - - # wrap the method with our retry handler - rate_limited_method: Callable[..., Any] = ( - self._make_rate_limited_confluence_method(name, self._credentials_provider) - ) - - return rate_limited_method - - def _try_one_by_one_for_paginated_url( - self, - url_suffix: str, - initial_start: int, - limit: int, - ) -> Generator[dict[str, Any], None, str | None]: - """ - Go through `limit` items, starting at `initial_start` one by one (e.g. using - `limit=1` for each call). - - If we encounter an error, we skip the item and try the next one. We will return - the items we were able to retrieve successfully. - - Returns the expected next url_suffix. Returns None if it thinks we've hit the end. - - TODO (chris): make this yield failures as well as successes. - TODO (chris): make this work for confluence cloud somehow. - """ - if self._is_cloud: - raise RuntimeError("This method is not implemented for Confluence Cloud.") - - found_empty_page = False - temp_url_suffix = url_suffix - - for ind in range(limit): - try: - temp_url_suffix = update_param_in_path( - url_suffix, "start", str(initial_start + ind) - ) - temp_url_suffix = update_param_in_path(temp_url_suffix, "limit", "1") - logger.info("Making recovery confluence call to %s", temp_url_suffix) - raw_response = self.get(path=temp_url_suffix, advanced_mode=True) - raw_response.raise_for_status() - - latest_results = raw_response.json().get("results", []) - yield from latest_results - - if not latest_results: - # no more results, break out of the loop - logger.info( - "No results found for call '%s'Stopping pagination.", - temp_url_suffix, - ) - found_empty_page = True - break - except Exception: - logger.exception( - "Error in confluence call to %s. Continuing.", - temp_url_suffix, - ) - - if found_empty_page: - return None - - # if we got here, we successfully tried `limit` items - return update_param_in_path(url_suffix, "start", str(initial_start + limit)) - - def _paginate_url( - self, - url_suffix: str, - limit: int | None = None, - # Called with the next url to use to get the next page - next_page_callback: Callable[[str], None] | None = None, - force_offset_pagination: bool = False, - ) -> Iterator[dict[str, Any]]: - """ - This will paginate through the top level query. - """ - if not limit: - limit = _DEFAULT_PAGINATION_LIMIT - - current_limit = limit - url_suffix = update_param_in_path(url_suffix, "limit", str(current_limit)) - - while url_suffix: - logger.debug("Making confluence call to %s", url_suffix) - try: - # Only pass params if they're not already in the URL to avoid duplicate - # params accumulating. Confluence's _links.next already includes these. - params = {} - if "body-format=" not in url_suffix: - params["body-format"] = "atlas_doc_format" - if "expand=" not in url_suffix: - params["expand"] = "body.atlas_doc_format" - - raw_response = self.get( - path=url_suffix, - advanced_mode=True, - params=params, - ) - except Exception as e: - logger.exception("Error in confluence call to %s", url_suffix) - raise e - - try: - raw_response.raise_for_status() - except Exception as e: - logger.warning("Error in confluence call to %s", url_suffix) - - # If the problematic expansion is in the url, replace it - # with the replacement expansion and try again - # If that fails, raise the error - if _PROBLEMATIC_EXPANSIONS in url_suffix: - logger.warning( - "Replacing %s with %s and trying again.", - _PROBLEMATIC_EXPANSIONS, - _REPLACEMENT_EXPANSIONS, - ) - url_suffix = url_suffix.replace( - _PROBLEMATIC_EXPANSIONS, - _REPLACEMENT_EXPANSIONS, - ) - continue - - # CONFCLOUD-77618 / 76424: typed signal so the perm-sync - # caller can restart in per-page restriction-fetch mode. - if ( - _ANCESTOR_RESTRICTIONS_EXPAND_PREFIX in url_suffix - and _is_confcloud_77618_response(raw_response) - ): - raise Confcloud77618Error( - url=url_suffix, body=raw_response.text - ) from e - - if raw_response.status_code in _SERVER_ERROR_CODES: - # Try reducing the page size -- Confluence often times out - # on large result sets (especially Cloud 504s). - if current_limit > _MINIMUM_PAGINATION_LIMIT: - old_limit = current_limit - current_limit = max( - current_limit // 2, _MINIMUM_PAGINATION_LIMIT - ) - logger.warning( - "Confluence returned %s. " - "Reducing limit from %s to %s " - "and retrying.", - raw_response.status_code, - old_limit, - current_limit, - ) - url_suffix = update_param_in_path( - url_suffix, "limit", str(current_limit) - ) - continue - - # Limit reduction exhausted -- for Server, fall back to - # one-by-one offset pagination as a last resort. - if not self._is_cloud: - initial_start = get_start_param_from_url(url_suffix) - # this will just yield the successful items from the batch - new_url_suffix = yield from self._try_one_by_one_for_paginated_url( - url_suffix, - initial_start=initial_start, - limit=current_limit, - ) - # this means we ran into an empty page - if new_url_suffix is None: - if next_page_callback: - next_page_callback("") - break - - url_suffix = new_url_suffix - continue - - logger.exception( - "Error in confluence call to %s " - "after reducing limit to %s.\n" - "Raw Response Text: %s\n" - "Error: %s\n", - url_suffix, - current_limit, - raw_response.text, - e, - ) - raise - - logger.exception( - "Error in confluence call to %s \n" - "Raw Response Text: %s \n" - "Full Response: %s \n" - "Error: %s \n", - url_suffix, - raw_response.text, - raw_response.__dict__, - e, - ) - raise - - try: - next_response = raw_response.json() - except Exception as e: - logger.exception( - "Failed to parse response as JSON. Response: %s", - raw_response.__dict__, - ) - raise e - - # Yield the results individually. - results = cast(list[dict[str, Any]], next_response.get("results", [])) - - # #4129: DC silently caps page size and under-counts the - # ``start`` it embeds in ``_links.next``; re-derive it - # ourselves. Manual yielding (not ``yield from``) so we can - # fire ``next_page_callback`` before the last yield -- - # otherwise the iterator may never resume. - old_url_suffix = url_suffix - next_start = get_start_param_from_url(old_url_suffix) + len(results) - url_suffix = cast(str, next_response.get("_links", {}).get("next", "")) - if url_suffix and current_limit != limit: - url_suffix = update_param_in_path( - url_suffix, "limit", str(current_limit) - ) - if url_suffix and not self._is_cloud and results: - url_suffix = update_param_in_path(url_suffix, "start", str(next_start)) - - for i, result in enumerate(results): - if i == len(results) - 1: - if url_suffix and next_page_callback: - next_page_callback(url_suffix) - elif force_offset_pagination: - url_suffix = update_param_in_path( - old_url_suffix, "start", str(next_start) - ) - - yield result - - # we've observed that Confluence sometimes returns a next link despite giving - # 0 results. This is a bug with Confluence, so we need to check for it and - # stop paginating. - if url_suffix and not results: - logger.info( - "No results found for call '%s' despite next link being present. Stopping pagination.", - old_url_suffix, - ) - break - - def build_cql_url(self, cql: str, expand: str | None = None) -> str: - expand_string = f"&expand={expand}" if expand else "" - return f"rest/api/content/search?cql={cql}{expand_string}" - - def fetch_content_read_restrictions( - self, - content_id: str, - ) -> dict[str, Any] | None: - """Fetch a single page's restrictions via the dedicated - ``content/{id}/restriction/byOperation`` endpoint. Returns - ``None`` on 403/404 so unreadable ancestors (drafts owned by - another user) resolve as "no inheritable restriction here". - ``advanced_mode=True`` bypasses the rate-limit wrapper's 7x - 403-retry loop which would otherwise burn ~70s per draft.""" - path = f"rest/api/content/{quote(content_id, safe='')}/restriction/byOperation" - response: requests.Response = self.get(path, advanced_mode=True) - if response.status_code in (403, 404): - return None - response.raise_for_status() - body = response.json() - return cast(dict[str, Any], body or {}) - - def paginated_cql_retrieval( - self, - cql: str, - expand: str | None = None, - limit: int | None = None, - ) -> Iterator[dict[str, Any]]: - """ - The content/search endpoint can be used to fetch pages, attachments, and comments. - """ - cql_url = self.build_cql_url(cql, expand) - yield from self._paginate_url(cql_url, limit) - - def paginated_page_retrieval( - self, - cql_url: str, - limit: int, - # Called with the next url to use to get the next page - next_page_callback: Callable[[str], None] | None = None, - ) -> Iterator[dict[str, Any]]: - """ - Error handling (and testing) wrapper for _paginate_url, - because the current approach to page retrieval involves handling the - next page links manually. - """ - try: - yield from self._paginate_url( - cql_url, limit=limit, next_page_callback=next_page_callback - ) - except Exception as e: - logger.exception("Error in paginated_page_retrieval: %s", e) - raise e - - def cql_paginate_all_expansions( - self, - cql: str, - expand: str | None = None, - limit: int | None = None, - ) -> Iterator[dict[str, Any]]: - """Paginate the top-level query, then each `_links.next` discovered - in the expansions.""" - - def _traverse_and_update(data: dict | list) -> None: - if isinstance(data, dict): - next_url = data.get("_links", {}).get("next") - if next_url and "results" in data: - data["results"].extend(self._paginate_url(next_url, limit=limit)) - - for value in data.values(): - _traverse_and_update(value) - elif isinstance(data, list): - for item in data: - _traverse_and_update(item) - - for confluence_object in self.paginated_cql_retrieval(cql, expand, limit): - _traverse_and_update(confluence_object) - yield confluence_object - - def paginated_cql_user_retrieval( - self, - expand: str | None = None, - limit: int | None = None, - ) -> Iterator[ConfluenceUser]: - """ - The search/user endpoint can be used to fetch users. - It's a separate endpoint from the content/search endpoint used only for users. - Otherwise it's very similar to the content/search endpoint. - """ - - # this is needed since there is a live bug with Confluence Server/Data Center - # where not all users are returned by the APIs. This is a workaround needed until - # that is patched. - if self._confluence_user_profiles_override: - yield from self._confluence_user_profiles_override - - elif self._is_cloud: - cql = "type=user" - url = "rest/api/search/user" - expand_string = f"&expand={expand}" if expand else "" - url += f"?cql={cql}{expand_string}" - for user_result in self._paginate_url( - url, limit, force_offset_pagination=True - ): - # Example response: - # { - # 'user': { - # 'type': 'known', - # 'accountId': '712020:35e60fbb-d0f3-4c91-b8c1-f2dd1d69462d', - # 'accountType': 'atlassian', - # 'email': 'chris@danswer.ai', - # 'publicName': 'Chris Weaver', - # 'profilePicture': { - # 'path': '/wiki/aa-avatar/712020:35e60fbb-d0f3-4c91-b8c1-f2dd1d69462d', - # 'width': 48, - # 'height': 48, - # 'isDefault': False - # }, - # 'displayName': 'Chris Weaver', - # 'isExternalCollaborator': False, - # '_expandable': { - # 'operations': '', - # 'personalSpace': '' - # }, - # '_links': { - # 'self': 'https://danswerai.atlassian.net/wiki/rest/api/user?accountId=712020:35e60fbb-d0f3-4c91-b8c1-f2dd1d69462d' - # } - # }, - # 'title': 'Chris Weaver', - # 'excerpt': '', - # 'url': '/people/712020:35e60fbb-d0f3-4c91-b8c1-f2dd1d69462d', - # 'breadcrumbs': [], - # 'entityType': 'user', - # 'iconCssClass': 'aui-icon content-type-profile', - # 'lastModified': '2025-02-18T04:08:03.579Z', - # 'score': 0.0 - # } - user = user_result["user"] - yield ConfluenceUser( - user_id=user["accountId"], - username=None, - display_name=user["displayName"], - email=user.get("email"), - type=user["accountType"], - ) - else: - for user in self._paginate_url("rest/api/user/list", limit): - yield ConfluenceUser( - user_id=user["userKey"], - username=user["username"], - display_name=user["displayName"], - email=None, - type=user.get("type", "user"), - ) - - def paginated_groups_by_user_retrieval( - self, - user_id: str, # accountId in Cloud, userKey in Server - limit: int | None = None, - ) -> Iterator[dict[str, Any]]: - """ - This is not an SQL like query. - It's a confluence specific endpoint that can be used to fetch groups. - """ - user_field = "accountId" if self._is_cloud else "key" - user_value = user_id - # Server uses userKey (but calls it key during the API call), Cloud uses accountId - user_query = f"{user_field}={quote(user_value)}" - - url = f"rest/api/user/memberof?{user_query}" - yield from self._paginate_url(url, limit, force_offset_pagination=True) - - def paginated_groups_retrieval( - self, - limit: int | None = None, - ) -> Iterator[dict[str, Any]]: - """ - This is not an SQL like query. - It's a confluence specific endpoint that can be used to fetch groups. - """ - yield from self._paginate_url("rest/api/group", limit) - - def paginated_group_members_retrieval( - self, - group_name: str, - limit: int | None = None, - ) -> Iterator[dict[str, Any]]: - """ - This is not an SQL like query. - It's a confluence specific endpoint that can be used to fetch the members of a group. - THIS DOESN'T WORK FOR SERVER because it breaks when there is a slash in the group name. - E.g. neither "test/group" nor "test%2Fgroup" works for confluence. - """ - group_name = quote(group_name) - yield from self._paginate_url(f"rest/api/group/{group_name}/member", limit) - - def get_all_space_permissions_server( - self, - space_key: str, - ) -> list[dict[str, Any]]: - """ - Fetches a space's permissions via the legacy JSON-RPC API. - - This is the only space-permissions API available on Confluence Data - Center < 9.1.0. DC 9.1.0+ ships a proper REST API at - /rest/api/space/{spaceKey}/permissions (CONFSERVER-78176) which is - preferred wherever available; this method is the fallback for older - Server / Data Center deployments. - - Failure modes handled here: - - - HTTP 401: the JSON-RPC plugin is disabled. Confluence Admin -> - General Configuration -> Further Configuration -> Enable - "Remote API (XML-RPC & SOAP)". - - HTTP 200 with a non-JSON body (Confluence 7.7+): "Secure - Administrator Sessions" / WebSudo is intercepting admin JSON-RPC - calls and serving the login HTML or a WebSudoRequiredException - page instead of a JSON-RPC envelope. We surface the actual HTTP - status, Content-Type, and a body snippet so the admin can confirm - which of the documented failure modes they're hitting (rather - than guessing) and act on it. - - We use atlassian-python-api's `advanced_mode=True` to get the raw - requests.Response back. Without it, the library's _response_handler - catches the JSON parse error and silently coerces the body to None, - which throws away every signal we'd need to debug the failure. - Trade-off: the library no longer raises HTTPError on 4xx/5xx in - advanced mode, so this call no longer benefits from the - __getattr__ wrapper's retry-on-5xx; we call raise_for_status - ourselves to preserve the "blow up on server error" behavior. - """ - url = "rpc/json-rpc/confluenceservice-v2" - data = { - "jsonrpc": "2.0", - "method": "getSpacePermissionSets", - "id": 7, - "params": [space_key], - } - response: requests.Response = self.post(url, data=data, advanced_mode=True) - - if response.status_code == 401: - raise HTTPError( - "Unauthorized (401) when calling JSON-RPC API for space permissions. " - "This is likely because the Remote API is disabled. " - "To fix: Confluence Admin -> General Configuration -> Further Configuration " - "-> Enable 'Remote API (XML-RPC & SOAP)'", - response=response, - ) - response.raise_for_status() - - try: - payload = response.json() - except ValueError: - content_type = response.headers.get("Content-Type", "") - body_snippet = response.text[:_JSONRPC_ERROR_BODY_SNIPPET_CHARS] - raise ConnectorValidationError( - f"Confluence JSON-RPC returned a non-JSON response for space " - f"'{space_key}' (HTTP {response.status_code}, " - f"Content-Type={content_type}). This typically happens on " - "Confluence Server / Data Center 7.7+ when 'Secure " - "Administrator Sessions' (WebSudo) intercepts admin JSON-RPC " - "calls. To fix, either (1) disable Secure Administrator " - "Sessions in General Configuration -> Security Configuration, " - "or (2) upgrade to Confluence Data Center 9.1+ where the REST " - f"space-permissions API replaces JSON-RPC. See " - f"{_WEBSUDO_KB_URL}\n" - f"Response body (first {_JSONRPC_ERROR_BODY_SNIPPET_CHARS} " - f"chars): {body_snippet!r}" - ) - - logger.debug("jsonrpc response: %s", payload) - if not payload.get("result"): - logger.warning( - "No jsonrpc response for space permissions for space %s\nResponse: %s", - space_key, - payload, - ) - - return payload.get("result", []) - - def get_server_version(self) -> tuple[int, int] | None: - """Returns the (major, minor) version of the upstream Confluence - Data Center instance, or None for Cloud or when the probe fails. - - Probed once per OnyxConfluence instance via Atlassian's - documented "Server Information" endpoint - (/rest/api/server-information). The result is cached on the - instance, including the negative result, so a one-off network - blip doesn't cause us to re-probe on every space-permissions - sync. - - Used to gate features that only exist on newer DC versions, such - as the REST space-permissions API introduced in DC 9.1.0 - (CONFSERVER-78176). When the probe fails (returns None), callers - intentionally fall back to the legacy JSON-RPC path, on the - assumption that probe failure most often correlates with older - DC builds where the REST permissions API isn't available - anyway. Most callers should prefer the higher-level feature - predicates (e.g. supports_rest_space_permissions) over - comparing the version tuple directly. - """ - if self._is_cloud: - return None - if self._server_version_probed: - return self._server_version - - self._server_version = self._probe_server_version() - self._server_version_probed = True - if self._server_version is not None: - logger.info( - "Detected Confluence Data Center version %s.%s", - self._server_version[0], - self._server_version[1], - ) - return self._server_version - - def _probe_server_version(self) -> tuple[int, int] | None: - try: - info = self.get(_DC_SERVER_INFORMATION_PATH) - except Exception as e: - logger.warning("Failed to probe Confluence server version: %s", e) - return None - if not isinstance(info, dict): - return None - version_str = info.get("version") or "" - return _parse_dc_version(version_str) - - def supports_rest_space_permissions(self) -> bool: - """Whether the upstream instance has the DC 9.1+ space-permissions - REST API (CONFSERVER-78176). Always False for Cloud (different API - surface, branched on `is_cloud` upstream of any version check) and - for DC instances older than 9.1.0 or where the version probe fails. - """ - version = self.get_server_version() - return ( - version is not None - and version >= _MIN_DC_VERSION_FOR_REST_SPACE_PERMISSIONS - ) - - def get_all_space_permissions_server_rest( - self, - space_key: str, - ) -> list[dict[str, Any]]: - """Confluence DC 9.1+ REST API for space permissions. - - GET /rest/api/space/{spaceKey}/permissions returns a flat list of - {operation, subject, spaceKey, spaceId} entries (CONFSERVER-78176). - - Failure modes: - - - 401: handled identically to the JSON-RPC path (token missing / - expired). - - 404: the endpoint isn't available on this Confluence DC version - (i.e. < 9.1.0). Surfaced as - ConfluenceRestSpacePermissionsNotAvailableError so the caller - can fall back to the legacy JSON-RPC path. - - 500: per CONFSERVER-99908, callers without - Confluence-admin/space-admin rights receive HTTP 500 (rather - than the more correct 403). Surfaced as - InsufficientPermissionsError with that ticket referenced so - the operator knows the actual remediation is "grant the bot - account admin", not "investigate a server-side bug". - """ - path = f"rest/api/space/{quote(space_key, safe='')}/permissions" - response: requests.Response = self.get(path, advanced_mode=True) - - if response.status_code == 404: - raise ConfluenceRestSpacePermissionsNotAvailableError( - f"REST space-permissions endpoint not available on this " - f"Confluence instance (HTTP 404 for space '{space_key}'). " - "The endpoint requires Confluence Data Center 9.1.0+." - ) - if response.status_code == 401: - raise HTTPError( - "Unauthorized (401) when calling REST space-permissions API. " - "The credential is missing or expired.", - response=response, - ) - if response.status_code == 500: - raise InsufficientPermissionsError( - f"Confluence returned HTTP 500 for " - f"GET /rest/api/space/{space_key}/permissions. Per " - "CONFSERVER-99908 this endpoint returns 500 (rather than " - "403) when the calling account lacks Confluence-admin or " - "space-admin rights. Grant the bot account admin " - "permissions on this space (or globally) and retry." - ) - response.raise_for_status() - - payload = response.json() - if not isinstance(payload, list): - logger.warning( - "Unexpected REST space-permissions payload shape for space " - "%s: expected list, got %s", - space_key, - type(payload).__name__, - ) - return [] - return payload - - def get_anonymous_space_permissions_server_rest( - self, - space_key: str, - ) -> list[dict[str, Any]]: - """Confluence DC 9.1+ anonymous space-permissions endpoint. - - GET /rest/api/space/{spaceKey}/permissions/anonymous returns the - operations the anonymous role has on the space. Distinct from the - bulk endpoint, which on the JSON-RPC path used to return an - anonymous "row" inline. - - 404 is treated as "no anonymous access" rather than fatal; some - 9.x patch versions had this endpoint missing or moved before it - stabilized, and "missing endpoint" should not be louder than - "no anonymous access" for our use case. - """ - path = f"rest/api/space/{quote(space_key, safe='')}/permissions/anonymous" - response: requests.Response = self.get(path, advanced_mode=True) - - if response.status_code == 404: - return [] - if response.status_code == 500: - # CONFSERVER-99908 again -- same remediation, different endpoint. - raise InsufficientPermissionsError( - f"Confluence returned HTTP 500 for " - f"GET /rest/api/space/{space_key}/permissions/anonymous. " - "Per CONFSERVER-99908 this endpoint returns 500 (rather " - "than 403) when the calling account lacks " - "Confluence-admin or space-admin rights." - ) - response.raise_for_status() - - payload = response.json() - if not isinstance(payload, list): - return [] - return payload - - def get_current_user(self, expand: str | None = None) -> Any: - """ - Implements a method that isn't in the third party client. - - Get information about the current user - :param expand: OPTIONAL expand for get status of user. - Possible param is "status". Results are "Active, Deactivated" - :return: Returns the user details - """ - - from atlassian.errors import ApiPermissionError - - url = "rest/api/user/current" - params = {} - if expand: - params["expand"] = expand - try: - response = self.get(url, params=params) - except HTTPError as e: - if e.response.status_code == 403: - raise ApiPermissionError( - "The calling user does not have permission", reason=e - ) - raise - return response def get_user_email_from_username__server( @@ -1384,189 +104,3 @@ def get_user_email_from_userkey__server( email = None _USER_KEY_TO_EMAIL_CACHE[cache_key] = email return _USER_KEY_TO_EMAIL_CACHE[cache_key] - - -def _parse_dc_version(version_str: str) -> tuple[int, int] | None: - """Parse 'X.Y.Z[...]' into (X, Y); returns None on malformed input.""" - if not version_str: - return None - parts = version_str.split(".") - if len(parts) < 2: - return None - try: - return (int(parts[0]), int(parts[1])) - except ValueError: - return None - - -def _get_user(confluence_client: OnyxConfluence, user_id: str) -> str: - """Get Confluence Display Name based on the account-id or userkey value - - Args: - user_id (str): The user id (i.e: the account-id or userkey) - confluence_client (Confluence): The Confluence Client - - Returns: - str: The User Display Name. 'Unknown User' if the user is deactivated or not found - """ - global _USER_ID_TO_DISPLAY_NAME_CACHE - cache_key = (confluence_client._url, user_id) - if _USER_ID_TO_DISPLAY_NAME_CACHE.get(cache_key) is None: - try: - result = confluence_client.get_user_details_by_userkey(user_id) - found_display_name = result.get("displayName") - except Exception: - found_display_name = None - - if not found_display_name: - try: - result = confluence_client.get_user_details_by_accountid(user_id) - found_display_name = result.get("displayName") - except Exception: - found_display_name = None - - _USER_ID_TO_DISPLAY_NAME_CACHE[cache_key] = found_display_name - - return _USER_ID_TO_DISPLAY_NAME_CACHE.get(cache_key) or _USER_NOT_FOUND - - -def sanitize_attachment_title(title: str) -> str: - """ - Sanitize the attachment title to be a valid HTML attribute. - """ - return title.replace("<", "_").replace(">", "_").replace(" ", "_").replace(":", "_") - - -def extract_text_from_confluence_html( - confluence_client: OnyxConfluence, - confluence_object: dict[str, Any], - fetched_titles: set[str], -) -> str: - """Parse a Confluence html page and replace the 'user Id' by the real - User Display Name - - Args: - confluence_object (dict): The confluence object as a dict - confluence_client (Confluence): Confluence client - fetched_titles (set[str]): The titles of the pages that have already been fetched - Returns: - str: loaded and formated Confluence page - """ - body = confluence_object["body"] - object_html = body.get("storage", body.get("view", {})).get("value") - - soup = bs4.BeautifulSoup(object_html, "html.parser") - - _remove_macro_stylings(soup=soup) - - for date_span in soup.findAll("span", {"class": "date-lozenger-container"}): - date_span.replaceWith(date_span.get_text()) - - for user in soup.findAll("ri:user"): - user_id = ( - user.attrs["ri:account-id"] - if "ri:account-id" in user.attrs - else user.get("ri:userkey") - ) - if not user_id: - logger.warning( - "ri:userkey not found in ri:user element. Found attrs: %s", - user.attrs, - ) - continue - # Include @ sign for tagging, more clear for LLM - user.replaceWith("@" + _get_user(confluence_client, user_id)) - - for html_page_reference in soup.findAll("ac:structured-macro"): - # Here, we only want to process page within page macros - if html_page_reference.attrs.get("ac:name") != "include": - continue - - page_data = html_page_reference.find("ri:page") - if not page_data: - logger.warning( - "Skipping retrieval of %s because because page data is missing", - html_page_reference, - ) - continue - - page_title = page_data.attrs.get("ri:content-title") - if not page_title: - # only fetch pages that have a title - logger.warning( - "Skipping retrieval of %s because it has no title", - html_page_reference, - ) - continue - - if page_title in fetched_titles: - # prevent recursive fetching of pages - logger.debug("Skipping %s because it has already been fetched", page_title) - continue - - fetched_titles.add(page_title) - - # Wrap this in a try-except because there are some pages that might not exist - try: - page_query = f"type=page and title='{quote(page_title)}'" - - page_contents: dict[str, Any] | None = None - # Confluence enforces title uniqueness, so we should only get one result here - for page in confluence_client.paginated_cql_retrieval( - cql=page_query, - expand="body.storage.value", - limit=1, - ): - page_contents = page - break - except Exception as e: - logger.warning( - "Error getting page contents for object %s: %s", - confluence_object, - e, - ) - continue - - if not page_contents: - continue - - text_from_page = extract_text_from_confluence_html( - confluence_client=confluence_client, - confluence_object=page_contents, - fetched_titles=fetched_titles, - ) - - html_page_reference.replaceWith(text_from_page) - - for html_link_body in soup.findAll("ac:link-body"): - # This extracts the text from inline links in the page so they can be - # represented in the document text as plain text - try: - text_from_link = html_link_body.text - html_link_body.replaceWith(f"(LINK TEXT: {text_from_link})") - except Exception as e: - logger.warning("Error processing ac:link-body: %s", e) - - for html_attachment in soup.findAll("ri:attachment"): - # This extracts the text from inline attachments in the page so they can be - # represented in the document text as plain text - try: - html_attachment.replaceWith( - f"{sanitize_attachment_title(html_attachment.attrs['ri:filename'])}" - ) # to be replaced later - except Exception as e: - logger.warning("Error processing ac:attachment: %s", e) - - return format_document_soup(soup) - - -def _remove_macro_stylings(soup: bs4.BeautifulSoup) -> None: - for macro_root in soup.findAll("ac:structured-macro"): - if not isinstance(macro_root, bs4.Tag): - continue - - macro_styling = macro_root.find(name="ac:parameter", attrs={"ac:name": "page"}) - if not macro_styling or not isinstance(macro_styling, bs4.Tag): - continue - - macro_styling.extract() diff --git a/backend/onyx/connectors/confluence/source_operations.py b/backend/onyx/connectors/confluence/source_operations.py new file mode 100644 index 0000000000..e6159d2f12 --- /dev/null +++ b/backend/onyx/connectors/confluence/source_operations.py @@ -0,0 +1,1654 @@ +"""Confluence source-operations gateway: every Confluence API call lives here. + +This is the single file in ``onyx/connectors/confluence/`` and +``ee/onyx/external_permissions/confluence/`` allowed to import ``atlassian`` +(the import-fence test enforces it). The gateway builds its clients lazily from +the credential plus the bound config fields (``wiki_base``, ``is_cloud``, +``scoped_token``), and exposes each remote call as a stamped operation that +returns plain data. + +``_OnyxConfluence`` is the transport: rate-limit retries, OAuth refresh, and the +pagination engine. The EE perm-sync modules still call it directly through the +temporary ``onyx_confluence`` re-export; #15454 moves those calls to operations. + +Pagination notes: the Cloud ``search/user`` and ``user/memberof`` endpoints use +offset pagination, while page retrieval uses cursors. The default for Cloud is +cursor pagination; pass ``force_offset_pagination`` for an API that does not +return ``_links.next``. +""" + +import json +import threading +import time +from collections.abc import Callable, Generator, Iterator +from datetime import datetime, timedelta, timezone +from enum import Enum +from typing import Any, TypeVar, cast +from urllib.parse import quote + +import requests +from atlassian import Confluence +from atlassian.errors import ApiError +from requests import HTTPError + +from onyx.configs.app_configs import ( + CONFLUENCE_CONNECTOR_USER_PROFILES_OVERRIDE, + OAUTH_CONFLUENCE_CLOUD_CLIENT_ID, + OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET, +) +from onyx.configs.constants import DocumentSource +from onyx.connectors.capabilities import CredentialCapability +from onyx.connectors.confluence.config import ConfluenceCredentialBinding +from onyx.connectors.confluence.models import ConfluenceUser +from onyx.connectors.confluence.user_profile_override import ( + process_confluence_user_profiles_override, +) +from onyx.connectors.confluence.utils import ( + _handle_http_error, + build_cql_url, + confluence_refresh_tokens, + get_start_param_from_url, + update_param_in_path, +) +from onyx.connectors.cross_connector_utils.miscellaneous_utils import scoped_url +from onyx.connectors.exceptions import ( + ConnectorValidationError, + InsufficientPermissionsError, +) +from onyx.connectors.interfaces import CredentialsProviderInterface +from onyx.connectors.source_operations import ( + OperationConsumes, + SourceOperations, + source_operation, +) +from onyx.redis.redis_pool import get_redis_client +from onyx.redis.tenant_redis_client import TenantRedisClient +from onyx.utils.logger import setup_logger + +logger = setup_logger() + +_T = TypeVar("_T") + +_UNTESTED = "Capability checks land in #15455." + +# Client settings for the connection probe and for all later calls. +_PROBE_KWARGS: dict[str, Any] = {"max_backoff_retries": 6, "max_backoff_seconds": 10} +_FINAL_KWARGS: dict[str, Any] = {"max_backoff_retries": 10, "max_backoff_seconds": 60} +# Timeout of the client behind ``fast=True`` operations (validation). +_FAST_TIMEOUT = 3 + +_FIND_PAGE_EXPAND = "body.storage.value" + +# Set by the Confluence Cloud OAuth finalize step. +_OAUTH_SITE_KEYS = ("cloud_name", "wiki_base") + +# https://jira.atlassian.com/browse/CONFCLOUD-76433 +_PROBLEMATIC_EXPANSIONS = "body.storage.value" +_REPLACEMENT_EXPANSIONS = "body.view.value" + +# CONFCLOUD-77618 / CONFCLOUD-76424: ancestor-restrictions expand on +# content/search 404s the whole batch when an ancestor is unreadable +# (draft / outdated / trashed). We detect the body signature and raise. +_ANCESTOR_RESTRICTIONS_EXPAND_PREFIX = "ancestors.restrictions.read.restrictions." +_CONFCLOUD_77618_404_BODY_SIGNATURES = ( + "No content with id", + "Cannot find content. Outdated version/old_draft/trashed", +) + +_USER_NOT_FOUND = "Unknown Confluence User" +# Keyed by (instance base url, user id): one worker process can serve several +# Confluence instances, and DC userkeys are unique only per instance. +_USER_ID_TO_DISPLAY_NAME_CACHE: dict[tuple[str, str], str | None] = {} +_DEFAULT_PAGINATION_LIMIT = 1000 +_MINIMUM_PAGINATION_LIMIT = 5 + +_SERVER_ERROR_CODES = {500, 502, 503, 504} +_FORBIDDEN_STATUS = 403 + +_CONFLUENCE_SPACES_API_V1 = "rest/api/space" +_CONFLUENCE_SPACES_API_V2 = "wiki/api/v2/spaces" + +# Atlassian KB documenting how Secure Administrator Sessions (WebSudo) breaks +# admin JSON-RPC calls. Surfaced in the validation error so admins can act on +# it without our help. +_WEBSUDO_KB_URL = ( + "https://support.atlassian.com/confluence/kb/" + "json-rpc-api-request-returns-websudorequiredexception-on-confluence/" +) +# Cap how much of an unparseable JSON-RPC response body we put in the error +# message. WebSudo / login HTML pages are well under this; the cap is a +# defense against a runaway response (e.g. a multi-MB error page) ending up +# in our logs and validation surface. +_JSONRPC_ERROR_BODY_SNIPPET_CHARS = 1000 + +# DC 9.1.0 is the first DC release with the REST API for space permissions +# (CONFSERVER-78176). Older DC versions still need the legacy JSON-RPC +# fallback. Server / Data Center only -- Cloud has its own permissions API +# and is branched on `is_cloud` upstream of any version check. +_MIN_DC_VERSION_FOR_REST_SPACE_PERMISSIONS: tuple[int, int] = (9, 1) + +# Atlassian's documented Confluence DC endpoint for build information, +# under the "Server Information" REST API group. Returns a JSON object +# whose top-level `version` field is the upstream Confluence version +# (e.g. "10.2.10"). Confirmed present in the v8.4, v9.3, and v10.x +# DC REST API references; we previously probed the Jira-style +# `/rest/api/serverInfo` slug, which 404s on Confluence DC 10.x. +_DC_SERVER_INFORMATION_PATH = "rest/api/server-information" + + +class Confcloud77618Error(Exception): + """Signal to the perm-sync caller that the ancestor-restrictions + expand 404'd on a draft / outdated / trashed ancestor and the run + must restart with per-page restriction lookups.""" + + def __init__(self, url: str, body: str) -> None: + super().__init__( + f"CONFCLOUD-77618: ancestor-restrictions expand 404 from " + f"{url}: {body[:500]}" + ) + self.url = url + self.body = body + + +class ConfluenceRetriesExhaustedError(RuntimeError): + """Every attempt of a Confluence call failed with a retryable error.""" + + def __init__(self, message: str, *, last_status_code: int | None) -> None: + super().__init__(message) + self.last_status_code = last_status_code + + +class ConfluenceRestSpacePermissionsNotAvailableError(Exception): + """Raised by REST-API space-permissions calls when the endpoint is missing + on the upstream Confluence DC instance (e.g. DC < 9.1.0 returning 404). + + Callers use this as a signal to fall back to the legacy JSON-RPC path. + """ + + +def _is_confcloud_77618_response(response: requests.Response) -> bool: + """Body-signature match for the CONFCLOUD-77618 / CONFCLOUD-76424 404 + so unrelated 404s still propagate.""" + if response.status_code != 404: + return False + body = response.text + return any(sig in body for sig in _CONFCLOUD_77618_404_BODY_SIGNATURES) + + +class _OnyxConfluence: + """Wraps the SDK ``Confluence`` client. + + Adds CQL pagination with expansions, which the SDK does not support well. + Every public SDK method reached through ``__getattr__`` runs inside + ``_call_with_retries`` (rate-limit retries and OAuth refresh). + """ + + CREDENTIAL_PREFIX = "connector:confluence:credential" + CREDENTIAL_TTL = 300 # 5 min + PROBE_TIMEOUT = 5 # 5 seconds + + def __init__( + self, + is_cloud: bool, + url: str, + credentials_provider: CredentialsProviderInterface, + timeout: int | None = None, + scoped_token: bool = False, + # should generally not be passed in, but making it overridable for + # easier testing + confluence_user_profiles_override: list[dict[str, str]] | None = ( + CONFLUENCE_CONNECTOR_USER_PROFILES_OVERRIDE + ), + # The resolved scoped-token API URL, so clients of one gateway share + # one tenant_info lookup. + scoped_api_url: str | None = None, + ) -> None: + self.base_url = url + if scoped_token: + url = scoped_api_url or scoped_url(url, "confluence") + + self._is_cloud = is_cloud + self._url = url.rstrip("/") + self._credentials_provider = credentials_provider + self.scoped_token = scoped_token + self.redis_client: TenantRedisClient | None = None + self.static_credentials: dict[str, Any] | None = None + if self._credentials_provider.is_dynamic(): + self.redis_client = get_redis_client( + tenant_id=credentials_provider.get_tenant_id() + ) + else: + self.static_credentials = self._credentials_provider.get_credentials() + + self._confluence = Confluence(url) + self.credential_key: str = ( + self.CREDENTIAL_PREFIX + + f":credential_{self._credentials_provider.get_provider_key()}" + ) + + self._kwargs: Any = None + + self.shared_base_kwargs: dict[str, str | int | bool] = { + "api_version": "cloud" if is_cloud else "latest", + "backoff_and_retry": False, + "cloud": is_cloud, + } + if timeout: + self.shared_base_kwargs["timeout"] = timeout + + self._confluence_user_profiles_override = ( + process_confluence_user_profiles_override(confluence_user_profiles_override) + if confluence_user_profiles_override + else None + ) + + # Cached result of the server-information probe, populated on + # first `get_server_version()` call. _server_version_probed=True + # with _server_version=None means "we tried and the probe + # failed", so we don't keep retrying every space-permissions sync. + self._server_version: tuple[int, int] | None = None + self._server_version_probed: bool = False + + def _renew_credentials(self) -> tuple[dict[str, Any], bool]: + """credential_json - the current json credentials + Returns a tuple + 1. The up to date credentials + 2. True if the credentials were updated + + This method is intended to be used within a distributed lock. + Lock, call this, update credentials if the tokens were refreshed, then release + """ + # static credentials are preloaded, so no locking/redis required + if self.static_credentials: + return self.static_credentials, False + + if not self.redis_client: + raise RuntimeError("self.redis_client is None") + + # dynamic credentials need locking + # check redis first, then fallback to the DB + credential_bytes = self.redis_client.get(self.credential_key) + if credential_bytes is not None: + credential_str = credential_bytes.decode("utf-8") + credential_json: dict[str, Any] = json.loads(credential_str) + else: + credential_json = self._credentials_provider.get_credentials() + + if "confluence_refresh_token" not in credential_json: + # static credentials ... cache them permanently and return + self.static_credentials = credential_json + return credential_json, False + + if not OAUTH_CONFLUENCE_CLOUD_CLIENT_ID: + raise RuntimeError("OAUTH_CONFLUENCE_CLOUD_CLIENT_ID must be set!") + + if not OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET: + raise RuntimeError("OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET must be set!") + + # check if we should refresh tokens. we're deciding to refresh halfway + # to expiration + now = datetime.now(timezone.utc) + created_at = datetime.fromisoformat(credential_json["created_at"]) + expires_in: int = credential_json["expires_in"] + renew_at = created_at + timedelta(seconds=expires_in // 2) + if now <= renew_at: + # cached/current credentials are reasonably up to date + return credential_json, False + + # we need to refresh + logger.info("Renewing Confluence Cloud credentials...") + new_credentials = { + # The refresh response has no site info; keep what the OAuth + # finalize step stored. + **{ + key: credential_json[key] + for key in _OAUTH_SITE_KEYS + if key in credential_json + }, + **confluence_refresh_tokens( + OAUTH_CONFLUENCE_CLOUD_CLIENT_ID, + OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET, + credential_json["cloud_id"], + credential_json["confluence_refresh_token"], + ), + } + + # store the new credentials to redis and to the db thru the provider + # redis: we use a 5 min TTL because we are given a 10 minute grace period + # when keys are rotated. it's easier to expire the cached credentials + # reasonably frequently rather than trying to handle strong synchronization + # between the db and redis everywhere the credentials might be updated + new_credential_str = json.dumps(new_credentials) + self.redis_client.set( + self.credential_key, new_credential_str, nx=True, ex=self.CREDENTIAL_TTL + ) + self._credentials_provider.set_credentials(new_credentials) + + return new_credentials, True + + @staticmethod + def _make_oauth2_dict(credentials: dict[str, Any]) -> dict[str, Any]: + oauth2_dict: dict[str, Any] = {} + if "confluence_refresh_token" in credentials: + oauth2_dict["client_id"] = OAUTH_CONFLUENCE_CLOUD_CLIENT_ID + oauth2_dict["token"] = {} + oauth2_dict["token"]["access_token"] = credentials[ + "confluence_access_token" + ] + return oauth2_dict + + def _build_spaces_url( + self, + is_v2: bool, + base_url: str, + limit: int, + space_keys: list[str] | None, + start: int | None = None, + ) -> str: + """Build URL for Confluence spaces API with query parameters.""" + key_param = "keys" if is_v2 else "spaceKey" + + params = [f"limit={limit}"] + if space_keys: + params.append(f"{key_param}={','.join(space_keys)}") + if start is not None and not is_v2: + params.append(f"start={start}") + + return f"{base_url}?{'&'.join(params)}" + + def _paginate_spaces_for_endpoint( + self, + is_v2: bool, + base_url: str, + limit: int, + space_keys: list[str] | None, + ) -> Iterator[dict[str, Any]]: + """Paginate spaces. Server stops on missing ``_links.next`` + (and empty ``results``, defensively). Don't stop on + ``len(results) < limit``: ``/rest/api/space`` on DC caps at + ``DefaultRestSpaceManager.MAX_SIZE`` (#4129). ``start`` is + re-derived locally; Confluence under-counts it on capped pages + and CONFSERVER-95272/-95312 returns records past the true end. + """ + start = 0 + url = self._build_spaces_url( + is_v2, base_url, limit, space_keys, start if not is_v2 else None + ) + + while url: + response = self.get(url, advanced_mode=True) + response.raise_for_status() + data = response.json() + + results = data.get("results", []) + if not results: + return + + yield from results + + next_link = data.get("_links", {}).get("next", "") + if not next_link: + return + + if is_v2: + url = next_link + else: + start += len(results) + url = self._build_spaces_url(is_v2, base_url, limit, space_keys, start) + + def retrieve_confluence_spaces( + self, + space_keys: list[str] | None = None, + limit: int = 50, + ) -> Iterator[dict[str, str]]: + """ + Retrieve spaces from Confluence using v2 API (Cloud) or v1 API (Server/fallback). + + Args: + space_keys: Optional list of space keys to filter by + limit: Results per page (default 50) + + Yields: + Space dictionaries with keys: id, key, name, type, status, etc. + + Note: + For Cloud instances, attempts v2 API first. If v2 returns 404, + automatically falls back to v1 API for compatibility with older instances. + """ + # Determine API version once + use_v2 = self._is_cloud and not self.scoped_token + base_url = _CONFLUENCE_SPACES_API_V2 if use_v2 else _CONFLUENCE_SPACES_API_V1 + + try: + yield from self._paginate_spaces_for_endpoint( + use_v2, base_url, limit, space_keys + ) + except HTTPError as e: + if e.response.status_code == 404 and use_v2: + logger.warning( + "v2 spaces API returned 404, falling back to v1 API. This may indicate an older Confluence Cloud instance." + ) + # Fallback to v1 + yield from self._paginate_spaces_for_endpoint( + False, _CONFLUENCE_SPACES_API_V1, limit, space_keys + ) + else: + raise + + def _probe_connection( + self, + **kwargs: Any, + ) -> None: + merged_kwargs = {**self.shared_base_kwargs, **kwargs} + # add special timeout to make sure that we don't hang indefinitely + merged_kwargs["timeout"] = self.PROBE_TIMEOUT + + with self._credentials_provider: + credentials, _ = self._renew_credentials() + if self.scoped_token: + # v2 endpoint doesn't always work with scoped tokens, use v1 + token = credentials["confluence_access_token"] + probe_url = f"{self.base_url}/{_CONFLUENCE_SPACES_API_V1}?limit=1" + try: + r = requests.get( + probe_url, + headers={"Authorization": f"Bearer {token}"}, + timeout=10, + ) + r.raise_for_status() + except HTTPError as e: + if e.response.status_code == 403: + logger.warning( + "scoped token authenticated but not valid for probe endpoint (spaces)" + ) + else: + if "WWW-Authenticate" in e.response.headers: + logger.warning( + "WWW-Authenticate: %s", + e.response.headers["WWW-Authenticate"], + ) + logger.warning("Full error: %s", e.response.text) + raise e + return + + # Initialize connection with probe timeout settings + self._confluence = self._initialize_connection_helper( + credentials, **merged_kwargs + ) + + # Retrieve first space to validate connection + spaces_iter = self.retrieve_confluence_spaces(limit=1) + first_space = next(spaces_iter, None) + + if not first_space: + raise RuntimeError( + f"No spaces found at {self._url}! Check your credentials and wiki_base and make sure is_cloud is set correctly." + ) + + logger.info("Confluence probe succeeded.") + + def _initialize_connection( + self, + **kwargs: Any, + ) -> None: + """Called externally to init the connection in a thread safe manner.""" + merged_kwargs = {**self.shared_base_kwargs, **kwargs} + with self._credentials_provider: + credentials, _ = self._renew_credentials() + self._confluence = self._initialize_connection_helper( + credentials, **merged_kwargs + ) + self._kwargs = merged_kwargs + + def _initialize_connection_helper( + self, + credentials: dict[str, Any], + **kwargs: Any, + ) -> Confluence: + """Called internally to init the connection. Distributed locking + to prevent multiple threads from modifying the credentials + must be handled around this function.""" + + confluence = None + + # probe connection with direct client, no retries + if "confluence_refresh_token" in credentials: + logger.info("Connecting to Confluence Cloud with OAuth Access Token.") + + oauth2_dict: dict[str, Any] = _OnyxConfluence._make_oauth2_dict(credentials) + url = f"https://api.atlassian.com/ex/confluence/{credentials['cloud_id']}" + confluence = Confluence(url=url, oauth2=oauth2_dict, **kwargs) + else: + logger.info( + "Connecting to Confluence with Personal Access Token as user: %s", + credentials["confluence_username"], + ) + if self._is_cloud: + confluence = Confluence( + url=self._url, + username=credentials["confluence_username"], + password=credentials["confluence_access_token"], + **kwargs, + ) + else: + confluence = Confluence( + url=self._url, + token=credentials["confluence_access_token"], + **kwargs, + ) + + return confluence + + def _sdk_url(self) -> str: + """The SDK client's API root URL.""" + return self._confluence.url + + # https://developer.atlassian.com/cloud/confluence/rate-limiting/ + # This uses the native rate limiting option provided by the + # confluence client and otherwise applies a simpler set of error handling. + def _call_with_retries( + self, call: Callable[[Confluence], _T], *, retry_forbidden: bool = True + ) -> _T: + """Runs ``call`` on the SDK client with OAuth refresh and retries. + + Retries a 403 (Confluence Server rate limit), 429 and 5xx as + ``_handle_http_error`` allows. ``retry_forbidden=False`` raises a 403 at + once. Raises ``ConfluenceRetriesExhaustedError`` when every attempt + failed with a retryable error. + """ + MAX_RETRIES = 5 + + TIMEOUT = 600 + timeout_at = time.monotonic() + TIMEOUT + last_status_code: int | None = None + + for attempt in range(MAX_RETRIES): + if time.monotonic() > timeout_at: + raise TimeoutError( + f"Confluence call attempts took longer than {TIMEOUT} seconds." + ) + + # we're relying more on the client to rate limit itself + # and applying our own retries in a more specific set of circumstances + try: + with self._credentials_provider: + credentials, renewed = self._renew_credentials() + if renewed: + self._confluence = self._initialize_connection_helper( + credentials, **self._kwargs + ) + return call(self._confluence) + + except HTTPError as e: + if e.response is not None: + last_status_code = e.response.status_code + if not retry_forbidden and last_status_code == _FORBIDDEN_STATUS: + raise + delay_until = _handle_http_error(e, attempt, MAX_RETRIES) + logger.warning( + "HTTPError in confluence call. Retrying in %s seconds...", + max(delay_until - time.monotonic(), 0), + ) + while time.monotonic() < delay_until: + # in the future, check a signal here to exit + time.sleep(1) + except AttributeError as e: + # Some error within the Confluence library, unclear why it fails. + # Users reported it to be intermittent, so just retry + if attempt == MAX_RETRIES - 1: + raise e + + logger.exception( + "Confluence Client raised an AttributeError. Retrying..." + ) + time.sleep(5) + raise ConfluenceRetriesExhaustedError( + f"Confluence call failed after {MAX_RETRIES} attempts " + f"(last status: {last_status_code}).", + last_status_code=last_status_code, + ) + + def _make_rate_limited_confluence_method(self, name: str) -> Callable[..., Any]: + def call_method(confluence: Confluence, *args: Any, **kwargs: Any) -> Any: + attr = getattr(confluence, name, None) # ods: ignore[getattr] + if attr is None: + # The underlying Confluence client doesn't have this attribute + raise AttributeError( + f"'{type(self).__name__}' object has no attribute '{name}'" + ) + return attr(*args, **kwargs) + + def wrapped_call(*args: Any, **kwargs: Any) -> Any: + return self._call_with_retries( + lambda confluence: call_method(confluence, *args, **kwargs) + ) + + return wrapped_call + + def __getattr__(self, name: str) -> Any: + """Dynamically intercept attribute/method access.""" + attr = getattr(self._confluence, name, None) # ods: ignore[getattr] + if attr is None: + # The underlying Confluence client doesn't have this attribute + raise AttributeError( + f"'{type(self).__name__}' object has no attribute '{name}'" + ) + + # If it's not a method, just return it after ensuring token validity + if not callable(attr): + return attr + + # skip methods that start with "_" + if name.startswith("_"): + return attr + + # wrap the method with our retry handler + return self._make_rate_limited_confluence_method(name) + + def _try_one_by_one_for_paginated_url( + self, + url_suffix: str, + initial_start: int, + limit: int, + ) -> Generator[dict[str, Any], None, str | None]: + """ + Go through `limit` items, starting at `initial_start` one by one (e.g. using + `limit=1` for each call). + + If we encounter an error, we skip the item and try the next one. We will return + the items we were able to retrieve successfully. + + Returns the expected next url_suffix. Returns None if it thinks we've hit the end. + + TODO(chris): make this yield failures as well as successes. + TODO(chris): make this work for confluence cloud somehow. + """ + if self._is_cloud: + raise RuntimeError("This method is not implemented for Confluence Cloud.") + + found_empty_page = False + temp_url_suffix = url_suffix + + for ind in range(limit): + try: + temp_url_suffix = update_param_in_path( + url_suffix, "start", str(initial_start + ind) + ) + temp_url_suffix = update_param_in_path(temp_url_suffix, "limit", "1") + logger.info("Making recovery confluence call to %s", temp_url_suffix) + raw_response = self.get(path=temp_url_suffix, advanced_mode=True) + raw_response.raise_for_status() + + latest_results = raw_response.json().get("results", []) + yield from latest_results + + if not latest_results: + # no more results, break out of the loop + logger.info( + "No results found for call '%s'Stopping pagination.", + temp_url_suffix, + ) + found_empty_page = True + break + except Exception: + logger.exception( + "Error in confluence call to %s. Continuing.", + temp_url_suffix, + ) + + if found_empty_page: + return None + + # if we got here, we successfully tried `limit` items + return update_param_in_path(url_suffix, "start", str(initial_start + limit)) + + def _paginate_url( + self, + url_suffix: str, + limit: int | None = None, + # Called with the next url to use to get the next page + next_page_callback: Callable[[str], None] | None = None, + force_offset_pagination: bool = False, + ) -> Iterator[dict[str, Any]]: + """ + This will paginate through the top level query. + """ + if not limit: + limit = _DEFAULT_PAGINATION_LIMIT + + current_limit = limit + url_suffix = update_param_in_path(url_suffix, "limit", str(current_limit)) + + while url_suffix: + logger.debug("Making confluence call to %s", url_suffix) + try: + # Only pass params if they're not already in the URL to avoid duplicate + # params accumulating. Confluence's _links.next already includes these. + params = {} + if "body-format=" not in url_suffix: + params["body-format"] = "atlas_doc_format" + if "expand=" not in url_suffix: + params["expand"] = "body.atlas_doc_format" + + raw_response = self.get( + path=url_suffix, + advanced_mode=True, + params=params, + ) + except Exception as e: + logger.exception("Error in confluence call to %s", url_suffix) + raise e + + try: + raw_response.raise_for_status() + except Exception as e: + logger.warning("Error in confluence call to %s", url_suffix) + + # If the problematic expansion is in the url, replace it + # with the replacement expansion and try again + # If that fails, raise the error + if _PROBLEMATIC_EXPANSIONS in url_suffix: + logger.warning( + "Replacing %s with %s and trying again.", + _PROBLEMATIC_EXPANSIONS, + _REPLACEMENT_EXPANSIONS, + ) + url_suffix = url_suffix.replace( + _PROBLEMATIC_EXPANSIONS, + _REPLACEMENT_EXPANSIONS, + ) + continue + + # CONFCLOUD-77618 / 76424: typed signal so the perm-sync + # caller can restart in per-page restriction-fetch mode. + if ( + _ANCESTOR_RESTRICTIONS_EXPAND_PREFIX in url_suffix + and _is_confcloud_77618_response(raw_response) + ): + raise Confcloud77618Error( + url=url_suffix, body=raw_response.text + ) from e + + if raw_response.status_code in _SERVER_ERROR_CODES: + # Try reducing the page size -- Confluence often times out + # on large result sets (especially Cloud 504s). + if current_limit > _MINIMUM_PAGINATION_LIMIT: + old_limit = current_limit + current_limit = max( + current_limit // 2, _MINIMUM_PAGINATION_LIMIT + ) + logger.warning( + "Confluence returned %s. " + "Reducing limit from %s to %s " + "and retrying.", + raw_response.status_code, + old_limit, + current_limit, + ) + url_suffix = update_param_in_path( + url_suffix, "limit", str(current_limit) + ) + continue + + # Limit reduction exhausted -- for Server, fall back to + # one-by-one offset pagination as a last resort. + if not self._is_cloud: + initial_start = get_start_param_from_url(url_suffix) + # this will just yield the successful items from the batch + new_url_suffix = yield from self._try_one_by_one_for_paginated_url( + url_suffix, + initial_start=initial_start, + limit=current_limit, + ) + # this means we ran into an empty page + if new_url_suffix is None: + if next_page_callback: + next_page_callback("") + break + + url_suffix = new_url_suffix + continue + + logger.exception( + "Error in confluence call to %s " + "after reducing limit to %s.\n" + "Raw Response Text: %s\n" + "Error: %s\n", + url_suffix, + current_limit, + raw_response.text, + e, + ) + raise + + logger.exception( + "Error in confluence call to %s \n" + "Raw Response Text: %s \n" + "Full Response: %s \n" + "Error: %s \n", + url_suffix, + raw_response.text, + raw_response.__dict__, + e, + ) + raise + + try: + next_response = raw_response.json() + except Exception as e: + logger.exception( + "Failed to parse response as JSON. Response: %s", + raw_response.__dict__, + ) + raise e + + # Yield the results individually. + results = cast(list[dict[str, Any]], next_response.get("results", [])) + + # #4129: DC silently caps page size and under-counts the + # ``start`` it embeds in ``_links.next``; re-derive it + # ourselves. Manual yielding (not ``yield from``) so we can + # fire ``next_page_callback`` before the last yield -- + # otherwise the iterator may never resume. + old_url_suffix = url_suffix + next_start = get_start_param_from_url(old_url_suffix) + len(results) + url_suffix = cast(str, next_response.get("_links", {}).get("next", "")) + if url_suffix and current_limit != limit: + url_suffix = update_param_in_path( + url_suffix, "limit", str(current_limit) + ) + if url_suffix and not self._is_cloud and results: + url_suffix = update_param_in_path(url_suffix, "start", str(next_start)) + + for i, result in enumerate(results): + if i == len(results) - 1: + if url_suffix and next_page_callback: + next_page_callback(url_suffix) + elif force_offset_pagination: + url_suffix = update_param_in_path( + old_url_suffix, "start", str(next_start) + ) + + yield result + + # we've observed that Confluence sometimes returns a next link despite giving + # 0 results. This is a bug with Confluence, so we need to check for it and + # stop paginating. + if url_suffix and not results: + logger.info( + "No results found for call '%s' despite next link being present. Stopping pagination.", + old_url_suffix, + ) + break + + def fetch_content_read_restrictions( + self, + content_id: str, + ) -> dict[str, Any] | None: + """Fetch a single page's restrictions via the dedicated + ``content/{id}/restriction/byOperation`` endpoint. Returns + ``None`` on 403/404 so unreadable ancestors (drafts owned by + another user) resolve as "no inheritable restriction here". + ``advanced_mode=True`` bypasses the rate-limit wrapper's 7x + 403-retry loop which would otherwise burn ~70s per draft.""" + path = f"rest/api/content/{quote(content_id, safe='')}/restriction/byOperation" + response: requests.Response = self.get(path, advanced_mode=True) + if response.status_code in (403, 404): + return None + response.raise_for_status() + body = response.json() + return cast(dict[str, Any], body or {}) + + def paginated_cql_retrieval( + self, + cql: str, + expand: str | None = None, + limit: int | None = None, + ) -> Iterator[dict[str, Any]]: + """ + The content/search endpoint can be used to fetch pages, attachments, and comments. + """ + cql_url = build_cql_url(cql, expand) + yield from self._paginate_url(cql_url, limit) + + def paginated_page_retrieval( + self, + cql_url: str, + limit: int, + # Called with the next url to use to get the next page + next_page_callback: Callable[[str], None] | None = None, + ) -> Iterator[dict[str, Any]]: + """ + Error handling (and testing) wrapper for _paginate_url, + because the current approach to page retrieval involves handling the + next page links manually. + """ + try: + yield from self._paginate_url( + cql_url, limit=limit, next_page_callback=next_page_callback + ) + except Exception as e: + logger.exception("Error in paginated_page_retrieval: %s", e) + raise e + + def cql_paginate_all_expansions( + self, + cql: str, + expand: str | None = None, + limit: int | None = None, + ) -> Iterator[dict[str, Any]]: + """Paginate the top-level query, then each `_links.next` discovered + in the expansions.""" + + def _traverse_and_update(data: dict | list) -> None: + if isinstance(data, dict): + next_url = data.get("_links", {}).get("next") + if next_url and "results" in data: + data["results"].extend(self._paginate_url(next_url, limit=limit)) + + for value in data.values(): + _traverse_and_update(value) + elif isinstance(data, list): + for item in data: + _traverse_and_update(item) + + for confluence_object in self.paginated_cql_retrieval(cql, expand, limit): + _traverse_and_update(confluence_object) + yield confluence_object + + def paginated_cql_user_retrieval( + self, + expand: str | None = None, + limit: int | None = None, + ) -> Iterator[ConfluenceUser]: + """ + The search/user endpoint can be used to fetch users. + It's a separate endpoint from the content/search endpoint used only for users. + Otherwise it's very similar to the content/search endpoint. + """ + + # this is needed since there is a live bug with Confluence Server/Data Center + # where not all users are returned by the APIs. This is a workaround needed until + # that is patched. + if self._confluence_user_profiles_override: + yield from self._confluence_user_profiles_override + + elif self._is_cloud: + cql = "type=user" + url = "rest/api/search/user" + expand_string = f"&expand={expand}" if expand else "" + url += f"?cql={cql}{expand_string}" + for user_result in self._paginate_url( + url, limit, force_offset_pagination=True + ): + # Example response: + # { + # 'user': { + # 'type': 'known', + # 'accountId': '712020:35e60fbb-d0f3-4c91-b8c1-f2dd1d69462d', + # 'accountType': 'atlassian', + # 'email': 'chris@danswer.ai', + # 'publicName': 'Chris Weaver', + # 'profilePicture': { + # 'path': '/wiki/aa-avatar/712020:35e60fbb-d0f3-4c91-b8c1-f2dd1d69462d', + # 'width': 48, + # 'height': 48, + # 'isDefault': False + # }, + # 'displayName': 'Chris Weaver', + # 'isExternalCollaborator': False, + # '_expandable': { + # 'operations': '', + # 'personalSpace': '' + # }, + # '_links': { + # 'self': 'https://danswerai.atlassian.net/wiki/rest/api/user?accountId=712020:35e60fbb-d0f3-4c91-b8c1-f2dd1d69462d' + # } + # }, + # 'title': 'Chris Weaver', + # 'excerpt': '', + # 'url': '/people/712020:35e60fbb-d0f3-4c91-b8c1-f2dd1d69462d', + # 'breadcrumbs': [], + # 'entityType': 'user', + # 'iconCssClass': 'aui-icon content-type-profile', + # 'lastModified': '2025-02-18T04:08:03.579Z', + # 'score': 0.0 + # } + user = user_result["user"] + yield ConfluenceUser( + user_id=user["accountId"], + username=None, + display_name=user["displayName"], + email=user.get("email"), + type=user["accountType"], + ) + else: + for user in self._paginate_url("rest/api/user/list", limit): + yield ConfluenceUser( + user_id=user["userKey"], + username=user["username"], + display_name=user["displayName"], + email=None, + type=user.get("type", "user"), + ) + + def paginated_groups_by_user_retrieval( + self, + user_id: str, # accountId in Cloud, userKey in Server + limit: int | None = None, + ) -> Iterator[dict[str, Any]]: + """ + This is not an SQL like query. + It's a confluence specific endpoint that can be used to fetch groups. + """ + user_field = "accountId" if self._is_cloud else "key" + user_value = user_id + # Server uses userKey (but calls it key during the API call), Cloud uses accountId + user_query = f"{user_field}={quote(user_value)}" + + url = f"rest/api/user/memberof?{user_query}" + yield from self._paginate_url(url, limit, force_offset_pagination=True) + + def get_all_space_permissions_server( + self, + space_key: str, + ) -> list[dict[str, Any]]: + """ + Fetches a space's permissions via the legacy JSON-RPC API. + + This is the only space-permissions API available on Confluence Data + Center < 9.1.0. DC 9.1.0+ ships a proper REST API at + /rest/api/space/{spaceKey}/permissions (CONFSERVER-78176) which is + preferred wherever available; this method is the fallback for older + Server / Data Center deployments. + + Failure modes handled here: + + - HTTP 401: the JSON-RPC plugin is disabled. Confluence Admin -> + General Configuration -> Further Configuration -> Enable + "Remote API (XML-RPC & SOAP)". + - HTTP 200 with a non-JSON body (Confluence 7.7+): "Secure + Administrator Sessions" / WebSudo is intercepting admin JSON-RPC + calls and serving the login HTML or a WebSudoRequiredException + page instead of a JSON-RPC envelope. We surface the actual HTTP + status, Content-Type, and a body snippet so the admin can confirm + which of the documented failure modes they're hitting (rather + than guessing) and act on it. + + We use atlassian-python-api's `advanced_mode=True` to get the raw + requests.Response back. Without it, the library's _response_handler + catches the JSON parse error and silently coerces the body to None, + which throws away every signal we'd need to debug the failure. + Trade-off: the library no longer raises HTTPError on 4xx/5xx in + advanced mode, so this call no longer benefits from the + __getattr__ wrapper's retry-on-5xx; we call raise_for_status + ourselves to preserve the "blow up on server error" behavior. + """ + url = "rpc/json-rpc/confluenceservice-v2" + data = { + "jsonrpc": "2.0", + "method": "getSpacePermissionSets", + "id": 7, + "params": [space_key], + } + response: requests.Response = self.post(url, data=data, advanced_mode=True) + + if response.status_code == 401: + raise HTTPError( + "Unauthorized (401) when calling JSON-RPC API for space permissions. " + "This is likely because the Remote API is disabled. " + "To fix: Confluence Admin -> General Configuration -> Further Configuration " + "-> Enable 'Remote API (XML-RPC & SOAP)'", + response=response, + ) + response.raise_for_status() + + try: + payload = response.json() + except ValueError: + content_type = response.headers.get("Content-Type", "") + body_snippet = response.text[:_JSONRPC_ERROR_BODY_SNIPPET_CHARS] + raise ConnectorValidationError( + f"Confluence JSON-RPC returned a non-JSON response for space " + f"'{space_key}' (HTTP {response.status_code}, " + f"Content-Type={content_type}). This typically happens on " + "Confluence Server / Data Center 7.7+ when 'Secure " + "Administrator Sessions' (WebSudo) intercepts admin JSON-RPC " + "calls. To fix, either (1) disable Secure Administrator " + "Sessions in General Configuration -> Security Configuration, " + "or (2) upgrade to Confluence Data Center 9.1+ where the REST " + f"space-permissions API replaces JSON-RPC. See " + f"{_WEBSUDO_KB_URL}\n" + f"Response body (first {_JSONRPC_ERROR_BODY_SNIPPET_CHARS} " + f"chars): {body_snippet!r}" + ) + + logger.debug("jsonrpc response: %s", payload) + if not payload.get("result"): + logger.warning( + "No jsonrpc response for space permissions for space %s\nResponse: %s", + space_key, + payload, + ) + + return payload.get("result", []) + + def get_server_version(self) -> tuple[int, int] | None: + """Returns the (major, minor) version of the upstream Confluence + Data Center instance, or None for Cloud or when the probe fails. + + Probed once per client instance via Atlassian's + documented "Server Information" endpoint + (/rest/api/server-information). The result is cached on the + instance, including the negative result, so a one-off network + blip doesn't cause us to re-probe on every space-permissions + sync. + + Used to gate features that only exist on newer DC versions, such + as the REST space-permissions API introduced in DC 9.1.0 + (CONFSERVER-78176). When the probe fails (returns None), callers + intentionally fall back to the legacy JSON-RPC path, on the + assumption that probe failure most often correlates with older + DC builds where the REST permissions API isn't available + anyway. Most callers should prefer the higher-level feature + predicates (e.g. supports_rest_space_permissions) over + comparing the version tuple directly. + """ + if self._is_cloud: + return None + if self._server_version_probed: + return self._server_version + + self._server_version = self._probe_server_version() + self._server_version_probed = True + if self._server_version is not None: + logger.info( + "Detected Confluence Data Center version %s.%s", + self._server_version[0], + self._server_version[1], + ) + return self._server_version + + def _probe_server_version(self) -> tuple[int, int] | None: + try: + info = self.get(_DC_SERVER_INFORMATION_PATH) + except Exception as e: + logger.warning("Failed to probe Confluence server version: %s", e) + return None + if not isinstance(info, dict): + return None + version_str = info.get("version") or "" + return _parse_dc_version(version_str) + + def supports_rest_space_permissions(self) -> bool: + """Whether the upstream instance has the DC 9.1+ space-permissions + REST API (CONFSERVER-78176). Always False for Cloud (different API + surface, branched on `is_cloud` upstream of any version check) and + for DC instances older than 9.1.0 or where the version probe fails. + """ + version = self.get_server_version() + return ( + version is not None + and version >= _MIN_DC_VERSION_FOR_REST_SPACE_PERMISSIONS + ) + + def get_all_space_permissions_server_rest( + self, + space_key: str, + ) -> list[dict[str, Any]]: + """Confluence DC 9.1+ REST API for space permissions. + + GET /rest/api/space/{spaceKey}/permissions returns a flat list of + {operation, subject, spaceKey, spaceId} entries (CONFSERVER-78176). + + Failure modes: + + - 401: handled identically to the JSON-RPC path (token missing / + expired). + - 404: the endpoint isn't available on this Confluence DC version + (i.e. < 9.1.0). Surfaced as + ConfluenceRestSpacePermissionsNotAvailableError so the caller + can fall back to the legacy JSON-RPC path. + - 500: per CONFSERVER-99908, callers without + Confluence-admin/space-admin rights receive HTTP 500 (rather + than the more correct 403). Surfaced as + InsufficientPermissionsError with that ticket referenced so + the operator knows the actual remediation is "grant the bot + account admin", not "investigate a server-side bug". + """ + path = f"rest/api/space/{quote(space_key, safe='')}/permissions" + response: requests.Response = self.get(path, advanced_mode=True) + + if response.status_code == 404: + raise ConfluenceRestSpacePermissionsNotAvailableError( + f"REST space-permissions endpoint not available on this " + f"Confluence instance (HTTP 404 for space '{space_key}'). " + "The endpoint requires Confluence Data Center 9.1.0+." + ) + if response.status_code == 401: + raise HTTPError( + "Unauthorized (401) when calling REST space-permissions API. " + "The credential is missing or expired.", + response=response, + ) + if response.status_code == 500: + raise InsufficientPermissionsError( + f"Confluence returned HTTP 500 for " + f"GET /rest/api/space/{space_key}/permissions. Per " + "CONFSERVER-99908 this endpoint returns 500 (rather than " + "403) when the calling account lacks Confluence-admin or " + "space-admin rights. Grant the bot account admin " + "permissions on this space (or globally) and retry." + ) + response.raise_for_status() + + payload = response.json() + if not isinstance(payload, list): + logger.warning( + "Unexpected REST space-permissions payload shape for space " + "%s: expected list, got %s", + space_key, + type(payload).__name__, + ) + return [] + return payload + + def get_anonymous_space_permissions_server_rest( + self, + space_key: str, + ) -> list[dict[str, Any]]: + """Confluence DC 9.1+ anonymous space-permissions endpoint. + + GET /rest/api/space/{spaceKey}/permissions/anonymous returns the + operations the anonymous role has on the space. Distinct from the + bulk endpoint, which on the JSON-RPC path used to return an + anonymous "row" inline. + + 404 is treated as "no anonymous access" rather than fatal; some + 9.x patch versions had this endpoint missing or moved before it + stabilized, and "missing endpoint" should not be louder than + "no anonymous access" for our use case. + """ + path = f"rest/api/space/{quote(space_key, safe='')}/permissions/anonymous" + response: requests.Response = self.get(path, advanced_mode=True) + + if response.status_code == 404: + return [] + if response.status_code == 500: + # CONFSERVER-99908 again -- same remediation, different endpoint. + raise InsufficientPermissionsError( + f"Confluence returned HTTP 500 for " + f"GET /rest/api/space/{space_key}/permissions/anonymous. " + "Per CONFSERVER-99908 this endpoint returns 500 (rather " + "than 403) when the calling account lacks " + "Confluence-admin or space-admin rights." + ) + response.raise_for_status() + + payload = response.json() + if not isinstance(payload, list): + return [] + return payload + + +def _parse_dc_version(version_str: str) -> tuple[int, int] | None: + """Parse 'X.Y.Z[...]' into (X, Y); returns None on malformed input.""" + if not version_str: + return None + parts = version_str.split(".") + if len(parts) < 2: + return None + try: + return (int(parts[0]), int(parts[1])) + except ValueError: + return None + + +def _get_user(confluence_client: _OnyxConfluence, user_id: str) -> str: + """Returns the display name for an account id (Cloud) or userkey (DC). + + Returns ``_USER_NOT_FOUND`` if the user is deactivated or not found. + """ + cache_key = (confluence_client._url, user_id) + if _USER_ID_TO_DISPLAY_NAME_CACHE.get(cache_key) is None: + try: + result = confluence_client.get_user_details_by_userkey(user_id) + found_display_name = result.get("displayName") + except Exception: + found_display_name = None + + if not found_display_name: + try: + result = confluence_client.get_user_details_by_accountid(user_id) + found_display_name = result.get("displayName") + except Exception: + found_display_name = None + + _USER_ID_TO_DISPLAY_NAME_CACHE[cache_key] = found_display_name + + return _USER_ID_TO_DISPLAY_NAME_CACHE.get(cache_key) or _USER_NOT_FOUND + + +class ConfluenceSpaceNotFoundError(Exception): + """The space key does not exist, or the credential cannot read the space.""" + + +class ConfluenceProbeVariant(str, Enum): + """Token type of the connection probe: scoped API tokens use another path.""" + + SCOPED = "scoped" + UNSCOPED = "unscoped" + + +class ConfluenceSearchVariant(str, Enum): + """Permission class of a CQL content search. + + ``content`` reads bodies, ``slim`` reads metadata only, and + ``with_restrictions`` also expands read restrictions, which needs the + permission-read scope. + """ + + CONTENT = "content" + SLIM = "slim" + WITH_RESTRICTIONS = "with_restrictions" + + +_SEARCH_VARIANTS = tuple(ConfluenceSearchVariant) + + +class ConfluenceSourceOperations(SourceOperations): + source = DocumentSource.CONFLUENCE + sdk_modules = ("atlassian",) + # The gateway reads only the credential-bound fields (site URL, Cloud or + # Data Center, scoped token). + config_keys = frozenset(ConfluenceCredentialBinding.model_fields) + + _cached_client: _OnyxConfluence | None = None + _cached_fast_client: _OnyxConfluence | None = None + _cached_scoped_api_url: str | None = None + + def _client_build_lock(self) -> threading.Lock: + """This gateway's lock against duplicate builds; a build makes remote + calls for scoped tokens (tenant_info), so concurrent first calls would + repeat them. Per gateway, so a slow site does not block other gateways. + ``dict.setdefault`` is atomic under the GIL, so no guard lock is needed.""" + return vars(self).setdefault("_build_lock", threading.Lock()) + + def _binding(self) -> ConfluenceCredentialBinding: + return ConfluenceCredentialBinding.model_validate( + self.connector_specific_config or {} + ) + + def _scoped_api_url(self, wiki_base: str) -> str: + if self._cached_scoped_api_url is None: + self._cached_scoped_api_url = scoped_url(wiki_base, "confluence") + return self._cached_scoped_api_url + + def _build_client(self, *, fast: bool) -> _OnyxConfluence: + binding = self._binding() + wiki_base = binding.wiki_base.rstrip("/") + return _OnyxConfluence( + is_cloud=binding.is_cloud, + url=wiki_base, + credentials_provider=self.credentials_provider, + timeout=_FAST_TIMEOUT if fast else None, + scoped_token=binding.scoped_token, + scoped_api_url=( + self._scoped_api_url(wiki_base) if binding.scoped_token else None + ), + ) + + def _client(self, *, fast: bool = False) -> _OnyxConfluence: + cached = self._cached_fast_client if fast else self._cached_client + if cached is not None: + return cached + with self._client_build_lock(): + cached = self._cached_fast_client if fast else self._cached_client + if cached is None: + cached = self._build_client(fast=fast) + cached._initialize_connection(**_FINAL_KWARGS) + if fast: + self._cached_fast_client = cached + else: + self._cached_client = cached + return cached + + def _legacy_client(self, *, fast: bool = False) -> _OnyxConfluence: + """Temporary: the perm-sync paths still pass the client to the EE + modules. #15454 moves those calls to operations and deletes this.""" + return self._client(fast=fast) + + @source_operation( + capabilities={CredentialCapability.INDEXING}, + consumes=OperationConsumes.CREDENTIAL, + variants=tuple(ConfluenceProbeVariant), + untested=_UNTESTED, + ) + def probe_site(self, *, variant: ConfluenceProbeVariant) -> None: + """Proves the credential works for the site. Scoped tokens resolve the + cloud id (tenant_info) and read one space over v1; other tokens read + one space. Raises on failure.""" + scoped = self._binding().scoped_token + if (variant == ConfluenceProbeVariant.SCOPED) != scoped: + raise ValueError( + f"probe_site variant {variant.value!r} does not match " + f"scoped_token={scoped}." + ) + with self._client_build_lock(): + client = self._cached_client or self._build_client(fast=False) + client._probe_connection(**_PROBE_KWARGS) + client._initialize_connection(**_FINAL_KWARGS) + self._cached_client = client + + @source_operation( + capabilities={ + CredentialCapability.INDEXING, + CredentialCapability.DOC_PERMISSION_SYNC, + }, + consumes=OperationConsumes.CREDENTIAL, + untested=_UNTESTED, + ) + def list_spaces( + self, + *, + space_keys: list[str] | None = None, + limit: int = 50, + fast: bool = False, + ) -> Iterator[dict[str, Any]]: + """Yields the visible spaces (v2 API for OAuth Cloud, else v1).""" + return self._client(fast=fast).retrieve_confluence_spaces( + space_keys=space_keys, limit=limit + ) + + @source_operation( + capabilities={CredentialCapability.INDEXING}, + consumes=OperationConsumes.BOTH, + untested=_UNTESTED, + ) + def get_space(self, *, space_key: str, fast: bool = False) -> dict[str, Any]: + """Raises ``ConfluenceSpaceNotFoundError`` if the space does not exist + or the credential cannot read it, and ``ConfluenceRetriesExhaustedError`` + when every retry failed or Confluence returned no body.""" + try: + space: dict[str, Any] | None = self._client(fast=fast).get_space(space_key) + except ApiError as e: + raise ConfluenceSpaceNotFoundError(str(e)) from e + if space is None: + raise ConfluenceRetriesExhaustedError( + f"Confluence returned no body for space {space_key}.", + last_status_code=None, + ) + return space + + @source_operation( + capabilities={ + CredentialCapability.INDEXING, + CredentialCapability.DOC_PERMISSION_SYNC, + }, + consumes=OperationConsumes.BOTH, + variants=_SEARCH_VARIANTS, + untested=_UNTESTED, + ) + def search_pages( + self, + *, + variant: ConfluenceSearchVariant, # noqa: ARG002 + cql: str, + expand: str, + limit: int | None = None, + follow_expansion_links: bool = False, + ) -> Iterator[dict[str, Any]]: + """Yields the pages a CQL query matches. ``follow_expansion_links`` also + pages through the ``_links.next`` of expanded fields.""" + return _search( + self._client(), + cql=cql, + expand=expand, + limit=limit, + follow_expansion_links=follow_expansion_links, + ) + + @source_operation( + capabilities={CredentialCapability.INDEXING}, + consumes=OperationConsumes.BOTH, + untested=_UNTESTED, + ) + def search_pages_from_url( + self, + *, + url: str, + limit: int, + next_page_callback: Callable[[str], None], + ) -> Iterator[dict[str, Any]]: + """Yields pages from a content-search URL (a start URL or a checkpoint's + ``_links.next``). Calls ``next_page_callback`` with each next URL.""" + return self._client().paginated_page_retrieval( + cql_url=url, limit=limit, next_page_callback=next_page_callback + ) + + @source_operation( + capabilities={ + CredentialCapability.INDEXING, + CredentialCapability.DOC_PERMISSION_SYNC, + }, + consumes=OperationConsumes.BOTH, + variants=_SEARCH_VARIANTS, + untested=_UNTESTED, + ) + def search_attachments( + self, + *, + variant: ConfluenceSearchVariant, # noqa: ARG002 + cql: str, + expand: str, + limit: int | None = None, + follow_expansion_links: bool = False, + ) -> Iterator[dict[str, Any]]: + """Yields the attachments a CQL query matches.""" + return _search( + self._client(), + cql=cql, + expand=expand, + limit=limit, + follow_expansion_links=follow_expansion_links, + ) + + @source_operation( + capabilities={CredentialCapability.INDEXING}, + consumes=OperationConsumes.BOTH, + untested=_UNTESTED, + ) + def search_comments(self, *, cql: str, expand: str) -> Iterator[dict[str, Any]]: + """Yields the comments a CQL query matches.""" + return self._client().paginated_cql_retrieval(cql=cql, expand=expand) + + @source_operation( + capabilities={CredentialCapability.INDEXING}, + consumes=OperationConsumes.CREDENTIAL, + untested=_UNTESTED, + ) + def find_page_by_title(self, *, title: str) -> dict[str, Any] | None: + """Returns the page with this title and its storage body, if any. + Confluence enforces unique titles, so one result is enough.""" + return next( + self._client().paginated_cql_retrieval( + cql=f"type=page and title='{quote(title)}'", + expand=_FIND_PAGE_EXPAND, + limit=1, + ), + None, + ) + + @source_operation( + capabilities={CredentialCapability.INDEXING}, + consumes=OperationConsumes.CREDENTIAL, + untested=_UNTESTED, + ) + def download_attachment( + self, *, attachment: dict[str, Any], parent_content_id: str | None + ) -> bytes: + """Returns the attachment file. Uses the retry and OAuth refresh wrapper, + but does not retry a 403: on a download it is a permission refusal. + + Raises ``ValueError`` if no download link can be built (Cloud needs the + parent content id), ``HTTPError`` on a non-retryable error status, and + ``ConfluenceRetriesExhaustedError`` when every retry failed. + """ + client = self._client() + link = _attachment_download_link( + client._sdk_url(), attachment, parent_content_id, client._is_cloud + ) + logger.info( + "Downloading attachment: title=%s link=%s", attachment["title"], link + ) + + def download(confluence: Confluence) -> bytes: + response = confluence.session.get(link) + response.raise_for_status() + return response.content + + return client._call_with_retries(download, retry_forbidden=False) + + @source_operation( + capabilities={CredentialCapability.INDEXING}, + consumes=OperationConsumes.CREDENTIAL, + untested=_UNTESTED, + ) + def get_user_display_name(self, *, user_id: str) -> str: + """Returns the display name of a userkey (DC) or account id (Cloud), or + "Unknown Confluence User". No variants: the lookup tries the userkey + endpoint and then the account-id endpoint for every id.""" + return _get_user(self._client(), user_id) + + +def _search( + client: _OnyxConfluence, + *, + cql: str, + expand: str, + limit: int | None, + follow_expansion_links: bool, +) -> Iterator[dict[str, Any]]: + if follow_expansion_links: + return client.cql_paginate_all_expansions(cql=cql, expand=expand, limit=limit) + return client.paginated_cql_retrieval(cql=cql, expand=expand, limit=limit) + + +def _attachment_download_link( + api_url: str, + attachment: dict[str, Any], + parent_content_id: str | None, + is_cloud: bool, +) -> str: + if not is_cloud: + return api_url + attachment["_links"]["download"] + # https://developer.atlassian.com/cloud/confluence/rest/v1/api-group-content---attachments/#api-wiki-rest-api-content-id-child-attachment-attachmentid-download-get + if not parent_content_id: + raise ValueError( + "parent_content_id is required to download attachments from " + "Confluence Cloud." + ) + return ( + api_url + + f"/rest/api/content/{parent_content_id}/child/attachment/{attachment['id']}/download" + ) diff --git a/backend/onyx/connectors/confluence/utils.py b/backend/onyx/connectors/confluence/utils.py index 55cdb0c982..ae0f116c2c 100644 --- a/backend/onyx/connectors/confluence/utils.py +++ b/backend/onyx/connectors/confluence/utils.py @@ -1,12 +1,12 @@ import math import time -from collections.abc import Callable from datetime import datetime, timedelta, timezone from io import BytesIO from pathlib import Path -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, Any from urllib.parse import parse_qs, quote, urljoin, urlparse +import bs4 import requests from pydantic import BaseModel @@ -18,13 +18,16 @@ from onyx.configs.app_configs import ( from onyx.configs.constants import FileOrigin from onyx.file_processing.extract_file_text import extract_file_text, get_file_ext from onyx.file_processing.file_types import OnyxFileExtensions, OnyxMimeTypes +from onyx.file_processing.html_utils import format_document_soup from onyx.file_processing.image_utils import store_image_and_create_section from onyx.utils.datetime import datetime_to_utc from onyx.utils.logger import setup_logger from onyx.utils.retry_after import parse_retry_after_seconds if TYPE_CHECKING: - from onyx.connectors.confluence.onyx_confluence import OnyxConfluence + from onyx.connectors.confluence.source_operations import ( + ConfluenceSourceOperations, + ) logger = setup_logger() @@ -71,38 +74,11 @@ class AttachmentProcessingResult(BaseModel): error: str | None = None -def _make_attachment_link( - confluence_client: "OnyxConfluence", - attachment: dict[str, Any], - parent_content_id: str | None, - is_cloud: bool, -) -> str | None: - download_link = "" - - if is_cloud: - # https://developer.atlassian.com/cloud/confluence/rest/v1/api-group-content---attachments/#api-wiki-rest-api-content-id-child-attachment-attachmentid-download-get - if not parent_content_id: - logger.warning( - "parent_content_id is required to download attachments from Confluence Cloud!" - ) - return None - - download_link = ( - confluence_client.url - + f"/rest/api/content/{parent_content_id}/child/attachment/{attachment['id']}/download" - ) - else: - download_link = confluence_client.url + attachment["_links"]["download"] - - return download_link - - def process_attachment( - confluence_client: "OnyxConfluence", + source_operations: "ConfluenceSourceOperations", attachment: dict[str, Any], parent_content_id: str | None, allow_images: bool, - is_cloud: bool, ) -> AttachmentProcessingResult: """ Processes a Confluence attachment. If it's a document, extracts text, @@ -119,14 +95,6 @@ def process_attachment( error=f"Unsupported file type: {media_type}", ) - attachment_link = _make_attachment_link( - confluence_client, attachment, parent_content_id, is_cloud - ) - if not attachment_link: - return AttachmentProcessingResult( - text=None, file_name=None, error="Failed to make attachment link" - ) - attachment_size = attachment["extensions"]["fileSize"] if media_type.startswith("image/"): @@ -140,7 +108,7 @@ def process_attachment( if attachment_size > CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD: logger.warning( "Skipping %s due to size. size=%s threshold=%s", - attachment_link, + attachment["title"], attachment_size, CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD, ) @@ -150,28 +118,9 @@ def process_attachment( error=f"Attachment text too long: {attachment_size} chars", ) - logger.info( - "Downloading attachment: title=%s length=%s link=%s", - attachment["title"], - attachment_size, - attachment_link, + raw_bytes = source_operations.download_attachment( + attachment=attachment, parent_content_id=parent_content_id ) - - # Download the attachment - resp: requests.Response = confluence_client._session.get(attachment_link) - if resp.status_code != 200: - logger.warning( - "Failed to fetch %s with status code %s", - attachment_link, - resp.status_code, - ) - return AttachmentProcessingResult( - text=None, - file_name=None, - error=f"Attachment download status code is {resp.status_code}", - ) - - raw_bytes = resp.content if not raw_bytes: return AttachmentProcessingResult( text=None, file_name=None, error="attachment.content is None" @@ -179,9 +128,7 @@ def process_attachment( # Process image attachments if media_type.startswith("image/"): - return _process_image_attachment( - confluence_client, attachment, raw_bytes, media_type - ) + return _process_image_attachment(attachment, raw_bytes, media_type) # Process document attachments try: @@ -211,7 +158,6 @@ def process_attachment( def _process_image_attachment( - confluence_client: "OnyxConfluence", # noqa: ARG001 attachment: dict[str, Any], raw_bytes: bytes, media_type: str, @@ -237,11 +183,10 @@ def _process_image_attachment( def convert_attachment_to_content( - confluence_client: "OnyxConfluence", + source_operations: "ConfluenceSourceOperations", attachment: dict[str, Any], page_id: str, allow_images: bool, - is_cloud: bool, ) -> tuple[str | None, str | None] | None: """ Facade function which: @@ -259,9 +204,7 @@ def convert_attachment_to_content( ) return None - result = process_attachment( - confluence_client, attachment, page_id, allow_images, is_cloud - ) + result = process_attachment(source_operations, attachment, page_id, allow_images) if result.error is not None: logger.warning( "Attachment %s encountered error: %s", attachment["title"], result.error @@ -295,6 +238,12 @@ def build_confluence_document_id( return final_url +def build_cql_url(cql: str, expand: str | None = None) -> str: + """Returns the content-search path for a CQL query.""" + expand_string = f"&expand={expand}" if expand else "" + return f"rest/api/content/search?cql={cql}{expand_string}" + + def datetime_from_string(datetime_string: str) -> datetime: return datetime_to_utc(datetime.fromisoformat(datetime_string)) @@ -337,52 +286,6 @@ def confluence_refresh_tokens( return new_credentials -F = TypeVar("F", bound=Callable[..., Any]) - - -# https://developer.atlassian.com/cloud/confluence/rate-limiting/ -# this uses the native rate limiting option provided by the -# confluence client and otherwise applies a simpler set of error handling -def handle_confluence_rate_limit(confluence_call: F) -> F: - def wrapped_call(*args: list[Any], **kwargs: Any) -> Any: - MAX_RETRIES = 5 - - TIMEOUT = 600 - timeout_at = time.monotonic() + TIMEOUT - - for attempt in range(MAX_RETRIES): - if time.monotonic() > timeout_at: - raise TimeoutError( - f"Confluence call attempts took longer than {TIMEOUT} seconds." - ) - - try: - # we're relying more on the client to rate limit itself - # and applying our own retries in a more specific set of circumstances - return confluence_call(*args, **kwargs) - except requests.HTTPError as e: - delay_until = _handle_http_error(e, attempt, MAX_RETRIES) - logger.warning( - "HTTPError in confluence call. Retrying in %s seconds...", - delay_until, - ) - while time.monotonic() < delay_until: - # in the future, check a signal here to exit - time.sleep(1) - except AttributeError as e: - # Some error within the Confluence library, unclear why it fails. - # Users reported it to be intermittent, so just retry - if attempt == MAX_RETRIES - 1: - raise e - - logger.exception( - "Confluence Client raised an AttributeError. Retrying..." - ) - time.sleep(5) - - return cast(F, wrapped_call) - - def _handle_http_error(e: requests.HTTPError, attempt: int, max_retries: int) -> int: MIN_DELAY = 2 MAX_DELAY = 60 @@ -403,7 +306,7 @@ def _handle_http_error(e: requests.HTTPError, attempt: int, max_retries: int) -> "403 error. This sometimes happens when we hit Confluence rate limits. Retrying in %s seconds...", FORBIDDEN_RETRY_DELAY, ) - return FORBIDDEN_RETRY_DELAY + return math.ceil(time.monotonic() + FORBIDDEN_RETRY_DELAY) raise e @@ -478,3 +381,135 @@ def update_param_in_path(path: str, param: str, value: str) -> str: + "?" + "&".join(f"{k}={quote(v[0])}" for k, v in query_params.items()) ) + + +def sanitize_attachment_title(title: str) -> str: + """ + Sanitize the attachment title to be a valid HTML attribute. + """ + return title.replace("<", "_").replace(">", "_").replace(" ", "_").replace(":", "_") + + +def extract_text_from_confluence_html( + source_operations: "ConfluenceSourceOperations", + confluence_object: dict[str, Any], + fetched_titles: set[str], +) -> str: + """Parse a Confluence html page and replace the 'user Id' by the real + User Display Name + + Args: + source_operations: The Confluence source-operations gateway + confluence_object (dict): The confluence object as a dict + fetched_titles (set[str]): The titles of the pages that have already been fetched + Returns: + str: loaded and formated Confluence page + """ + body = confluence_object["body"] + object_html = body.get("storage", body.get("view", {})).get("value") + + soup = bs4.BeautifulSoup(object_html, "html.parser") + + _remove_macro_stylings(soup=soup) + + for date_span in soup.findAll("span", {"class": "date-lozenger-container"}): + date_span.replaceWith(date_span.get_text()) + + for user in soup.findAll("ri:user"): + user_id = ( + user.attrs["ri:account-id"] + if "ri:account-id" in user.attrs + else user.get("ri:userkey") + ) + if not user_id: + logger.warning( + "ri:userkey not found in ri:user element. Found attrs: %s", + user.attrs, + ) + continue + # Include @ sign for tagging, more clear for LLM + user.replaceWith("@" + source_operations.get_user_display_name(user_id=user_id)) + + for html_page_reference in soup.findAll("ac:structured-macro"): + # Here, we only want to process page within page macros + if html_page_reference.attrs.get("ac:name") != "include": + continue + + page_data = html_page_reference.find("ri:page") + if not page_data: + logger.warning( + "Skipping retrieval of %s because because page data is missing", + html_page_reference, + ) + continue + + page_title = page_data.attrs.get("ri:content-title") + if not page_title: + # only fetch pages that have a title + logger.warning( + "Skipping retrieval of %s because it has no title", + html_page_reference, + ) + continue + + if page_title in fetched_titles: + # prevent recursive fetching of pages + logger.debug("Skipping %s because it has already been fetched", page_title) + continue + + fetched_titles.add(page_title) + + # Wrap this in a try-except because there are some pages that might not exist + try: + page_contents = source_operations.find_page_by_title(title=page_title) + except Exception as e: + logger.warning( + "Error getting page contents for object %s: %s", + confluence_object, + e, + ) + continue + + if not page_contents: + continue + + text_from_page = extract_text_from_confluence_html( + source_operations=source_operations, + confluence_object=page_contents, + fetched_titles=fetched_titles, + ) + + html_page_reference.replaceWith(text_from_page) + + for html_link_body in soup.findAll("ac:link-body"): + # This extracts the text from inline links in the page so they can be + # represented in the document text as plain text + try: + text_from_link = html_link_body.text + html_link_body.replaceWith(f"(LINK TEXT: {text_from_link})") + except Exception as e: + logger.warning("Error processing ac:link-body: %s", e) + + for html_attachment in soup.findAll("ri:attachment"): + # This extracts the text from inline attachments in the page so they can be + # represented in the document text as plain text + try: + html_attachment.replaceWith( + f"{sanitize_attachment_title(html_attachment.attrs['ri:filename'])}" + ) # to be replaced later + except Exception as e: + logger.warning("Error processing ac:attachment: %s", e) + + return format_document_soup(soup) + + +def _remove_macro_stylings(soup: bs4.BeautifulSoup) -> None: + for macro_root in soup.findAll("ac:structured-macro"): + if not isinstance(macro_root, bs4.Tag): + continue + + macro_styling = macro_root.find(name="ac:parameter", attrs={"ac:name": "page"}) + if not macro_styling or not isinstance(macro_styling, bs4.Tag): + continue + + macro_styling.extract() diff --git a/backend/onyx/connectors/cross_connector_utils/miscellaneous_utils.py b/backend/onyx/connectors/cross_connector_utils/miscellaneous_utils.py index 477cc1289e..1492e075b4 100644 --- a/backend/onyx/connectors/cross_connector_utils/miscellaneous_utils.py +++ b/backend/onyx/connectors/cross_connector_utils/miscellaneous_utils.py @@ -206,18 +206,6 @@ def get_cloudId(base_url: str) -> str: return response.json()["cloudId"] -def credential_uses_scoped_token(credentials: dict[str, Any]) -> bool: - """Whether an Atlassian credential says its API token has scopes. - - The credential form stores a boolean; the edit form can store text, so - "true" counts and "false" does not. - """ - value = credentials.get("scoped_token") - if isinstance(value, str): - return value.strip().lower() == "true" - return value is True - - def scoped_url(url: str, product: str) -> str: parsed = urlparse(url) base_url = parsed.scheme + "://" + parsed.netloc diff --git a/backend/onyx/connectors/jira/connector.py b/backend/onyx/connectors/jira/connector.py index 66b481e3d4..ea80a792fe 100644 --- a/backend/onyx/connectors/jira/connector.py +++ b/backend/onyx/connectors/jira/connector.py @@ -20,7 +20,6 @@ from onyx.configs.app_configs import ( ) from onyx.configs.constants import DocumentSource from onyx.connectors.cross_connector_utils.miscellaneous_utils import ( - credential_uses_scoped_token, is_atlassian_date_error, time_str_to_utc, ) @@ -705,11 +704,6 @@ class JiraConnector( return project_key def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: - # The credential now carries the flag; older connectors set it in - # their own config. - self.scoped_token = self.scoped_token or credential_uses_scoped_token( - credentials - ) self._jira_client = build_jira_client( credentials=credentials, jira_base=self.jira_base, diff --git a/backend/tests/daily/connectors/confluence/test_confluence_user_email_overrides.py b/backend/tests/daily/connectors/confluence/test_confluence_user_email_overrides.py index 377d2046f2..5b1e9491c1 100644 --- a/backend/tests/daily/connectors/confluence/test_confluence_user_email_overrides.py +++ b/backend/tests/daily/connectors/confluence/test_confluence_user_email_overrides.py @@ -1,7 +1,8 @@ import types from unittest.mock import patch -from onyx.connectors.confluence.onyx_confluence import ConfluenceUser, OnyxConfluence +from onyx.connectors.confluence.models import ConfluenceUser +from onyx.connectors.confluence.onyx_confluence import OnyxConfluence from onyx.connectors.interfaces import CredentialsProviderInterface diff --git a/backend/tests/unit/onyx/connectors/confluence/confluence_gateway_fakes.py b/backend/tests/unit/onyx/connectors/confluence/confluence_gateway_fakes.py new file mode 100644 index 0000000000..cb9ac29330 --- /dev/null +++ b/backend/tests/unit/onyx/connectors/confluence/confluence_gateway_fakes.py @@ -0,0 +1,23 @@ +"""Builds a Confluence gateway around a fake transport client for unit tests.""" + +from typing import Any +from unittest import mock + +from onyx.connectors.confluence.source_operations import ConfluenceSourceOperations +from onyx.connectors.interfaces import CredentialsProviderInterface + + +def gateway_with_client( + client: Any, + *, + wiki_base: str = "https://example.atlassian.net/wiki", + is_cloud: bool = False, +) -> ConfluenceSourceOperations: + """Returns a gateway whose main and fast clients are both ``client``.""" + gateway = ConfluenceSourceOperations( + credentials_provider=mock.Mock(spec=CredentialsProviderInterface), + connector_specific_config={"wiki_base": wiki_base, "is_cloud": is_cloud}, + ) + gateway._cached_client = client + gateway._cached_fast_client = client + return gateway diff --git a/backend/tests/unit/onyx/connectors/confluence/test_attachment_pagination_400.py b/backend/tests/unit/onyx/connectors/confluence/test_attachment_pagination_400.py index 921403adca..b8f7169ffc 100644 --- a/backend/tests/unit/onyx/connectors/confluence/test_attachment_pagination_400.py +++ b/backend/tests/unit/onyx/connectors/confluence/test_attachment_pagination_400.py @@ -20,8 +20,14 @@ from onyx.connectors.confluence.connector import ( ConfluenceConnector, _extract_page_id_from_url, ) -from onyx.connectors.confluence.onyx_confluence import OnyxConfluence +from onyx.connectors.confluence.source_operations import ConfluenceSearchVariant +from onyx.connectors.confluence.source_operations import ( + _OnyxConfluence as OnyxConfluence, +) from onyx.connectors.models import ConnectorFailure, Document, DocumentFailure +from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import ( + gateway_with_client, +) _PAGE = { "id": "111", @@ -73,9 +79,9 @@ def _fetch_with_pagination_error( with ( mock.patch.object( ConfluenceConnector, - "confluence_client", + "source_operations", new_callable=mock.PropertyMock, - return_value=fake_client, + return_value=gateway_with_client(fake_client), ), mock.patch.object( connector, "_maybe_yield_page_hierarchy_node", return_value=None @@ -155,9 +161,9 @@ def _run_reindex(attachment_error_code: int | None) -> list[Any]: with ( mock.patch.object( ConfluenceConnector, - "confluence_client", + "source_operations", new_callable=mock.PropertyMock, - return_value=fake_client, + return_value=gateway_with_client(fake_client), ), mock.patch.object( connector, "_yield_space_hierarchy_nodes", return_value=iter([]) @@ -209,12 +215,12 @@ def _slim_attachments_with(side_effects: list[Any]) -> tuple[list[Any], mock.Moc with mock.patch.object( ConfluenceConnector, - "confluence_client", + "source_operations", new_callable=mock.PropertyMock, - return_value=fake_client, + return_value=gateway_with_client(fake_client), ): results = connector._retrieve_attachments_for_slim_page( - "111", "space", None, None + "111", ConfluenceSearchVariant.SLIM, "space", None, None ) return results, fake_client diff --git a/backend/tests/unit/onyx/connectors/confluence/test_confcloud_77618_fallback.py b/backend/tests/unit/onyx/connectors/confluence/test_confcloud_77618_fallback.py index a95a2e2928..fc193fe488 100644 --- a/backend/tests/unit/onyx/connectors/confluence/test_confcloud_77618_fallback.py +++ b/backend/tests/unit/onyx/connectors/confluence/test_confcloud_77618_fallback.py @@ -19,12 +19,17 @@ from onyx.connectors.confluence.access import ( get_page_restrictions_with_per_ancestor_fetch as get_page_restrictions_with_per_ancestor_fetch_shim, ) from onyx.connectors.confluence.connector import ConfluenceConnector -from onyx.connectors.confluence.onyx_confluence import ( +from onyx.connectors.confluence.source_operations import ( Confcloud77618Error, - OnyxConfluence, _is_confcloud_77618_response, ) +from onyx.connectors.confluence.source_operations import ( + _OnyxConfluence as OnyxConfluence, +) from onyx.connectors.interfaces import CredentialsProviderInterface +from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import ( + gateway_with_client, +) _CONFCLOUD_77618_BODY = ( '{"statusCode":404,"data":{"authorized":false,"valid":false,' @@ -361,9 +366,9 @@ def test_pruning_expand_skips_restrictions_but_keeps_hierarchy( with mock.patch.object( ConfluenceConnector, - "confluence_client", + "source_operations", new_callable=mock.PropertyMock, - return_value=fake_client, + return_value=gateway_with_client(fake_client), ): with mock.patch.object( confluence_connector, diff --git a/backend/tests/unit/onyx/connectors/confluence/test_confluence_attachment_links.py b/backend/tests/unit/onyx/connectors/confluence/test_confluence_attachment_links.py index 3129b17ccc..2e5b648c9d 100644 --- a/backend/tests/unit/onyx/connectors/confluence/test_confluence_attachment_links.py +++ b/backend/tests/unit/onyx/connectors/confluence/test_confluence_attachment_links.py @@ -5,8 +5,13 @@ from unittest import mock import pytest from onyx.connectors.confluence.connector import ConfluenceConnector -from onyx.connectors.confluence.onyx_confluence import OnyxConfluence +from onyx.connectors.confluence.source_operations import ( + _OnyxConfluence as OnyxConfluence, +) from onyx.connectors.models import Document +from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import ( + gateway_with_client, +) _PAGE_ID = "111" _ATTACHMENT_PAGE_ID = "222" @@ -70,9 +75,9 @@ def test_attachment_section_link_uses_platform_specific_url( with ( mock.patch.object( ConfluenceConnector, - "confluence_client", + "source_operations", new_callable=mock.PropertyMock, - return_value=confluence_client, + return_value=gateway_with_client(confluence_client), ), mock.patch( "onyx.connectors.confluence.connector.convert_attachment_to_content", @@ -101,9 +106,9 @@ def test_attachment_failure_uses_attachment_document_id() -> None: with ( mock.patch.object( ConfluenceConnector, - "confluence_client", + "source_operations", new_callable=mock.PropertyMock, - return_value=confluence_client, + return_value=gateway_with_client(confluence_client), ), mock.patch( "onyx.connectors.confluence.connector.convert_attachment_to_content", diff --git a/backend/tests/unit/onyx/connectors/confluence/test_confluence_checkpointing.py b/backend/tests/unit/onyx/connectors/confluence/test_confluence_checkpointing.py index e956bb2cd6..2e9c4b45c4 100644 --- a/backend/tests/unit/onyx/connectors/confluence/test_confluence_checkpointing.py +++ b/backend/tests/unit/onyx/connectors/confluence/test_confluence_checkpointing.py @@ -12,7 +12,9 @@ from onyx.connectors.confluence.connector import ( ConfluenceCheckpoint, ConfluenceConnector, ) -from onyx.connectors.confluence.onyx_confluence import OnyxConfluence +from onyx.connectors.confluence.source_operations import ( + _OnyxConfluence as OnyxConfluence, +) from onyx.connectors.exceptions import ( CredentialExpiredError, InsufficientPermissionsError, @@ -25,6 +27,9 @@ from onyx.connectors.models import ( HierarchyNode, SlimDocument, ) +from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import ( + gateway_with_client, +) from tests.unit.onyx.connectors.utils import ( load_everything_from_checkpoint_connector, load_everything_from_checkpoint_connector_from_checkpoint, @@ -69,8 +74,7 @@ def confluence_connector( batch_size=2, ) # Initialize the client directly - connector._confluence_client = mock_confluence_client - connector._low_timeout_confluence_client = mock_confluence_client + connector._source_operations = gateway_with_client(mock_confluence_client) with patch("onyx.connectors.confluence.connector._SLIM_DOC_BATCH_SIZE", 2): yield connector @@ -151,8 +155,7 @@ def test_load_from_checkpoint_happy_path( ) # Mock paginated_cql_retrieval to return our mock pages - confluence_client = confluence_connector._confluence_client - assert confluence_client is not None, "bad test setup" + confluence_client = confluence_connector.source_operations._client() # Mock space retrieval for hierarchy nodes (called at start of first batch) confluence_client.retrieve_confluence_spaces = MagicMock( @@ -222,8 +225,7 @@ def test_load_from_checkpoint_with_page_processing_error( mock_page2 = create_mock_page(id="2", title="Page 2") # Mock paginated_cql_retrieval to return our mock pages - confluence_client = confluence_connector._confluence_client - assert confluence_client is not None, "bad test setup" + confluence_client = confluence_connector.source_operations._client() # Mock space retrieval for hierarchy nodes (called at start of first batch) confluence_client.retrieve_confluence_spaces = MagicMock( @@ -317,8 +319,7 @@ def test_retrieve_all_slim_docs_perm_sync( mock_page2 = create_mock_page(id="2") # Mock paginated_cql_retrieval to return our mock pages - confluence_client = confluence_connector._confluence_client - assert confluence_client is not None, "bad test setup" + confluence_client = confluence_connector.source_operations._client() # Mock space retrieval for hierarchy nodes confluence_client.retrieve_confluence_spaces = MagicMock( @@ -389,7 +390,7 @@ def test_validate_connector_settings_errors( error = HTTPError(response=MagicMock(status_code=status_code)) with patch( - "onyx.connectors.confluence.onyx_confluence.OnyxConfluence.retrieve_confluence_spaces" + "onyx.connectors.confluence.source_operations._OnyxConfluence.retrieve_confluence_spaces" ) as mock_retrieve: mock_retrieve.side_effect = error @@ -402,7 +403,7 @@ def test_validate_connector_settings_success( confluence_connector: ConfluenceConnector, ) -> None: """Test successful validation""" - low_client = confluence_connector.low_timeout_confluence_client + low_client = confluence_connector.source_operations._client(fast=True) with ( patch.object( low_client, @@ -442,8 +443,7 @@ def test_checkpoint_progress( ) # Mock paginated_cql_retrieval to return our mock pages - confluence_client = confluence_connector._confluence_client - assert confluence_client is not None, "bad test setup" + confluence_client = confluence_connector.source_operations._client() # Mock space retrieval for hierarchy nodes (called at start of first batch) confluence_client.retrieve_confluence_spaces = MagicMock( diff --git a/backend/tests/unit/onyx/connectors/confluence/test_confluence_failure_document_id.py b/backend/tests/unit/onyx/connectors/confluence/test_confluence_failure_document_id.py index 21a1a44957..fabb1e5873 100644 --- a/backend/tests/unit/onyx/connectors/confluence/test_confluence_failure_document_id.py +++ b/backend/tests/unit/onyx/connectors/confluence/test_confluence_failure_document_id.py @@ -27,8 +27,13 @@ from onyx.connectors.confluence.connector import ( ConfluenceConnector, _extract_page_id_from_url, ) -from onyx.connectors.confluence.onyx_confluence import OnyxConfluence +from onyx.connectors.confluence.source_operations import ( + _OnyxConfluence as OnyxConfluence, +) from onyx.connectors.models import ConnectorFailure, Document +from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import ( + gateway_with_client, +) _PAGE_ID = "555" @@ -47,7 +52,7 @@ def _make_connector() -> ConfluenceConnector: ) confluence_client = mock.Mock(spec=OnyxConfluence) confluence_client.paginated_cql_retrieval.return_value = iter([]) - connector._confluence_client = confluence_client + connector._source_operations = gateway_with_client(confluence_client) return connector diff --git a/backend/tests/unit/onyx/connectors/confluence/test_confluence_source_operations.py b/backend/tests/unit/onyx/connectors/confluence/test_confluence_source_operations.py new file mode 100644 index 0000000000..8661dfc0d2 --- /dev/null +++ b/backend/tests/unit/onyx/connectors/confluence/test_confluence_source_operations.py @@ -0,0 +1,237 @@ +from types import SimpleNamespace +from typing import Any +from unittest import mock + +import pytest +import requests +from atlassian.errors import ApiError + +from onyx.connectors.confluence import source_operations +from onyx.connectors.confluence import utils as confluence_utils +from onyx.connectors.confluence.connector import ConfluenceConnector +from onyx.connectors.confluence.source_operations import ( + ConfluenceProbeVariant, + ConfluenceRetriesExhaustedError, + ConfluenceSourceOperations, + ConfluenceSpaceNotFoundError, + _OnyxConfluence, +) +from onyx.connectors.exceptions import ConnectorValidationError +from onyx.connectors.interfaces import CredentialsProviderInterface +from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import ( + gateway_with_client, +) + +_WIKI_BASE = "https://confluence.example.com" +_ATTACHMENT: dict[str, Any] = { + "id": "att1", + "title": "spec.pdf", + "_links": {"download": "/download/attachments/1/spec.pdf"}, +} + + +def _provider() -> mock.Mock: + provider = mock.Mock(spec=CredentialsProviderInterface) + provider.is_dynamic.return_value = False + provider.get_credentials.return_value = { + "confluence_username": "user", + "confluence_access_token": "token", + } + provider.get_provider_key.return_value = "key" + provider.__enter__ = mock.Mock(return_value=None) + provider.__exit__ = mock.Mock(return_value=None) + return provider + + +def _response(status_code: int, content: bytes = b"") -> requests.Response: + response = requests.Response() + response.status_code = status_code + response._content = content + response.url = _WIKI_BASE + return response + + +def _sdk_client(*responses: requests.Response) -> mock.Mock: + sdk = mock.Mock() + sdk.url = _WIKI_BASE + sdk.session.get.side_effect = list(responses) + return sdk + + +def _transport(sdk: mock.Mock, is_cloud: bool = False) -> _OnyxConfluence: + client = _OnyxConfluence( + is_cloud=is_cloud, url=_WIKI_BASE, credentials_provider=_provider() + ) + client._confluence = sdk + client._kwargs = client.shared_base_kwargs + return client + + +@pytest.fixture +def fake_clock(monkeypatch: pytest.MonkeyPatch) -> None: + """Makes retry waits instant: sleep advances a fake monotonic clock.""" + now = 0.0 + + def monotonic() -> float: + return now + + def sleep(seconds: float) -> None: + nonlocal now + now += seconds + + fake_time = SimpleNamespace(monotonic=monotonic, sleep=sleep) + monkeypatch.setattr(source_operations, "time", fake_time) + monkeypatch.setattr(confluence_utils, "time", fake_time) + + +@pytest.mark.usefixtures("fake_clock") +def test_download_attachment_retries_server_errors() -> None: + # Precondition. + sdk = _sdk_client(_response(503), _response(200, b"file bytes")) + gateway = gateway_with_client(_transport(sdk), wiki_base=_WIKI_BASE) + + # Under test. + content = gateway.download_attachment(attachment=_ATTACHMENT, parent_content_id="1") + + # Postcondition. + assert content == b"file bytes" + assert sdk.session.get.call_count == 2 + sdk.session.get.assert_called_with(_WIKI_BASE + "/download/attachments/1/spec.pdf") + + +def test_download_attachment_uses_the_refreshed_client() -> None: + # Precondition. + stale_sdk = _sdk_client(_response(200, b"stale")) + fresh_sdk = _sdk_client(_response(200, b"fresh")) + client = _transport(stale_sdk) + gateway = gateway_with_client(client, wiki_base=_WIKI_BASE) + + # Under test. + with ( + mock.patch.object( + client, "_renew_credentials", return_value=({"new": "creds"}, True) + ), + mock.patch.object( + client, "_initialize_connection_helper", return_value=fresh_sdk + ) as rebuild, + ): + content = gateway.download_attachment( + attachment=_ATTACHMENT, parent_content_id="1" + ) + + # Postcondition. + assert content == b"fresh" + rebuild.assert_called_once() + stale_sdk.session.get.assert_not_called() + + +def test_download_attachment_raises_on_client_error() -> None: + # Precondition. + sdk = _sdk_client(_response(404)) + gateway = gateway_with_client(_transport(sdk), wiki_base=_WIKI_BASE) + + # Under test / postcondition. + with pytest.raises(requests.HTTPError): + gateway.download_attachment(attachment=_ATTACHMENT, parent_content_id="1") + assert sdk.session.get.call_count == 1 + + +def test_download_attachment_does_not_retry_a_403() -> None: + # Precondition. + sdk = _sdk_client(_response(403), _response(200, b"file bytes")) + gateway = gateway_with_client(_transport(sdk), wiki_base=_WIKI_BASE) + + # Under test / postcondition. + with pytest.raises(requests.HTTPError) as raised: + gateway.download_attachment(attachment=_ATTACHMENT, parent_content_id="1") + assert raised.value.response is not None + assert raised.value.response.status_code == 403 + assert sdk.session.get.call_count == 1 + + +@pytest.mark.usefixtures("fake_clock") +def test_exhausted_retries_raise_with_the_last_status() -> None: + # Precondition. + client = _transport(_sdk_client()) + + def always_rate_limited(_confluence: Any) -> bytes: + response = _response(429) + raise requests.HTTPError(response=response) + + # Under test / postcondition. + with pytest.raises(ConfluenceRetriesExhaustedError) as raised: + client._call_with_retries(always_rate_limited) + assert raised.value.last_status_code == 429 + + +def test_download_attachment_cloud_needs_parent_id() -> None: + gateway = gateway_with_client( + _transport(_sdk_client(), is_cloud=True), is_cloud=True + ) + + with pytest.raises(ValueError): + gateway.download_attachment(attachment=_ATTACHMENT, parent_content_id=None) + + +@pytest.mark.parametrize("scoped_token", [False, True]) +def test_connector_probes_the_site_once(scoped_token: bool) -> None: + """One probe and one tenant_info lookup per connector, even though + validation builds a second, low-timeout client.""" + # Precondition. + connector = ConfluenceConnector( + wiki_base=_WIKI_BASE, is_cloud=True, space="KEY", scoped_token=scoped_token + ) + + with ( + mock.patch.object(_OnyxConfluence, "_probe_connection") as probe, + mock.patch.object( + source_operations, + "scoped_url", + return_value="https://api.atlassian.com/ex/confluence/cloud-id", + ) as resolve_scoped_url, + mock.patch.object( + ConfluenceSourceOperations, + "list_spaces", + return_value=iter([{"key": "KEY"}]), + ), + mock.patch.object( + _OnyxConfluence, "get_space", create=True, return_value={"key": "KEY"} + ), + ): + # Under test. + connector.set_credentials_provider(_provider()) + connector.validate_connector_settings() + + # Postcondition. + probe.assert_called_once() + assert resolve_scoped_url.call_count == (1 if scoped_token else 0) + gateway = connector.source_operations + assert gateway._cached_client is not None + assert gateway._cached_fast_client is not None + assert gateway._cached_client is not gateway._cached_fast_client + + +def test_probe_site_rejects_a_mismatched_variant() -> None: + gateway = ConfluenceSourceOperations( + credentials_provider=_provider(), + connector_specific_config={"wiki_base": _WIKI_BASE, "is_cloud": False}, + ) + + with pytest.raises(ValueError): + gateway.probe_site(variant=ConfluenceProbeVariant.SCOPED) + + +def test_get_space_translates_the_sdk_error() -> None: + # Precondition. + client = mock.Mock(spec=_OnyxConfluence) + client.get_space = mock.Mock(side_effect=ApiError("no such space")) + gateway = gateway_with_client(client) + connector = ConfluenceConnector(wiki_base=_WIKI_BASE, is_cloud=False, space="X") + connector._source_operations = gateway + + # Under test / postcondition. + with pytest.raises(ConfluenceSpaceNotFoundError): + gateway.get_space(space_key="X") + client.retrieve_confluence_spaces.return_value = iter([{"key": "Y"}]) + with pytest.raises(ConnectorValidationError, match="Invalid Confluence space key"): + connector.validate_connector_settings() diff --git a/backend/tests/unit/onyx/connectors/confluence/test_confluence_user_cache_instance_isolation.py b/backend/tests/unit/onyx/connectors/confluence/test_confluence_user_cache_instance_isolation.py index 829053d662..e03adc19b6 100644 --- a/backend/tests/unit/onyx/connectors/confluence/test_confluence_user_cache_instance_isolation.py +++ b/backend/tests/unit/onyx/connectors/confluence/test_confluence_user_cache_instance_isolation.py @@ -8,6 +8,7 @@ run in the same multi-tenant worker process.""" from unittest.mock import MagicMock import onyx.connectors.confluence.onyx_confluence as onyx_confluence +import onyx.connectors.confluence.source_operations as source_operations def _client(url: str) -> MagicMock: @@ -64,13 +65,13 @@ def test_userkey_email_cache_is_instance_isolated() -> None: def test_display_name_cache_is_instance_isolated() -> None: - onyx_confluence._USER_ID_TO_DISPLAY_NAME_CACHE.clear() + source_operations._USER_ID_TO_DISPLAY_NAME_CACHE.clear() user_id = "user-1" client_a = _client("https://a.example.com") client_a.get_user_details_by_userkey.return_value = {"displayName": "Alice A"} client_b = _client("https://b.example.com") client_b.get_user_details_by_userkey.return_value = {"displayName": "Bob B"} - assert onyx_confluence._get_user(client_a, user_id) == "Alice A" + assert source_operations._get_user(client_a, user_id) == "Alice A" # same id on a different instance must not see instance A's display name - assert onyx_confluence._get_user(client_b, user_id) == "Bob B" + assert source_operations._get_user(client_b, user_id) == "Bob B" diff --git a/backend/tests/unit/onyx/connectors/confluence/test_extract_text.py b/backend/tests/unit/onyx/connectors/confluence/test_extract_text.py index 00d020e297..d6ab6c9ba9 100644 --- a/backend/tests/unit/onyx/connectors/confluence/test_extract_text.py +++ b/backend/tests/unit/onyx/connectors/confluence/test_extract_text.py @@ -1,16 +1,20 @@ from unittest import mock -from onyx.connectors.confluence.onyx_confluence import extract_text_from_confluence_html +from onyx.connectors.confluence.source_operations import ConfluenceSourceOperations +from onyx.connectors.confluence.utils import extract_text_from_confluence_html +from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import ( + gateway_with_client, +) def _make_confluence_object(html: str) -> dict: return {"body": {"storage": {"value": html}}} -def _make_mock_client() -> mock.Mock: +def _make_gateway() -> ConfluenceSourceOperations: client = mock.Mock() client.paginated_cql_retrieval.return_value = iter([]) - return client + return gateway_with_client(client) def test_date_lozenge_text_is_preserved() -> None: @@ -21,7 +25,7 @@ def test_date_lozenge_text_is_preserved() -> None: " at noon.

" ) result = extract_text_from_confluence_html( - confluence_client=_make_mock_client(), + source_operations=_make_gateway(), confluence_object=_make_confluence_object(html), fetched_titles=set(), ) @@ -32,7 +36,7 @@ def test_page_without_date_lozenge_unaffected() -> None: """Pages with no date lozenge spans are processed normally.""" html = "

No dates here.

" result = extract_text_from_confluence_html( - confluence_client=_make_mock_client(), + source_operations=_make_gateway(), confluence_object=_make_confluence_object(html), fetched_titles=set(), ) diff --git a/backend/tests/unit/onyx/connectors/confluence/test_include_attachments_skip.py b/backend/tests/unit/onyx/connectors/confluence/test_include_attachments_skip.py index 6c9e7c8766..b932940ae5 100644 --- a/backend/tests/unit/onyx/connectors/confluence/test_include_attachments_skip.py +++ b/backend/tests/unit/onyx/connectors/confluence/test_include_attachments_skip.py @@ -6,8 +6,13 @@ from typing import Any from unittest import mock from onyx.connectors.confluence.connector import ConfluenceConnector -from onyx.connectors.confluence.onyx_confluence import OnyxConfluence +from onyx.connectors.confluence.source_operations import ( + _OnyxConfluence as OnyxConfluence, +) from onyx.connectors.models import SlimDocument +from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import ( + gateway_with_client, +) _PAGE_CQL = "PAGE_CQL" _ATTACHMENT_CQL = "ATTACHMENT_CQL" @@ -58,9 +63,9 @@ def _collect_slim_doc_ids( with ( mock.patch.object( ConfluenceConnector, - "confluence_client", + "source_operations", new_callable=mock.PropertyMock, - return_value=fake_client, + return_value=gateway_with_client(fake_client), ), mock.patch.object( connector, "_yield_space_hierarchy_nodes", return_value=iter([]) @@ -124,9 +129,9 @@ def test_main_pass_skips_attachment_fetch_when_disabled() -> None: with mock.patch.object( ConfluenceConnector, - "confluence_client", + "source_operations", new_callable=mock.PropertyMock, - return_value=fake_client, + return_value=gateway_with_client(fake_client), ): docs, failures = connector._fetch_page_attachments(_PAGE) diff --git a/backend/tests/unit/onyx/connectors/confluence/test_onyx_confluence.py b/backend/tests/unit/onyx/connectors/confluence/test_onyx_confluence.py index 3b00bea919..4095834149 100644 --- a/backend/tests/unit/onyx/connectors/confluence/test_onyx_confluence.py +++ b/backend/tests/unit/onyx/connectors/confluence/test_onyx_confluence.py @@ -7,12 +7,17 @@ import requests from requests import HTTPError from onyx.connectors.confluence import onyx_confluence as onyx_confluence_module +from onyx.connectors.confluence import source_operations as source_operations_module from onyx.connectors.confluence.onyx_confluence import ( + get_user_email_from_userkey__server, +) +from onyx.connectors.confluence.source_operations import ( _DEFAULT_PAGINATION_LIMIT, _MINIMUM_PAGINATION_LIMIT, ConfluenceRestSpacePermissionsNotAvailableError, - OnyxConfluence, - get_user_email_from_userkey__server, +) +from onyx.connectors.confluence.source_operations import ( + _OnyxConfluence as OnyxConfluence, ) from onyx.connectors.exceptions import ( ConnectorValidationError, @@ -1493,16 +1498,16 @@ def test_token_refresh_keeps_the_oauth_site(monkeypatch: pytest.MonkeyPatch) -> redis_client = mock.MagicMock() redis_client.get.return_value = None monkeypatch.setattr( - onyx_confluence_module, "get_redis_client", lambda **_: redis_client + source_operations_module, "get_redis_client", lambda **_: redis_client ) monkeypatch.setattr( - onyx_confluence_module, "OAUTH_CONFLUENCE_CLOUD_CLIENT_ID", "id" + source_operations_module, "OAUTH_CONFLUENCE_CLOUD_CLIENT_ID", "id" ) monkeypatch.setattr( - onyx_confluence_module, "OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET", "secret" + source_operations_module, "OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET", "secret" ) monkeypatch.setattr( - onyx_confluence_module, + source_operations_module, "confluence_refresh_tokens", lambda *_: { "confluence_access_token": "new-access", diff --git a/backend/tests/unit/onyx/connectors/confluence/test_rate_limit_handler.py b/backend/tests/unit/onyx/connectors/confluence/test_rate_limit_handler.py deleted file mode 100644 index 4313117fc6..0000000000 --- a/backend/tests/unit/onyx/connectors/confluence/test_rate_limit_handler.py +++ /dev/null @@ -1,63 +0,0 @@ -from unittest.mock import Mock - -import pytest -from requests import HTTPError - -from onyx.connectors.confluence.utils import handle_confluence_rate_limit - - -@pytest.fixture -def mock_confluence_call() -> Mock: - return Mock() - - -# ***** Checking call count to sleep() won't correctly reflect test correctness -# especially since we really need to sleep multiple times and check for -# abort signals moving forward. Disabling this test for now until we come up with -# a better way forward. - -# @pytest.mark.parametrize( -# "status_code,text,retry_after", -# [ -# (429, "Rate limit exceeded", "5"), -# (200, "Rate limit exceeded", None), -# (429, "Some other error", "5"), -# ], -# ) -# def test_rate_limit_handling( -# mock_confluence_call: Mock, status_code: int, text: str, retry_after: str | None -# ) -> None: -# with patch("time.sleep") as mock_sleep: -# mock_confluence_call.side_effect = [ -# HTTPError( -# response=Mock( -# status_code=status_code, -# text=text, -# headers={"Retry-After": retry_after} if retry_after else {}, -# ) -# ), -# ] * 2 + ["Success"] - -# handled_call = make_confluence_call_handle_rate_limit(mock_confluence_call) -# result = handled_call() - -# assert result == "Success" -# assert mock_confluence_call.call_count == 3 -# assert mock_sleep.call_count == 2 -# if retry_after: -# mock_sleep.assert_called_with(int(retry_after)) - - -# NOTE(rkuo): This tests an older version of rate limiting that is being deprecated -# and probably should go away soon. -def test_non_rate_limit_error(mock_confluence_call: Mock) -> None: - mock_confluence_call.side_effect = HTTPError( - response=Mock(status_code=500, text="Internal Server Error") - ) - - handled_call = handle_confluence_rate_limit(mock_confluence_call) - - with pytest.raises(HTTPError): - handled_call() - - assert mock_confluence_call.call_count == 5 diff --git a/backend/tests/unit/onyx/connectors/confluence/test_slim_doc_image_skip.py b/backend/tests/unit/onyx/connectors/confluence/test_slim_doc_image_skip.py index b51bc2cfb5..e2f1c2d25c 100644 --- a/backend/tests/unit/onyx/connectors/confluence/test_slim_doc_image_skip.py +++ b/backend/tests/unit/onyx/connectors/confluence/test_slim_doc_image_skip.py @@ -14,8 +14,13 @@ from unittest import mock import pytest from onyx.connectors.confluence.connector import ConfluenceConnector -from onyx.connectors.confluence.onyx_confluence import OnyxConfluence +from onyx.connectors.confluence.source_operations import ( + _OnyxConfluence as OnyxConfluence, +) from onyx.connectors.models import SlimDocument +from tests.unit.onyx.connectors.confluence.confluence_gateway_fakes import ( + gateway_with_client, +) _PAGE_CQL = "PAGE_CQL" _ATTACHMENT_CQL = "ATTACHMENT_CQL" @@ -75,9 +80,9 @@ def _collect_slim_doc_ids(connector: ConfluenceConnector) -> list[str]: with ( mock.patch.object( ConfluenceConnector, - "confluence_client", + "source_operations", new_callable=mock.PropertyMock, - return_value=fake_client, + return_value=gateway_with_client(fake_client), ), mock.patch.object( connector, "_yield_space_hierarchy_nodes", return_value=iter([]) diff --git a/backend/tests/unit/onyx/connectors/cross_connector_utils/test_miscellaneous_utils.py b/backend/tests/unit/onyx/connectors/cross_connector_utils/test_miscellaneous_utils.py index 999cf53abc..a0843e7a8f 100644 --- a/backend/tests/unit/onyx/connectors/cross_connector_utils/test_miscellaneous_utils.py +++ b/backend/tests/unit/onyx/connectors/cross_connector_utils/test_miscellaneous_utils.py @@ -2,10 +2,7 @@ import datetime import pytest -from onyx.connectors.cross_connector_utils.miscellaneous_utils import ( - credential_uses_scoped_token, - time_str_to_utc, -) +from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc def test_time_str_to_utc() -> None: @@ -54,23 +51,3 @@ def test_time_str_to_utc_raises_on_impossible_dates() -> None: ): with pytest.raises(ValueError): time_str_to_utc(bad) - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - (True, True), - (False, False), - ("true", True), - ("True", True), - ("false", False), - ("", False), - (None, False), - ], -) -def test_credential_uses_scoped_token(value: object, expected: bool) -> None: - assert credential_uses_scoped_token({"scoped_token": value}) is expected - - -def test_credential_uses_scoped_token_without_the_key() -> None: - assert credential_uses_scoped_token({}) is False diff --git a/pyproject.toml b/pyproject.toml index 8a7adca1e9..19495c83c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -380,7 +380,7 @@ known-first-party = ["onyx", "ee", "tests", "shared_configs", "model_server"] "backend/onyx/chat/process_message.py" = ["C901"] # 47 "backend/onyx/connectors/blob/connector.py" = ["C901"] # 24 "backend/onyx/connectors/confluence/connector.py" = ["C901"] # 22 -"backend/onyx/connectors/confluence/onyx_confluence.py" = ["C901"] # 22 +"backend/onyx/connectors/confluence/source_operations.py" = ["C901"] # 22 "backend/onyx/connectors/connector_runner.py" = ["C901"] # 26 "backend/onyx/connectors/drupal_wiki/connector.py" = ["C901"] # 21 "backend/onyx/connectors/github/connector.py" = ["C901"] # 28