mirror of
https://github.com/onyx-dot-app/onyx.git
synced 2026-10-23 03:42:59 -05:00
* first cut at redis * some new helper functions for the db * ignore kombu tables in alembic migrations (used by celery) * multiline commands for readability, add vespa_metadata_sync queue to worker * typo fix * fix returning tuple fields * add constants * fix _get_access_for_document * docstrings! * fix double function declaration and typing * fix type hinting * add a global redis pool * Add get_document function * use task_logger in various celery tasks * add celeryconfig.py to simplify configuration. Will be used in a subsequent commit * Add celery redis helper. used in a subsequent PR * kombu warning getting spammy since celery is not self managing its queue in Postgres any more * add last_modified and last_synced to documents * fix task naming convention * use celeryconfig.py * the big one. adds queues and tasks, updates functions to use the queues with priorities, etc * change vespa index log line to debug * mypy fixes * update alembic migration * fix fence ordering, rename to "monitor", fix fetch_versioned_implementation call * mypy * switch to monotonic time * fix startup dependencies on redis * rebase alembic migration * kombu cleanup - fail silently * mypy * add redis_host environment override * update REDIS_HOST env var in docker-compose.dev.yml * update the rest of the docker files * harden indexing-status endpoint against db changes happening in the background. Needs further improvement but OK for now. * allow no task syncs to run because we create certain objects with no entries but initially marked as out of date * add back writing to vespa on indexing * update contributing guide * backporting fixes from background_deletion * renaming cache to cache_volume * add redis password to various deployments * try setting up pr testing for helm * fix indent * hopefully this release version actually exists * fix command line option to --chart-dirs * fetch-depth 0 * edit values.yaml * try setting ct working directory * bypass testing only on change for now * move files and lint them * update helm testing * some issues suggest using --config works * add vespa repo * add postgresql repo * increase timeout * try amd64 runner * fix redis password reference * add comment to helm chart testing workflow * rename helm testing workflow to disable it * adding clarifying comments * address code review * missed a file * remove commented warning ... just not needed --------- Co-authored-by: Richard Kuo <rkuo@rkuo.com>
122 lines
4.0 KiB
Python
122 lines
4.0 KiB
Python
import functools
|
|
import importlib
|
|
from typing import Any
|
|
from typing import TypeVar
|
|
|
|
from danswer.configs.app_configs import ENTERPRISE_EDITION_ENABLED
|
|
from danswer.utils.logger import setup_logger
|
|
|
|
logger = setup_logger()
|
|
|
|
|
|
class DanswerVersion:
|
|
def __init__(self) -> None:
|
|
self._is_ee = False
|
|
|
|
def set_ee(self) -> None:
|
|
self._is_ee = True
|
|
|
|
def get_is_ee_version(self) -> bool:
|
|
return self._is_ee
|
|
|
|
|
|
global_version = DanswerVersion()
|
|
|
|
|
|
def set_is_ee_based_on_env_variable() -> None:
|
|
if ENTERPRISE_EDITION_ENABLED and not global_version.get_is_ee_version():
|
|
logger.notice("Enterprise Edition enabled")
|
|
global_version.set_ee()
|
|
|
|
|
|
@functools.lru_cache(maxsize=128)
|
|
def fetch_versioned_implementation(module: str, attribute: str) -> Any:
|
|
"""
|
|
Fetches a versioned implementation of a specified attribute from a given module.
|
|
This function first checks if the application is running in an Enterprise Edition (EE)
|
|
context. If so, it attempts to import the attribute from the EE-specific module.
|
|
If the module or attribute is not found, it falls back to the default module or
|
|
raises the appropriate exception depending on the context.
|
|
|
|
Args:
|
|
module (str): The name of the module from which to fetch the attribute.
|
|
attribute (str): The name of the attribute to fetch from the module.
|
|
|
|
Returns:
|
|
Any: The fetched implementation of the attribute.
|
|
|
|
Raises:
|
|
ModuleNotFoundError: If the module cannot be found and the error is not related to
|
|
the Enterprise Edition fallback logic.
|
|
|
|
Logs:
|
|
Logs debug information about the fetching process and warnings if the versioned
|
|
implementation cannot be found or loaded.
|
|
"""
|
|
logger.debug("Fetching versioned implementation for %s.%s", module, attribute)
|
|
is_ee = global_version.get_is_ee_version()
|
|
|
|
module_full = f"ee.{module}" if is_ee else module
|
|
try:
|
|
return getattr(importlib.import_module(module_full), attribute)
|
|
except ModuleNotFoundError as e:
|
|
logger.warning(
|
|
"Failed to fetch versioned implementation for %s.%s: %s",
|
|
module_full,
|
|
attribute,
|
|
e,
|
|
)
|
|
|
|
if is_ee:
|
|
if "ee.danswer" not in str(e):
|
|
# If it's a non Danswer related import failure, this is likely because
|
|
# a dependent library has not been installed. Should raise this failure
|
|
# instead of letting the server start up
|
|
raise e
|
|
|
|
# Use the MIT version as a fallback, this allows us to develop MIT
|
|
# versions independently and later add additional EE functionality
|
|
# similar to feature flagging
|
|
return getattr(importlib.import_module(module), attribute)
|
|
|
|
raise
|
|
|
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
def fetch_versioned_implementation_with_fallback(
|
|
module: str, attribute: str, fallback: T
|
|
) -> T:
|
|
"""
|
|
Attempts to fetch a versioned implementation of a specified attribute from a given module.
|
|
If the attempt fails (e.g., due to an import error or missing attribute), the function logs
|
|
a warning and returns the provided fallback implementation.
|
|
|
|
Args:
|
|
module (str): The name of the module from which to fetch the attribute.
|
|
attribute (str): The name of the attribute to fetch from the module.
|
|
fallback (T): The fallback implementation to return if fetching the attribute fails.
|
|
|
|
Returns:
|
|
T: The fetched implementation if successful, otherwise the provided fallback.
|
|
"""
|
|
try:
|
|
return fetch_versioned_implementation(module, attribute)
|
|
except Exception:
|
|
return fallback
|
|
|
|
|
|
def noop_fallback(*args: Any, **kwargs: Any) -> None:
|
|
"""
|
|
A no-op (no operation) fallback function that accepts any arguments but does nothing.
|
|
This is often used as a default or placeholder callback function.
|
|
|
|
Args:
|
|
*args (Any): Positional arguments, which are ignored.
|
|
**kwargs (Any): Keyword arguments, which are ignored.
|
|
|
|
Returns:
|
|
None
|
|
"""
|