Files
onyx/backend/danswer/utils/variable_functionality.py
T
rkuo-danswerandRichard Kuo f1c5e80f17 Feature/background processing (#2275)
* first cut at redis

* some new helper functions for the db

* ignore kombu tables in alembic migrations (used by celery)

* multiline commands for readability, add vespa_metadata_sync queue to worker

* typo fix

* fix returning tuple fields

* add constants

* fix _get_access_for_document

* docstrings!

* fix double function declaration and typing

* fix type hinting

* add a global redis pool

* Add get_document function

* use task_logger in various celery tasks

* add celeryconfig.py to simplify configuration. Will be used in a subsequent commit

* Add celery redis helper. used in a subsequent PR

* kombu warning getting spammy since celery is not self managing its queue in Postgres any more

* add last_modified and last_synced to documents

* fix task naming convention

* use celeryconfig.py

* the big one. adds queues and tasks, updates functions to use the queues with priorities, etc

* change vespa index log line to debug

* mypy fixes

* update alembic migration

* fix fence ordering, rename to "monitor", fix fetch_versioned_implementation call

* mypy

* switch to monotonic time

* fix startup dependencies on redis

* rebase alembic migration

* kombu cleanup - fail silently

* mypy

* add redis_host environment override

* update REDIS_HOST env var in docker-compose.dev.yml

* update the rest of the docker files

* harden indexing-status endpoint against db changes happening in the background.  Needs further improvement but OK for now.

* allow no task syncs to run because we create certain objects with no entries but initially marked as out of date

* add back writing to vespa on indexing

* update contributing guide

* backporting fixes from background_deletion

* renaming cache to cache_volume

* add redis password to various deployments

* try setting up pr testing for helm

* fix indent

* hopefully this release version actually exists

* fix command line option to --chart-dirs

* fetch-depth 0

* edit values.yaml

* try setting ct working directory

* bypass testing only on change for now

* move files and lint them

* update helm testing

* some issues suggest using --config works

* add vespa repo

* add postgresql repo

* increase timeout

* try amd64 runner

* fix redis password reference

* add comment to helm chart testing workflow

* rename helm testing workflow to disable it

* adding clarifying comments

* address code review

* missed a file

* remove commented warning ... just not needed

---------

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
2024-09-10 16:28:19 +00:00

122 lines
4.0 KiB
Python

import functools
import importlib
from typing import Any
from typing import TypeVar
from danswer.configs.app_configs import ENTERPRISE_EDITION_ENABLED
from danswer.utils.logger import setup_logger
logger = setup_logger()
class DanswerVersion:
def __init__(self) -> None:
self._is_ee = False
def set_ee(self) -> None:
self._is_ee = True
def get_is_ee_version(self) -> bool:
return self._is_ee
global_version = DanswerVersion()
def set_is_ee_based_on_env_variable() -> None:
if ENTERPRISE_EDITION_ENABLED and not global_version.get_is_ee_version():
logger.notice("Enterprise Edition enabled")
global_version.set_ee()
@functools.lru_cache(maxsize=128)
def fetch_versioned_implementation(module: str, attribute: str) -> Any:
"""
Fetches a versioned implementation of a specified attribute from a given module.
This function first checks if the application is running in an Enterprise Edition (EE)
context. If so, it attempts to import the attribute from the EE-specific module.
If the module or attribute is not found, it falls back to the default module or
raises the appropriate exception depending on the context.
Args:
module (str): The name of the module from which to fetch the attribute.
attribute (str): The name of the attribute to fetch from the module.
Returns:
Any: The fetched implementation of the attribute.
Raises:
ModuleNotFoundError: If the module cannot be found and the error is not related to
the Enterprise Edition fallback logic.
Logs:
Logs debug information about the fetching process and warnings if the versioned
implementation cannot be found or loaded.
"""
logger.debug("Fetching versioned implementation for %s.%s", module, attribute)
is_ee = global_version.get_is_ee_version()
module_full = f"ee.{module}" if is_ee else module
try:
return getattr(importlib.import_module(module_full), attribute)
except ModuleNotFoundError as e:
logger.warning(
"Failed to fetch versioned implementation for %s.%s: %s",
module_full,
attribute,
e,
)
if is_ee:
if "ee.danswer" not in str(e):
# If it's a non Danswer related import failure, this is likely because
# a dependent library has not been installed. Should raise this failure
# instead of letting the server start up
raise e
# Use the MIT version as a fallback, this allows us to develop MIT
# versions independently and later add additional EE functionality
# similar to feature flagging
return getattr(importlib.import_module(module), attribute)
raise
T = TypeVar("T")
def fetch_versioned_implementation_with_fallback(
module: str, attribute: str, fallback: T
) -> T:
"""
Attempts to fetch a versioned implementation of a specified attribute from a given module.
If the attempt fails (e.g., due to an import error or missing attribute), the function logs
a warning and returns the provided fallback implementation.
Args:
module (str): The name of the module from which to fetch the attribute.
attribute (str): The name of the attribute to fetch from the module.
fallback (T): The fallback implementation to return if fetching the attribute fails.
Returns:
T: The fetched implementation if successful, otherwise the provided fallback.
"""
try:
return fetch_versioned_implementation(module, attribute)
except Exception:
return fallback
def noop_fallback(*args: Any, **kwargs: Any) -> None:
"""
A no-op (no operation) fallback function that accepts any arguments but does nothing.
This is often used as a default or placeholder callback function.
Args:
*args (Any): Positional arguments, which are ignored.
**kwargs (Any): Keyword arguments, which are ignored.
Returns:
None
"""