From b89f63e511a0d8446aec25a02de7c4d7d2704bc4 Mon Sep 17 00:00:00 2001 From: roshan <38771624+rohoswagger@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:30:39 +0000 Subject: [PATCH] feat(usage): add PDF review pack to exports (#13872) --- .../reporting/usage_export_generation.py | 49 +- .../server/reporting/usage_report_branding.py | 79 +++ .../server/reporting/usage_report_data.py | 171 +++++ .../onyx/server/reporting/usage_report_pdf.py | 628 ++++++++++++++++++ backend/requirements/default.txt | 8 +- .../tests/reporting/test_usage_export_api.py | 7 + .../reporting/test_usage_report_branding.py | 88 +++ .../reporting/test_usage_report_data.py | 281 ++++++++ docs/usage/usage-reports.md | 273 ++++++++ pyproject.toml | 2 + uv.lock | 15 + 11 files changed, 1599 insertions(+), 2 deletions(-) create mode 100644 backend/ee/onyx/server/reporting/usage_report_branding.py create mode 100644 backend/ee/onyx/server/reporting/usage_report_data.py create mode 100644 backend/ee/onyx/server/reporting/usage_report_pdf.py create mode 100644 backend/tests/unit/ee/onyx/server/reporting/test_usage_report_branding.py create mode 100644 backend/tests/unit/ee/onyx/server/reporting/test_usage_report_data.py create mode 100644 docs/usage/usage-reports.md diff --git a/backend/ee/onyx/server/reporting/usage_export_generation.py b/backend/ee/onyx/server/reporting/usage_export_generation.py index c94c3ac691..a874d9e985 100644 --- a/backend/ee/onyx/server/reporting/usage_export_generation.py +++ b/backend/ee/onyx/server/reporting/usage_export_generation.py @@ -4,6 +4,7 @@ import uuid import zipfile from collections.abc import Iterable from datetime import datetime, timedelta, timezone +from io import BytesIO from fastapi_users_db_sqlalchemy import UUID_ID from sqlalchemy import cast @@ -19,6 +20,9 @@ from ee.onyx.server.reporting.usage_export_models import ( UsageReportMetadata, UserSkeleton, ) +from ee.onyx.server.reporting.usage_report_branding import load_report_branding +from ee.onyx.server.reporting.usage_report_data import build_usage_report_data +from ee.onyx.server.reporting.usage_report_pdf import render_usage_report_pdf from onyx.configs.constants import FileOrigin from onyx.db.models import User from onyx.db.user_usage import UsageExportRow, iter_usage_export @@ -184,6 +188,31 @@ def generate_usage_breakdown_report( return file_id +def generate_usage_report_pdf( + db_session: Session, + file_store: FileStore, + report_id: str, + period: tuple[datetime, datetime] | None, + rows: list[UsageExportRow], +) -> str: + """Render the review pack PDF and store it. Returns the file id.""" + file_name = f"{report_id}_review_pack" + + # The queried bounds are half-open; only a given period gets the extra day. + display_start, display_end = period if period else _normalize_period(None) + + data = build_usage_report_data(db_session, rows, display_start, display_end) + branding = load_report_branding(file_store) + pdf_bytes = render_usage_report_pdf(data, branding) + + return file_store.save_file( + content=BytesIO(pdf_bytes), + display_name=file_name, + file_origin=FileOrigin.GENERATED_REPORT, + file_type="application/pdf", + ) + + def create_new_usage_report( db_session: Session, user_id: UUID_ID | None, # None = auto-generated @@ -204,12 +233,24 @@ def create_new_usage_report( intermediate_file_ids.append(users_file_id) query_start, query_end = normalized_period - usage_rows = iter_usage_export(db_session, query_start, query_end) + # The CSV and PDF must use the same rows so their totals reconcile. + usage_rows = list(iter_usage_export(db_session, query_start, query_end)) usage_breakdown_file_id = generate_usage_breakdown_report( file_store, report_id, usage_rows ) intermediate_file_ids.append(usage_breakdown_file_id) + # A render failure must not cost the admin their CSV export. + pdf_file_id: str | None = None + try: + pdf_file_id = generate_usage_report_pdf( + db_session, file_store, report_id, period, usage_rows + ) + except Exception: + logger.exception("Failed to render usage report PDF; continuing without it") + else: + intermediate_file_ids.append(pdf_file_id) + # Re-check just before writing the final report: the API-level check # happens before this (async) task runs, so a second request with the # same client-supplied report_id can slip past it while this task is @@ -239,6 +280,12 @@ def create_new_usage_report( ) zip_file.writestr("usage_by_user.csv", usage_breakdown_tmpfile.read()) + if pdf_file_id is not None: + pdf_tmpfile = file_store.read_file( + pdf_file_id, mode="b", use_tempfile=True + ) + zip_file.writestr("usage_report.pdf", pdf_tmpfile.read()) + zip_buffer.seek(0) # store zip blob to file_store diff --git a/backend/ee/onyx/server/reporting/usage_report_branding.py b/backend/ee/onyx/server/reporting/usage_report_branding.py new file mode 100644 index 0000000000..7d97157fa5 --- /dev/null +++ b/backend/ee/onyx/server/reporting/usage_report_branding.py @@ -0,0 +1,79 @@ +"""Branding for the usage report review pack, from enterprise settings.""" + +from pathlib import Path + +from pydantic import BaseModel + +from ee.onyx.server.enterprise_settings.store import ( + get_logo_filename, + get_logotype_filename, + load_runtime_settings, +) +from onyx.configs.constants import ONYX_DEFAULT_APPLICATION_NAME +from onyx.file_store.file_store import FileStore +from onyx.utils.logger import setup_logger + +logger = setup_logger() + +_FALLBACK_LOGO = Path(__file__).parents[4] / "static" / "images" / "logotype.png" + +_SUPPORTED_LOGO_TYPES = ("image/png", "image/jpeg", "image/jpg", "image/gif") + + +class ReportBranding(BaseModel): + application_name: str + # Raster bytes ReportLab can draw. None means the pack sets the name as a + # wordmark instead, which is right when a deployment has a custom logo we + # cannot draw: its own name beats another company's mark. + logo: bytes | None = None + + +def _read_logo(file_store: FileStore, file_id: str) -> bytes | None: + try: + stored = file_store.get_file_with_mime_type(file_id) + except Exception: + logger.exception("Failed to read logo %s for the usage report", file_id) + return None + + if stored is None: + return None + + # ReportLab cannot rasterize SVG, the common upload. + if stored.mime_type not in _SUPPORTED_LOGO_TYPES: + logger.info( + "Usage report cannot draw logo %s of type %s", file_id, stored.mime_type + ) + return None + + return stored.data + + +def load_report_branding(file_store: FileStore) -> ReportBranding: + settings = load_runtime_settings() + name = settings.application_name or ONYX_DEFAULT_APPLICATION_NAME + + has_custom_logo = settings.use_custom_logotype or settings.use_custom_logo + logo: bytes | None = None + if settings.use_custom_logotype: + logo = _read_logo(file_store, get_logotype_filename()) + if logo is None and settings.use_custom_logo: + logo = _read_logo(file_store, get_logo_filename()) + + if logo is None and has_custom_logo: + # Their logo exists but cannot be drawn, so fall through to the + # wordmark. Stamping the bundled mark here would ship our brand on + # their report. + logger.warning( + "Usage report rendering %s as a wordmark: the configured logo is " + "not a raster image ReportLab can draw", + name, + ) + return ReportBranding(application_name=name, logo=None) + + if logo is None: + try: + logo = _FALLBACK_LOGO.read_bytes() + except OSError: + logger.exception("Usage report could not read the fallback logo") + + return ReportBranding(application_name=name, logo=logo) diff --git a/backend/ee/onyx/server/reporting/usage_report_data.py b/backend/ee/onyx/server/reporting/usage_report_data.py new file mode 100644 index 0000000000..d7b9215c9b --- /dev/null +++ b/backend/ee/onyx/server/reporting/usage_report_data.py @@ -0,0 +1,171 @@ +"""Aggregates behind the usage report review pack.""" + +from collections import defaultdict +from datetime import datetime + +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from ee.onyx.db.license import user_counts_toward_seats +from onyx.db.api_key import is_api_key_email_address +from onyx.db.user_usage import DELETED_USER_EXPORT_EMAIL, UsageExportRow +from onyx.db.users import get_all_users + +TOP_USER_LIMIT = 10 +TOP_ENTRY_LIMIT = 8 +DORMANT_USER_LIMIT = 25 +UNLABELED_FLOW = "other" + + +class NamedSpend(BaseModel): + name: str + cost_cents: float + input_tokens: int + output_tokens: int + + @property + def total_tokens(self) -> int: + return self.input_tokens + self.output_tokens + + +class DailySpend(BaseModel): + day: str # YYYY-MM-DD + cost_cents: float + active_users: int + + +class UsageReportData(BaseModel): + period_start: datetime + period_end: datetime + + total_cost_cents: float + total_input_tokens: int + total_output_tokens: int + total_cache_read_tokens: int + + licensed_users: int + # Everyone who used it, including people since deactivated, so this can + # exceed licensed_users. `seated_active_users` is the subset holding a seat. + active_users: int + seated_active_users: int + dormant_users: list[str] + + top_users: list[NamedSpend] + by_model: list[NamedSpend] + by_flow: list[NamedSpend] + daily: list[DailySpend] + + @property + def dormant_user_count(self) -> int: + return len(self.dormant_users) + + @property + def cost_per_active_user_cents(self) -> float: + if not self.active_users: + return 0.0 + return self.total_cost_cents / self.active_users + + @property + def has_usage(self) -> bool: + return bool(self.daily) + + +def _top_n(spend_by_name: dict[str, NamedSpend], limit: int) -> list[NamedSpend]: + ordered = sorted(spend_by_name.values(), key=lambda s: s.cost_cents, reverse=True) + if len(ordered) <= limit: + return ordered + + head, tail = ordered[:limit], ordered[limit:] + # Folding a single entry hides a name and saves no space. + if len(tail) == 1: + return ordered + + remainder = NamedSpend( + name=f"Other ({len(tail)})", + cost_cents=sum(s.cost_cents for s in tail), + input_tokens=sum(s.input_tokens for s in tail), + output_tokens=sum(s.output_tokens for s in tail), + ) + return head + [remainder] + + +def build_usage_report_data( + db_session: Session, + rows: list[UsageExportRow], + period_start: datetime, + period_end: datetime, +) -> UsageReportData: + """`rows` is the list written to usage_by_user.csv, so the two cannot + diverge. The period is the admin's requested bounds, for display.""" + by_user: dict[str, NamedSpend] = {} + by_model: dict[str, NamedSpend] = {} + by_flow: dict[str, NamedSpend] = {} + daily_cost: dict[str, float] = defaultdict(float) + daily_users: dict[str, set[str]] = defaultdict(set) + + total_cost = 0.0 + total_input = 0 + total_output = 0 + total_cache_read = 0 + active_emails: set[str] = set() + + for row in rows: + for bucket, key in ( + (by_user, row.email), + (by_model, row.model), + (by_flow, row.flow or UNLABELED_FLOW), + ): + entry = bucket.get(key) + if entry is None: + entry = NamedSpend( + name=key, cost_cents=0.0, input_tokens=0, output_tokens=0 + ) + bucket[key] = entry + entry.cost_cents += row.cost_cents + entry.input_tokens += row.input_tokens + entry.output_tokens += row.output_tokens + + total_cost += row.cost_cents + total_input += row.input_tokens + total_output += row.output_tokens + total_cache_read += row.cache_read_tokens + + daily_cost[row.day] += row.cost_cents + # Their spend still counts toward totals so the pack reconciles with the + # CSV, but neither is a person. + if row.email != DELETED_USER_EXPORT_EMAIL and not is_api_key_email_address( + row.email + ): + active_emails.add(row.email) + daily_users[row.day].add(row.email) + + # Must match license enforcement, or this disagrees with what is billed. + users = get_all_users(db_session, include_api_key_users=False) + seat_emails = {user.email for user in users if user_counts_toward_seats(user)} + dormant = sorted(seat_emails - active_emails) + + daily = [ + DailySpend( + day=day, + cost_cents=daily_cost[day], + active_users=len(daily_users[day]), + ) + for day in sorted(daily_cost) + ] + + return UsageReportData( + period_start=period_start, + period_end=period_end, + total_cost_cents=total_cost, + total_input_tokens=total_input, + total_output_tokens=total_output, + total_cache_read_tokens=total_cache_read, + licensed_users=len(seat_emails), + active_users=len(active_emails), + seated_active_users=len(active_emails & seat_emails), + dormant_users=dormant, + top_users=_top_n(by_user, TOP_USER_LIMIT), + by_model=_top_n(by_model, TOP_ENTRY_LIMIT), + by_flow=_top_n(by_flow, TOP_ENTRY_LIMIT), + daily=daily, + ) diff --git a/backend/ee/onyx/server/reporting/usage_report_pdf.py b/backend/ee/onyx/server/reporting/usage_report_pdf.py new file mode 100644 index 0000000000..414cf40d3d --- /dev/null +++ b/backend/ee/onyx/server/reporting/usage_report_pdf.py @@ -0,0 +1,628 @@ +"""Renders the usage report review pack as a PDF. + +The `ty: ignore`s below are load-bearing: ReportLab types `chart.data` from a +sample literal and populates `valueAxis.labels` dynamically, so neither is +resolvable by a static checker. +""" + +from io import BytesIO +from xml.sax.saxutils import escape + +from reportlab.graphics.charts.barcharts import VerticalBarChart +from reportlab.graphics.charts.linecharts import HorizontalLineChart +from reportlab.graphics.shapes import Drawing +from reportlab.lib import colors +from reportlab.lib.enums import TA_LEFT +from reportlab.lib.pagesizes import LETTER +from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet +from reportlab.lib.units import inch +from reportlab.lib.utils import ImageReader +from reportlab.pdfgen import canvas +from reportlab.platypus import ( + Flowable, + Image, + KeepTogether, + PageBreak, + Paragraph, + SimpleDocTemplate, + Spacer, + Table, + TableStyle, +) + +from ee.onyx.server.reporting.usage_report_branding import ReportBranding +from ee.onyx.server.reporting.usage_report_data import ( + DORMANT_USER_LIMIT, + NamedSpend, + UsageReportData, +) +from onyx.configs.constants import DANSWER_API_KEY_PREFIX, UNNAMED_KEY_PLACEHOLDER +from onyx.db.api_key import is_api_key_email_address +from onyx.utils.logger import setup_logger + +logger = setup_logger() + +_INK = colors.HexColor("#1c1c1c") # onyx-ink-95 +_ACCENT = colors.HexColor("#286df8") # action-selection-05 / blue-50 +_BODY = colors.HexColor("#54545d") # stone-60, 7.5:1 on white +_HAIRLINE = colors.HexColor("#e6e6e9") # stone-10 +_SURFACE = colors.HexColor("#f0f0f1") # stone-05 + +_CONTENT_WIDTH = LETTER[0] - 2 * inch +_MAX_AXIS_LABELS = 12 +_LOGO_MAX_W, _LOGO_MAX_H = 2.0 * inch, 0.5 * inch + + +def _dollars(cents: float) -> str: + return f"${cents / 100:,.2f}" + + +def _thousands(value: int) -> str: + return f"{value:,}" + + +def _display_name(name: str) -> str: + """Render an API key by its name instead of its synthetic address. + + Each API key owns a `User` row whose email is + `API_KEY__@onyxapikey.ai`, so per-key spend already + aggregates correctly. Only the label needs help. A no-op for every other + name, since none of them carry the API-key domain. + """ + if not is_api_key_email_address(name): + return name + + # The key's name can itself contain "@", so split on the last one. + local_part = name.rsplit("@", 1)[0] + # Stored emails are lowercased by a DB check constraint, so the prefix + # cannot be matched case-sensitively against the constant. + if local_part.lower().startswith(DANSWER_API_KEY_PREFIX.lower()): + local_part = local_part[len(DANSWER_API_KEY_PREFIX) :] + return f"{local_part or UNNAMED_KEY_PLACEHOLDER} (API key)" + + +def _styles() -> dict[str, ParagraphStyle]: + base = getSampleStyleSheet() + return { + "cover_title": ParagraphStyle( + "CoverTitle", + parent=base["Title"], + fontName="Helvetica-Bold", + fontSize=32, + leading=36, + textColor=_INK, + alignment=TA_LEFT, + spaceAfter=6, + ), + "wordmark": ParagraphStyle( + "Wordmark", + parent=base["Normal"], + fontName="Helvetica-Bold", + fontSize=19, + leading=23, + textColor=_INK, + ), + "cover_period": ParagraphStyle( + "CoverPeriod", + parent=base["Normal"], + fontName="Helvetica", + fontSize=13, + leading=18, + textColor=_BODY, + ), + "lede": ParagraphStyle( + "Lede", + parent=base["Normal"], + fontName="Helvetica", + fontSize=11.5, + leading=17, + textColor=_INK, + ), + "heading": ParagraphStyle( + "Heading", + parent=base["Heading2"], + fontName="Helvetica-Bold", + fontSize=14, + leading=18, + textColor=_INK, + spaceBefore=22, + spaceAfter=2, + keepWithNext=1, + ), + "subheading": ParagraphStyle( + "Subheading", + parent=base["Normal"], + fontName="Helvetica", + fontSize=9.5, + leading=13, + textColor=_BODY, + spaceAfter=10, + keepWithNext=1, + ), + "note": ParagraphStyle( + "Note", + parent=base["Normal"], + fontName="Helvetica", + fontSize=8.5, + leading=12, + textColor=_BODY, + spaceBefore=4, + ), + } + + +def _logo_flowable(branding: ReportBranding) -> Flowable | None: + if not branding.logo: + return None + try: + reader = ImageReader(BytesIO(branding.logo)) + src_w, src_h = reader.getSize() + except Exception: + logger.exception( + "Usage report could not render the configured logo for %s", + branding.application_name, + ) + return None + if not src_w or not src_h: + return None + + scale = min(_LOGO_MAX_W / src_w, _LOGO_MAX_H / src_h) + return Image( + BytesIO(branding.logo), width=src_w * scale, height=src_h * scale, mask="auto" + ) + + +class _NumberedCanvas(canvas.Canvas): + """Stamps "page N of M" once the total is known. + + The total only exists after the last page is laid out, so pages are held + back until save. The cover is deliberately left unnumbered. + """ + + def __init__(self, *args: object, **kwargs: object) -> None: + super().__init__(*args, **kwargs) + self._pages: list[dict[str, object]] = [] + + def showPage(self) -> None: + self._pages.append(dict(self.__dict__)) + self._startPage() + + def save(self) -> None: + total = len(self._pages) + for number, state in enumerate(self._pages, start=1): + self.__dict__.update(state) + if number > 1: + self._draw_folio(number, total) + super().showPage() + super().save() + + def _draw_folio(self, number: int, total: int) -> None: + width = self._pagesize[0] + self.setFont("Helvetica", 8.5) + self.setFillColor(_BODY) + self.drawRightString(width - inch, 0.6 * inch, f"{number} of {total}") + + +class _Rule(Flowable): + def __init__(self, width: float, color: colors.Color = _HAIRLINE) -> None: + super().__init__() + self.width, self.height, self.color = width, 1, color + + def draw(self) -> None: + self.canv.setStrokeColor(self.color) + self.canv.setLineWidth(1) + self.canv.line(0, 0, self.width, 0) + + +class _SeatMeter(Flowable): + """Seats in use against seats bought.""" + + def __init__(self, active: int, licensed: int, unseated: int, width: float) -> None: + super().__init__() + self.active, self.licensed, self.unseated = active, licensed, unseated + self.width, self.height = width, 54 + + def draw(self) -> None: + c = self.canv + bar_h, bar_y = 14, 20 + ratio = min(1.0, self.active / self.licensed) if self.licensed else 0.0 + + c.setFillColor(_SURFACE) + c.roundRect(0, bar_y, self.width, bar_h, 3, stroke=0, fill=1) + if ratio > 0: + c.setFillColor(_ACCENT) + c.roundRect( + 0, bar_y, max(3.0, self.width * ratio), bar_h, 3, stroke=0, fill=1 + ) + + c.setFillColor(_INK) + c.setFont("Helvetica-Bold", 11) + c.drawString( + 0, bar_y + bar_h + 8, f"{self.active} of {self.licensed} seats active" + ) + + c.setFillColor(_BODY) + c.setFont("Helvetica", 9) + idle = self.licensed - self.active + caption = ( + f"{ratio:.0%} in use · {idle} seats idle" + if idle + else f"{ratio:.0%} of licensed seats in use" + ) + if self.unseated: + caption += f" · {self.unseated} used it without a seat" + c.drawString(0, bar_y - 13, caption) + + +def _headline(data: UsageReportData) -> Table: + figures = [ + (_thousands(data.active_users), "People using it"), + (_dollars(data.total_cost_cents), "Total spend"), + (_dollars(data.cost_per_active_user_cents), "Cost per active person"), + ] + value_style = ParagraphStyle( + "Figure", + fontName="Helvetica-Bold", + fontSize=23, + leading=26, + textColor=_INK, + ) + label_style = ParagraphStyle( + "FigureLabel", + fontName="Helvetica", + fontSize=9, + leading=12, + textColor=_BODY, + ) + row = [ + [Paragraph(v, value_style) for v, _ in figures], + [Paragraph(label, label_style) for _, label in figures], + ] + col = _CONTENT_WIDTH / 3 + table = Table(row, colWidths=[col] * 3, hAlign="LEFT") + table.setStyle( + TableStyle( + [ + ("VALIGN", (0, 0), (-1, -1), "BOTTOM"), + ("TOPPADDING", (0, 0), (-1, 0), 0), + ("BOTTOMPADDING", (0, 0), (-1, 0), 2), + ("TOPPADDING", (0, 1), (-1, 1), 0), + ("LEFTPADDING", (0, 0), (-1, -1), 0), + ("RIGHTPADDING", (0, 0), (-1, -1), 12), + ] + ) + ) + return table + + +def _table(rows: list[list[str]], col_widths: list[float]) -> Table: + table = Table(rows, colWidths=col_widths, repeatRows=1, hAlign="LEFT") + table.setStyle( + TableStyle( + [ + # Header and body typography. + ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"), + ("FONTNAME", (0, 1), (-1, -1), "Helvetica"), + ("FONTSIZE", (0, 0), (-1, -1), 9), + ("TEXTCOLOR", (0, 0), (-1, 0), _INK), + ("TEXTCOLOR", (0, 1), (-1, -1), _BODY), + ("TEXTCOLOR", (0, 1), (0, -1), _INK), + # Text columns left-align; numeric columns right-align. + ("ALIGN", (1, 0), (-1, -1), "RIGHT"), + ("ALIGN", (0, 0), (0, -1), "LEFT"), + # Separate the header and each body row. + ("LINEBELOW", (0, 0), (-1, 0), 1, _INK), + ("LINEBELOW", (0, 1), (-1, -2), 0.5, _HAIRLINE), + # Keep rows readable without changing column width. + ("TOPPADDING", (0, 0), (-1, -1), 7), + ("BOTTOMPADDING", (0, 0), (-1, -1), 7), + ("LEFTPADDING", (0, 0), (-1, -1), 0), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ] + ) + ) + return table + + +def _axis_labels(days: list[str]) -> list[str]: + """Keep at most `_MAX_AXIS_LABELS` ticks, blanking the rest.""" + step = max(1, (len(days) + _MAX_AXIS_LABELS - 1) // _MAX_AXIS_LABELS) + # Drop the year: the period is already stated on the cover. + return [day[5:] if index % step == 0 else "" for index, day in enumerate(days)] + + +def _style_axes(chart: HorizontalLineChart | VerticalBarChart) -> None: + chart.categoryAxis.labels.fontName = "Helvetica" + chart.categoryAxis.labels.fontSize = 7 + chart.categoryAxis.labels.fillColor = _BODY + chart.categoryAxis.strokeColor = _HAIRLINE + chart.valueAxis.labels.fontName = "Helvetica" # ty: ignore[unresolved-attribute] + chart.valueAxis.labels.fontSize = 7 # ty: ignore[unresolved-attribute] + chart.valueAxis.labels.fillColor = _BODY # ty: ignore[unresolved-attribute] + chart.valueAxis.strokeColor = _HAIRLINE + chart.valueAxis.valueMin = 0 + chart.valueAxis.gridStrokeColor = _HAIRLINE + chart.valueAxis.gridStrokeWidth = 0.5 + chart.valueAxis.visibleGrid = True + + +def _spend_over_time(data: UsageReportData) -> Drawing: + drawing = Drawing(_CONTENT_WIDTH, 168) + chart = HorizontalLineChart() + chart.x, chart.y = 42, 28 + chart.width, chart.height = int(_CONTENT_WIDTH - 56), 122 + chart.data = [[point.cost_cents / 100 for point in data.daily]] # ty: ignore[invalid-assignment] + chart.categoryAxis.categoryNames = _axis_labels([p.day for p in data.daily]) + _style_axes(chart) + chart.lines[0].strokeColor = _ACCENT + chart.lines[0].strokeWidth = 1.6 + drawing.add(chart) + return drawing + + +def _active_users_over_time(data: UsageReportData) -> Drawing: + drawing = Drawing(_CONTENT_WIDTH, 168) + chart = HorizontalLineChart() + chart.x, chart.y = 42, 28 + chart.width, chart.height = int(_CONTENT_WIDTH - 56), 122 + chart.data = [[float(point.active_users) for point in data.daily]] # ty: ignore[invalid-assignment] + chart.categoryAxis.categoryNames = _axis_labels([p.day for p in data.daily]) + _style_axes(chart) + chart.lines[0].strokeColor = _INK + chart.lines[0].strokeWidth = 1.6 + drawing.add(chart) + return drawing + + +def _spend_by_model(data: UsageReportData) -> Drawing: + drawing = Drawing(_CONTENT_WIDTH, 170) + chart = VerticalBarChart() + chart.x, chart.y = 42, 38 + chart.width, chart.height = int(_CONTENT_WIDTH - 56), 112 + chart.data = [[entry.cost_cents / 100 for entry in data.by_model]] # ty: ignore[invalid-assignment] + chart.categoryAxis.categoryNames = [ + _display_name(entry.name) for entry in data.by_model + ] + _style_axes(chart) + chart.categoryAxis.labels.angle = 20 + chart.categoryAxis.labels.dy = -8 + chart.bars[0].fillColor = _ACCENT + chart.bars[0].strokeColor = None + chart.barSpacing = 2 + drawing.add(chart) + return drawing + + +def _cover( + data: UsageReportData, + branding: ReportBranding, + styles: dict[str, ParagraphStyle], +) -> list[Flowable]: + period = ( + f"{data.period_start.date().isoformat()} to " + f"{data.period_end.date().isoformat()} (UTC)" + ) + story: list[Flowable] = [] + + logo = _logo_flowable(branding) + if logo is not None: + logo.hAlign = "LEFT" + story += [logo, Spacer(1, 30)] + else: + story += [ + Paragraph(escape(branding.application_name), styles["wordmark"]), + Spacer(1, 26), + ] + + story += [ + Paragraph("Usage report", styles["cover_title"]), + Paragraph(period, styles["cover_period"]), + Spacer(1, 26), + _Rule(_CONTENT_WIDTH, _INK), + Spacer(1, 22), + _headline(data), + Spacer(1, 30), + ] + + if data.licensed_users: + story += [ + _SeatMeter( + data.seated_active_users, + data.licensed_users, + data.active_users - data.seated_active_users, + _CONTENT_WIDTH, + ) + ] + + story += [ + Spacer(1, 30), + Paragraph(_summary_sentence(data, branding.application_name), styles["lede"]), + ] + + if data.by_model: + story += [ + Spacer(1, 26), + Paragraph("Top models by spend", styles["subheading"]), + _spend_table("Model", data.by_model[:3]), + ] + + return story + + +def _summary_sentence(data: UsageReportData, application_name: str) -> str: + """Returns Paragraph markup, so every interpolated name is escaped.""" + people = "1 person" if data.active_users == 1 else f"{data.active_users} people" + parts = [ + f"{people} used {escape(application_name)} in this period, at a total cost " + f"of {_dollars(data.total_cost_cents)}." + ] + if data.by_flow: + flow = escape(_display_name(data.by_flow[0].name)) + parts.append(f"Most of that ran through {flow}.") + if data.dormant_user_count: + share = ( + data.dormant_user_count / data.licensed_users + if data.licensed_users + else 0.0 + ) + parts.append( + f"{data.dormant_user_count} of {data.licensed_users} licensed seats " + f"({share:.0%}) went unused and are candidates to reassign." + ) + return " ".join(parts) + + +def _spend_table(label: str, entries: list[NamedSpend]) -> Table: + rows: list[list[str]] = [[label, "Spend (USD)", "Tokens"]] + rows += [ + [ + _display_name(entry.name), + _dollars(entry.cost_cents), + _thousands(entry.total_tokens), + ] + for entry in entries + ] + widths = [_CONTENT_WIDTH * 0.5, _CONTENT_WIDTH * 0.25, _CONTENT_WIDTH * 0.25] + return _table(rows, widths) + + +def _section( + heading: str, subheading: str, styles: dict[str, ParagraphStyle] +) -> list[Flowable]: + return [ + Paragraph(heading, styles["heading"]), + Paragraph(subheading, styles["subheading"]), + ] + + +def _charted_section( + heading: str, + subheading: str, + chart: Drawing, + styles: dict[str, ParagraphStyle], +) -> Flowable: + """`keepWithNext` does not reach into a KeepTogether, so the heading must + travel inside the group or it strands at the page foot.""" + return KeepTogether( + [ + Paragraph(heading, styles["heading"]), + Paragraph(subheading, styles["subheading"]), + chart, + ] + ) + + +def render_usage_report_pdf(data: UsageReportData, branding: ReportBranding) -> bytes: + styles = _styles() + buffer = BytesIO() + doc = SimpleDocTemplate( + buffer, + pagesize=LETTER, + leftMargin=inch, + rightMargin=inch, + topMargin=0.9 * inch, + bottomMargin=0.9 * inch, + title=f"{branding.application_name} usage report", + author=branding.application_name, + # Byte-identical output for identical input. + invariant=1, + ) + + story: list[Flowable] = [] + + if not data.has_usage: + logo = _logo_flowable(branding) + if logo is not None: + logo.hAlign = "LEFT" + story += [logo, Spacer(1, 30)] + else: + story += [ + Paragraph(escape(branding.application_name), styles["wordmark"]), + Spacer(1, 26), + ] + story += [ + Paragraph("Usage report", styles["cover_title"]), + Paragraph( + f"{data.period_start.date().isoformat()} to " + f"{data.period_end.date().isoformat()} (UTC)", + styles["cover_period"], + ), + Spacer(1, 24), + Paragraph( + "No recorded usage in this period. If the deployment was active, " + "the usage rollup may have started after the period began.", + styles["lede"], + ), + ] + doc.build(story, canvasmaker=_NumberedCanvas) + return buffer.getvalue() + + story += _cover(data, branding, styles) + story += [PageBreak()] + + story += [ + _charted_section( + "Adoption", + "Distinct people who sent at least one message each day.", + _active_users_over_time(data), + styles, + ), + _charted_section( + "Spend over time", + "Daily cost across every model and surface.", + _spend_over_time(data), + styles, + ), + _charted_section( + "Where the spend goes", + "Cost by model for the period.", + _spend_by_model(data), + styles, + ), + Spacer(1, 6), + _spend_table("Model", data.by_model), + ] + + story += _section("Heaviest users", "The people driving most of the cost.", styles) + story += [_spend_table("User", data.top_users)] + + story += _section("Spend by surface", "Where the work happens.", styles) + story += [_spend_table("Flow", data.by_flow)] + + story += _section( + "Seats not in use", + "Licensed people who sent nothing this period. Reclaim, retrain, or " + "drop them at renewal.", + styles, + ) + if not data.dormant_users: + story.append( + Paragraph("Every licensed seat was used this period.", styles["lede"]) + ) + else: + shown = data.dormant_users[:DORMANT_USER_LIMIT] + rows: list[list[str]] = [["User"]] + [[email] for email in shown] + story.append(_table(rows, [_CONTENT_WIDTH])) + remaining = data.dormant_user_count - len(shown) + if remaining > 0: + story.append( + Paragraph( + f"{remaining} more idle seats. See users.csv for the full list.", + styles["note"], + ) + ) + + story += [ + Spacer(1, 20), + _Rule(_CONTENT_WIDTH), + Spacer(1, 8), + Paragraph( + "Spend from deleted users and API keys is included in every total " + "and attributed separately. Neither counts as a person or a seat. " + "Days are UTC.", + styles["note"], + ), + ] + + doc.build(story, canvasmaker=_NumberedCanvas) + return buffer.getvalue() diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt index 4c0bd13d0d..b71eb75ba6 100644 --- a/backend/requirements/default.txt +++ b/backend/requirements/default.txt @@ -531,6 +531,7 @@ charset-normalizer==3.4.4 \ # htmldate # markitdown # pdfminer-six + # reportlab # requests # trafilatura # unstructured @@ -2021,7 +2022,9 @@ pillow==12.3.0 \ --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 - # via python-pptx + # via + # python-pptx + # reportlab platformdirs==4.5.0 \ --hash=sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312 \ --hash=sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3 @@ -2672,6 +2675,9 @@ regex==2025.11.3 \ # dateparser # nltk # tiktoken +reportlab==5.0.0 \ + --hash=sha256:9d5a3affa84919e1111ede580031266a570e93b1ce388219621347965ff1d93c \ + --hash=sha256:e4494a0c6623ae213bb856fba523171b2b54a7bf629fda02d5e525a7b899a784 requests==2.33.0 \ --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b \ --hash=sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652 diff --git a/backend/tests/integration/tests/reporting/test_usage_export_api.py b/backend/tests/integration/tests/reporting/test_usage_export_api.py index e3420181cd..87f36c3889 100644 --- a/backend/tests/integration/tests/reporting/test_usage_export_api.py +++ b/backend/tests/integration/tests/reporting/test_usage_export_api.py @@ -290,6 +290,13 @@ class TestUsageExportAPI: assert "chat_messages.csv" in file_names assert "users.csv" in file_names assert "usage_by_user.csv" in file_names + assert "usage_report.pdf" in file_names + + with zip_file.open("usage_report.pdf") as pdf_file: + pdf_bytes = pdf_file.read() + assert pdf_bytes.startswith(b"%PDF-") + assert len(pdf_bytes) > 1000 + # Verify usage_by_user.csv has the expected columns. The seeded # chat history doesn't record UserUsage rows, so there's no data # to assert on, just the header shape. diff --git a/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_branding.py b/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_branding.py new file mode 100644 index 0000000000..9010c38d61 --- /dev/null +++ b/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_branding.py @@ -0,0 +1,88 @@ +"""Which logo the review pack renders under.""" + +from unittest.mock import MagicMock, patch + +from ee.onyx.server.enterprise_settings.models import EnterpriseSettings +from ee.onyx.server.reporting.usage_report_branding import ( + ReportBranding, + load_report_branding, +) +from onyx.utils.file import FileWithMimeType + +_LOGOTYPE = b"logotype-bytes" +_LOGO = b"logo-bytes" + + +def _file_store(stored: dict[str, FileWithMimeType]) -> MagicMock: + store = MagicMock() + store.get_file_with_mime_type.side_effect = lambda file_id: stored.get(file_id) + return store + + +def _load( + settings: EnterpriseSettings, stored: dict[str, FileWithMimeType] +) -> ReportBranding: + with patch( + "ee.onyx.server.reporting.usage_report_branding.load_runtime_settings", + return_value=settings, + ): + return load_report_branding(_file_store(stored)) + + +def test_logotype_wins_over_the_square_mark() -> None: + branding = _load( + EnterpriseSettings( + application_name="Acme", use_custom_logo=True, use_custom_logotype=True + ), + { + "__logotype__": FileWithMimeType(data=_LOGOTYPE, mime_type="image/png"), + "__logo__": FileWithMimeType(data=_LOGO, mime_type="image/png"), + }, + ) + + assert branding.logo == _LOGOTYPE + assert branding.application_name == "Acme" + + +def test_falls_back_to_the_square_mark_when_no_logotype() -> None: + branding = _load( + EnterpriseSettings(use_custom_logo=True, use_custom_logotype=True), + {"__logo__": FileWithMimeType(data=_LOGO, mime_type="image/png")}, + ) + + assert branding.logo == _LOGO + + +def test_an_undrawable_logo_becomes_a_wordmark_not_our_mark() -> None: + """ReportLab cannot draw SVG. Their own name beats stamping the Onyx mark + on a report they forward to their leadership.""" + branding = _load( + EnterpriseSettings(application_name="Acme", use_custom_logotype=True), + {"__logotype__": FileWithMimeType(data=b"", mime_type="image/svg+xml")}, + ) + + assert branding.logo is None + assert branding.application_name == "Acme" + + +def test_bundled_logo_is_used_when_nothing_is_uploaded() -> None: + branding = _load(EnterpriseSettings(), {}) + + assert branding.logo is not None + assert branding.logo.startswith(b"\x89PNG") + + +def test_a_file_store_failure_falls_back_to_the_wordmark() -> None: + """A configured custom logo we cannot read still must not become our mark.""" + store = MagicMock() + store.get_file_with_mime_type.side_effect = RuntimeError("object storage down") + + with patch( + "ee.onyx.server.reporting.usage_report_branding.load_runtime_settings", + return_value=EnterpriseSettings( + application_name="Acme", use_custom_logotype=True + ), + ): + branding = load_report_branding(store) + + assert branding.logo is None diff --git a/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_data.py b/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_data.py new file mode 100644 index 0000000000..331b497b1f --- /dev/null +++ b/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_data.py @@ -0,0 +1,281 @@ +"""Aggregation behind the usage report review pack.""" + +from datetime import datetime, timezone +from io import BytesIO +from unittest.mock import MagicMock, patch + +import pytest +from PIL import Image as PILImage +from pypdf import PdfReader + +from ee.onyx.server.reporting.usage_report_branding import ReportBranding +from ee.onyx.server.reporting.usage_report_data import ( + TOP_USER_LIMIT, + UsageReportData, + build_usage_report_data, +) +from ee.onyx.server.reporting.usage_report_pdf import ( + _axis_labels, + _display_name, + render_usage_report_pdf, +) +from onyx.db.enums import AccountType +from onyx.db.models import User +from onyx.db.user_usage import DELETED_USER_EXPORT_EMAIL, UsageExportRow + +_BRANDING = ReportBranding(application_name="Acme Intelligence", logo=None) + +PERIOD_START = datetime(2026, 7, 1, tzinfo=timezone.utc) +PERIOD_END = datetime(2026, 7, 31, tzinfo=timezone.utc) + + +def _row( + email: str, + cost: float = 10.0, + day: str = "2026-07-01", + flow: str = "chat", +) -> UsageExportRow: + return UsageExportRow( + email=email, + model="gpt-5", + flow=flow, + provider="openai", + day=day, + input_tokens=100, + output_tokens=50, + cache_read_tokens=10, + cost_cents=cost, + ) + + +def _user( + email: str, + is_active: bool = True, + account_type: AccountType = AccountType.STANDARD, +) -> User: + user = User() + user.email = email + user.is_active = is_active + user.account_type = account_type + return user + + +def _build(rows: list[UsageExportRow], users: list[User]) -> UsageReportData: + with patch( + "ee.onyx.server.reporting.usage_report_data.get_all_users", return_value=users + ): + return build_usage_report_data( + db_session=MagicMock(), + rows=rows, + period_start=PERIOD_START, + period_end=PERIOD_END, + ) + + +def test_totals_reconcile_with_the_rows() -> None: + """The pack's totals must equal the CSV's, including deleted-user spend.""" + rows = [ + _row("a@x.com", 10.0), + _row("b@x.com", 5.5), + _row(DELETED_USER_EXPORT_EMAIL, 4.5), + ] + + data = _build(rows, [_user("a@x.com"), _user("b@x.com")]) + + assert data.total_cost_cents == pytest.approx(20.0) + assert sum(e.cost_cents for e in data.by_model) == pytest.approx(20.0) + assert sum(e.cost_cents for e in data.by_flow) == pytest.approx(20.0) + assert sum(e.cost_cents for e in data.top_users) == pytest.approx(20.0) + assert data.total_input_tokens == 300 + + +def test_deleted_user_counts_toward_spend_but_is_not_a_person() -> None: + rows = [_row("a@x.com"), _row(DELETED_USER_EXPORT_EMAIL)] + + data = _build(rows, [_user("a@x.com")]) + + assert data.active_users == 1 + assert data.daily[0].active_users == 1 + assert data.total_cost_cents == pytest.approx(20.0) + + +def test_api_key_usage_is_not_an_active_user() -> None: + """Seats exclude API-key users, so activity must exclude them too, or + active_users can exceed licensed_users.""" + rows = [_row("a@x.com"), _row("somekey@onyxapikey.ai")] + + data = _build(rows, [_user("a@x.com")]) + + assert data.active_users == 1 + assert data.active_users <= data.licensed_users + + +def test_unlabeled_flow_is_grouped_as_other() -> None: + data = _build([_row("a@x.com", flow="")], [_user("a@x.com")]) + + assert [(entry.name, entry.cost_cents) for entry in data.by_flow] == [ + ("other", 10.0) + ] + + +def test_service_accounts_do_not_hold_a_seat() -> None: + users = [ + _user("human@x.com"), + _user("bot@x.com", account_type=AccountType.SERVICE_ACCOUNT), + _user("gone@x.com", is_active=False), + ] + + data = _build([_row("human@x.com")], users) + + assert data.licensed_users == 1 + assert data.seated_active_users == 1 + assert data.dormant_users == [] + + +def test_a_deactivated_user_is_active_but_holds_no_seat() -> None: + """Someone who used it mid-period and was deactivated before the report ran + counts as a person, not as an occupied seat. The meter must not read + "2 of 1 seats active".""" + rows = [_row("stayed@x.com"), _row("departed@x.com")] + users = [_user("stayed@x.com"), _user("departed@x.com", is_active=False)] + + data = _build(rows, users) + + assert data.active_users == 2 + assert data.licensed_users == 1 + assert data.seated_active_users == 1 + assert data.seated_active_users <= data.licensed_users + + +def test_dormant_seats_are_named() -> None: + users = [_user("active@x.com"), _user("idle@x.com")] + + data = _build([_row("active@x.com")], users) + + assert data.licensed_users == 2 + assert data.active_users == 1 + assert data.dormant_users == ["idle@x.com"] + + +def test_top_users_folds_the_tail_and_preserves_the_total() -> None: + rows = [_row(f"u{i}@x.com", cost=float(i + 1)) for i in range(TOP_USER_LIMIT + 3)] + + data = _build(rows, []) + + assert len(data.top_users) == TOP_USER_LIMIT + 1 + assert data.top_users[-1].name == "Other (3)" + assert sum(e.cost_cents for e in data.top_users) == pytest.approx( + data.total_cost_cents + ) + + +def test_a_single_extra_user_is_named_rather_than_folded() -> None: + """Folding one entry hides a name and saves no space.""" + rows = [_row(f"u{i}@x.com", cost=float(i + 1)) for i in range(TOP_USER_LIMIT + 1)] + + data = _build(rows, []) + + assert not any(e.name.startswith("Other") for e in data.top_users) + + +def test_api_key_spend_is_labeled_by_key_name() -> None: + """Per-key spend stays in the breakdown; only the label is cleaned up.""" + # A DB check constraint lowercases stored emails, so this is the real shape. + assert ( + _display_name("api_key__nightly-sync@2f3c8a10-uuid.onyxapikey.ai") + == "nightly-sync (API key)" + ) + assert ( + _display_name("API_KEY__nightly-sync@2f3c8a10-uuid.onyxapikey.ai") + == "nightly-sync (API key)" + ) + assert ( + _display_name("api_key__sync@corp@2f3c8a10-uuid.onyxapikey.ai") + == "sync@corp (API key)" + ) + assert _display_name("human@corp.com") == "human@corp.com" + assert _display_name("gpt-5") == "gpt-5" + assert _display_name(DELETED_USER_EXPORT_EMAIL) == DELETED_USER_EXPORT_EMAIL + + +def test_api_key_spend_still_reconciles_after_relabeling() -> None: + rows = [_row("a@x.com", 10.0), _row("api_key__bot@uuid.onyxapikey.ai", 90.0)] + + data = _build(rows, [_user("a@x.com")]) + + assert data.total_cost_cents == pytest.approx(100.0) + assert sum(e.cost_cents for e in data.top_users) == pytest.approx(100.0) + assert data.active_users == 1 + + +def test_zero_usage_still_renders_a_pdf() -> None: + data = _build([], [_user("idle@x.com")]) + + assert not data.has_usage + assert data.cost_per_active_user_cents == 0.0 + assert render_usage_report_pdf(data, _BRANDING).startswith(b"%PDF-") + + +def test_application_name_is_not_parsed_as_markup() -> None: + """ReportLab Paragraph parses a markup subset: an unescaped name with a tag + silently drops text, injects formatting, or raises and loses the PDF.""" + data = _build([_row("a@x.com")], [_user("a@x.com")]) + + for name in ["Tools Us", "Acme Corp", 'X Y']: + branding = ReportBranding(application_name=name, logo=None) + pdf = render_usage_report_pdf(data, branding) + assert pdf.startswith(b"%PDF-") + + +def _png(width: int = 40, height: int = 10) -> bytes: + buffer = BytesIO() + PILImage.new("RGBA", (width, height), (0, 0, 0, 255)).save(buffer, format="PNG") + return buffer.getvalue() + + +def test_the_cover_is_unnumbered_and_later_pages_are_not() -> None: + """A folio on the cover, or an off-by-one, corrupts every generated pack.""" + rows = [_row(f"u{i}@x.com", cost=float(i + 1)) for i in range(12)] + data = _build(rows, [_user(f"u{i}@x.com") for i in range(12)]) + + reader = PdfReader(BytesIO(render_usage_report_pdf(data, _BRANDING))) + total = len(reader.pages) + + assert total > 1 + assert f"1 of {total}" not in reader.pages[0].extract_text() + for number in range(2, total + 1): + assert f"{number} of {total}" in reader.pages[number - 1].extract_text() + + +def test_render_is_deterministic() -> None: + data = _build([_row("a@x.com")], [_user("a@x.com")]) + + assert render_usage_report_pdf(data, _BRANDING) == render_usage_report_pdf( + data, _BRANDING + ) + + +def test_render_is_deterministic_with_an_embedded_logo() -> None: + """The default path embeds a logo, so determinism must hold with one.""" + data = _build([_row("a@x.com")], [_user("a@x.com")]) + branding = ReportBranding(application_name="Acme", logo=_png()) + + assert render_usage_report_pdf(data, branding) == render_usage_report_pdf( + data, branding + ) + + +def test_unreadable_logo_still_produces_a_pdf() -> None: + """A corrupt upload must cost the branding, not the report.""" + data = _build([_row("a@x.com")], [_user("a@x.com")]) + + for logo in (b"not-an-image", b""): + branding = ReportBranding(application_name="Acme", logo=logo) + assert render_usage_report_pdf(data, branding).startswith(b"%PDF-") + + +def test_axis_labels_never_exceed_the_display_limit() -> None: + for day_count in (0, 1, 12, 13, 23, 24, 25, 30, 365): + days = [f"2026-07-{day + 1:02d}" for day in range(day_count)] + + assert sum(bool(label) for label in _axis_labels(days)) <= 12 diff --git a/docs/usage/usage-reports.md b/docs/usage/usage-reports.md new file mode 100644 index 0000000000..553ecfbe47 --- /dev/null +++ b/docs/usage/usage-reports.md @@ -0,0 +1,273 @@ +# Usage reports: what they are for + +This document defines what an Onyx usage report must tell an organization admin, +and why. It starts from the admin's job, not from the data we happen to store. + +The rule that governs every decision here: **a number belongs in the report only +if the admin can finish the sentence "so I will...".** If no action follows, cut +the number. + +## Where we are today + +`create_new_usage_report` builds a zip with three CSVs and a PDF review pack: + +| File | Contents | +| -------------------- | ------------------------------------------------------------------------------------- | +| `chat_messages.csv` | One row per message: session, user, flow, time, agent, email, tokens, model | +| `users.csv` | `user_id`, `is_active` | +| `usage_by_user.csv` | Per user, per day, per model/flow/provider: tokens, cache reads, cost | +| `usage_report.pdf` | Summary of spend, adoption, seats, and usage attribution | + +The raw CSV files are still a data dump. They have three problems: + +1. **It answers no question.** The admin must build every pivot. +2. **It has no dimensions the admin budgets by.** No team, no agent, no source. +3. **It joins badly.** `users.csv` has no email, so it cannot join to the other files. + +The raw export is still valuable. It is just the wrong artifact for every job. + +## The admin's job + +An org admin is accountable for three things: + +1. **Money.** They signed the contract. They own the spend. +2. **Adoption.** They championed the rollout. Someone will ask if it worked. +3. **Risk.** If the tool leaks or misbehaves, it lands on them. + +Every useful metric comes from one of these. The sections below derive the +metrics from the decisions. + +### Decision: how many seats do I buy at renewal? + +The admin needs a seat ledger, not a message count. + +- Licensed seats, provisioned seats, and active seats. +- A named list of users who hold a seat and did not use it in 30 days. +- The inverse list: users who hit rate limits. + +The named lists are the deliverable. The admin acts on them directly. They +reclaim a seat, retrain the person, or drop the seat at renewal. + +### Decision: am I overspending, and what can I cut? + +Total cost drives no action. Cost **concentration** does. + +- Share of spend from the top 5 users, the top 3 agents, and the top model. +- Cost per active user per month. This is the one number finance accepts. +- Spend sent to an expensive model for work a cheap model handles. +- Money already saved by prompt caching. We store `cache_read_tokens` today. + +### Decision: who pays for it? + +Cost split by team or user group. Most orgs must charge the cost back, or at +least explain the invoice internally. An admin who cannot attribute cost to a +cost center cannot grow the deployment. This blocks expansion. + +`User__UserGroup` already gives us the join. + +### Decision: did the rollout work? + +Message volume is a trap. It rises when three people go heavy. + +Measure breadth first, then habit, then depth: + +- Distinct humans who used Onyx in the period. +- How many use it every week, and whether that count rises. +- Multi-turn sessions versus one-question-and-leave. +- The funnel: invited, first message, five messages, weekly habit. + +The drop-off point in the funnel tells the admin what to fix. A drop before the +first message means onboarding. A drop after it means answer quality. + +### Decision: is the quality good? + +The admin cannot read conversations. Give them proxies, and give them trends. +Nobody knows what a good absolute thumbs-down rate is. + +- Negative feedback rate. +- Regeneration rate. +- Sessions abandoned after one answer. +- Answers where retrieval returned nothing. + +### Decision: what do I do next to improve it? + +This section is the most actionable, and it does not exist today. + +- Connectors that are indexed but never cited. +- Topics that people ask often and Onyx answers badly. +- Agents that nobody uses. + +Each item maps to one concrete action: add a source, write a document, delete +an agent. + +### Decision: am I exposed? + +Admins do not want a security console here. They want a short "look at this" +list, with names on it. + +- A user whose usage jumped 10x. +- Access to sensitive sources. +- Traffic from a service account or API key, not a human. +- Activity from an employee who left and kept an account. + +## The flagship: the knowledge gap report + +Every vendor can report spend and logins. Only Onyx knows **what the +organization tries to learn and fails to find.** + +A monthly artifact that lists the top unanswered questions, clustered by topic, +with the missing sources named, is worth more than the full cost breakdown. It +tells the admin something about their own company that they cannot get anywhere +else. It also makes the strongest renewal argument that exists. + +Treat this as the headline of the report, not an extra tab. + +## Three artifacts, not one zip + +The current report tries to be one thing. The admin needs three, at three +cadences. This is the main structural change. + +| Artifact | Cadence | Form | Purpose | +| ----------------------- | --------- | --------------------------------- | -------------------------------- | +| **Pulse** | Monthly | Pushed to email or Slack, no download | Tell the admin if anything changed | +| **Review pack** | Quarterly | A PDF the admin forwards to their boss | Defend the spend and the rollout | +| **Investigation export**| On demand | The raw CSVs we build today | Answer a specific question, feed BI | + +The pulse must be pushed. An artifact that needs a download and a spreadsheet +gets read once. + +## The one screen + +If the pulse were a single screen, it holds eight items: + +1. Active users, and the change. +2. Spend, and the change. +3. Cost per active user. +4. Percent of seats dormant, with the list. +5. Top 3 cost concentrations. +6. Quality trend, one line. +7. Top 5 unanswered topics. +8. One anomaly callout. + +Everything else lives one click deeper. + +## The PDF review pack + +The review pack must be a PDF. An admin forwards it to a VP or a CFO. Those +people do not open a zip of CSVs, and they do not log in to an admin panel. The +PDF is the artifact that travels. + +### Build it in pure Python with ReportLab + +Use **ReportLab** (BSD licensed). Write the document in Python. Do not template +markdown, do not render HTML, and do not drive a browser. + +ReportLab supplies everything the pack needs: + +- `platypus` flows content into pages. It splits long tables across pages and + repeats the header row. +- `graphics.charts` draws bar, line, and pie charts as native PDF vectors. +- The PDF base-14 fonts (Helvetica, Times) need no embedding. A brand font + works too, but vendor the TTF in the repo. Never fetch a font at render time. + +Measured on a representative pack (40-row table across 2 pages, one line chart, +one bar chart): **13 ms, 4.4 KB**. No subprocess, no browser. + +### The shared layer is the data model, not the text + +Build one typed aggregate object, and give each output its own renderer: + +| Output | Renderer | +| ------------ | --------------------------------- | +| PDF | ReportLab document builder | +| Email digest | HTML with inlined styles | +| Slack digest | Slack blocks | +| CSV rollups | The existing writers | + +All four read the same aggregate object, so the numbers always agree. Sharing a +data model is stronger than sharing a markdown string. Markdown cannot express a +chart, a page break, or a repeated table header, so it would have leaked layout +concerns into the shared layer anyway. + +### Pipeline + +1. Query the aggregates into the typed object. Same queries as the CSV rollups. +2. Build the ReportLab document from that object. +3. Save the PDF to the file store next to the zip, under the same `report_id`. + +This runs in the existing Celery report task. + +### Alternatives considered + +- **Chromium through Playwright.** It works, and it needs no new dependency, + because the image already installs Chromium (`backend/Dockerfile:124`) and + already launches it for the web connector. It also renders correctly with no + network (verified below). It still loses: a browser launch costs about 470 ms + and a few hundred MB of RSS, versus 13 ms for ReportLab, and it drags a + browser process, a template layer, and hand-written SVG into the report path. +- **WeasyPrint.** Adds a Python dependency plus pango system libraries, and + still makes us write CSS to control pagination. +- **Markdown plus Jinja2.** A lossy intermediate format. It cannot express + charts or pagination, so every hard part would still need solving elsewhere. + +### Air-gap status + +Verified inside the shipped image with `docker run --network none`: + +- Chromium launches and prints a PDF with tables, inline SVG, and real fonts. + Text extracts correctly. So the browser path is air-gap safe if we ever want + it. +- ReportLab needs no network by construction, and the base-14 fonts live inside + the PDF spec. + +Note that the air-gap CI job only starts `api_server`, `inference_model_server`, +and `minio`. It does not exercise the background worker or a browser. Any +air-gap claim for a new render path needs its own check. + +### Constraints to respect + +- Determinism. The same period must produce the same PDF. Do not stamp a + render timestamp inside the content body. +- Size. Cap the page count. The pack is a summary, not the export. +- Anonymized mode. The PDF must honor it, the same as the CSVs. +- Failure isolation. A PDF failure must not fail the zip. Generate them + independently. + +## Anti-metrics + +Do not put these in a report. They look informative and drive no decision: + +- Total messages, total tokens, total sessions. +- Average response time. +- Any cumulative all-time count. + +## What the raw export still needs + +The investigation export stays. Fix its defects: + +- Add `summary.csv`, `manifest.json` (schema version, period, timezone, row + counts, generator version), and a `README.md` that defines every column. +- Add pre-built rollups: by team, by agent, by model, by day. +- Fix `users.csv`: email, role, groups, created date, last active, seat state. +- State the units. Name the currency. Snapshot the price table, so the numbers + stay reproducible after prices change. +- Warn when the period is incomplete. If the usage rollup started after the + period start, say so on the report. Otherwise a partial month reads as a + full one. +- Support an anonymized mode. Some EU customers cannot legally receive per + person usage. Hash the email with a per-report salt. + +## Build order + +Ordered by value per unit of work. The first two are independent of each other. + +1. **Seat ledger.** Fix `users.csv` and derive the dormant-seat list. Small + change, and it makes every other file joinable. +2. **Knowledge gap report.** The differentiated artifact. +3. **Team and agent dimensions** on the cost breakdown. Unblocks chargeback. +4. **Summary, manifest, and README** on the export. +5. **Extend the PDF review pack.** The shipped pack summarizes spend and + adoption by person, model, and flow. Add the missing dimensions from steps + 1 to 3 as they become available. +6. **Scheduled pulse** to email or Slack. The Celery beat already exists, and + the existing typed aggregate object supplies the content. diff --git a/pyproject.toml b/pyproject.toml index 40d8265304..ee65af339e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,6 +99,8 @@ backend = [ "pywikibot==11.4.2", "readerwriterlock==1.0.9", "redis==5.0.8", + # PDF generation for the usage report review pack. + "reportlab==5.0.0", "requests==2.33.0", "requests-oauthlib==2.0.0", "simple-salesforce==1.12.6", diff --git a/uv.lock b/uv.lock index d27dfd39bd..fda8c3d5ac 100644 --- a/uv.lock +++ b/uv.lock @@ -4350,6 +4350,7 @@ backend = [ { name = "rapidfuzz" }, { name = "readerwriterlock" }, { name = "redis" }, + { name = "reportlab" }, { name = "requests" }, { name = "requests-oauthlib" }, { name = "sendgrid" }, @@ -4524,6 +4525,7 @@ backend = [ { name = "rapidfuzz", specifier = "==3.14.5" }, { name = "readerwriterlock", specifier = "==1.0.9" }, { name = "redis", specifier = "==5.0.8" }, + { name = "reportlab", specifier = "==5.0.0" }, { name = "requests", specifier = "==2.33.0" }, { name = "requests-oauthlib", specifier = "==2.0.0" }, { name = "sendgrid", specifier = "==6.12.5" }, @@ -6271,6 +6273,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/6f/832c2023a8bd8414c93452bd8b43bf61cedfa5b9575f70c06fb911e51a29/release_tag-0.5.2-py3-none-win_arm64.whl", hash = "sha256:5f26b008e0be0c7a122acd8fcb1bb5c822f38e77fed0c0bf6c550cc226c6bf14", size = 1203191, upload-time = "2026-03-11T00:27:29.789Z" }, ] +[[package]] +name = "reportlab" +version = "5.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/41/d6/4b7b0cf56880eb96533e607967be6a939e344675601e033d113a0bfa1f4e/reportlab-5.0.0.tar.gz", hash = "sha256:e4494a0c6623ae213bb856fba523171b2b54a7bf629fda02d5e525a7b899a784", size = 3701928, upload-time = "2026-06-18T11:34:31.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/07/70085c17a369605f15e301d10ab902115019b1126c7253d964afc230c7d6/reportlab-5.0.0-py3-none-any.whl", hash = "sha256:9d5a3affa84919e1111ede580031266a570e93b1ce388219621347965ff1d93c", size = 1956710, upload-time = "2026-06-18T11:34:29.07Z" }, +] + [[package]] name = "requests" version = "2.33.0"