feat(usage): add PDF review pack to exports (#13872)

This commit is contained in:
roshan authored and GitHub committed 2026-08-12 16:30:39 +00:00
1 parent b8365496c5
commit b89f63e511
11 files changed
+1599 -2

No files matched your search

@@ -4,6 +4,7 @@ import uuid
import zipfile import zipfile
from collections.abc import Iterable from collections.abc import Iterable
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from io import BytesIO
from fastapi_users_db_sqlalchemy import UUID_ID from fastapi_users_db_sqlalchemy import UUID_ID
from sqlalchemy import cast from sqlalchemy import cast
@@ -19,6 +20,9 @@ from ee.onyx.server.reporting.usage_export_models import (
UsageReportMetadata, UsageReportMetadata,
UserSkeleton, UserSkeleton,
) )
from ee.onyx.server.reporting.usage_report_branding import load_report_branding
from ee.onyx.server.reporting.usage_report_data import build_usage_report_data
from ee.onyx.server.reporting.usage_report_pdf import render_usage_report_pdf
from onyx.configs.constants import FileOrigin from onyx.configs.constants import FileOrigin
from onyx.db.models import User from onyx.db.models import User
from onyx.db.user_usage import UsageExportRow, iter_usage_export from onyx.db.user_usage import UsageExportRow, iter_usage_export
@@ -184,6 +188,31 @@ def generate_usage_breakdown_report(
return file_id return file_id
def generate_usage_report_pdf(
db_session: Session,
file_store: FileStore,
report_id: str,
period: tuple[datetime, datetime] | None,
rows: list[UsageExportRow],
) -> str:
"""Render the review pack PDF and store it. Returns the file id."""
file_name = f"{report_id}_review_pack"
# The queried bounds are half-open; only a given period gets the extra day.
display_start, display_end = period if period else _normalize_period(None)
data = build_usage_report_data(db_session, rows, display_start, display_end)
branding = load_report_branding(file_store)
pdf_bytes = render_usage_report_pdf(data, branding)
return file_store.save_file(
content=BytesIO(pdf_bytes),
display_name=file_name,
file_origin=FileOrigin.GENERATED_REPORT,
file_type="application/pdf",
)
def create_new_usage_report( def create_new_usage_report(
db_session: Session, db_session: Session,
user_id: UUID_ID | None, # None = auto-generated user_id: UUID_ID | None, # None = auto-generated
@@ -204,12 +233,24 @@ def create_new_usage_report(
intermediate_file_ids.append(users_file_id) intermediate_file_ids.append(users_file_id)
query_start, query_end = normalized_period query_start, query_end = normalized_period
usage_rows = iter_usage_export(db_session, query_start, query_end) # The CSV and PDF must use the same rows so their totals reconcile.
usage_rows = list(iter_usage_export(db_session, query_start, query_end))
usage_breakdown_file_id = generate_usage_breakdown_report( usage_breakdown_file_id = generate_usage_breakdown_report(
file_store, report_id, usage_rows file_store, report_id, usage_rows
) )
intermediate_file_ids.append(usage_breakdown_file_id) intermediate_file_ids.append(usage_breakdown_file_id)
# A render failure must not cost the admin their CSV export.
pdf_file_id: str | None = None
try:
pdf_file_id = generate_usage_report_pdf(
db_session, file_store, report_id, period, usage_rows
)
except Exception:
logger.exception("Failed to render usage report PDF; continuing without it")
else:
intermediate_file_ids.append(pdf_file_id)
# Re-check just before writing the final report: the API-level check # Re-check just before writing the final report: the API-level check
# happens before this (async) task runs, so a second request with the # happens before this (async) task runs, so a second request with the
# same client-supplied report_id can slip past it while this task is # same client-supplied report_id can slip past it while this task is
@@ -239,6 +280,12 @@ def create_new_usage_report(
) )
zip_file.writestr("usage_by_user.csv", usage_breakdown_tmpfile.read()) zip_file.writestr("usage_by_user.csv", usage_breakdown_tmpfile.read())
if pdf_file_id is not None:
pdf_tmpfile = file_store.read_file(
pdf_file_id, mode="b", use_tempfile=True
)
zip_file.writestr("usage_report.pdf", pdf_tmpfile.read())
zip_buffer.seek(0) zip_buffer.seek(0)
# store zip blob to file_store # store zip blob to file_store
@@ -0,0 +1,79 @@
"""Branding for the usage report review pack, from enterprise settings."""
from pathlib import Path
from pydantic import BaseModel
from ee.onyx.server.enterprise_settings.store import (
get_logo_filename,
get_logotype_filename,
load_runtime_settings,
)
from onyx.configs.constants import ONYX_DEFAULT_APPLICATION_NAME
from onyx.file_store.file_store import FileStore
from onyx.utils.logger import setup_logger
logger = setup_logger()
_FALLBACK_LOGO = Path(__file__).parents[4] / "static" / "images" / "logotype.png"
_SUPPORTED_LOGO_TYPES = ("image/png", "image/jpeg", "image/jpg", "image/gif")
class ReportBranding(BaseModel):
application_name: str
# Raster bytes ReportLab can draw. None means the pack sets the name as a
# wordmark instead, which is right when a deployment has a custom logo we
# cannot draw: its own name beats another company's mark.
logo: bytes | None = None
def _read_logo(file_store: FileStore, file_id: str) -> bytes | None:
try:
stored = file_store.get_file_with_mime_type(file_id)
except Exception:
logger.exception("Failed to read logo %s for the usage report", file_id)
return None
if stored is None:
return None
# ReportLab cannot rasterize SVG, the common upload.
if stored.mime_type not in _SUPPORTED_LOGO_TYPES:
logger.info(
"Usage report cannot draw logo %s of type %s", file_id, stored.mime_type
)
return None
return stored.data
def load_report_branding(file_store: FileStore) -> ReportBranding:
settings = load_runtime_settings()
name = settings.application_name or ONYX_DEFAULT_APPLICATION_NAME
has_custom_logo = settings.use_custom_logotype or settings.use_custom_logo
logo: bytes | None = None
if settings.use_custom_logotype:
logo = _read_logo(file_store, get_logotype_filename())
if logo is None and settings.use_custom_logo:
logo = _read_logo(file_store, get_logo_filename())
if logo is None and has_custom_logo:
# Their logo exists but cannot be drawn, so fall through to the
# wordmark. Stamping the bundled mark here would ship our brand on
# their report.
logger.warning(
"Usage report rendering %s as a wordmark: the configured logo is "
"not a raster image ReportLab can draw",
name,
)
return ReportBranding(application_name=name, logo=None)
if logo is None:
try:
logo = _FALLBACK_LOGO.read_bytes()
except OSError:
logger.exception("Usage report could not read the fallback logo")
return ReportBranding(application_name=name, logo=logo)
@@ -0,0 +1,171 @@
"""Aggregates behind the usage report review pack."""
from collections import defaultdict
from datetime import datetime
from pydantic import BaseModel
from sqlalchemy.orm import Session
from ee.onyx.db.license import user_counts_toward_seats
from onyx.db.api_key import is_api_key_email_address
from onyx.db.user_usage import DELETED_USER_EXPORT_EMAIL, UsageExportRow
from onyx.db.users import get_all_users
TOP_USER_LIMIT = 10
TOP_ENTRY_LIMIT = 8
DORMANT_USER_LIMIT = 25
UNLABELED_FLOW = "other"
class NamedSpend(BaseModel):
name: str
cost_cents: float
input_tokens: int
output_tokens: int
@property
def total_tokens(self) -> int:
return self.input_tokens + self.output_tokens
class DailySpend(BaseModel):
day: str # YYYY-MM-DD
cost_cents: float
active_users: int
class UsageReportData(BaseModel):
period_start: datetime
period_end: datetime
total_cost_cents: float
total_input_tokens: int
total_output_tokens: int
total_cache_read_tokens: int
licensed_users: int
# Everyone who used it, including people since deactivated, so this can
# exceed licensed_users. `seated_active_users` is the subset holding a seat.
active_users: int
seated_active_users: int
dormant_users: list[str]
top_users: list[NamedSpend]
by_model: list[NamedSpend]
by_flow: list[NamedSpend]
daily: list[DailySpend]
@property
def dormant_user_count(self) -> int:
return len(self.dormant_users)
@property
def cost_per_active_user_cents(self) -> float:
if not self.active_users:
return 0.0
return self.total_cost_cents / self.active_users
@property
def has_usage(self) -> bool:
return bool(self.daily)
def _top_n(spend_by_name: dict[str, NamedSpend], limit: int) -> list[NamedSpend]:
ordered = sorted(spend_by_name.values(), key=lambda s: s.cost_cents, reverse=True)
if len(ordered) <= limit:
return ordered
head, tail = ordered[:limit], ordered[limit:]
# Folding a single entry hides a name and saves no space.
if len(tail) == 1:
return ordered
remainder = NamedSpend(
name=f"Other ({len(tail)})",
cost_cents=sum(s.cost_cents for s in tail),
input_tokens=sum(s.input_tokens for s in tail),
output_tokens=sum(s.output_tokens for s in tail),
)
return head + [remainder]
def build_usage_report_data(
db_session: Session,
rows: list[UsageExportRow],
period_start: datetime,
period_end: datetime,
) -> UsageReportData:
"""`rows` is the list written to usage_by_user.csv, so the two cannot
diverge. The period is the admin's requested bounds, for display."""
by_user: dict[str, NamedSpend] = {}
by_model: dict[str, NamedSpend] = {}
by_flow: dict[str, NamedSpend] = {}
daily_cost: dict[str, float] = defaultdict(float)
daily_users: dict[str, set[str]] = defaultdict(set)
total_cost = 0.0
total_input = 0
total_output = 0
total_cache_read = 0
active_emails: set[str] = set()
for row in rows:
for bucket, key in (
(by_user, row.email),
(by_model, row.model),
(by_flow, row.flow or UNLABELED_FLOW),
):
entry = bucket.get(key)
if entry is None:
entry = NamedSpend(
name=key, cost_cents=0.0, input_tokens=0, output_tokens=0
)
bucket[key] = entry
entry.cost_cents += row.cost_cents
entry.input_tokens += row.input_tokens
entry.output_tokens += row.output_tokens
total_cost += row.cost_cents
total_input += row.input_tokens
total_output += row.output_tokens
total_cache_read += row.cache_read_tokens
daily_cost[row.day] += row.cost_cents
# Their spend still counts toward totals so the pack reconciles with the
# CSV, but neither is a person.
if row.email != DELETED_USER_EXPORT_EMAIL and not is_api_key_email_address(
row.email
):
active_emails.add(row.email)
daily_users[row.day].add(row.email)
# Must match license enforcement, or this disagrees with what is billed.
users = get_all_users(db_session, include_api_key_users=False)
seat_emails = {user.email for user in users if user_counts_toward_seats(user)}
dormant = sorted(seat_emails - active_emails)
daily = [
DailySpend(
day=day,
cost_cents=daily_cost[day],
active_users=len(daily_users[day]),
)
for day in sorted(daily_cost)
]
return UsageReportData(
period_start=period_start,
period_end=period_end,
total_cost_cents=total_cost,
total_input_tokens=total_input,
total_output_tokens=total_output,
total_cache_read_tokens=total_cache_read,
licensed_users=len(seat_emails),
active_users=len(active_emails),
seated_active_users=len(active_emails & seat_emails),
dormant_users=dormant,
top_users=_top_n(by_user, TOP_USER_LIMIT),
by_model=_top_n(by_model, TOP_ENTRY_LIMIT),
by_flow=_top_n(by_flow, TOP_ENTRY_LIMIT),
daily=daily,
)
@@ -0,0 +1,628 @@
"""Renders the usage report review pack as a PDF.
The `ty: ignore`s below are load-bearing: ReportLab types `chart.data` from a
sample literal and populates `valueAxis.labels` dynamically, so neither is
resolvable by a static checker.
"""
from io import BytesIO
from xml.sax.saxutils import escape
from reportlab.graphics.charts.barcharts import VerticalBarChart
from reportlab.graphics.charts.linecharts import HorizontalLineChart
from reportlab.graphics.shapes import Drawing
from reportlab.lib import colors
from reportlab.lib.enums import TA_LEFT
from reportlab.lib.pagesizes import LETTER
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
from reportlab.lib.units import inch
from reportlab.lib.utils import ImageReader
from reportlab.pdfgen import canvas
from reportlab.platypus import (
Flowable,
Image,
KeepTogether,
PageBreak,
Paragraph,
SimpleDocTemplate,
Spacer,
Table,
TableStyle,
)
from ee.onyx.server.reporting.usage_report_branding import ReportBranding
from ee.onyx.server.reporting.usage_report_data import (
DORMANT_USER_LIMIT,
NamedSpend,
UsageReportData,
)
from onyx.configs.constants import DANSWER_API_KEY_PREFIX, UNNAMED_KEY_PLACEHOLDER
from onyx.db.api_key import is_api_key_email_address
from onyx.utils.logger import setup_logger
logger = setup_logger()
_INK = colors.HexColor("#1c1c1c") # onyx-ink-95
_ACCENT = colors.HexColor("#286df8") # action-selection-05 / blue-50
_BODY = colors.HexColor("#54545d") # stone-60, 7.5:1 on white
_HAIRLINE = colors.HexColor("#e6e6e9") # stone-10
_SURFACE = colors.HexColor("#f0f0f1") # stone-05
_CONTENT_WIDTH = LETTER[0] - 2 * inch
_MAX_AXIS_LABELS = 12
_LOGO_MAX_W, _LOGO_MAX_H = 2.0 * inch, 0.5 * inch
def _dollars(cents: float) -> str:
return f"${cents / 100:,.2f}"
def _thousands(value: int) -> str:
return f"{value:,}"
def _display_name(name: str) -> str:
"""Render an API key by its name instead of its synthetic address.
Each API key owns a `User` row whose email is
`API_KEY__<key name>@<uuid>onyxapikey.ai`, so per-key spend already
aggregates correctly. Only the label needs help. A no-op for every other
name, since none of them carry the API-key domain.
"""
if not is_api_key_email_address(name):
return name
# The key's name can itself contain "@", so split on the last one.
local_part = name.rsplit("@", 1)[0]
# Stored emails are lowercased by a DB check constraint, so the prefix
# cannot be matched case-sensitively against the constant.
if local_part.lower().startswith(DANSWER_API_KEY_PREFIX.lower()):
local_part = local_part[len(DANSWER_API_KEY_PREFIX) :]
return f"{local_part or UNNAMED_KEY_PLACEHOLDER} (API key)"
def _styles() -> dict[str, ParagraphStyle]:
base = getSampleStyleSheet()
return {
"cover_title": ParagraphStyle(
"CoverTitle",
parent=base["Title"],
fontName="Helvetica-Bold",
fontSize=32,
leading=36,
textColor=_INK,
alignment=TA_LEFT,
spaceAfter=6,
),
"wordmark": ParagraphStyle(
"Wordmark",
parent=base["Normal"],
fontName="Helvetica-Bold",
fontSize=19,
leading=23,
textColor=_INK,
),
"cover_period": ParagraphStyle(
"CoverPeriod",
parent=base["Normal"],
fontName="Helvetica",
fontSize=13,
leading=18,
textColor=_BODY,
),
"lede": ParagraphStyle(
"Lede",
parent=base["Normal"],
fontName="Helvetica",
fontSize=11.5,
leading=17,
textColor=_INK,
),
"heading": ParagraphStyle(
"Heading",
parent=base["Heading2"],
fontName="Helvetica-Bold",
fontSize=14,
leading=18,
textColor=_INK,
spaceBefore=22,
spaceAfter=2,
keepWithNext=1,
),
"subheading": ParagraphStyle(
"Subheading",
parent=base["Normal"],
fontName="Helvetica",
fontSize=9.5,
leading=13,
textColor=_BODY,
spaceAfter=10,
keepWithNext=1,
),
"note": ParagraphStyle(
"Note",
parent=base["Normal"],
fontName="Helvetica",
fontSize=8.5,
leading=12,
textColor=_BODY,
spaceBefore=4,
),
}
def _logo_flowable(branding: ReportBranding) -> Flowable | None:
if not branding.logo:
return None
try:
reader = ImageReader(BytesIO(branding.logo))
src_w, src_h = reader.getSize()
except Exception:
logger.exception(
"Usage report could not render the configured logo for %s",
branding.application_name,
)
return None
if not src_w or not src_h:
return None
scale = min(_LOGO_MAX_W / src_w, _LOGO_MAX_H / src_h)
return Image(
BytesIO(branding.logo), width=src_w * scale, height=src_h * scale, mask="auto"
)
class _NumberedCanvas(canvas.Canvas):
"""Stamps "page N of M" once the total is known.
The total only exists after the last page is laid out, so pages are held
back until save. The cover is deliberately left unnumbered.
"""
def __init__(self, *args: object, **kwargs: object) -> None:
super().__init__(*args, **kwargs)
self._pages: list[dict[str, object]] = []
def showPage(self) -> None:
self._pages.append(dict(self.__dict__))
self._startPage()
def save(self) -> None:
total = len(self._pages)
for number, state in enumerate(self._pages, start=1):
self.__dict__.update(state)
if number > 1:
self._draw_folio(number, total)
super().showPage()
super().save()
def _draw_folio(self, number: int, total: int) -> None:
width = self._pagesize[0]
self.setFont("Helvetica", 8.5)
self.setFillColor(_BODY)
self.drawRightString(width - inch, 0.6 * inch, f"{number} of {total}")
class _Rule(Flowable):
def __init__(self, width: float, color: colors.Color = _HAIRLINE) -> None:
super().__init__()
self.width, self.height, self.color = width, 1, color
def draw(self) -> None:
self.canv.setStrokeColor(self.color)
self.canv.setLineWidth(1)
self.canv.line(0, 0, self.width, 0)
class _SeatMeter(Flowable):
"""Seats in use against seats bought."""
def __init__(self, active: int, licensed: int, unseated: int, width: float) -> None:
super().__init__()
self.active, self.licensed, self.unseated = active, licensed, unseated
self.width, self.height = width, 54
def draw(self) -> None:
c = self.canv
bar_h, bar_y = 14, 20
ratio = min(1.0, self.active / self.licensed) if self.licensed else 0.0
c.setFillColor(_SURFACE)
c.roundRect(0, bar_y, self.width, bar_h, 3, stroke=0, fill=1)
if ratio > 0:
c.setFillColor(_ACCENT)
c.roundRect(
0, bar_y, max(3.0, self.width * ratio), bar_h, 3, stroke=0, fill=1
)
c.setFillColor(_INK)
c.setFont("Helvetica-Bold", 11)
c.drawString(
0, bar_y + bar_h + 8, f"{self.active} of {self.licensed} seats active"
)
c.setFillColor(_BODY)
c.setFont("Helvetica", 9)
idle = self.licensed - self.active
caption = (
f"{ratio:.0%} in use · {idle} seats idle"
if idle
else f"{ratio:.0%} of licensed seats in use"
)
if self.unseated:
caption += f" · {self.unseated} used it without a seat"
c.drawString(0, bar_y - 13, caption)
def _headline(data: UsageReportData) -> Table:
figures = [
(_thousands(data.active_users), "People using it"),
(_dollars(data.total_cost_cents), "Total spend"),
(_dollars(data.cost_per_active_user_cents), "Cost per active person"),
]
value_style = ParagraphStyle(
"Figure",
fontName="Helvetica-Bold",
fontSize=23,
leading=26,
textColor=_INK,
)
label_style = ParagraphStyle(
"FigureLabel",
fontName="Helvetica",
fontSize=9,
leading=12,
textColor=_BODY,
)
row = [
[Paragraph(v, value_style) for v, _ in figures],
[Paragraph(label, label_style) for _, label in figures],
]
col = _CONTENT_WIDTH / 3
table = Table(row, colWidths=[col] * 3, hAlign="LEFT")
table.setStyle(
TableStyle(
[
("VALIGN", (0, 0), (-1, -1), "BOTTOM"),
("TOPPADDING", (0, 0), (-1, 0), 0),
("BOTTOMPADDING", (0, 0), (-1, 0), 2),
("TOPPADDING", (0, 1), (-1, 1), 0),
("LEFTPADDING", (0, 0), (-1, -1), 0),
("RIGHTPADDING", (0, 0), (-1, -1), 12),
]
)
)
return table
def _table(rows: list[list[str]], col_widths: list[float]) -> Table:
table = Table(rows, colWidths=col_widths, repeatRows=1, hAlign="LEFT")
table.setStyle(
TableStyle(
[
# Header and body typography.
("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"),
("FONTNAME", (0, 1), (-1, -1), "Helvetica"),
("FONTSIZE", (0, 0), (-1, -1), 9),
("TEXTCOLOR", (0, 0), (-1, 0), _INK),
("TEXTCOLOR", (0, 1), (-1, -1), _BODY),
("TEXTCOLOR", (0, 1), (0, -1), _INK),
# Text columns left-align; numeric columns right-align.
("ALIGN", (1, 0), (-1, -1), "RIGHT"),
("ALIGN", (0, 0), (0, -1), "LEFT"),
# Separate the header and each body row.
("LINEBELOW", (0, 0), (-1, 0), 1, _INK),
("LINEBELOW", (0, 1), (-1, -2), 0.5, _HAIRLINE),
# Keep rows readable without changing column width.
("TOPPADDING", (0, 0), (-1, -1), 7),
("BOTTOMPADDING", (0, 0), (-1, -1), 7),
("LEFTPADDING", (0, 0), (-1, -1), 0),
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
]
)
)
return table
def _axis_labels(days: list[str]) -> list[str]:
"""Keep at most `_MAX_AXIS_LABELS` ticks, blanking the rest."""
step = max(1, (len(days) + _MAX_AXIS_LABELS - 1) // _MAX_AXIS_LABELS)
# Drop the year: the period is already stated on the cover.
return [day[5:] if index % step == 0 else "" for index, day in enumerate(days)]
def _style_axes(chart: HorizontalLineChart | VerticalBarChart) -> None:
chart.categoryAxis.labels.fontName = "Helvetica"
chart.categoryAxis.labels.fontSize = 7
chart.categoryAxis.labels.fillColor = _BODY
chart.categoryAxis.strokeColor = _HAIRLINE
chart.valueAxis.labels.fontName = "Helvetica" # ty: ignore[unresolved-attribute]
chart.valueAxis.labels.fontSize = 7 # ty: ignore[unresolved-attribute]
chart.valueAxis.labels.fillColor = _BODY # ty: ignore[unresolved-attribute]
chart.valueAxis.strokeColor = _HAIRLINE
chart.valueAxis.valueMin = 0
chart.valueAxis.gridStrokeColor = _HAIRLINE
chart.valueAxis.gridStrokeWidth = 0.5
chart.valueAxis.visibleGrid = True
def _spend_over_time(data: UsageReportData) -> Drawing:
drawing = Drawing(_CONTENT_WIDTH, 168)
chart = HorizontalLineChart()
chart.x, chart.y = 42, 28
chart.width, chart.height = int(_CONTENT_WIDTH - 56), 122
chart.data = [[point.cost_cents / 100 for point in data.daily]] # ty: ignore[invalid-assignment]
chart.categoryAxis.categoryNames = _axis_labels([p.day for p in data.daily])
_style_axes(chart)
chart.lines[0].strokeColor = _ACCENT
chart.lines[0].strokeWidth = 1.6
drawing.add(chart)
return drawing
def _active_users_over_time(data: UsageReportData) -> Drawing:
drawing = Drawing(_CONTENT_WIDTH, 168)
chart = HorizontalLineChart()
chart.x, chart.y = 42, 28
chart.width, chart.height = int(_CONTENT_WIDTH - 56), 122
chart.data = [[float(point.active_users) for point in data.daily]] # ty: ignore[invalid-assignment]
chart.categoryAxis.categoryNames = _axis_labels([p.day for p in data.daily])
_style_axes(chart)
chart.lines[0].strokeColor = _INK
chart.lines[0].strokeWidth = 1.6
drawing.add(chart)
return drawing
def _spend_by_model(data: UsageReportData) -> Drawing:
drawing = Drawing(_CONTENT_WIDTH, 170)
chart = VerticalBarChart()
chart.x, chart.y = 42, 38
chart.width, chart.height = int(_CONTENT_WIDTH - 56), 112
chart.data = [[entry.cost_cents / 100 for entry in data.by_model]] # ty: ignore[invalid-assignment]
chart.categoryAxis.categoryNames = [
_display_name(entry.name) for entry in data.by_model
]
_style_axes(chart)
chart.categoryAxis.labels.angle = 20
chart.categoryAxis.labels.dy = -8
chart.bars[0].fillColor = _ACCENT
chart.bars[0].strokeColor = None
chart.barSpacing = 2
drawing.add(chart)
return drawing
def _cover(
data: UsageReportData,
branding: ReportBranding,
styles: dict[str, ParagraphStyle],
) -> list[Flowable]:
period = (
f"{data.period_start.date().isoformat()} to "
f"{data.period_end.date().isoformat()} (UTC)"
)
story: list[Flowable] = []
logo = _logo_flowable(branding)
if logo is not None:
logo.hAlign = "LEFT"
story += [logo, Spacer(1, 30)]
else:
story += [
Paragraph(escape(branding.application_name), styles["wordmark"]),
Spacer(1, 26),
]
story += [
Paragraph("Usage report", styles["cover_title"]),
Paragraph(period, styles["cover_period"]),
Spacer(1, 26),
_Rule(_CONTENT_WIDTH, _INK),
Spacer(1, 22),
_headline(data),
Spacer(1, 30),
]
if data.licensed_users:
story += [
_SeatMeter(
data.seated_active_users,
data.licensed_users,
data.active_users - data.seated_active_users,
_CONTENT_WIDTH,
)
]
story += [
Spacer(1, 30),
Paragraph(_summary_sentence(data, branding.application_name), styles["lede"]),
]
if data.by_model:
story += [
Spacer(1, 26),
Paragraph("Top models by spend", styles["subheading"]),
_spend_table("Model", data.by_model[:3]),
]
return story
def _summary_sentence(data: UsageReportData, application_name: str) -> str:
"""Returns Paragraph markup, so every interpolated name is escaped."""
people = "1 person" if data.active_users == 1 else f"{data.active_users} people"
parts = [
f"{people} used {escape(application_name)} in this period, at a total cost "
f"of {_dollars(data.total_cost_cents)}."
]
if data.by_flow:
flow = escape(_display_name(data.by_flow[0].name))
parts.append(f"Most of that ran through {flow}.")
if data.dormant_user_count:
share = (
data.dormant_user_count / data.licensed_users
if data.licensed_users
else 0.0
)
parts.append(
f"{data.dormant_user_count} of {data.licensed_users} licensed seats "
f"({share:.0%}) went unused and are candidates to reassign."
)
return " ".join(parts)
def _spend_table(label: str, entries: list[NamedSpend]) -> Table:
rows: list[list[str]] = [[label, "Spend (USD)", "Tokens"]]
rows += [
[
_display_name(entry.name),
_dollars(entry.cost_cents),
_thousands(entry.total_tokens),
]
for entry in entries
]
widths = [_CONTENT_WIDTH * 0.5, _CONTENT_WIDTH * 0.25, _CONTENT_WIDTH * 0.25]
return _table(rows, widths)
def _section(
heading: str, subheading: str, styles: dict[str, ParagraphStyle]
) -> list[Flowable]:
return [
Paragraph(heading, styles["heading"]),
Paragraph(subheading, styles["subheading"]),
]
def _charted_section(
heading: str,
subheading: str,
chart: Drawing,
styles: dict[str, ParagraphStyle],
) -> Flowable:
"""`keepWithNext` does not reach into a KeepTogether, so the heading must
travel inside the group or it strands at the page foot."""
return KeepTogether(
[
Paragraph(heading, styles["heading"]),
Paragraph(subheading, styles["subheading"]),
chart,
]
)
def render_usage_report_pdf(data: UsageReportData, branding: ReportBranding) -> bytes:
styles = _styles()
buffer = BytesIO()
doc = SimpleDocTemplate(
buffer,
pagesize=LETTER,
leftMargin=inch,
rightMargin=inch,
topMargin=0.9 * inch,
bottomMargin=0.9 * inch,
title=f"{branding.application_name} usage report",
author=branding.application_name,
# Byte-identical output for identical input.
invariant=1,
)
story: list[Flowable] = []
if not data.has_usage:
logo = _logo_flowable(branding)
if logo is not None:
logo.hAlign = "LEFT"
story += [logo, Spacer(1, 30)]
else:
story += [
Paragraph(escape(branding.application_name), styles["wordmark"]),
Spacer(1, 26),
]
story += [
Paragraph("Usage report", styles["cover_title"]),
Paragraph(
f"{data.period_start.date().isoformat()} to "
f"{data.period_end.date().isoformat()} (UTC)",
styles["cover_period"],
),
Spacer(1, 24),
Paragraph(
"No recorded usage in this period. If the deployment was active, "
"the usage rollup may have started after the period began.",
styles["lede"],
),
]
doc.build(story, canvasmaker=_NumberedCanvas)
return buffer.getvalue()
story += _cover(data, branding, styles)
story += [PageBreak()]
story += [
_charted_section(
"Adoption",
"Distinct people who sent at least one message each day.",
_active_users_over_time(data),
styles,
),
_charted_section(
"Spend over time",
"Daily cost across every model and surface.",
_spend_over_time(data),
styles,
),
_charted_section(
"Where the spend goes",
"Cost by model for the period.",
_spend_by_model(data),
styles,
),
Spacer(1, 6),
_spend_table("Model", data.by_model),
]
story += _section("Heaviest users", "The people driving most of the cost.", styles)
story += [_spend_table("User", data.top_users)]
story += _section("Spend by surface", "Where the work happens.", styles)
story += [_spend_table("Flow", data.by_flow)]
story += _section(
"Seats not in use",
"Licensed people who sent nothing this period. Reclaim, retrain, or "
"drop them at renewal.",
styles,
)
if not data.dormant_users:
story.append(
Paragraph("Every licensed seat was used this period.", styles["lede"])
)
else:
shown = data.dormant_users[:DORMANT_USER_LIMIT]
rows: list[list[str]] = [["User"]] + [[email] for email in shown]
story.append(_table(rows, [_CONTENT_WIDTH]))
remaining = data.dormant_user_count - len(shown)
if remaining > 0:
story.append(
Paragraph(
f"{remaining} more idle seats. See users.csv for the full list.",
styles["note"],
)
)
story += [
Spacer(1, 20),
_Rule(_CONTENT_WIDTH),
Spacer(1, 8),
Paragraph(
"Spend from deleted users and API keys is included in every total "
"and attributed separately. Neither counts as a person or a seat. "
"Days are UTC.",
styles["note"],
),
]
doc.build(story, canvasmaker=_NumberedCanvas)
return buffer.getvalue()
+7 -1
View File
@@ -531,6 +531,7 @@ charset-normalizer==3.4.4 \
# htmldate # htmldate
# markitdown # markitdown
# pdfminer-six # pdfminer-six
# reportlab
# requests # requests
# trafilatura # trafilatura
# unstructured # unstructured
@@ -2021,7 +2022,9 @@ pillow==12.3.0 \
--hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \
--hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \
--hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198
# via python-pptx # via
# python-pptx
# reportlab
platformdirs==4.5.0 \ platformdirs==4.5.0 \
--hash=sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312 \ --hash=sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312 \
--hash=sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3 --hash=sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3
@@ -2672,6 +2675,9 @@ regex==2025.11.3 \
# dateparser # dateparser
# nltk # nltk
# tiktoken # tiktoken
reportlab==5.0.0 \
--hash=sha256:9d5a3affa84919e1111ede580031266a570e93b1ce388219621347965ff1d93c \
--hash=sha256:e4494a0c6623ae213bb856fba523171b2b54a7bf629fda02d5e525a7b899a784
requests==2.33.0 \ requests==2.33.0 \
--hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b \ --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b \
--hash=sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652 --hash=sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652
@@ -290,6 +290,13 @@ class TestUsageExportAPI:
assert "chat_messages.csv" in file_names assert "chat_messages.csv" in file_names
assert "users.csv" in file_names assert "users.csv" in file_names
assert "usage_by_user.csv" in file_names assert "usage_by_user.csv" in file_names
assert "usage_report.pdf" in file_names
with zip_file.open("usage_report.pdf") as pdf_file:
pdf_bytes = pdf_file.read()
assert pdf_bytes.startswith(b"%PDF-")
assert len(pdf_bytes) > 1000
# Verify usage_by_user.csv has the expected columns. The seeded # Verify usage_by_user.csv has the expected columns. The seeded
# chat history doesn't record UserUsage rows, so there's no data # chat history doesn't record UserUsage rows, so there's no data
# to assert on, just the header shape. # to assert on, just the header shape.
@@ -0,0 +1,88 @@
"""Which logo the review pack renders under."""
from unittest.mock import MagicMock, patch
from ee.onyx.server.enterprise_settings.models import EnterpriseSettings
from ee.onyx.server.reporting.usage_report_branding import (
ReportBranding,
load_report_branding,
)
from onyx.utils.file import FileWithMimeType
_LOGOTYPE = b"logotype-bytes"
_LOGO = b"logo-bytes"
def _file_store(stored: dict[str, FileWithMimeType]) -> MagicMock:
store = MagicMock()
store.get_file_with_mime_type.side_effect = lambda file_id: stored.get(file_id)
return store
def _load(
settings: EnterpriseSettings, stored: dict[str, FileWithMimeType]
) -> ReportBranding:
with patch(
"ee.onyx.server.reporting.usage_report_branding.load_runtime_settings",
return_value=settings,
):
return load_report_branding(_file_store(stored))
def test_logotype_wins_over_the_square_mark() -> None:
branding = _load(
EnterpriseSettings(
application_name="Acme", use_custom_logo=True, use_custom_logotype=True
),
{
"__logotype__": FileWithMimeType(data=_LOGOTYPE, mime_type="image/png"),
"__logo__": FileWithMimeType(data=_LOGO, mime_type="image/png"),
},
)
assert branding.logo == _LOGOTYPE
assert branding.application_name == "Acme"
def test_falls_back_to_the_square_mark_when_no_logotype() -> None:
branding = _load(
EnterpriseSettings(use_custom_logo=True, use_custom_logotype=True),
{"__logo__": FileWithMimeType(data=_LOGO, mime_type="image/png")},
)
assert branding.logo == _LOGO
def test_an_undrawable_logo_becomes_a_wordmark_not_our_mark() -> None:
"""ReportLab cannot draw SVG. Their own name beats stamping the Onyx mark
on a report they forward to their leadership."""
branding = _load(
EnterpriseSettings(application_name="Acme", use_custom_logotype=True),
{"__logotype__": FileWithMimeType(data=b"<svg/>", mime_type="image/svg+xml")},
)
assert branding.logo is None
assert branding.application_name == "Acme"
def test_bundled_logo_is_used_when_nothing_is_uploaded() -> None:
branding = _load(EnterpriseSettings(), {})
assert branding.logo is not None
assert branding.logo.startswith(b"\x89PNG")
def test_a_file_store_failure_falls_back_to_the_wordmark() -> None:
"""A configured custom logo we cannot read still must not become our mark."""
store = MagicMock()
store.get_file_with_mime_type.side_effect = RuntimeError("object storage down")
with patch(
"ee.onyx.server.reporting.usage_report_branding.load_runtime_settings",
return_value=EnterpriseSettings(
application_name="Acme", use_custom_logotype=True
),
):
branding = load_report_branding(store)
assert branding.logo is None
@@ -0,0 +1,281 @@
"""Aggregation behind the usage report review pack."""
from datetime import datetime, timezone
from io import BytesIO
from unittest.mock import MagicMock, patch
import pytest
from PIL import Image as PILImage
from pypdf import PdfReader
from ee.onyx.server.reporting.usage_report_branding import ReportBranding
from ee.onyx.server.reporting.usage_report_data import (
TOP_USER_LIMIT,
UsageReportData,
build_usage_report_data,
)
from ee.onyx.server.reporting.usage_report_pdf import (
_axis_labels,
_display_name,
render_usage_report_pdf,
)
from onyx.db.enums import AccountType
from onyx.db.models import User
from onyx.db.user_usage import DELETED_USER_EXPORT_EMAIL, UsageExportRow
_BRANDING = ReportBranding(application_name="Acme Intelligence", logo=None)
PERIOD_START = datetime(2026, 7, 1, tzinfo=timezone.utc)
PERIOD_END = datetime(2026, 7, 31, tzinfo=timezone.utc)
def _row(
email: str,
cost: float = 10.0,
day: str = "2026-07-01",
flow: str = "chat",
) -> UsageExportRow:
return UsageExportRow(
email=email,
model="gpt-5",
flow=flow,
provider="openai",
day=day,
input_tokens=100,
output_tokens=50,
cache_read_tokens=10,
cost_cents=cost,
)
def _user(
email: str,
is_active: bool = True,
account_type: AccountType = AccountType.STANDARD,
) -> User:
user = User()
user.email = email
user.is_active = is_active
user.account_type = account_type
return user
def _build(rows: list[UsageExportRow], users: list[User]) -> UsageReportData:
with patch(
"ee.onyx.server.reporting.usage_report_data.get_all_users", return_value=users
):
return build_usage_report_data(
db_session=MagicMock(),
rows=rows,
period_start=PERIOD_START,
period_end=PERIOD_END,
)
def test_totals_reconcile_with_the_rows() -> None:
"""The pack's totals must equal the CSV's, including deleted-user spend."""
rows = [
_row("a@x.com", 10.0),
_row("b@x.com", 5.5),
_row(DELETED_USER_EXPORT_EMAIL, 4.5),
]
data = _build(rows, [_user("a@x.com"), _user("b@x.com")])
assert data.total_cost_cents == pytest.approx(20.0)
assert sum(e.cost_cents for e in data.by_model) == pytest.approx(20.0)
assert sum(e.cost_cents for e in data.by_flow) == pytest.approx(20.0)
assert sum(e.cost_cents for e in data.top_users) == pytest.approx(20.0)
assert data.total_input_tokens == 300
def test_deleted_user_counts_toward_spend_but_is_not_a_person() -> None:
rows = [_row("a@x.com"), _row(DELETED_USER_EXPORT_EMAIL)]
data = _build(rows, [_user("a@x.com")])
assert data.active_users == 1
assert data.daily[0].active_users == 1
assert data.total_cost_cents == pytest.approx(20.0)
def test_api_key_usage_is_not_an_active_user() -> None:
"""Seats exclude API-key users, so activity must exclude them too, or
active_users can exceed licensed_users."""
rows = [_row("a@x.com"), _row("somekey@onyxapikey.ai")]
data = _build(rows, [_user("a@x.com")])
assert data.active_users == 1
assert data.active_users <= data.licensed_users
def test_unlabeled_flow_is_grouped_as_other() -> None:
data = _build([_row("a@x.com", flow="")], [_user("a@x.com")])
assert [(entry.name, entry.cost_cents) for entry in data.by_flow] == [
("other", 10.0)
]
def test_service_accounts_do_not_hold_a_seat() -> None:
users = [
_user("human@x.com"),
_user("bot@x.com", account_type=AccountType.SERVICE_ACCOUNT),
_user("gone@x.com", is_active=False),
]
data = _build([_row("human@x.com")], users)
assert data.licensed_users == 1
assert data.seated_active_users == 1
assert data.dormant_users == []
def test_a_deactivated_user_is_active_but_holds_no_seat() -> None:
"""Someone who used it mid-period and was deactivated before the report ran
counts as a person, not as an occupied seat. The meter must not read
"2 of 1 seats active"."""
rows = [_row("stayed@x.com"), _row("departed@x.com")]
users = [_user("stayed@x.com"), _user("departed@x.com", is_active=False)]
data = _build(rows, users)
assert data.active_users == 2
assert data.licensed_users == 1
assert data.seated_active_users == 1
assert data.seated_active_users <= data.licensed_users
def test_dormant_seats_are_named() -> None:
users = [_user("active@x.com"), _user("idle@x.com")]
data = _build([_row("active@x.com")], users)
assert data.licensed_users == 2
assert data.active_users == 1
assert data.dormant_users == ["idle@x.com"]
def test_top_users_folds_the_tail_and_preserves_the_total() -> None:
rows = [_row(f"u{i}@x.com", cost=float(i + 1)) for i in range(TOP_USER_LIMIT + 3)]
data = _build(rows, [])
assert len(data.top_users) == TOP_USER_LIMIT + 1
assert data.top_users[-1].name == "Other (3)"
assert sum(e.cost_cents for e in data.top_users) == pytest.approx(
data.total_cost_cents
)
def test_a_single_extra_user_is_named_rather_than_folded() -> None:
"""Folding one entry hides a name and saves no space."""
rows = [_row(f"u{i}@x.com", cost=float(i + 1)) for i in range(TOP_USER_LIMIT + 1)]
data = _build(rows, [])
assert not any(e.name.startswith("Other") for e in data.top_users)
def test_api_key_spend_is_labeled_by_key_name() -> None:
"""Per-key spend stays in the breakdown; only the label is cleaned up."""
# A DB check constraint lowercases stored emails, so this is the real shape.
assert (
_display_name("api_key__nightly-sync@2f3c8a10-uuid.onyxapikey.ai")
== "nightly-sync (API key)"
)
assert (
_display_name("API_KEY__nightly-sync@2f3c8a10-uuid.onyxapikey.ai")
== "nightly-sync (API key)"
)
assert (
_display_name("api_key__sync@corp@2f3c8a10-uuid.onyxapikey.ai")
== "sync@corp (API key)"
)
assert _display_name("human@corp.com") == "human@corp.com"
assert _display_name("gpt-5") == "gpt-5"
assert _display_name(DELETED_USER_EXPORT_EMAIL) == DELETED_USER_EXPORT_EMAIL
def test_api_key_spend_still_reconciles_after_relabeling() -> None:
rows = [_row("a@x.com", 10.0), _row("api_key__bot@uuid.onyxapikey.ai", 90.0)]
data = _build(rows, [_user("a@x.com")])
assert data.total_cost_cents == pytest.approx(100.0)
assert sum(e.cost_cents for e in data.top_users) == pytest.approx(100.0)
assert data.active_users == 1
def test_zero_usage_still_renders_a_pdf() -> None:
data = _build([], [_user("idle@x.com")])
assert not data.has_usage
assert data.cost_per_active_user_cents == 0.0
assert render_usage_report_pdf(data, _BRANDING).startswith(b"%PDF-")
def test_application_name_is_not_parsed_as_markup() -> None:
"""ReportLab Paragraph parses a markup subset: an unescaped name with a tag
silently drops text, injects formatting, or raises and loses the PDF."""
data = _build([_row("a@x.com")], [_user("a@x.com")])
for name in ["Tools <R> Us", "Acme <b>Corp", 'X <font color="red">Y</font>']:
branding = ReportBranding(application_name=name, logo=None)
pdf = render_usage_report_pdf(data, branding)
assert pdf.startswith(b"%PDF-")
def _png(width: int = 40, height: int = 10) -> bytes:
buffer = BytesIO()
PILImage.new("RGBA", (width, height), (0, 0, 0, 255)).save(buffer, format="PNG")
return buffer.getvalue()
def test_the_cover_is_unnumbered_and_later_pages_are_not() -> None:
"""A folio on the cover, or an off-by-one, corrupts every generated pack."""
rows = [_row(f"u{i}@x.com", cost=float(i + 1)) for i in range(12)]
data = _build(rows, [_user(f"u{i}@x.com") for i in range(12)])
reader = PdfReader(BytesIO(render_usage_report_pdf(data, _BRANDING)))
total = len(reader.pages)
assert total > 1
assert f"1 of {total}" not in reader.pages[0].extract_text()
for number in range(2, total + 1):
assert f"{number} of {total}" in reader.pages[number - 1].extract_text()
def test_render_is_deterministic() -> None:
data = _build([_row("a@x.com")], [_user("a@x.com")])
assert render_usage_report_pdf(data, _BRANDING) == render_usage_report_pdf(
data, _BRANDING
)
def test_render_is_deterministic_with_an_embedded_logo() -> None:
"""The default path embeds a logo, so determinism must hold with one."""
data = _build([_row("a@x.com")], [_user("a@x.com")])
branding = ReportBranding(application_name="Acme", logo=_png())
assert render_usage_report_pdf(data, branding) == render_usage_report_pdf(
data, branding
)
def test_unreadable_logo_still_produces_a_pdf() -> None:
"""A corrupt upload must cost the branding, not the report."""
data = _build([_row("a@x.com")], [_user("a@x.com")])
for logo in (b"not-an-image", b""):
branding = ReportBranding(application_name="Acme", logo=logo)
assert render_usage_report_pdf(data, branding).startswith(b"%PDF-")
def test_axis_labels_never_exceed_the_display_limit() -> None:
for day_count in (0, 1, 12, 13, 23, 24, 25, 30, 365):
days = [f"2026-07-{day + 1:02d}" for day in range(day_count)]
assert sum(bool(label) for label in _axis_labels(days)) <= 12
+273
View File
@@ -0,0 +1,273 @@
# Usage reports: what they are for
This document defines what an Onyx usage report must tell an organization admin,
and why. It starts from the admin's job, not from the data we happen to store.
The rule that governs every decision here: **a number belongs in the report only
if the admin can finish the sentence "so I will...".** If no action follows, cut
the number.
## Where we are today
`create_new_usage_report` builds a zip with three CSVs and a PDF review pack:
| File | Contents |
| -------------------- | ------------------------------------------------------------------------------------- |
| `chat_messages.csv` | One row per message: session, user, flow, time, agent, email, tokens, model |
| `users.csv` | `user_id`, `is_active` |
| `usage_by_user.csv` | Per user, per day, per model/flow/provider: tokens, cache reads, cost |
| `usage_report.pdf` | Summary of spend, adoption, seats, and usage attribution |
The raw CSV files are still a data dump. They have three problems:
1. **It answers no question.** The admin must build every pivot.
2. **It has no dimensions the admin budgets by.** No team, no agent, no source.
3. **It joins badly.** `users.csv` has no email, so it cannot join to the other files.
The raw export is still valuable. It is just the wrong artifact for every job.
## The admin's job
An org admin is accountable for three things:
1. **Money.** They signed the contract. They own the spend.
2. **Adoption.** They championed the rollout. Someone will ask if it worked.
3. **Risk.** If the tool leaks or misbehaves, it lands on them.
Every useful metric comes from one of these. The sections below derive the
metrics from the decisions.
### Decision: how many seats do I buy at renewal?
The admin needs a seat ledger, not a message count.
- Licensed seats, provisioned seats, and active seats.
- A named list of users who hold a seat and did not use it in 30 days.
- The inverse list: users who hit rate limits.
The named lists are the deliverable. The admin acts on them directly. They
reclaim a seat, retrain the person, or drop the seat at renewal.
### Decision: am I overspending, and what can I cut?
Total cost drives no action. Cost **concentration** does.
- Share of spend from the top 5 users, the top 3 agents, and the top model.
- Cost per active user per month. This is the one number finance accepts.
- Spend sent to an expensive model for work a cheap model handles.
- Money already saved by prompt caching. We store `cache_read_tokens` today.
### Decision: who pays for it?
Cost split by team or user group. Most orgs must charge the cost back, or at
least explain the invoice internally. An admin who cannot attribute cost to a
cost center cannot grow the deployment. This blocks expansion.
`User__UserGroup` already gives us the join.
### Decision: did the rollout work?
Message volume is a trap. It rises when three people go heavy.
Measure breadth first, then habit, then depth:
- Distinct humans who used Onyx in the period.
- How many use it every week, and whether that count rises.
- Multi-turn sessions versus one-question-and-leave.
- The funnel: invited, first message, five messages, weekly habit.
The drop-off point in the funnel tells the admin what to fix. A drop before the
first message means onboarding. A drop after it means answer quality.
### Decision: is the quality good?
The admin cannot read conversations. Give them proxies, and give them trends.
Nobody knows what a good absolute thumbs-down rate is.
- Negative feedback rate.
- Regeneration rate.
- Sessions abandoned after one answer.
- Answers where retrieval returned nothing.
### Decision: what do I do next to improve it?
This section is the most actionable, and it does not exist today.
- Connectors that are indexed but never cited.
- Topics that people ask often and Onyx answers badly.
- Agents that nobody uses.
Each item maps to one concrete action: add a source, write a document, delete
an agent.
### Decision: am I exposed?
Admins do not want a security console here. They want a short "look at this"
list, with names on it.
- A user whose usage jumped 10x.
- Access to sensitive sources.
- Traffic from a service account or API key, not a human.
- Activity from an employee who left and kept an account.
## The flagship: the knowledge gap report
Every vendor can report spend and logins. Only Onyx knows **what the
organization tries to learn and fails to find.**
A monthly artifact that lists the top unanswered questions, clustered by topic,
with the missing sources named, is worth more than the full cost breakdown. It
tells the admin something about their own company that they cannot get anywhere
else. It also makes the strongest renewal argument that exists.
Treat this as the headline of the report, not an extra tab.
## Three artifacts, not one zip
The current report tries to be one thing. The admin needs three, at three
cadences. This is the main structural change.
| Artifact | Cadence | Form | Purpose |
| ----------------------- | --------- | --------------------------------- | -------------------------------- |
| **Pulse** | Monthly | Pushed to email or Slack, no download | Tell the admin if anything changed |
| **Review pack** | Quarterly | A PDF the admin forwards to their boss | Defend the spend and the rollout |
| **Investigation export**| On demand | The raw CSVs we build today | Answer a specific question, feed BI |
The pulse must be pushed. An artifact that needs a download and a spreadsheet
gets read once.
## The one screen
If the pulse were a single screen, it holds eight items:
1. Active users, and the change.
2. Spend, and the change.
3. Cost per active user.
4. Percent of seats dormant, with the list.
5. Top 3 cost concentrations.
6. Quality trend, one line.
7. Top 5 unanswered topics.
8. One anomaly callout.
Everything else lives one click deeper.
## The PDF review pack
The review pack must be a PDF. An admin forwards it to a VP or a CFO. Those
people do not open a zip of CSVs, and they do not log in to an admin panel. The
PDF is the artifact that travels.
### Build it in pure Python with ReportLab
Use **ReportLab** (BSD licensed). Write the document in Python. Do not template
markdown, do not render HTML, and do not drive a browser.
ReportLab supplies everything the pack needs:
- `platypus` flows content into pages. It splits long tables across pages and
repeats the header row.
- `graphics.charts` draws bar, line, and pie charts as native PDF vectors.
- The PDF base-14 fonts (Helvetica, Times) need no embedding. A brand font
works too, but vendor the TTF in the repo. Never fetch a font at render time.
Measured on a representative pack (40-row table across 2 pages, one line chart,
one bar chart): **13 ms, 4.4 KB**. No subprocess, no browser.
### The shared layer is the data model, not the text
Build one typed aggregate object, and give each output its own renderer:
| Output | Renderer |
| ------------ | --------------------------------- |
| PDF | ReportLab document builder |
| Email digest | HTML with inlined styles |
| Slack digest | Slack blocks |
| CSV rollups | The existing writers |
All four read the same aggregate object, so the numbers always agree. Sharing a
data model is stronger than sharing a markdown string. Markdown cannot express a
chart, a page break, or a repeated table header, so it would have leaked layout
concerns into the shared layer anyway.
### Pipeline
1. Query the aggregates into the typed object. Same queries as the CSV rollups.
2. Build the ReportLab document from that object.
3. Save the PDF to the file store next to the zip, under the same `report_id`.
This runs in the existing Celery report task.
### Alternatives considered
- **Chromium through Playwright.** It works, and it needs no new dependency,
because the image already installs Chromium (`backend/Dockerfile:124`) and
already launches it for the web connector. It also renders correctly with no
network (verified below). It still loses: a browser launch costs about 470 ms
and a few hundred MB of RSS, versus 13 ms for ReportLab, and it drags a
browser process, a template layer, and hand-written SVG into the report path.
- **WeasyPrint.** Adds a Python dependency plus pango system libraries, and
still makes us write CSS to control pagination.
- **Markdown plus Jinja2.** A lossy intermediate format. It cannot express
charts or pagination, so every hard part would still need solving elsewhere.
### Air-gap status
Verified inside the shipped image with `docker run --network none`:
- Chromium launches and prints a PDF with tables, inline SVG, and real fonts.
Text extracts correctly. So the browser path is air-gap safe if we ever want
it.
- ReportLab needs no network by construction, and the base-14 fonts live inside
the PDF spec.
Note that the air-gap CI job only starts `api_server`, `inference_model_server`,
and `minio`. It does not exercise the background worker or a browser. Any
air-gap claim for a new render path needs its own check.
### Constraints to respect
- Determinism. The same period must produce the same PDF. Do not stamp a
render timestamp inside the content body.
- Size. Cap the page count. The pack is a summary, not the export.
- Anonymized mode. The PDF must honor it, the same as the CSVs.
- Failure isolation. A PDF failure must not fail the zip. Generate them
independently.
## Anti-metrics
Do not put these in a report. They look informative and drive no decision:
- Total messages, total tokens, total sessions.
- Average response time.
- Any cumulative all-time count.
## What the raw export still needs
The investigation export stays. Fix its defects:
- Add `summary.csv`, `manifest.json` (schema version, period, timezone, row
counts, generator version), and a `README.md` that defines every column.
- Add pre-built rollups: by team, by agent, by model, by day.
- Fix `users.csv`: email, role, groups, created date, last active, seat state.
- State the units. Name the currency. Snapshot the price table, so the numbers
stay reproducible after prices change.
- Warn when the period is incomplete. If the usage rollup started after the
period start, say so on the report. Otherwise a partial month reads as a
full one.
- Support an anonymized mode. Some EU customers cannot legally receive per
person usage. Hash the email with a per-report salt.
## Build order
Ordered by value per unit of work. The first two are independent of each other.
1. **Seat ledger.** Fix `users.csv` and derive the dormant-seat list. Small
change, and it makes every other file joinable.
2. **Knowledge gap report.** The differentiated artifact.
3. **Team and agent dimensions** on the cost breakdown. Unblocks chargeback.
4. **Summary, manifest, and README** on the export.
5. **Extend the PDF review pack.** The shipped pack summarizes spend and
adoption by person, model, and flow. Add the missing dimensions from steps
1 to 3 as they become available.
6. **Scheduled pulse** to email or Slack. The Celery beat already exists, and
the existing typed aggregate object supplies the content.
+2
View File
@@ -99,6 +99,8 @@ backend = [
"pywikibot==11.4.2", "pywikibot==11.4.2",
"readerwriterlock==1.0.9", "readerwriterlock==1.0.9",
"redis==5.0.8", "redis==5.0.8",
# PDF generation for the usage report review pack.
"reportlab==5.0.0",
"requests==2.33.0", "requests==2.33.0",
"requests-oauthlib==2.0.0", "requests-oauthlib==2.0.0",
"simple-salesforce==1.12.6", "simple-salesforce==1.12.6",
Generated
+15
View File
@@ -4350,6 +4350,7 @@ backend = [
{ name = "rapidfuzz" }, { name = "rapidfuzz" },
{ name = "readerwriterlock" }, { name = "readerwriterlock" },
{ name = "redis" }, { name = "redis" },
{ name = "reportlab" },
{ name = "requests" }, { name = "requests" },
{ name = "requests-oauthlib" }, { name = "requests-oauthlib" },
{ name = "sendgrid" }, { name = "sendgrid" },
@@ -4524,6 +4525,7 @@ backend = [
{ name = "rapidfuzz", specifier = "==3.14.5" }, { name = "rapidfuzz", specifier = "==3.14.5" },
{ name = "readerwriterlock", specifier = "==1.0.9" }, { name = "readerwriterlock", specifier = "==1.0.9" },
{ name = "redis", specifier = "==5.0.8" }, { name = "redis", specifier = "==5.0.8" },
{ name = "reportlab", specifier = "==5.0.0" },
{ name = "requests", specifier = "==2.33.0" }, { name = "requests", specifier = "==2.33.0" },
{ name = "requests-oauthlib", specifier = "==2.0.0" }, { name = "requests-oauthlib", specifier = "==2.0.0" },
{ name = "sendgrid", specifier = "==6.12.5" }, { name = "sendgrid", specifier = "==6.12.5" },
@@ -6271,6 +6273,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/dd/6f/832c2023a8bd8414c93452bd8b43bf61cedfa5b9575f70c06fb911e51a29/release_tag-0.5.2-py3-none-win_arm64.whl", hash = "sha256:5f26b008e0be0c7a122acd8fcb1bb5c822f38e77fed0c0bf6c550cc226c6bf14", size = 1203191, upload-time = "2026-03-11T00:27:29.789Z" }, { url = "https://files.pythonhosted.org/packages/dd/6f/832c2023a8bd8414c93452bd8b43bf61cedfa5b9575f70c06fb911e51a29/release_tag-0.5.2-py3-none-win_arm64.whl", hash = "sha256:5f26b008e0be0c7a122acd8fcb1bb5c822f38e77fed0c0bf6c550cc226c6bf14", size = 1203191, upload-time = "2026-03-11T00:27:29.789Z" },
] ]
[[package]]
name = "reportlab"
version = "5.0.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "charset-normalizer" },
{ name = "pillow" },
]
sdist = { url = "https://files.pythonhosted.org/packages/41/d6/4b7b0cf56880eb96533e607967be6a939e344675601e033d113a0bfa1f4e/reportlab-5.0.0.tar.gz", hash = "sha256:e4494a0c6623ae213bb856fba523171b2b54a7bf629fda02d5e525a7b899a784", size = 3701928, upload-time = "2026-06-18T11:34:31.145Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/a3/07/70085c17a369605f15e301d10ab902115019b1126c7253d964afc230c7d6/reportlab-5.0.0-py3-none-any.whl", hash = "sha256:9d5a3affa84919e1111ede580031266a570e93b1ce388219621347965ff1d93c", size = 1956710, upload-time = "2026-06-18T11:34:29.07Z" },
]
[[package]] [[package]]
name = "requests" name = "requests"
version = "2.33.0" version = "2.33.0"