journald: maintain message counters in a memory mapped file in /run/

journald now counts received log messages by priority and by transport
in $RUNTIME_DIRECTORY/counters, which it keeps mapped for its whole
runtime. The counters hence survive restarts of journald, but are reset
on reboot.

Messages are counted in each transport right after their priority is
known, i.e. before any filtering (LogLevelMax=, LogFilterPatterns=,
MaxLevelStore=, Storage=none) or rate limiting is applied. Kernel
messages are only counted after the kernel seqnum check, so that
messages seen before a restart are not counted twice. journald's own
messages are not counted.

This introduces a JournalTransport enum for the transports we receive
messages on, whose string names match the _TRANSPORT= field.

src/journal/journald-audit.c

@@ -12,6 +12,7 @@
 #include "iovec-util.h"
 #include "journal-internal.h"
 #include "journald-audit.h"
+#include "journald-counters.h"
 #include "journald-manager.h"
 #include "log.h"
 #include "log-ratelimit.h"
@@ -407,6 +408,8 @@ void process_audit_string(Manager *m, int type, const char *data, size_t size) {
 
         map_all_fields(p, map_fields_kernel, "_AUDIT_FIELD_", true, iovec, &n, n + N_IOVEC_AUDIT_FIELDS);
 
+        manager_count_message(m, JOURNAL_TRANSPORT_AUDIT, LOG_NOTICE);
+
         manager_dispatch_message(m, iovec, n, ELEMENTSOF(iovec), NULL,
                                  TIMEVAL_STORE((usec_t) seconds * USEC_PER_SEC + (usec_t) msec * USEC_PER_MSEC),
                                  LOG_NOTICE, 0);

src/journal/journald-counters.c

@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "alloc-util.h"
+#include "counters-file.h"
+#include "journald-counters.h"
+#include "journald-manager.h"
+#include "log.h"
+#include "path-util.h"
+
+int manager_map_counters(Manager *m) {
+        int r;
+
+        assert(m);
+        assert(!m->counters);
+
+        _cleanup_free_ char *fn = path_join(m->runtime_directory, "counters");
+        if (!fn)
+                return log_oom();
+
+        r = counters_file_map(fn, sizeof(JournalCounters), JOURNAL_COUNTERS_VERSION, (void**) &m->counters);
+        if (r < 0)
+                return log_warning_errno(r, "Failed to map message counters file '%s', ignoring: %m", fn);
+
+        return 0;
+}
+
+void manager_count_message(Manager *m, JournalTransport t, int priority) {
+        assert(m);
+        assert(t >= 0 && t < _JOURNAL_TRANSPORT_MAX);
+
+        if (!m->counters)
+                return;
+
+        m->counters->n_priority[LOG_PRI(priority)]++;
+        m->counters->n_transport[t]++;
+}

src/journal/journald-counters.h

@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include <syslog.h>
+
+#include "journald-forward.h"
+#include "journald-transport.h"
+
+#define JOURNAL_COUNTERS_VERSION UINT64_C(1)
+
+/* This structure is kept in $RUNTIME_DIRECTORY/counters and is mapped by journald for its whole runtime.
+ * It contains monotonic counters of received log messages, which hence survive journald restarts, but not
+ * reboots. Native endian, not a stable interface. */
+typedef struct JournalCounters {
+        uint64_t version;
+        uint64_t n_priority[LOG_DEBUG + 1];            /* indexed by LOG_PRI() */
+        uint64_t n_transport[_JOURNAL_TRANSPORT_MAX];  /* indexed by JournalTransport */
+} JournalCounters;
+
+int manager_map_counters(Manager *m);
+
+void manager_count_message(Manager *m, JournalTransport t, int priority);

src/journal/journald-forward.h

@@ -11,6 +11,7 @@ typedef enum SplitMode SplitMode;
 typedef enum AuditSetMode AuditSetMode;
 typedef struct JournalCompressOptions JournalCompressOptions;
 typedef struct JournalConfig JournalConfig;
+typedef struct JournalCounters JournalCounters;
 
 typedef struct Manager Manager;
 typedef struct StreamSyncReq StreamSyncReq;

src/journal/journald-kmsg.c

@@ -17,6 +17,7 @@
 #include "format-util.h"
 #include "iovec-util.h"
 #include "journal-internal.h"
+#include "journald-counters.h"
 #include "journald-kmsg.h"
 #include "journald-manager.h"
 #include "journald-sync.h"
@@ -277,9 +278,14 @@ void dev_kmsg_record(Manager *m, char *p, size_t l) {
         xsprintf(syslog_facility, "SYSLOG_FACILITY=%i", LOG_FAC(priority));
         iovec[n++] = IOVEC_MAKE_STRING(syslog_facility);
 
-        if (LOG_FAC(priority) == LOG_KERN)
+        /* Messages are counted below, i.e. after the kernel seqnum check above (so that messages we already
+         * saw before a restart of journald are not counted twice), but before any filtering or rate
+         * limiting. Our own messages are not counted. */
+        if (LOG_FAC(priority) == LOG_KERN) {
+                manager_count_message(m, JOURNAL_TRANSPORT_KERNEL, priority);
+
                 iovec[n++] = IOVEC_MAKE_STRING("SYSLOG_IDENTIFIER=kernel");
-        else {
+        } else {
                 _cleanup_free_ char *identifier = NULL;
                 pid_t pid;
 
@@ -294,7 +300,8 @@ void dev_kmsg_record(Manager *m, char *p, size_t l) {
                         saved_log_max_level = log_get_max_level();
                         c = m->my_context;
                         log_set_max_level(LOG_NULL);
-                }
+                } else
+                        manager_count_message(m, JOURNAL_TRANSPORT_KERNEL, priority);
 
                 if (identifier) {
                         syslog_identifier = strjoin("SYSLOG_IDENTIFIER=", identifier);

src/journal/journald-manager.c

@@ -37,6 +37,7 @@
 #include "journald-audit.h"
 #include "journald-config.h"
 #include "journald-context.h"
+#include "journald-counters.h"
 #include "journald-kmsg.h"
 #include "journald-manager.h"
 #include "journald-native.h"
@@ -2349,6 +2350,8 @@ int manager_init(Manager *m) {
 
         (void) mkdir_p(m->runtime_directory, 0755);
 
+        (void) manager_map_counters(m);
+
         m->user_journals = ordered_hashmap_new(&journal_file_hash_ops_offline_close);
         if (!m->user_journals)
                 return log_oom();
@@ -2596,6 +2599,7 @@ Manager* manager_free(Manager *m) {
 
         m->seqnum = munmap_safe(m->seqnum, sizeof(*m->seqnum));
         m->kernel_seqnum = munmap_safe(m->kernel_seqnum, sizeof(*m->kernel_seqnum));
+        m->counters = munmap_safe(m->counters, sizeof(*m->counters));
 
         free(m->buffer);
         free(m->cgroup_root);

src/journal/journald-manager.h

@@ -69,6 +69,7 @@ typedef struct Manager {
         OrderedHashmap *user_journals;
 
         SeqnumData *seqnum;
+        JournalCounters *counters;
 
         char *buffer;
 

src/journal/journald-native.c

@@ -14,6 +14,7 @@
 #include "journald-client.h"
 #include "journald-console.h"
 #include "journald-context.h"
+#include "journald-counters.h"
 #include "journald-kmsg.h"
 #include "journald-manager.h"
 #include "journald-native.h"
@@ -261,6 +262,9 @@ static int manager_process_entry(
 
         r = 0; /* Success, we read the message. */
 
+        /* Count before any filtering or rate limiting */
+        manager_count_message(m, JOURNAL_TRANSPORT_NATIVE, priority);
+
         if (!client_context_test_priority(context, priority))
                 goto finish;
 

src/journal/journald-stream.c

@@ -19,6 +19,7 @@
 #include "journald-client.h"
 #include "journald-console.h"
 #include "journald-context.h"
+#include "journald-counters.h"
 #include "journald-kmsg.h"
 #include "journald-manager.h"
 #include "journald-stream.h"
@@ -241,10 +242,13 @@ static int stdout_stream_log(
         if (s->level_prefix)
                 syslog_parse_priority(&p, &priority, false);
 
-        if (!client_context_test_priority(s->context, priority))
+        if (isempty(p))
                 return 0;
 
-        if (isempty(p))
+        /* Count before any filtering or rate limiting */
+        manager_count_message(s->manager, JOURNAL_TRANSPORT_STREAM, priority);
+
+        if (!client_context_test_priority(s->context, priority))
                 return 0;
 
         r = client_context_check_keep_log(s->context, p, strlen(p));

src/journal/journald-syslog.c

@@ -15,6 +15,7 @@
 #include "journald-client.h"
 #include "journald-console.h"
 #include "journald-context.h"
+#include "journald-counters.h"
 #include "journald-kmsg.h"
 #include "journald-manager.h"
 #include "journald-syslog.h"
@@ -399,6 +400,9 @@ void manager_process_syslog_message(
 
         syslog_parse_priority(&msg, &priority, true);
 
+        /* Count before any filtering or rate limiting */
+        manager_count_message(m, JOURNAL_TRANSPORT_SYSLOG, priority);
+
         if (!client_context_test_priority(context, priority))
                 return;
 

src/journal/journald-transport.c

@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "journald-transport.h"
+#include "string-table.h"
+
+static const char* const journal_transport_table[_JOURNAL_TRANSPORT_MAX] = {
+        [JOURNAL_TRANSPORT_SYSLOG] = "syslog",
+        [JOURNAL_TRANSPORT_NATIVE] = "journal",
+        [JOURNAL_TRANSPORT_STREAM] = "stdout",
+        [JOURNAL_TRANSPORT_AUDIT]  = "audit",
+        [JOURNAL_TRANSPORT_KERNEL] = "kernel",
+};
+
+DEFINE_STRING_TABLE_LOOKUP(journal_transport, JournalTransport);

src/journal/journald-transport.h

@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include "journald-forward.h"
+
+/* The transports we receive log messages on, whose string names match the _TRANSPORT= field. This is
+ * used as index into JournalCounters.n_transport[], hence only ever append new entries, and bump
+ * JOURNAL_COUNTERS_VERSION when doing so. */
+typedef enum JournalTransport {
+        JOURNAL_TRANSPORT_SYSLOG,
+        JOURNAL_TRANSPORT_NATIVE,
+        JOURNAL_TRANSPORT_STREAM,
+        JOURNAL_TRANSPORT_AUDIT,
+        JOURNAL_TRANSPORT_KERNEL,
+        _JOURNAL_TRANSPORT_MAX,
+        _JOURNAL_TRANSPORT_INVALID = -EINVAL,
+} JournalTransport;
+
+DECLARE_STRING_TABLE_LOOKUP(journal_transport, JournalTransport);

src/journal/meson.build

@@ -10,6 +10,7 @@ systemd_journald_export_sources = files(
         'journald-config.c',
         'journald-console.c',
         'journald-context.c',
+        'journald-counters.c',
         'journald-kmsg.c',
         'journald-manager.c',
         'journald-native.c',
@@ -18,6 +19,7 @@ systemd_journald_export_sources = files(
         'journald-stream.c',
         'journald-sync.c',
         'journald-syslog.c',
+        'journald-transport.c',
         'journald-varlink.c',
         'journald-wall.c',
 )

src/journal/test-journald-tables.c

@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later */
 
 #include "journald-config.h"
+#include "journald-transport.h"
 #include "test-tables.h"
 #include "tests.h"
 
@@ -9,6 +10,7 @@ int main(int argc, char **argv) {
 
         test_table(SplitMode, split_mode, SPLIT);
         test_table(Storage, storage, STORAGE);
+        test_table(JournalTransport, journal_transport, JOURNAL_TRANSPORT);
 
         return EXIT_SUCCESS;
 }