Wave 8: UDP, QUIC, IKEv2, tunnels, the cleartext inventory, and the docs

cryptomon/analysis.py

@@ -92,6 +92,14 @@
 
 # Symmetric strength before Grover, read off the IANA ciphersuite name.
 SYMMETRIC_BITS = (
+    # First, and deliberately: RFC 4543's ENCR_NULL_AUTH_AES_GMAC is *null
+    # encryption* with a GMAC for integrity, and its IKEv2 transform name is
+    # `ENCR_NULL_AUTH_AES_256_GMAC`. Matching is by substring in this order,
+    # so without this entry `AES_256` wins and an IPsec SA that encrypts
+    # nothing at all is reported as "AES_256, 256 bits, 128 after Grover".
+    # For a tool whose headline distinction is encrypted versus cleartext,
+    # that is the worst single answer it could give.
+    ('NULL_AUTH', 0),
     ('AES_256', 256), ('AES_128', 128), ('CHACHA20', 256),
     ('CAMELLIA_256', 256), ('CAMELLIA_128', 128),
     ('ARIA_256', 256), ('ARIA_128', 128),

deploy/systemd/api.env.example

@@ -32,7 +32,14 @@ DB_NAME=cryptomon
 # with each other, and the nginx config is arranged so that disagreement
 # gives a 404 on the first page rather than a site whose links are quietly
 # one level out. See the long comment at the top of cryptomon.conf.
-ROOT_PATH=/cryptomon
+# Leave this EMPTY unless a reverse proxy really is mounting the
+# service under a subpath. create-service.sh installs this file
+# verbatim while --nginx is opt-in, so a value here reaches a
+# service that nothing is proxying: /docs then fetches
+# /cryptomon/openapi.json and 404s, and the redirect after an
+# upload points at /cryptomon/analyse/reports/<id>, also a 404.
+# Uncomment it at the same moment you put the nginx config in place.
+#ROOT_PATH=/cryptomon
 
 # Which addresses uvicorn will believe X-Forwarded-For and X-Forwarded-Proto
 # from. 127.0.0.1 is uvicorn's own default and is right when nginx is on this

docker/README.md

@@ -103,7 +103,9 @@ Both published ports bind `127.0.0.1`. MongoDB is published too, on
 `127.0.0.1:27017`, because `mongosh` from the host is how this project's own
 documentation says to look at the data and because the sensor profile
 (below) runs with host networking and cannot resolve a compose service name.
-Set `MONGO_PORT=` empty in `.env` if you want neither.
+If you want neither, comment out the `ports:` entry for `mongo` in
+`compose.yaml`. Setting `MONGO_PORT=` empty does *not* unpublish it --
+compose's `:-` substitutes the default for an empty value too.
 
 ### Exposure
 

docker/compose.yaml

@@ -53,8 +53,11 @@ services:
       # reasons: `mongosh` and `mongoimport` from the host are how this
       # project's own documentation says to look at the data, and the
       # sensor runs with `network_mode: host` and so cannot resolve `mongo`.
-      # Set MONGO_PORT= (empty) in .env to drop the publication entirely if
-      # you use neither.
+      # To drop the publication entirely if you use neither, comment this
+      # line out. Setting MONGO_PORT= empty does NOT do it: `:-` substitutes
+      # the default for an empty value as well as an unset one, so the
+      # database would stay published on 27017 while the operator believed
+      # otherwise -- which is a worse outcome than no knob at all.
       - "${MONGO_BIND:-127.0.0.1}:${MONGO_PORT:-27017}:27017"
     volumes:
       - mongo-data:/data/db

fapi/app/stats.py

@@ -700,9 +700,7 @@ async def alerts(collection, window, limit=None):
         {'description': '$tls.alerts.description',
          'from_client': '$tls.alerts.from_client'},
         limit, unwind='$tls.alerts')
-    buckets = _sorted([{'name': _alert_name(row.get('_id') or {}),
-                        'count': int(row.get('count') or 0)}
-                       for row in rows])
+    buckets = _sorted([_alert_bucket(row) for row in rows])
     notes = [
         "total is documents; the buckets sum to {0} alerts, because one "
         "handshake can carry more than one.".format(counted),
@@ -715,19 +713,28 @@ async def alerts(collection, window, limit=None):
     return _envelope('alerts', window, documents, buckets, other, notes)
 
 
-def _alert_name(key):
+def _alert_bucket(row):
     """
-    One grouped alert as `handshake_failure (server)`.
-
-    Same spelling as `pcapscan.export._alert_label`, minus the level, which
-    this panel does not group on.
+    One grouped alert, with the direction in **both** places it is needed.
+
+    In the name, spelled exactly as `pcapscan.export._alert_label` spells it,
+    so a CSV cell and a dashboard bar read the same. And as its own
+    `direction` key, because a consumer that wants to put the sender in its
+    own column should not have to parse it back out of a display string --
+    the dashboard did exactly that and got it wrong, printing "the
+    aggregation is not reporting the direction" directly above rows that
+    plainly said "(server)". Two spellings of one fact is a bug waiting to
+    happen; one fact carried twice, deliberately, is not.
     """
+    key = row.get('_id') or {}
     description = key.get('description')
     name = NONE if description is None else describe_description(description)
+    bucket = {'name': name, 'count': int(row.get('count') or 0)}
     sender = key.get('from_client')
-    if sender is None:
-        return name
-    return '{0} ({1})'.format(name, 'client' if sender else 'server')
+    if sender is not None:
+        bucket['direction'] = 'client' if sender else 'server'
+        bucket['name'] = '{0} ({1})'.format(name, bucket['direction'])
+    return bucket
 
 
 async def ech(collection, window, limit=None):
@@ -796,8 +803,19 @@ async def timeline(collection, window, limit=None, bucket=None):
         if verdict in point['verdicts']:
             point['verdicts'][verdict] += count
 
-    series = _gap_fill(points, seconds)
+    series, dropped = _gap_fill(points, seconds)
     notes = []
+    if dropped:
+        # Said, not swallowed. The series is bounded, so something had to go;
+        # a chart that quietly starts later than it claims to is the kind of
+        # unlabelled missing data this project has twice been caught by.
+        notes.append(
+            "The {0} oldest buckets were dropped to stay within {1} points; "
+            "the series starts at {2:.0f}. A single document with a broken "
+            "timestamp can stretch the span this way -- check `span` on "
+            "/stats/overview if that looks wrong.".format(
+                dropped, MAX_SERIES_POINTS,
+                series[0]['t'] if series else 0))
     if widened:
         notes.append(
             "Buckets were widened to {0} seconds; the window asked for more "
@@ -825,16 +843,27 @@ def _gap_fill(points, seconds):
     Bounded by the data rather than by the window on purpose: a `hours=0`
     request against a collection holding one afternoon should draw that
     afternoon, not two years of zeros leading up to it.
+
+    Returns (series, dropped). When the span is too long to draw, the
+    **oldest** buckets are the ones dropped, not the newest. That direction
+    is the whole point: a single document with a broken `ts` -- epoch 0, or a
+    capture whose clock ran slow -- drags `min(points)` back to 1970, and
+    filling forward from there spends the entire budget on empty buckets and
+    then stops before reaching today. The page would show years of zeros and
+    silently omit the traffic somebody opened it to see. `dropped` is
+    returned rather than swallowed so the caller can say so.
     """
     if not points:
-        return []
-    at = min(points)
-    last = max(points)
+        return [], 0
+    first, last = min(points), max(points)
+    span = int((last - first) // seconds) + 1
+    dropped = max(0, span - MAX_SERIES_POINTS)
+    at = first + dropped * seconds
     series = []
-    while at <= last and len(series) <= MAX_SERIES_POINTS:
+    while at <= last and len(series) < MAX_SERIES_POINTS:
         series.append(points.get(at) or _empty_point(at))
         at += seconds
-    return series
+    return series, dropped
 
 
 def _choose_seconds(window, bucket):

fapi/app/templates/dashboard.html

@@ -193,15 +193,22 @@ <h2>No data in this window</h2>
 {% set kex = overview.key_exchanges_performed or 0 %}
 {% set docs = overview.documents or 0 %}
 {% set safe = overview.quantum_safe or 0 %}
-{% set rollup = ch.verdict_rollup((panels['key-exchange'] or {}).buckets) %}
+{# The rollup only sees the buckets the panel returned, which `limit` may
+   have truncated; `other` is the remainder. Feeding a truncated stack a
+   full-population denominator draws a bar that does not reach its own end
+   and understates every share on it -- the one thing this bar exists not to
+   do. So the shortfall is carried as its own segment. #}
+{% set kexpanel = panels['key-exchange'] or {} %}
+{% set rollup = ch.verdict_rollup(kexpanel.buckets) %}
+{% set truncated = kexpanel.other or 0 %}
 {% set share_of_kex = ch.percent(safe, kex) %}
 {% set share_of_all = ch.percent(safe, docs) %}
 
 <div class="card">
   <h2>Quantum-safe key exchange</h2>
-  {{ ch.ratio_bar(ch.headline_parts(rollup, docs - kex),
+  {{ ch.ratio_bar(ch.headline_parts(rollup, docs - kex + truncated),
        'Every session in the window, split by how its key exchange was assessed; '
-       'the last segment performed none at all.',
+       'the last segment performed none at all, or was left out by the row limit.',
        docs) }}
   <div class="headline">
     <div><div class="v">{{ '{:,}'.format(safe) }}</div>

fapi/app/templates/dashboard_panels.html

@@ -134,14 +134,24 @@ <h2>{{ titles.get(name, name) }}</h2>
 {% set total_certs = rollup | map(attribute=1) | sum %}
 {% set safe_certs = rollup | selectattr(0, 'in', ['post-quantum', 'hybrid']) | map(attribute=1) | sum %}
 {% set classical_certs = rollup | selectattr(0, 'equalto', 'classical') | map(attribute=1) | sum %}
-{% if total_certs and not safe_certs %}
+{# `data.other` is what the panel's `limit` left out, and the rollup cannot
+   see it. An absolute claim over a partial count is a wrong claim, so when
+   anything was truncated this says so instead of asserting. #}
+{% if total_certs and not safe_certs and not data.other %}
 <p class="note bad">
   <strong>Not one certificate in this window is quantum-safe.</strong>
   {{ '{:,} of {:,} are classical'.format(classical_certs, total_certs) }}{% if total_certs - classical_certs %}, and {{ '{:,} could not be classified'.format(total_certs - classical_certs) }} from what the capture contained{% endif %}. A key exchange can
   be made quantum-safe by enabling a hybrid group on two endpoints; a
   certificate cannot, because every issuer above it has to change first.
   This is the number with the longest lead time on it.
 </p>
+{% elif total_certs and not safe_certs and data.other %}
+<p class="note bad">
+  <strong>No quantum-safe certificate among the {{ '{:,}'.format(total_certs) }}
+  shown</strong>, but {{ '{:,}'.format(data.other) }} more were left out by the
+  row limit and are not counted here. Raise the limit to make this a complete
+  statement.
+</p>
 {% endif %}
 {% endif %}
 

fapi/app/templates/upload.html

@@ -29,7 +29,8 @@ <h1>Analyse a capture</h1>
          accept=".pcap,.pcapng,.gz,application/vnd.tcpdump.pcap" required>
   <p class="note">
     pcap or pcapng, gzipped or not, up to
-    {{ '%.3g'|format(max_bytes / 1048576) }} MiB.
+    {{ '{:,.0f}'.format(max_bytes / 1048576) if max_bytes >= 1048576
+       else '{:,.0f} KiB'.format(max_bytes / 1024) }}{{ ' MiB' if max_bytes >= 1048576 }}.
   </p>
   {% if key_required %}
   <p class="note">

pcapscan/cbom.py

@@ -208,7 +208,11 @@ def primitive_for(entry):
     if entry['kind'] == 'cipher':
         # An IKEv2 ENCR transform names one concrete cipher rather than a
         # suite, so the schema's own word for it is available: the AEAD modes
-        # are `ae` and the rest are block ciphers.
+        # are `ae` and the rest are block ciphers -- except RFC 4543's
+        # ENCR_NULL_AUTH_AES_GMAC, which occupies the ENCR slot while
+        # encrypting nothing, and is a MAC.
+        if _is_mac_only(entry['name']):
+            return PRIMITIVE_MAC
         return PRIMITIVE_AE if _is_aead(entry['name']) else PRIMITIVE_BLOCK
     if entry['kind'] == 'prf':
         return PRIMITIVE_KDF
@@ -219,9 +223,16 @@ def primitive_for(entry):
 
 def _is_aead(name):
     flat = _flat(name)
+    if 'nullauth' in flat:
+        return False        # RFC 4543: integrity only, nothing is encrypted
     return any(mode in flat for mode in ('gcm', 'ccm', 'poly1305', 'mgm'))
 
 
+def _is_mac_only(name):
+    """RFC 4543's ENCR_NULL_AUTH_AES_GMAC: a MAC wearing a cipher's slot."""
+    return 'nullauth' in _flat(name)
+
+
 def functions_for(entry):
     if entry['kind'] == 'key-exchange':
         if entry['verdict'] in (POST_QUANTUM, HYBRID):
@@ -230,6 +241,8 @@ def functions_for(entry):
     if entry['kind'] in ('signature', 'certificate-key'):
         return ['sign', 'verify']
     if entry['kind'] == 'cipher':
+        if _is_mac_only(entry['name']):
+            return ['tag']
         return ['encrypt', 'decrypt']
     if entry['kind'] == 'prf':
         return ['keyderive']

pcapscan/cleartext.py

@@ -283,6 +283,11 @@ def _is_multicast(address, groups):
 # needs to chain it to a trust anchor or to prove a negative answer.
 DNSSEC_TYPES = frozenset({43, 46, 47, 48, 50, 51, 59, 60})
 
+RRSIG_TYPE = 46                 # the only record that authenticates an RRset
+# 0/1 flags that accumulate across a flow rather than freezing at the first
+# datagram. Emitted as ints rather than bools, so they need naming here.
+STICKY_FLAGS = frozenset({'dnssec_ok', 'dnssec_signed'})
+
 OPT_TYPE = 41                   # the EDNS0 pseudo-record (RFC 6891)
 DNSSEC_OK = 0x8000              # the DO bit, in the OPT record's TTL field
 
@@ -528,7 +533,16 @@ def _dnssec_state(message):
                            '{1}, DO {2}'.format(rclass, (ttl >> 16) & 0xFF,
                                                 'set' if ttl & DNSSEC_OK
                                                 else 'clear'))
-        elif rtype in DNSSEC_TYPES:
+        elif rtype == RRSIG_TYPE:
+            # RRSIG and nothing else. The other DNSSEC record types are
+            # *material* -- DNSKEY is a public key, DS is a delegation
+            # pointer, NSEC proves non-existence -- and a response carrying
+            # a DNSKEY with no RRSIG over it is exactly as unauthenticated
+            # as any other answer. Treating the whole family as proof of
+            # signing meant a plain `dig DNSKEY example.com` came back
+            # labelled "authenticated against the zone key", which is a
+            # claim this tool should never make when it has not seen a
+            # signature.
             signed = True
     for _labels, qtype, _qclass in message.questions:
         if qtype in DNSSEC_TYPES:
@@ -1946,8 +1960,47 @@ def _merge_facts(self, protocol, facts):
             elif isinstance(value, int) and name in ('questions', 'responses',
                                                      'names'):
                 kept[name] = kept.get(name, 0) + value
-            else:
-                kept.setdefault(name, value)
+            elif name in STICKY_FLAGS:
+                # 0/1 flags that must latch on rather than freeze. These are
+                # emitted as `int(...)` rather than bool, so they miss the
+                # bool branch above and used to hit `setdefault` -- which
+                # pinned them to whatever the *first* datagram said. A DNS
+                # flow of (query with DO set) then (response carrying an
+                # RRSIG) therefore reported `dnssec_signed: 0`, having just
+                # seen the signature.
+                kept[name] = max(kept.get(name, 0), value)
+            elif kept.get(name) is None:
+                # A later real value replaces a None; a later None never
+                # replaces a real value. `setdefault` did neither: it pinned
+                # the field to whatever the *first* datagram said, and DHCP's
+                # `message_type` is legitimately None on a datagram with no
+                # option 53, so the flow then reported None for ever. Note
+                # this still records the key when the first value is None --
+                # "seen, and empty" and "never seen" are different answers.
+                kept[name] = value
+
+    def _protection_reason(self, protocol, protection):
+        """Why this flow earned its verdict, and from which protocol."""
+        exact = self.protection_reasons.get((protocol, protection))
+        if exact:
+            return exact
+        # Whichever protocol earned the verdict. Sorted by how much of the
+        # flow it accounts for, then by name, rather than taken in insertion
+        # order -- otherwise which protocol gets credited for a shared
+        # verdict depends on the order datagrams happened to arrive in, and
+        # the same capture could describe the same flow two ways.
+        candidates = [(other, reason)
+                      for (other, found), reason in
+                      self.protection_reasons.items()
+                      if found == protection and reason]
+        if not candidates:
+            return None
+        other, reason = max(
+            candidates, key=lambda item: (self.protocols.get(item[0], 0),
+                                          item[0]))
+        if other == protocol:
+            return reason
+        return '{0} ({1})'.format(reason, other)
 
     def finish(self):
         """
@@ -1971,8 +2024,16 @@ def finish(self):
             'confidence': self.confidence.get(protocol),
             'reason': self.reasons.get(protocol),
             'protection': protection,
-            'protection_reason': self.protection_reasons.get(
-                (protocol, protection)),
+            # Looked up by the verdict alone when the flow's most-common
+            # protocol did not earn it. A mixed flow is expected here --
+            # three authenticated STUN datagrams plus one MD5-authenticated
+            # NTP datagram is `obsolete` on NTP's account, not STUN's -- and
+            # keying on (protocol, protection) meant that pair had never
+            # been recorded, so the reason came back None and the verdict
+            # read as though STUN had earned it. The verdict is about the
+            # flow; so is its reason.
+            'protection_reason': self._protection_reason(protocol,
+                                                         protection),
             'datagrams': self.datagrams,
             'bytes': self.octets,
             'identified': sum(self.protocols.values()),

pcapscan/datagrams.py

@@ -260,7 +260,21 @@ def finish(self):
             self._retire(flow)
         self.flows.clear()
         for flow in sorted(self.done, key=lambda f: f.first_ts or 0.0):
-            for document in flow.handler.finish() or ():
+            try:
+                documents = list(flow.handler.finish() or ())
+            except Exception:               # noqa: BLE001
+                # Guarded for the same reason `push` and `_detect` are, and
+                # it matters more here than in either: `finish` is where the
+                # expensive parsing happens -- reassembled CRYPTO bytes split
+                # into handshake messages, a hello parsed, a certificate
+                # chain walked -- all over input somebody else chose. An
+                # unguarded raise here does not cost one datagram, it
+                # propagates out of SessionBuilder.finish() and past the
+                # CLI's `except (OSError, CaptureError)`, so the whole
+                # capture ends in a traceback with no report at all.
+                self.stats['finish_error_' + flow.handler.name] += 1
+                continue
+            for document in documents:
                 document.setdefault('ptype', 'session')
                 document.setdefault('ts', flow.first_ts)
                 document.setdefault(