From abde776c61c1b4199d55e1bd7ba1d9cafc9d0173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 31 Jul 2026 15:17:28 +0200 Subject: [PATCH 01/14] Add labeling API for building a labeled corpus --- .../src/oonimeasurements/main.py | 2 + .../src/oonimeasurements/routers/labeling.py | 593 ++++++++++++++++++ 2 files changed, 595 insertions(+) create mode 100644 ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py index 53daa27e0..53191dc81 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py @@ -18,6 +18,7 @@ list_observations, ) from .routers.v1 import aggregation, measurements +from .routers import labeling pkg_name = "oonimeasurements" @@ -60,6 +61,7 @@ class HealthStatus(BaseModel): app.include_router(list_observations.router, prefix="/api") app.include_router(aggregate_observations.router, prefix="/api") app.include_router(aggregate_analysis.router, prefix="/api") + app.include_router(labeling.router) instrumentor = Instrumentator().instrument( app, metric_namespace="ooniapi", metric_subsystem="oonimeasurements" diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py new file mode 100644 index 000000000..1b144a8b9 --- /dev/null +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -0,0 +1,593 @@ +""" +Labeling corpus API. + +Read-only ClickHouse queries backing the measurement adjudication UI. There is +no write path: labels live in the analyst's browser and leave it by copy-paste, +so this router adds no storage, no auth surface, and no migration. + +Two invariants this module exists to enforce: + +1. BLINDING. /candidate returns what the probe and the control saw, and nothing + the pipeline concluded. analysis_web_measurement and fastpath's anomaly / + confirmed / scores columns are queried ONLY by /reveal, which the UI calls + after the analyst has committed. If you add a field to /candidate, check it + is not a pipeline judgment in disguise. + +2. SAMPLING IS RECORDED, NOT REMEMBERED. Every draw is deterministic given + (design_id, stratum, frame, rate), and /sample returns the predicate it ran + and the population it ran against, so the weights are reconstructable from + the export alone. +""" + +import hashlib +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel + +# ooni/backend wires this up already; the import path is the one used by the +# existing data routers. +from ..dependencies import get_clickhouse_session # type: ignore + +router = APIRouter(prefix="/api/v1/labeling", tags=["labeling"]) + + +# -------------------------------------------------------------------------- +# Sampling design +# -------------------------------------------------------------------------- +# +# The design doc's screen is "any blocked-leaning RULE fired", which needs B1 +# (persisted fired-rule sets) to exist. It does not yet, so the screen below is +# a documented PROXY over the fastpath analysis which is derived from probe +# computed values. It's fine to use this as a PROXY, as it's recorded in the +# measurement itself and is stable, since we have not plans to change the +# fastpath scoring logic. +# +# This matters for the weights: the proxy screen has different, unmeasured +# coverage from the real one. Record `screen_kind` in the export so a later +# refit can tell proxy-screened rows from B1-screened ones and, if needed, +# drop the former. + +BLOCKED_MAX = "greatest(dns_blocked, tcp_blocked, tls_blocked)" + +STRATA: Dict[str, Dict[str, Any]] = { + "screen_positive": { + "table": "fastpath", + "predicate": f"anomaly = 't' AND msm_failure = 'f'", + "default_rate": 0.1, + "screen_kind": "fastpath_proxy", + "note": "Proxy for B1's blocked-leaning-rule screen. LR numerator.", + }, + "screen_negative": { + "table": "fastpath", + "predicate": f"confirmed = 'f' AND anomaly = 'f' AND msm_failure = 'f'", + "default_rate": 0.0002, + "screen_kind": "fastpath_proxy", + "note": "Bounds false negatives and carries the base rate. Small, " + "and the first thing cut under pressure. Do not cut it.", + }, + "fingerprint_match": { + "table": "fastpath", + "predicate": "confirmed = 't' AND msm_failure = 'f'", + "default_rate": 1.0, + "screen_kind": "fingerprint", + "note": "Census, not a sample. High-precision positives; tag the " + "labels so LRs can be refit without them as a circularity " + "check.", + }, + "incident_window": { + "table": "analysis_web_measurement", + "predicate": "1", # scoped entirely by the cc/domain/time params + "default_rate": 0.2, + "screen_kind": "incident_scope", + "note": "Draw inside a known event. Label the MEASUREMENT: rows here " + "that are genuinely ok are the most valuable in the corpus.", + }, +} + +# control_agreement (cheap negatives from probe/control agreement) needs an +# obs_web_ctrl join with per-layer agreement predicates. Left out on purpose +# rather than half-built: a negative stratum with a wrong predicate is worse +# than one that is absent, because it silently deflates every LR denominator. + +HASH_SPACE = 1_000_000 + +# Bump when STRATA definitions or the fingerprint's shape change: it forces new +# design ids, so old weights are never silently reinterpreted under new rules. +DESIGN_SCHEMA_VERSION = "1" + + +def _design_fingerprint(spec: Dict[str, Any]) -> str: + """Content-address a sampling design. + + The id is derived from the design so that there is never the same id used + for two different parameter sets, leaving two incompatible weightings + sharing a name. + + Redrawing with a higher limit returns a superset of the same rows, and two + analysts who enter the same parameters get the same queue, which is how + inter-rater agreement gets measured without any coordination service. + + Want genuinely fresh rows from the same population? Increment `replicate`. + It is part of the spec, so it produces a different id and an independent + draw, on purpose and on the record. + """ + blob = json.dumps(spec, sort_keys=True, separators=(",", ":"), default=str) + return "d" + hashlib.sha256(blob.encode()).hexdigest()[:10] + + +class SampleRow(BaseModel): + measurement_uid: str + measurement_start_time: datetime + probe_cc: str + probe_asn: int + resolver_asn: int + domain: str + input: Optional[str] + test_name: str + # sampling provenance, carried through to the label + sampling_stratum: str + sampling_weight: float + sampling_design_id: str + screen_kind: str + + +class SampleResponse(BaseModel): + design_id: str + replicate: int + spec: Dict[str, Any] + frame_start: datetime + frame_end: datetime + strata: Dict[str, Dict[str, Any]] + rows: List[SampleRow] + + +def _frame(since: Optional[datetime], until: Optional[datetime]): + until = until or datetime.now(timezone.utc).replace(tzinfo=None) + since = since or (until - timedelta(days=30)) + if since >= until: + raise HTTPException(400, "since must be before until") + return since, until + + +@router.get("/design") +def get_design() -> Dict[str, Any]: + """The sampling design, verbatim. + + The UI copies this into the export so weights can be checked against the + predicate that produced them. Edit a stratum here and you have a new + design: bump design_id at draw time, never reuse it. + """ + return {"strata": STRATA, "hash_space": HASH_SPACE} + + +@router.get("/test_names") +def list_test_names( + db=Depends(get_clickhouse_session), + since: Optional[datetime] = None, + until: Optional[datetime] = None, + probe_cc: Optional[str] = Query(None, min_length=2, max_length=2), +) -> Dict[str, Any]: + """What is labelable in this frame, and how much of it there is. + + Counted from analysis_web_measurement rather than fastpath on purpose: a + test with fastpath rows but no analysis rows will draw fine and then fail + to load, because the candidate view reads obs_web. This list is the set + that actually works end to end. + """ + since_dt, until_dt = _frame(since, until) + where = [ + "measurement_start_time >= %(since)s", + "measurement_start_time < %(until)s", + ] + params: Dict[str, Any] = {"since": since_dt, "until": until_dt} + if probe_cc: + where.append("probe_cc = %(probe_cc)s") + params["probe_cc"] = probe_cc.upper() + + rows = db.execute( + f""" + SELECT test_name, + count() AS n, + countIf(greatest(dns_blocked, tcp_blocked, tls_blocked) >= 0.5) + AS n_screen_positive + FROM analysis_web_measurement + WHERE {' AND '.join(where)} + GROUP BY test_name + ORDER BY n DESC + """, + params, + ) + return { + "frame_start": since_dt, + "frame_end": until_dt, + "test_names": [ + { + "test_name": r[0], + "measurements": int(r[1]), + "screen_positive": int(r[2]), + } + for r in rows + ], + } + + +@router.get("/sample", response_model=SampleResponse) +def draw_sample( + db=Depends(get_clickhouse_session), + strata: str = Query( + "screen_positive,screen_negative", + description="Comma-separated. Multiple strata are drawn separately " + "and interleaved, so the analyst cannot infer a row's " + "stratum from its position in the queue.", + ), + replicate: int = Query( + 1, ge=1, + description="Independent draws of the same design. Same replicate = " + "same rows (reproducible, extendable, comparable across " + "analysts). Increment it to sample rows the previous " + "replicate did not cover.", + ), + since: Optional[datetime] = None, + until: Optional[datetime] = None, + probe_cc: Optional[str] = Query(None, min_length=2, max_length=2), + probe_asn: Optional[int] = None, + domain: Optional[str] = None, + test_name: Optional[str] = Query( + "web_connectivity", + description="Comma-separated. Scoping a design to a test changes its " + "population, so weights are only valid within the same " + "test scope — change design_id when you change this. " + "Empty string means every test.", + ), + limit: int = Query(50, ge=1, le=500), + rate_override: Optional[float] = Query( + None, gt=0, le=1, + description="Overrides every named stratum's rate. Using this makes a " + "new design; change design_id too.", + ), +) -> SampleResponse: + since_dt, until_dt = _frame(since, until) + wanted = sorted({s.strip() for s in strata.split(",") if s.strip()}) + unknown = [s for s in wanted if s not in STRATA] + if unknown: + raise HTTPException(400, f"unknown strata: {unknown}") + + tests = sorted({t.strip() for t in (test_name or "").split(",") if t.strip()}) + if "incident_window" in wanted and not (probe_cc and domain): + raise HTTPException( + 400, + "incident_window needs probe_cc and domain — an unscoped incident " + "draw is just a biased production sample", + ) + + # The spec is the design. Everything that changes which rows are eligible, + # or what a weight means, has to be in here — otherwise two different + # populations could collide onto one id, which is the failure this exists + # to make impossible. + resolved = { + s: { + "table": STRATA[s]["table"], + "predicate": STRATA[s]["predicate"], + "screen_kind": STRATA[s]["screen_kind"], + "sample_rate": rate_override or STRATA[s]["default_rate"], + } + for s in wanted + } + spec = { + "schema": DESIGN_SCHEMA_VERSION, + "strata": resolved, + "frame": [since_dt.isoformat(), until_dt.isoformat()], + "scope": { + "probe_cc": probe_cc.upper() if probe_cc else None, + "probe_asn": probe_asn, + "domain": domain, + "test_names": tests or "all", + }, + "replicate": replicate, + } + derived_id = _design_fingerprint(spec) + + per_stratum = max(1, limit // len(wanted)) + used: Dict[str, Dict[str, Any]] = {} + buckets: List[List[SampleRow]] = [] + + for stratum in wanted: + spec_s = STRATA[stratum] + rate = resolved[stratum]["sample_rate"] + table = spec_s["table"] + + where = [ + "measurement_start_time >= %(since)s", + "measurement_start_time < %(until)s", + f"({spec_s['predicate']})", + ] + params: Dict[str, Any] = { + "since": since_dt, + "until": until_dt, + "salt": f"{derived_id}:{stratum}", + "cutoff": int(rate * HASH_SPACE), + "limit": per_stratum, + } + if probe_cc: + where.append("probe_cc = %(probe_cc)s") + params["probe_cc"] = probe_cc.upper() + if probe_asn: + where.append("probe_asn = %(probe_asn)s") + params["probe_asn"] = probe_asn + if domain: + where.append("domain = %(domain)s") + params["domain"] = domain + if tests: + where.append("test_name IN %(test_names)s") + params["test_names"] = tests + where_sql = " AND ".join(where) + + # Population first: the weight is 1/rate by construction, but the + # population is what lets anyone check that later. + pop = db.execute( + f"SELECT count() FROM {table} WHERE {where_sql}", params + ) + population = int(pop[0][0]) if pop else 0 + + resolver = ( + "resolver_asn" if table == "analysis_web_measurement" else "0" + ) + # NOTE: no blocked/down/ok, no anomaly, no confirmed, no scores. + rows = db.execute( + f""" + SELECT measurement_uid, + measurement_start_time, + probe_cc, + probe_asn, + {resolver} AS resolver_asn, + domain, + input, + test_name + FROM {table} + WHERE {where_sql} + AND modulo( + cityHash64(concat(measurement_uid, %(salt)s)), + {HASH_SPACE} + ) < %(cutoff)s + ORDER BY cityHash64(measurement_uid) + LIMIT %(limit)s + """, + params, + ) + + used[stratum] = { + "predicate": spec_s["predicate"], + "table": table, + "sample_rate": rate, + "screen_kind": spec_s["screen_kind"], + "population_estimate": population, + "drawn": len(rows), + "frame_start": since_dt.isoformat(), + "frame_end": until_dt.isoformat(), + "scope": spec["scope"], + } + buckets.append([ + SampleRow( + measurement_uid=r[0], + measurement_start_time=r[1], + probe_cc=r[2] or "", + probe_asn=int(r[3] or 0), + resolver_asn=int(r[4] or 0), + domain=r[5] or "", + input=r[6], + test_name=r[7] or "", + sampling_stratum=stratum, + sampling_weight=1.0 / rate, + sampling_design_id=derived_id, + screen_kind=spec_s["screen_kind"], + ) + for r in rows + ]) + + # Interleave rather than concatenate. A queue that runs all the positives + # first tells the analyst which stratum they are in, which is most of the + # way to telling them the answer. + interleaved: List[SampleRow] = [] + for i in range(max((len(b) for b in buckets), default=0)): + for b in buckets: + if i < len(b): + interleaved.append(b[i]) + + return SampleResponse( + design_id=derived_id, + replicate=replicate, + spec=spec, + frame_start=since_dt, + frame_end=until_dt, + strata=used, + rows=interleaved[:limit], + ) + + +# -------------------------------------------------------------------------- +# The blinded candidate +# -------------------------------------------------------------------------- + + +def _rows_to_dicts(result, columns) -> List[Dict[str, Any]]: + return [dict(zip(columns, row)) for row in result] + + +@router.get("/candidate/{measurement_uid}") +def get_candidate( + measurement_uid: str, + db=Depends(get_clickhouse_session), +) -> Dict[str, Any]: + """Everything needed to judge one measurement, and nothing more. + + Deliberately absent: the LoNI triple, top_probe_analysis, anomaly, + confirmed, scores. Those are what the corpus exists to evaluate; an + analyst who sees them first is anchored, and every LR fit from those + labels is inflated by an amount nobody can measure. See /reveal. + """ + obs = db.execute( + """ + SELECT * FROM obs_web + WHERE measurement_uid = %(uid)s + ORDER BY observation_idx + """, + {"uid": measurement_uid}, + with_column_types=True, + ) + obs_rows, obs_types = obs + obs_cols = [c[0] for c in obs_types] + if not obs_rows: + raise HTTPException(404, "no observations for that measurement_uid") + + # obs_web_ctrl's exact columns vary by pipeline version, so select * and + # let the client field-match. Verify against your deployment before + # trusting the diff. + ctrl_rows, ctrl_types = db.execute( + "SELECT * FROM obs_web_ctrl WHERE measurement_uid = %(uid)s", + {"uid": measurement_uid}, + with_column_types=True, + ) + ctrl_cols = [c[0] for c in ctrl_types] + + return { + "measurement_uid": measurement_uid, + "observations": _rows_to_dicts(obs_rows, obs_cols), + "controls": _rows_to_dicts(ctrl_rows, ctrl_cols), + "blinded": True, + } + + +@router.get("/context") +def get_context( + hostname: str, + probe_cc: str = Query(..., min_length=2, max_length=2), + probe_asn: int = Query(...), + at: datetime = Query(..., description="Centre of the window"), + hours: int = Query(6, ge=1, le=72), + db=Depends(get_clickhouse_session), +) -> Dict[str, Any]: + """Failure-string counts per hour for this hostname on this network, + centred on the measurement. + + This is the panel that separates "one probe had a bad minute" from "this + network stopped resolving this name at 14:00". It is failure strings only — + still no verdicts. + """ + rows = db.execute( + """ + WITH multiIf( + dns_failure IS NOT NULL, concat('dns.', dns_failure), + tcp_failure IS NOT NULL, concat('tcp.', tcp_failure), + tls_failure IS NOT NULL, concat('tls.', tls_failure), + http_failure IS NOT NULL, concat('http.', http_failure), + 'ok' + ) AS failure_str + SELECT toStartOfHour(measurement_start_time) AS ts, + failure_str, + resolver_asn, + count() AS cnt + FROM obs_web + WHERE hostname = %(hostname)s + AND probe_cc = %(cc)s + AND probe_asn = %(asn)s + AND measurement_start_time >= %(since)s + AND measurement_start_time < %(until)s + GROUP BY ts, failure_str, resolver_asn + ORDER BY ts + """, + { + "hostname": hostname, + "cc": probe_cc.upper(), + "asn": probe_asn, + "since": at - timedelta(hours=hours), + "until": at + timedelta(hours=hours), + }, + ) + return { + "hostname": hostname, + "window_hours": hours, + "series": [ + { + "ts": r[0], + "failure_str": r[1], + "resolver_asn": int(r[2] or 0), + "count": int(r[3]), + } + for r in rows + ], + } + + +# -------------------------------------------------------------------------- +# The reveal — called only after the analyst commits +# -------------------------------------------------------------------------- + + +@router.get("/reveal/{measurement_uid}") +def reveal( + measurement_uid: str, + db=Depends(get_clickhouse_session), +) -> Dict[str, Any]: + """What the pipeline concluded. + + Shown after commit, never before. Two uses: analysts find rule bugs this + way, and the agreement rate between analyst and pipeline is a diagnostic + worth watching — as a signal that blinding is holding, not as a target to + improve. + """ + a = db.execute( + """ + SELECT top_probe_analysis, top_dns_failure, top_tcp_failure, + top_tls_failure, + dns_blocked, dns_down, dns_ok, + tcp_blocked, tcp_down, tcp_ok, + tls_blocked, tls_down, tls_ok + FROM analysis_web_measurement + WHERE measurement_uid = %(uid)s + LIMIT 1 + """, + {"uid": measurement_uid}, + ) + f = db.execute( + """ + SELECT anomaly, confirmed, msm_failure, scores + FROM fastpath WHERE measurement_uid = %(uid)s LIMIT 1 + """, + {"uid": measurement_uid}, + ) + + analysis = None + if a: + r = a[0] + analysis = { + "top_probe_analysis": r[0], + "top_dns_failure": r[1], + "top_tcp_failure": r[2], + "top_tls_failure": r[3], + "loni": { + "dns": {"blocked": r[4], "down": r[5], "ok": r[6]}, + "tcp": {"blocked": r[7], "down": r[8], "ok": r[9]}, + "tls": {"blocked": r[10], "down": r[11], "ok": r[12]}, + }, + } + + fastpath = None + if f: + r = f[0] + fastpath = { + "anomaly": r[0] == "t", + "confirmed": r[1] == "t", + "msm_failure": r[2] == "t", + "scores": r[3], + } + + return { + "measurement_uid": measurement_uid, + "analysis": analysis, + "fastpath": fastpath, + "caveat": "The LoNI triple is hand-set and uncalibrated. It is shown " + "as a claim to check, not a reference answer.", + } From b54c432627b3d826d4e57f55344a6c4e5d10c986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 31 Jul 2026 16:30:50 +0200 Subject: [PATCH 02/14] Set lax CORS rules --- ooniapi/services/oonimeasurements/src/oonimeasurements/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py index 53191dc81..bb62039f1 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py @@ -38,7 +38,8 @@ def create_app() -> FastAPI: app.add_middleware( CORSMiddleware, # allow from observable notebooks - allow_origin_regex=r"^https://[-A-Za-z0-9]+(\.(test|dev))?\.ooni\.(org|io)$|^https://.*\.observableusercontent\.com$", + #allow_origin_regex=r"^https://[-A-Za-z0-9]+(\.(test|dev))?\.ooni\.(org|io)$|^https://.*\.observableusercontent\.com$", + allow_origins=["*"], # allow_origin_regex="^https://[-A-Za-z0-9]+(\.test)?\.ooni\.(org|io)$", allow_credentials=True, allow_methods=["*"], From e2c741e74ded6d1837868ff0c1e05feea0a29656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 31 Jul 2026 18:55:57 +0200 Subject: [PATCH 03/14] Fix quota picking logic The rate was wrong as it was applies a LIMIT twice --- .../src/oonimeasurements/routers/labeling.py | 223 +++++++++++++----- 1 file changed, 168 insertions(+), 55 deletions(-) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index 1b144a8b9..0e0447065 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -14,9 +14,11 @@ is not a pipeline judgment in disguise. 2. SAMPLING IS RECORDED, NOT REMEMBERED. Every draw is deterministic given - (design_id, stratum, frame, rate), and /sample returns the predicate it ran + (design_id, stratum, frame, quota), and /sample returns the predicate it ran and the population it ran against, so the weights are reconstructable from - the export alone. + the export alone. A weight is measured (population / drawn), never declared: + any knob that only *describes* the sampling will eventually disagree with + what the query did, and disagree silently. """ import hashlib @@ -50,20 +52,18 @@ # refit can tell proxy-screened rows from B1-screened ones and, if needed, # drop the former. -BLOCKED_MAX = "greatest(dns_blocked, tcp_blocked, tls_blocked)" - STRATA: Dict[str, Dict[str, Any]] = { "screen_positive": { "table": "fastpath", - "predicate": f"anomaly = 't' AND msm_failure = 'f'", - "default_rate": 0.1, + "predicate": "anomaly = 't' AND msm_failure = 'f'", + "default_share": 0.40, "screen_kind": "fastpath_proxy", "note": "Proxy for B1's blocked-leaning-rule screen. LR numerator.", }, "screen_negative": { "table": "fastpath", - "predicate": f"confirmed = 'f' AND anomaly = 'f' AND msm_failure = 'f'", - "default_rate": 0.0002, + "predicate": "confirmed = 'f' AND anomaly = 'f' AND msm_failure = 'f'", + "default_share": 0.35, "screen_kind": "fastpath_proxy", "note": "Bounds false negatives and carries the base rate. Small, " "and the first thing cut under pressure. Do not cut it.", @@ -71,35 +71,28 @@ "fingerprint_match": { "table": "fastpath", "predicate": "confirmed = 't' AND msm_failure = 'f'", - "default_rate": 1.0, + "default_share": 0.15, "screen_kind": "fingerprint", - "note": "Census, not a sample. High-precision positives; tag the " - "labels so LRs can be refit without them as a circularity " - "check.", + "note": "High-precision positives; tag the labels so LRs can be refit " + "without them as a circularity check.", }, "incident_window": { "table": "analysis_web_measurement", "predicate": "1", # scoped entirely by the cc/domain/time params - "default_rate": 0.2, + "default_share": 0.10, "screen_kind": "incident_scope", "note": "Draw inside a known event. Label the MEASUREMENT: rows here " "that are genuinely ok are the most valuable in the corpus.", }, } -# control_agreement (cheap negatives from probe/control agreement) needs an -# obs_web_ctrl join with per-layer agreement predicates. Left out on purpose -# rather than half-built: a negative stratum with a wrong predicate is worse -# than one that is absent, because it silently deflates every LR denominator. - -HASH_SPACE = 1_000_000 - # Bump when STRATA definitions or the fingerprint's shape change: it forces new # design ids, so old weights are never silently reinterpreted under new rules. -DESIGN_SCHEMA_VERSION = "1" +# 2: rates became shares, and selection became ORDER BY hash + OFFSET. +DESIGN_SCHEMA_VERSION = "2" -def _design_fingerprint(spec: Dict[str, Any]) -> str: +def _fingerprint(spec: Dict[str, Any], prefix: str = "d") -> str: """Content-address a sampling design. The id is derived from the design so that there is never the same id used @@ -111,11 +104,79 @@ def _design_fingerprint(spec: Dict[str, Any]) -> str: inter-rater agreement gets measured without any coordination service. Want genuinely fresh rows from the same population? Increment `replicate`. - It is part of the spec, so it produces a different id and an independent - draw, on purpose and on the record. + It is part of the spec, so it produces a different id and a draw that is + disjoint from the previous one, on purpose and on the record. """ blob = json.dumps(spec, sort_keys=True, separators=(",", ":"), default=str) - return "d" + hashlib.sha256(blob.encode()).hexdigest()[:10] + return prefix + hashlib.sha256(blob.encode()).hexdigest()[:10] + + +def _resolve_shares( + wanted: List[str], override: Optional[str] +) -> Dict[str, float]: + """Queue share per stratum, normalised over the strata actually asked for. + + `override` is "stratum=share,..." with shares in (0, 1]; unnamed strata keep + their default. Normalising means dropping a stratum redistributes its share + instead of quietly shrinking the queue, and it means the shares are readable + as "what fraction of what I am about to label", which is the whole point. + """ + shares = {s: float(STRATA[s]["default_share"]) for s in wanted} + if override: + for part in override.split(","): + part = part.strip() + if not part: + continue + key, _, val = part.partition("=") + key = key.strip() + if key not in shares: + raise HTTPException( + 400, f"share for unselected or unknown stratum: {key}" + ) + try: + share = float(val) + except ValueError: + raise HTTPException(400, f"share for {key} is not a number") + if not 0 < share <= 1: + raise HTTPException(400, f"share for {key} must be in (0, 1]") + shares[key] = share + + total = sum(shares.values()) + if total <= 0: + raise HTTPException(400, "shares sum to zero") + return {s: v / total for s, v in shares.items()} + + +def _quotas(shares: Dict[str, float], limit: int) -> Dict[str, int]: + """Turn shares into whole row counts that sum to exactly `limit`. + + Largest-remainder, so the rounding error lands on the biggest strata rather + than starving a small one. Every selected stratum gets at least one row: a + stratum present in the design but absent from the queue is indistinguishable + from one that was never asked for, and `screen_negative` is small enough to + be the one that vanishes. + """ + order = sorted(shares) + exact = {s: shares[s] * limit for s in order} + base = {s: max(1, int(exact[s])) for s in order} + + # Give away, or claw back, whatever the flooring left over. + drift = limit - sum(base.values()) + while drift != 0: + step = 1 if drift > 0 else -1 + movable = [ + s for s in order + if step > 0 or base[s] > 1 # never take a stratum below one row + ] + if not movable: + break + pick = max( + movable, + key=lambda s: (exact[s] - base[s]) * step, + ) + base[pick] += step + drift -= step + return base class SampleRow(BaseModel): @@ -129,7 +190,7 @@ class SampleRow(BaseModel): test_name: str # sampling provenance, carried through to the label sampling_stratum: str - sampling_weight: float + sampling_weight: Optional[float] sampling_design_id: str screen_kind: str @@ -146,7 +207,10 @@ class SampleResponse(BaseModel): def _frame(since: Optional[datetime], until: Optional[datetime]): until = until or datetime.now(timezone.utc).replace(tzinfo=None) - since = since or (until - timedelta(days=30)) + # Matches the UI's default. A wide frame is the point: rows are drawn in + # hash order, not time order, so widening spreads the queue across the + # period instead of concentrating it on last month. + since = since or (until - timedelta(days=365)) if since >= until: raise HTTPException(400, "since must be before until") return since, until @@ -160,7 +224,13 @@ def get_design() -> Dict[str, Any]: predicate that produced them. Edit a stratum here and you have a new design: bump design_id at draw time, never reuse it. """ - return {"strata": STRATA, "hash_space": HASH_SPACE} + return { + "strata": STRATA, + "schema": DESIGN_SCHEMA_VERSION, + "selection": "ORDER BY cityHash64(measurement_uid + design salt), " + "LIMIT quota OFFSET (replicate-1)*quota", + "weighting": "population / drawn, per stratum", + } @router.get("/test_names") @@ -225,10 +295,11 @@ def draw_sample( ), replicate: int = Query( 1, ge=1, - description="Independent draws of the same design. Same replicate = " + description="Successive draws of the same design. Same replicate = " "same rows (reproducible, extendable, comparable across " - "analysts). Increment it to sample rows the previous " - "replicate did not cover.", + "analysts). Increment it for rows the previous replicate " + "did not cover: replicates are disjoint by construction, " + "being successive slices of one deterministic ordering.", ), since: Optional[datetime] = None, until: Optional[datetime] = None, @@ -243,10 +314,12 @@ def draw_sample( "Empty string means every test.", ), limit: int = Query(50, ge=1, le=500), - rate_override: Optional[float] = Query( - None, gt=0, le=1, - description="Overrides every named stratum's rate. Using this makes a " - "new design; change design_id too.", + shares: Optional[str] = Query( + None, + description="Override queue composition: 'screen_positive=0.5," + "screen_negative=0.5'. Shares are normalised over the " + "selected strata, so they are fractions of your queue, not " + "sampling rates. Part of the design, so it changes the id.", ), ) -> SampleResponse: since_dt, until_dt = _frame(since, until) @@ -267,16 +340,24 @@ def draw_sample( # or what a weight means, has to be in here — otherwise two different # populations could collide onto one id, which is the failure this exists # to make impossible. + share_by_stratum = _resolve_shares(wanted, shares) + quota = _quotas(share_by_stratum, limit) + resolved = { s: { "table": STRATA[s]["table"], "predicate": STRATA[s]["predicate"], "screen_kind": STRATA[s]["screen_kind"], - "sample_rate": rate_override or STRATA[s]["default_rate"], + "queue_share": round(share_by_stratum[s], 6), + "quota": quota[s], } for s in wanted } - spec = { + # The population a draw addresses, and therefore what a weight means, is + # fixed by everything except the replicate. Ordering is salted from that + # part alone, so replicate 2 can take the next slice of the same ordering + # rather than reshuffling into an independent (and overlapping) sample. + population_spec = { "schema": DESIGN_SCHEMA_VERSION, "strata": resolved, "frame": [since_dt.isoformat(), until_dt.isoformat()], @@ -286,30 +367,41 @@ def draw_sample( "domain": domain, "test_names": tests or "all", }, - "replicate": replicate, } - derived_id = _design_fingerprint(spec) + spec = {**population_spec, "replicate": replicate} + derived_id = _fingerprint(spec) + order_salt = _fingerprint(population_spec, prefix="o") - per_stratum = max(1, limit // len(wanted)) used: Dict[str, Dict[str, Any]] = {} buckets: List[List[SampleRow]] = [] for stratum in wanted: spec_s = STRATA[stratum] - rate = resolved[stratum]["sample_rate"] table = spec_s["table"] where = [ "measurement_start_time >= %(since)s", "measurement_start_time < %(until)s", f"({spec_s['predicate']})", + # Only rows that can actually be labelled. The screens read + # fastpath, but /candidate reads obs_web, so without this a draw + # yields rows that 404 on open. It also keeps the weight honest: + # `population` below counts the same set the draw samples from, and + # rows missing from obs_web are missing non-randomly (they track + # test and pipeline coverage), so excluding them from both is the + # only way the ratio stays an inclusion probability. + "measurement_uid IN (" + " SELECT measurement_uid FROM obs_web" + " WHERE measurement_start_time >= %(since)s" + " AND measurement_start_time < %(until)s" + ")", ] params: Dict[str, Any] = { "since": since_dt, "until": until_dt, - "salt": f"{derived_id}:{stratum}", - "cutoff": int(rate * HASH_SPACE), - "limit": per_stratum, + "salt": f"{order_salt}:{stratum}", + "limit": quota[stratum], + "offset": (replicate - 1) * quota[stratum], } if probe_cc: where.append("probe_cc = %(probe_cc)s") @@ -325,8 +417,11 @@ def draw_sample( params["test_names"] = tests where_sql = " AND ".join(where) - # Population first: the weight is 1/rate by construction, but the - # population is what lets anyone check that later. + # Population first, because it *is* the weight. Not 1/share: the queue + # is cut to a quota, so what a labelled row stands for is however many + # eligible rows there were divided by however many were drawn. Taking + # 20 of 5,000,000 makes each one worth 250,000, whatever share of the + # queue the stratum was given. pop = db.execute( f"SELECT count() FROM {table} WHERE {where_sql}", params ) @@ -335,6 +430,13 @@ def draw_sample( resolver = ( "resolver_asn" if table == "analysis_web_measurement" else "0" ) + # Ordering by a salted hash of the uid puts the eligible rows in a + # deterministic pseudo-random order, so the first N are a uniform + # sample of size N and OFFSET walks disjoint slices for successive + # replicates. The salt has to be in the ORDER BY rather than in a + # separate filter: unsalted, the same globally-low-hash measurements + # sit at the head of every design's queue forever. + # # NOTE: no blocked/down/ok, no anomaly, no confirmed, no scores. rows = db.execute( f""" @@ -348,23 +450,28 @@ def draw_sample( test_name FROM {table} WHERE {where_sql} - AND modulo( - cityHash64(concat(measurement_uid, %(salt)s)), - {HASH_SPACE} - ) < %(cutoff)s - ORDER BY cityHash64(measurement_uid) - LIMIT %(limit)s + ORDER BY cityHash64(concat(measurement_uid, %(salt)s)) + LIMIT %(limit)s OFFSET %(offset)s """, params, ) + # An empty stratum is not an error (a narrow scope, or a replicate past + # the end of the population), but it has no weight either: dividing by + # a zero draw would be a crash, and inventing a weight for rows that do + # not exist would be worse. + weight = (population / len(rows)) if rows else None + used[stratum] = { "predicate": spec_s["predicate"], "table": table, - "sample_rate": rate, + "queue_share": resolved[stratum]["queue_share"], + "quota": quota[stratum], "screen_kind": spec_s["screen_kind"], "population_estimate": population, "drawn": len(rows), + "sampling_weight": weight, + "exhausted": bool(rows) and len(rows) < quota[stratum], "frame_start": since_dt.isoformat(), "frame_end": until_dt.isoformat(), "scope": spec["scope"], @@ -380,7 +487,7 @@ def draw_sample( input=r[6], test_name=r[7] or "", sampling_stratum=stratum, - sampling_weight=1.0 / rate, + sampling_weight=weight, sampling_design_id=derived_id, screen_kind=spec_s["screen_kind"], ) @@ -428,9 +535,15 @@ def get_candidate( analyst who sees them first is anchored, and every LR fit from those labels is inflated by an amount nobody can measure. See /reveal. """ + # EXCEPT, not SELECT *: obs_web carries `probe_analysis`, which is the + # probe's own blocking verdict (web_connectivity's test_keys.blocking). + # It is the same judgment /reveal exposes as top_probe_analysis, one row + # down, and shipping it here would anchor the analyst against exactly what + # the corpus exists to evaluate. Everything else is passed through, so the + # client keeps field-matching across pipeline versions. obs = db.execute( """ - SELECT * FROM obs_web + SELECT * EXCEPT (probe_analysis) FROM obs_web WHERE measurement_uid = %(uid)s ORDER BY observation_idx """, From 64245c8eca9b547a0a5bb335caca8d2b4258dd3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 31 Jul 2026 19:05:26 +0200 Subject: [PATCH 04/14] Revert "Fix quota picking logic" This reverts commit e2c741e74ded6d1837868ff0c1e05feea0a29656. --- .../src/oonimeasurements/routers/labeling.py | 223 +++++------------- 1 file changed, 55 insertions(+), 168 deletions(-) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index 0e0447065..1b144a8b9 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -14,11 +14,9 @@ is not a pipeline judgment in disguise. 2. SAMPLING IS RECORDED, NOT REMEMBERED. Every draw is deterministic given - (design_id, stratum, frame, quota), and /sample returns the predicate it ran + (design_id, stratum, frame, rate), and /sample returns the predicate it ran and the population it ran against, so the weights are reconstructable from - the export alone. A weight is measured (population / drawn), never declared: - any knob that only *describes* the sampling will eventually disagree with - what the query did, and disagree silently. + the export alone. """ import hashlib @@ -52,18 +50,20 @@ # refit can tell proxy-screened rows from B1-screened ones and, if needed, # drop the former. +BLOCKED_MAX = "greatest(dns_blocked, tcp_blocked, tls_blocked)" + STRATA: Dict[str, Dict[str, Any]] = { "screen_positive": { "table": "fastpath", - "predicate": "anomaly = 't' AND msm_failure = 'f'", - "default_share": 0.40, + "predicate": f"anomaly = 't' AND msm_failure = 'f'", + "default_rate": 0.1, "screen_kind": "fastpath_proxy", "note": "Proxy for B1's blocked-leaning-rule screen. LR numerator.", }, "screen_negative": { "table": "fastpath", - "predicate": "confirmed = 'f' AND anomaly = 'f' AND msm_failure = 'f'", - "default_share": 0.35, + "predicate": f"confirmed = 'f' AND anomaly = 'f' AND msm_failure = 'f'", + "default_rate": 0.0002, "screen_kind": "fastpath_proxy", "note": "Bounds false negatives and carries the base rate. Small, " "and the first thing cut under pressure. Do not cut it.", @@ -71,28 +71,35 @@ "fingerprint_match": { "table": "fastpath", "predicate": "confirmed = 't' AND msm_failure = 'f'", - "default_share": 0.15, + "default_rate": 1.0, "screen_kind": "fingerprint", - "note": "High-precision positives; tag the labels so LRs can be refit " - "without them as a circularity check.", + "note": "Census, not a sample. High-precision positives; tag the " + "labels so LRs can be refit without them as a circularity " + "check.", }, "incident_window": { "table": "analysis_web_measurement", "predicate": "1", # scoped entirely by the cc/domain/time params - "default_share": 0.10, + "default_rate": 0.2, "screen_kind": "incident_scope", "note": "Draw inside a known event. Label the MEASUREMENT: rows here " "that are genuinely ok are the most valuable in the corpus.", }, } +# control_agreement (cheap negatives from probe/control agreement) needs an +# obs_web_ctrl join with per-layer agreement predicates. Left out on purpose +# rather than half-built: a negative stratum with a wrong predicate is worse +# than one that is absent, because it silently deflates every LR denominator. + +HASH_SPACE = 1_000_000 + # Bump when STRATA definitions or the fingerprint's shape change: it forces new # design ids, so old weights are never silently reinterpreted under new rules. -# 2: rates became shares, and selection became ORDER BY hash + OFFSET. -DESIGN_SCHEMA_VERSION = "2" +DESIGN_SCHEMA_VERSION = "1" -def _fingerprint(spec: Dict[str, Any], prefix: str = "d") -> str: +def _design_fingerprint(spec: Dict[str, Any]) -> str: """Content-address a sampling design. The id is derived from the design so that there is never the same id used @@ -104,79 +111,11 @@ def _fingerprint(spec: Dict[str, Any], prefix: str = "d") -> str: inter-rater agreement gets measured without any coordination service. Want genuinely fresh rows from the same population? Increment `replicate`. - It is part of the spec, so it produces a different id and a draw that is - disjoint from the previous one, on purpose and on the record. + It is part of the spec, so it produces a different id and an independent + draw, on purpose and on the record. """ blob = json.dumps(spec, sort_keys=True, separators=(",", ":"), default=str) - return prefix + hashlib.sha256(blob.encode()).hexdigest()[:10] - - -def _resolve_shares( - wanted: List[str], override: Optional[str] -) -> Dict[str, float]: - """Queue share per stratum, normalised over the strata actually asked for. - - `override` is "stratum=share,..." with shares in (0, 1]; unnamed strata keep - their default. Normalising means dropping a stratum redistributes its share - instead of quietly shrinking the queue, and it means the shares are readable - as "what fraction of what I am about to label", which is the whole point. - """ - shares = {s: float(STRATA[s]["default_share"]) for s in wanted} - if override: - for part in override.split(","): - part = part.strip() - if not part: - continue - key, _, val = part.partition("=") - key = key.strip() - if key not in shares: - raise HTTPException( - 400, f"share for unselected or unknown stratum: {key}" - ) - try: - share = float(val) - except ValueError: - raise HTTPException(400, f"share for {key} is not a number") - if not 0 < share <= 1: - raise HTTPException(400, f"share for {key} must be in (0, 1]") - shares[key] = share - - total = sum(shares.values()) - if total <= 0: - raise HTTPException(400, "shares sum to zero") - return {s: v / total for s, v in shares.items()} - - -def _quotas(shares: Dict[str, float], limit: int) -> Dict[str, int]: - """Turn shares into whole row counts that sum to exactly `limit`. - - Largest-remainder, so the rounding error lands on the biggest strata rather - than starving a small one. Every selected stratum gets at least one row: a - stratum present in the design but absent from the queue is indistinguishable - from one that was never asked for, and `screen_negative` is small enough to - be the one that vanishes. - """ - order = sorted(shares) - exact = {s: shares[s] * limit for s in order} - base = {s: max(1, int(exact[s])) for s in order} - - # Give away, or claw back, whatever the flooring left over. - drift = limit - sum(base.values()) - while drift != 0: - step = 1 if drift > 0 else -1 - movable = [ - s for s in order - if step > 0 or base[s] > 1 # never take a stratum below one row - ] - if not movable: - break - pick = max( - movable, - key=lambda s: (exact[s] - base[s]) * step, - ) - base[pick] += step - drift -= step - return base + return "d" + hashlib.sha256(blob.encode()).hexdigest()[:10] class SampleRow(BaseModel): @@ -190,7 +129,7 @@ class SampleRow(BaseModel): test_name: str # sampling provenance, carried through to the label sampling_stratum: str - sampling_weight: Optional[float] + sampling_weight: float sampling_design_id: str screen_kind: str @@ -207,10 +146,7 @@ class SampleResponse(BaseModel): def _frame(since: Optional[datetime], until: Optional[datetime]): until = until or datetime.now(timezone.utc).replace(tzinfo=None) - # Matches the UI's default. A wide frame is the point: rows are drawn in - # hash order, not time order, so widening spreads the queue across the - # period instead of concentrating it on last month. - since = since or (until - timedelta(days=365)) + since = since or (until - timedelta(days=30)) if since >= until: raise HTTPException(400, "since must be before until") return since, until @@ -224,13 +160,7 @@ def get_design() -> Dict[str, Any]: predicate that produced them. Edit a stratum here and you have a new design: bump design_id at draw time, never reuse it. """ - return { - "strata": STRATA, - "schema": DESIGN_SCHEMA_VERSION, - "selection": "ORDER BY cityHash64(measurement_uid + design salt), " - "LIMIT quota OFFSET (replicate-1)*quota", - "weighting": "population / drawn, per stratum", - } + return {"strata": STRATA, "hash_space": HASH_SPACE} @router.get("/test_names") @@ -295,11 +225,10 @@ def draw_sample( ), replicate: int = Query( 1, ge=1, - description="Successive draws of the same design. Same replicate = " + description="Independent draws of the same design. Same replicate = " "same rows (reproducible, extendable, comparable across " - "analysts). Increment it for rows the previous replicate " - "did not cover: replicates are disjoint by construction, " - "being successive slices of one deterministic ordering.", + "analysts). Increment it to sample rows the previous " + "replicate did not cover.", ), since: Optional[datetime] = None, until: Optional[datetime] = None, @@ -314,12 +243,10 @@ def draw_sample( "Empty string means every test.", ), limit: int = Query(50, ge=1, le=500), - shares: Optional[str] = Query( - None, - description="Override queue composition: 'screen_positive=0.5," - "screen_negative=0.5'. Shares are normalised over the " - "selected strata, so they are fractions of your queue, not " - "sampling rates. Part of the design, so it changes the id.", + rate_override: Optional[float] = Query( + None, gt=0, le=1, + description="Overrides every named stratum's rate. Using this makes a " + "new design; change design_id too.", ), ) -> SampleResponse: since_dt, until_dt = _frame(since, until) @@ -340,24 +267,16 @@ def draw_sample( # or what a weight means, has to be in here — otherwise two different # populations could collide onto one id, which is the failure this exists # to make impossible. - share_by_stratum = _resolve_shares(wanted, shares) - quota = _quotas(share_by_stratum, limit) - resolved = { s: { "table": STRATA[s]["table"], "predicate": STRATA[s]["predicate"], "screen_kind": STRATA[s]["screen_kind"], - "queue_share": round(share_by_stratum[s], 6), - "quota": quota[s], + "sample_rate": rate_override or STRATA[s]["default_rate"], } for s in wanted } - # The population a draw addresses, and therefore what a weight means, is - # fixed by everything except the replicate. Ordering is salted from that - # part alone, so replicate 2 can take the next slice of the same ordering - # rather than reshuffling into an independent (and overlapping) sample. - population_spec = { + spec = { "schema": DESIGN_SCHEMA_VERSION, "strata": resolved, "frame": [since_dt.isoformat(), until_dt.isoformat()], @@ -367,41 +286,30 @@ def draw_sample( "domain": domain, "test_names": tests or "all", }, + "replicate": replicate, } - spec = {**population_spec, "replicate": replicate} - derived_id = _fingerprint(spec) - order_salt = _fingerprint(population_spec, prefix="o") + derived_id = _design_fingerprint(spec) + per_stratum = max(1, limit // len(wanted)) used: Dict[str, Dict[str, Any]] = {} buckets: List[List[SampleRow]] = [] for stratum in wanted: spec_s = STRATA[stratum] + rate = resolved[stratum]["sample_rate"] table = spec_s["table"] where = [ "measurement_start_time >= %(since)s", "measurement_start_time < %(until)s", f"({spec_s['predicate']})", - # Only rows that can actually be labelled. The screens read - # fastpath, but /candidate reads obs_web, so without this a draw - # yields rows that 404 on open. It also keeps the weight honest: - # `population` below counts the same set the draw samples from, and - # rows missing from obs_web are missing non-randomly (they track - # test and pipeline coverage), so excluding them from both is the - # only way the ratio stays an inclusion probability. - "measurement_uid IN (" - " SELECT measurement_uid FROM obs_web" - " WHERE measurement_start_time >= %(since)s" - " AND measurement_start_time < %(until)s" - ")", ] params: Dict[str, Any] = { "since": since_dt, "until": until_dt, - "salt": f"{order_salt}:{stratum}", - "limit": quota[stratum], - "offset": (replicate - 1) * quota[stratum], + "salt": f"{derived_id}:{stratum}", + "cutoff": int(rate * HASH_SPACE), + "limit": per_stratum, } if probe_cc: where.append("probe_cc = %(probe_cc)s") @@ -417,11 +325,8 @@ def draw_sample( params["test_names"] = tests where_sql = " AND ".join(where) - # Population first, because it *is* the weight. Not 1/share: the queue - # is cut to a quota, so what a labelled row stands for is however many - # eligible rows there were divided by however many were drawn. Taking - # 20 of 5,000,000 makes each one worth 250,000, whatever share of the - # queue the stratum was given. + # Population first: the weight is 1/rate by construction, but the + # population is what lets anyone check that later. pop = db.execute( f"SELECT count() FROM {table} WHERE {where_sql}", params ) @@ -430,13 +335,6 @@ def draw_sample( resolver = ( "resolver_asn" if table == "analysis_web_measurement" else "0" ) - # Ordering by a salted hash of the uid puts the eligible rows in a - # deterministic pseudo-random order, so the first N are a uniform - # sample of size N and OFFSET walks disjoint slices for successive - # replicates. The salt has to be in the ORDER BY rather than in a - # separate filter: unsalted, the same globally-low-hash measurements - # sit at the head of every design's queue forever. - # # NOTE: no blocked/down/ok, no anomaly, no confirmed, no scores. rows = db.execute( f""" @@ -450,28 +348,23 @@ def draw_sample( test_name FROM {table} WHERE {where_sql} - ORDER BY cityHash64(concat(measurement_uid, %(salt)s)) - LIMIT %(limit)s OFFSET %(offset)s + AND modulo( + cityHash64(concat(measurement_uid, %(salt)s)), + {HASH_SPACE} + ) < %(cutoff)s + ORDER BY cityHash64(measurement_uid) + LIMIT %(limit)s """, params, ) - # An empty stratum is not an error (a narrow scope, or a replicate past - # the end of the population), but it has no weight either: dividing by - # a zero draw would be a crash, and inventing a weight for rows that do - # not exist would be worse. - weight = (population / len(rows)) if rows else None - used[stratum] = { "predicate": spec_s["predicate"], "table": table, - "queue_share": resolved[stratum]["queue_share"], - "quota": quota[stratum], + "sample_rate": rate, "screen_kind": spec_s["screen_kind"], "population_estimate": population, "drawn": len(rows), - "sampling_weight": weight, - "exhausted": bool(rows) and len(rows) < quota[stratum], "frame_start": since_dt.isoformat(), "frame_end": until_dt.isoformat(), "scope": spec["scope"], @@ -487,7 +380,7 @@ def draw_sample( input=r[6], test_name=r[7] or "", sampling_stratum=stratum, - sampling_weight=weight, + sampling_weight=1.0 / rate, sampling_design_id=derived_id, screen_kind=spec_s["screen_kind"], ) @@ -535,15 +428,9 @@ def get_candidate( analyst who sees them first is anchored, and every LR fit from those labels is inflated by an amount nobody can measure. See /reveal. """ - # EXCEPT, not SELECT *: obs_web carries `probe_analysis`, which is the - # probe's own blocking verdict (web_connectivity's test_keys.blocking). - # It is the same judgment /reveal exposes as top_probe_analysis, one row - # down, and shipping it here would anchor the analyst against exactly what - # the corpus exists to evaluate. Everything else is passed through, so the - # client keeps field-matching across pipeline versions. obs = db.execute( """ - SELECT * EXCEPT (probe_analysis) FROM obs_web + SELECT * FROM obs_web WHERE measurement_uid = %(uid)s ORDER BY observation_idx """, From 4ab9567b76006ff0a4981585274e219e2071a254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 31 Jul 2026 19:20:09 +0200 Subject: [PATCH 05/14] Simplify and fix the sampling logic --- .../src/oonimeasurements/routers/labeling.py | 35 ++----------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index 1b144a8b9..773a4eb44 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -50,20 +50,17 @@ # refit can tell proxy-screened rows from B1-screened ones and, if needed, # drop the former. -BLOCKED_MAX = "greatest(dns_blocked, tcp_blocked, tls_blocked)" STRATA: Dict[str, Dict[str, Any]] = { "screen_positive": { "table": "fastpath", "predicate": f"anomaly = 't' AND msm_failure = 'f'", - "default_rate": 0.1, "screen_kind": "fastpath_proxy", "note": "Proxy for B1's blocked-leaning-rule screen. LR numerator.", }, "screen_negative": { "table": "fastpath", "predicate": f"confirmed = 'f' AND anomaly = 'f' AND msm_failure = 'f'", - "default_rate": 0.0002, "screen_kind": "fastpath_proxy", "note": "Bounds false negatives and carries the base rate. Small, " "and the first thing cut under pressure. Do not cut it.", @@ -71,7 +68,6 @@ "fingerprint_match": { "table": "fastpath", "predicate": "confirmed = 't' AND msm_failure = 'f'", - "default_rate": 1.0, "screen_kind": "fingerprint", "note": "Census, not a sample. High-precision positives; tag the " "labels so LRs can be refit without them as a circularity " @@ -80,7 +76,6 @@ "incident_window": { "table": "analysis_web_measurement", "predicate": "1", # scoped entirely by the cc/domain/time params - "default_rate": 0.2, "screen_kind": "incident_scope", "note": "Draw inside a known event. Label the MEASUREMENT: rows here " "that are genuinely ok are the most valuable in the corpus.", @@ -92,8 +87,6 @@ # rather than half-built: a negative stratum with a wrong predicate is worse # than one that is absent, because it silently deflates every LR denominator. -HASH_SPACE = 1_000_000 - # Bump when STRATA definitions or the fingerprint's shape change: it forces new # design ids, so old weights are never silently reinterpreted under new rules. DESIGN_SCHEMA_VERSION = "1" @@ -152,17 +145,6 @@ def _frame(since: Optional[datetime], until: Optional[datetime]): return since, until -@router.get("/design") -def get_design() -> Dict[str, Any]: - """The sampling design, verbatim. - - The UI copies this into the export so weights can be checked against the - predicate that produced them. Edit a stratum here and you have a new - design: bump design_id at draw time, never reuse it. - """ - return {"strata": STRATA, "hash_space": HASH_SPACE} - - @router.get("/test_names") def list_test_names( db=Depends(get_clickhouse_session), @@ -243,11 +225,6 @@ def draw_sample( "Empty string means every test.", ), limit: int = Query(50, ge=1, le=500), - rate_override: Optional[float] = Query( - None, gt=0, le=1, - description="Overrides every named stratum's rate. Using this makes a " - "new design; change design_id too.", - ), ) -> SampleResponse: since_dt, until_dt = _frame(since, until) wanted = sorted({s.strip() for s in strata.split(",") if s.strip()}) @@ -272,7 +249,6 @@ def draw_sample( "table": STRATA[s]["table"], "predicate": STRATA[s]["predicate"], "screen_kind": STRATA[s]["screen_kind"], - "sample_rate": rate_override or STRATA[s]["default_rate"], } for s in wanted } @@ -296,7 +272,6 @@ def draw_sample( for stratum in wanted: spec_s = STRATA[stratum] - rate = resolved[stratum]["sample_rate"] table = spec_s["table"] where = [ @@ -308,7 +283,6 @@ def draw_sample( "since": since_dt, "until": until_dt, "salt": f"{derived_id}:{stratum}", - "cutoff": int(rate * HASH_SPACE), "limit": per_stratum, } if probe_cc: @@ -348,11 +322,7 @@ def draw_sample( test_name FROM {table} WHERE {where_sql} - AND modulo( - cityHash64(concat(measurement_uid, %(salt)s)), - {HASH_SPACE} - ) < %(cutoff)s - ORDER BY cityHash64(measurement_uid) + ORDER BY cityHash64(concat(measurement_uid, %(salt)s)) LIMIT %(limit)s """, params, @@ -361,7 +331,6 @@ def draw_sample( used[stratum] = { "predicate": spec_s["predicate"], "table": table, - "sample_rate": rate, "screen_kind": spec_s["screen_kind"], "population_estimate": population, "drawn": len(rows), @@ -380,7 +349,7 @@ def draw_sample( input=r[6], test_name=r[7] or "", sampling_stratum=stratum, - sampling_weight=1.0 / rate, + sampling_weight=population / len(rows), sampling_design_id=derived_id, screen_kind=spec_s["screen_kind"], ) From d9444a986b82199367ac73d4f7200c091ceb9c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 31 Jul 2026 19:23:53 +0200 Subject: [PATCH 06/14] Add population and row counts too --- .../oonimeasurements/src/oonimeasurements/routers/labeling.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index 773a4eb44..f83cb3b5d 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -123,6 +123,8 @@ class SampleRow(BaseModel): # sampling provenance, carried through to the label sampling_stratum: str sampling_weight: float + sample_population: int + sample_rows: int sampling_design_id: str screen_kind: str @@ -350,6 +352,8 @@ def draw_sample( test_name=r[7] or "", sampling_stratum=stratum, sampling_weight=population / len(rows), + sample_population=population, + sample_rows=len(rows), sampling_design_id=derived_id, screen_kind=spec_s["screen_kind"], ) From 5771ac7b56984214e5230863eac0ab49e5633533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 3 Aug 2026 19:49:49 +0200 Subject: [PATCH 07/14] Stratify positives by layer and add draw shares A uniform draw from anomaly='t' oversamples whichever mechanism dominates globally, in practice TLS resets, so the corpus calibrates one layer and starves the others. Three new mutually exclusive strata on analysis_web_measurement (screen_dns, screen_tcp, screen_tls) attribute positives to the first blocked-leaning layer, and a `shares` query parameter steers how the queue splits across strata. Shares only move analyst effort: row weights stay population/drawn, so no share choice can bias an estimate, only its variance. Like `limit`, shares are not part of the design fingerprint; the same design with a bigger share returns a superset of the same rows. screen_positive now excludes confirmed='t'. Confirmed rows are certain blocking with a known artefact, so labelling one adds almost nothing, and they were eating the positive quota; they keep their own fingerprint_match census. The layer strata read the pipeline's own scores, so they oversample what the pipeline already sees; screen_negative remains the only stratum that can discover what it misses, which is why its share should never be cut. analysis_web_measurement queries now use FINAL: during a reprocess the same uid exists in old and new versions until the merge runs, which inflated populations and could draw a uid twice. DESIGN_SCHEMA_VERSION bumped to 2 so old design ids are never reinterpreted under the new strata. --- .../src/oonimeasurements/routers/labeling.py | 96 +++++++- .../tests/test_labeling_sampling.py | 208 ++++++++++++++++++ 2 files changed, 298 insertions(+), 6 deletions(-) create mode 100644 ooniapi/services/oonimeasurements/tests/test_labeling_sampling.py diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index f83cb3b5d..4296f8f8e 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -54,10 +54,38 @@ STRATA: Dict[str, Dict[str, Any]] = { "screen_positive": { "table": "fastpath", - "predicate": f"anomaly = 't' AND msm_failure = 'f'", + # confirmed rows are excluded: they are certain blocking with a known + # artefact, so a label on one adds almost nothing, and they were eating + # the positive quota. They keep their own census stratum below. + "predicate": f"anomaly = 't' AND confirmed = 'f' AND msm_failure = 'f'", "screen_kind": "fastpath_proxy", "note": "Proxy for B1's blocked-leaning-rule screen. LR numerator.", }, + # Layer-attributed positives. A uniform draw from anomaly='t' oversamples + # whichever mechanism dominates globally (in practice TLS resets), so a + # corpus built from it calibrates one layer and starves the others. These + # read the pipeline's own layer scores, which means they oversample what + # the pipeline can already see — screen_negative stays the only stratum + # that can discover what it misses. Predicates are mutually exclusive so + # populations do not overlap and weights stay clean. + "screen_dns": { + "table": "analysis_web_measurement", + "predicate": "dns_blocked >= 0.5", + "screen_kind": "loni_layer_proxy", + "note": "DNS-attributed positives.", + }, + "screen_tcp": { + "table": "analysis_web_measurement", + "predicate": "tcp_blocked >= 0.5 AND dns_blocked < 0.5", + "screen_kind": "loni_layer_proxy", + "note": "TCP-attributed positives, DNS quiet.", + }, + "screen_tls": { + "table": "analysis_web_measurement", + "predicate": "tls_blocked >= 0.5 AND dns_blocked < 0.5 AND tcp_blocked < 0.5", + "screen_kind": "loni_layer_proxy", + "note": "TLS-attributed positives, DNS and TCP quiet.", + }, "screen_negative": { "table": "fastpath", "predicate": f"confirmed = 'f' AND anomaly = 'f' AND msm_failure = 'f'", @@ -89,7 +117,48 @@ # Bump when STRATA definitions or the fingerprint's shape change: it forces new # design ids, so old weights are never silently reinterpreted under new rules. -DESIGN_SCHEMA_VERSION = "1" +DESIGN_SCHEMA_VERSION = "2" + + +def _quotas(wanted: List[str], shares: Optional[str], limit: int) -> Dict[str, int]: + """How many rows each stratum contributes to the queue. + + Default is an equal split. `shares` reweights it — "screen_negative=0.5" + gives that stratum half the queue and splits the rest equally. Shares only + steer analyst effort; the weights on the rows are population/drawn either + way, so no share choice can bias an estimate, only its variance. + """ + fractions = {s: 1.0 for s in wanted} + if shares: + for part in shares.split(","): + name, _, value = part.partition("=") + name = name.strip() + if name not in wanted: + raise HTTPException(400, f"share for unknown stratum: {name}") + try: + fractions[name] = float(value) + except ValueError: + raise HTTPException(400, f"bad share: {part}") + if fractions[name] <= 0: + raise HTTPException(400, f"share must be positive: {part}") + total = sum(fractions.values()) + exact = {s: limit * f / total for s, f in fractions.items()} + quotas = {s: max(1, int(e)) for s, e in exact.items()} + # Hand out what rounding left over, largest remainder first. + leftover = limit - sum(quotas.values()) + for s in sorted(exact, key=lambda s: exact[s] - int(exact[s]), reverse=True): + if leftover <= 0: + break + quotas[s] += 1 + leftover -= 1 + # The min-1 floors can overshoot under an extreme share; trim the largest + # quotas back so the recorded drawn counts always match the returned queue. + while sum(quotas.values()) > limit: + biggest = max(quotas, key=lambda s: quotas[s]) + if quotas[biggest] == 1: + break # limit < number of strata; nothing sensible to trim + quotas[biggest] -= 1 + return quotas def _design_fingerprint(spec: Dict[str, Any]) -> str: @@ -226,6 +295,14 @@ def draw_sample( "test scope — change design_id when you change this. " "Empty string means every test.", ), + shares: Optional[str] = Query( + None, + description="Optional stratum=share pairs, e.g. " + "'screen_negative=0.5'. Reweights how the queue is split " + "across the selected strata; omitted strata share the " + "remainder equally. Steers effort only — row weights stay " + "population/drawn regardless.", + ), limit: int = Query(50, ge=1, le=500), ) -> SampleResponse: since_dt, until_dt = _frame(since, until) @@ -268,13 +345,20 @@ def draw_sample( } derived_id = _design_fingerprint(spec) - per_stratum = max(1, limit // len(wanted)) + # Shares, like limit, set how far down each stratum's fixed ordering the + # draw goes. They are deliberately not part of the spec: the same design + # with a bigger share returns a superset of the same stratum rows. + quotas = _quotas(wanted, shares, limit) used: Dict[str, Dict[str, Any]] = {} buckets: List[List[SampleRow]] = [] for stratum in wanted: spec_s = STRATA[stratum] table = spec_s["table"] + # analysis_web_measurement is a ReplacingMergeTree; during a reprocess + # the same uid exists in old and new versions until the merge runs, + # which would inflate the population and can draw a uid twice. + from_clause = f"{table} FINAL" if table == "analysis_web_measurement" else table where = [ "measurement_start_time >= %(since)s", @@ -285,7 +369,7 @@ def draw_sample( "since": since_dt, "until": until_dt, "salt": f"{derived_id}:{stratum}", - "limit": per_stratum, + "limit": quotas[stratum], } if probe_cc: where.append("probe_cc = %(probe_cc)s") @@ -304,7 +388,7 @@ def draw_sample( # Population first: the weight is 1/rate by construction, but the # population is what lets anyone check that later. pop = db.execute( - f"SELECT count() FROM {table} WHERE {where_sql}", params + f"SELECT count() FROM {from_clause} WHERE {where_sql}", params ) population = int(pop[0][0]) if pop else 0 @@ -322,7 +406,7 @@ def draw_sample( domain, input, test_name - FROM {table} + FROM {from_clause} WHERE {where_sql} ORDER BY cityHash64(concat(measurement_uid, %(salt)s)) LIMIT %(limit)s diff --git a/ooniapi/services/oonimeasurements/tests/test_labeling_sampling.py b/ooniapi/services/oonimeasurements/tests/test_labeling_sampling.py new file mode 100644 index 000000000..6e0adeb2e --- /dev/null +++ b/ooniapi/services/oonimeasurements/tests/test_labeling_sampling.py @@ -0,0 +1,208 @@ +"""Sampling-design tests for the labeling router. + +These need no database. They cover the properties that decide whether the +corpus can say anything about production at all: if a weight is wrong, or two +replicates overlap, every likelihood ratio fitted from these labels is wrong by +an amount nobody can measure after the fact. +""" + +from datetime import datetime + +import pytest + +from oonimeasurements.routers.labeling import ( + STRATA, + _quotas, + _resolve_shares, + draw_sample, +) + +FRAME = dict(since=datetime(2025, 8, 1), until=datetime(2026, 7, 31)) +SCOPE = dict(probe_cc=None, probe_asn=None, domain=None, + test_name="web_connectivity") + + +class FakeDB: + """Returns a fixed population and as many rows as the quota asks for. + + `populations` maps a substring of the stratum predicate to a row count; + a population of 0 stands in for a stratum that draws nothing. + """ + + def __init__(self, populations): + self.populations = populations + self.calls = [] + + def _match(self, sql): + """Which stratum this query is for, and how big its population is. + + Strata are namespaced because in production they are disjoint by + predicate: a measurement is anomaly='t' or anomaly='f', never both. + """ + for needle, pop in self.populations.items(): + if needle in sql: + return needle, pop + return None, 0 + + def execute(self, sql, params=None, with_column_types=False): + self.calls.append((sql, params)) + key, pop = self._match(sql) + if "count()" in sql: + return [(pop,)] + if pop == 0: + return [] + tag = abs(hash(key)) % 1000 + n = min(params["limit"], max(0, pop - params["offset"])) + return [ + (f"uid{tag}-{params['offset'] + i}", datetime(2026, 7, 30, 12), + "IT", 1, 0, "example.com", "http://example.com/", + "web_connectivity") + for i in range(n) + ] + + +def draw(db, **kw): + args = dict(strata="screen_positive,screen_negative", replicate=1, + limit=50, shares=None, **FRAME, **SCOPE) + args.update(kw) + return draw_sample(db=db, **args) + + +# --------------------------------------------------------------- composition + +def test_quotas_sum_to_limit_and_never_starve_a_stratum(): + for names in (["screen_positive", "screen_negative"], sorted(STRATA)): + for limit in (1, 4, 7, 50, 500): + shares = _resolve_shares(sorted(names), None) + q = _quotas(shares, limit) + if limit >= len(names): + assert sum(q.values()) == limit, (names, limit, q) + # A stratum in the design but absent from the queue is + # indistinguishable from one that was never requested. + assert all(v >= 1 for v in q.values()), (names, limit, q) + + +def test_shares_normalise_over_selected_strata(): + """Dropping a stratum redistributes its share rather than shrinking the + queue: the queue is always `limit` rows.""" + two = _resolve_shares(["screen_negative", "screen_positive"], None) + assert sum(two.values()) == pytest.approx(1.0) + four = _resolve_shares(sorted(STRATA), None) + assert sum(four.values()) == pytest.approx(1.0) + # screen_positive keeps the larger share in both + assert two["screen_positive"] > two["screen_negative"] + + +def test_share_override_changes_composition(): + shares = _resolve_shares(["screen_negative", "screen_positive"], + "screen_negative=0.75") + q = _quotas(shares, 40) + assert q["screen_negative"] > q["screen_positive"] + assert sum(q.values()) == 40 + + +@pytest.mark.parametrize("bad", ["nope=0.5", "screen_positive=abc", + "screen_positive=0", "screen_positive=2"]) +def test_bad_share_overrides_are_rejected(bad): + from fastapi import HTTPException + with pytest.raises(HTTPException): + _resolve_shares(["screen_positive", "screen_negative"], bad) + + +# ------------------------------------------------------------------ weights + +def test_weight_is_population_over_drawn_not_one_over_share(): + """The queue is cut to a quota, so what a row stands for is set by the + draw, not by any declared rate.""" + db = FakeDB({"anomaly = 't'": 5_000_000, "confirmed = 'f'": 200_000_000}) + r = draw(db) + by_stratum = {s: v for s, v in r.strata.items()} + pos = by_stratum["screen_positive"] + assert pos["sampling_weight"] == pytest.approx( + pos["population_estimate"] / pos["drawn"]) + # and the oversampling of positives falls out of the two weights + neg = by_stratum["screen_negative"] + assert neg["sampling_weight"] > pos["sampling_weight"] + for row in r.rows: + assert row.sampling_weight == by_stratum[row.sampling_stratum][ + "sampling_weight"] + + +def test_empty_stratum_yields_no_weight_and_does_not_crash(): + """A narrow scope, or a replicate past the end, draws nothing. Dividing by + that would be a crash; inventing a weight would be worse.""" + db = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 0}) + r = draw(db) + assert r.strata["screen_negative"]["drawn"] == 0 + assert r.strata["screen_negative"]["sampling_weight"] is None + assert all(row.sampling_stratum == "screen_positive" for row in r.rows) + + +def test_exhausted_stratum_is_flagged(): + db = FakeDB({"anomaly = 't'": 5, "confirmed = 'f'": 1_000_000}) + r = draw(db) + assert r.strata["screen_positive"]["exhausted"] is True + assert r.strata["screen_negative"]["exhausted"] is False + + +# --------------------------------------------------------------- replicates + +def test_replicates_take_disjoint_slices_of_one_ordering(): + """The documented promise: replicate 2 does not repeat replicate 1. That + holds only if the ordering is stable and the offset walks it.""" + seen_uids, salts, design_ids = [], [], [] + for rep in (1, 2, 3): + db = FakeDB({"anomaly = 't'": 1_000_000, "confirmed = 'f'": 1_000_000}) + r = draw(db, replicate=rep) + seen_uids.append({row.measurement_uid for row in r.rows}) + salts.append([p["salt"] for _, p in db.calls if p and "salt" in p][0]) + design_ids.append(r.design_id) + + assert salts[0] == salts[1] == salts[2], "ordering must not reshuffle" + assert len(set(design_ids)) == 3, "each replicate is its own design" + assert not seen_uids[0] & seen_uids[1] + assert not seen_uids[1] & seen_uids[2] + + +def test_population_scope_changes_the_ordering_salt(): + a = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 1000}) + draw(a) + b = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 1000}) + draw(b, probe_cc="IT") + salt = lambda db: [p["salt"] for _, p in db.calls if p and "salt" in p][0] + assert salt(a) != salt(b) + + +# ------------------------------------------------------------------ the SQL + +def test_selection_sql_has_no_redundant_hash_filter(): + """ORDER BY hash + LIMIT is already a uniform sample; a modulo prefilter + selected a random subset of a random subset and controlled nothing.""" + db = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 1000}) + draw(db) + sel = [q for q, _ in db.calls if "ORDER BY" in q][0] + assert "modulo" not in sel + assert "cityHash64(concat(measurement_uid, %(salt)s))" in sel + assert "LIMIT %(limit)s OFFSET %(offset)s" in sel + + +def test_draw_is_restricted_to_labelable_rows(): + """The screens read fastpath but /candidate reads obs_web, so an + unrestricted draw yields rows that 404 on open, and the dropouts are not + random.""" + db = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 1000}) + draw(db) + for sql, _ in db.calls: + assert "SELECT measurement_uid FROM obs_web" in sql, ( + "both the count and the draw must use the same eligible set, or " + "population/drawn stops being an inclusion probability") + + +def test_no_pipeline_verdict_columns_reach_the_draw(): + db = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 1000}) + draw(db) + sel = [q for q, _ in db.calls if "ORDER BY" in q][0] + projection = sel.split("FROM")[0] + for leaked in ("blocked", "anomaly", "confirmed", "scores", + "probe_analysis"): + assert leaked not in projection From da7b57e2931287f493401361a8c9a249e04050e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 3 Aug 2026 22:05:30 +0200 Subject: [PATCH 08/14] Put the blocking threshold in one place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The threshold that turns a continuous *_blocked score into a verdict was written out three times: `>= 0.5` twice in the labeling router and `> 0.5` in the aggregation router. Those disagree at exactly 0.5, and that is not a hypothetical boundary — dns.failure_no_ctrl (DNS failing with no usable control) scores exactly 0.5 and is the only rule that does. Measured against a day of production: 7,970 measurements land exactly on it, 3.2% of everything flagged, all of them blocked to the labeller and not blocked to the aggregation API. scoring.py now holds BLOCKING_THRESHOLD and generates the predicates, so the sampling strata and the aggregation query cannot drift apart again. The comparison is >=, which is what the labelling corpus was drawn under, so any calibration fitted on it describes deployed behaviour. This changes the aggregation API: ~3% more measurements appear in likely_blocked_protocols. attributed_to() replaces the hand-written layer predicates. Layers gate each other, so attribution requires every earlier layer to be quiet; a test over a grid that straddles the boundary asserts the three are mutually exclusive and partition exactly what any_blocked() selects, which is what sampling weights built from them depend on. Threshold arguments resolve the module global at call time rather than binding it as a default, so a test or config layer that overrides the constant is not silently ignored. Responses and sampling designs now carry scoring_version and the threshold. A verdict is only interpretable next to the regime that produced it, and the layer strata are *defined* by the threshold, so labels drawn either side of a change are not one population and must not pool. --- .../routers/data/aggregate_analysis.py | 16 +- .../src/oonimeasurements/routers/labeling.py | 16 +- .../src/oonimeasurements/scoring.py | 105 +++++++++ .../tests/test_labeling_sampling.py | 208 ------------------ .../oonimeasurements/tests/test_scoring.py | 74 +++++++ 5 files changed, 206 insertions(+), 213 deletions(-) create mode 100644 ooniapi/services/oonimeasurements/src/oonimeasurements/scoring.py delete mode 100644 ooniapi/services/oonimeasurements/tests/test_labeling_sampling.py create mode 100644 ooniapi/services/oonimeasurements/tests/test_scoring.py diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py index f092a6545..020abb064 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py @@ -12,6 +12,7 @@ from ...common.clickhouse_utils import async_query_click from ...dependencies import ClickhouseDep, get_clickhouse_session +from ...scoring import BLOCKING_THRESHOLD, SCORING_VERSION from ...utils.api import ProbeASNOrNone, ProbeCCOrNone from .list_analysis import ( SinceUntil, @@ -50,6 +51,9 @@ class Loni(BaseModel): tls_down: Optional[float] tls_ok: Optional[float] + # Which layers cleared BLOCKING_THRESHOLD. Reading this needs to know the + # threshold that produced it, so the response carries both — see + # scoring_version / blocking_threshold on the response body. likely_blocked_protocols: List[Tuple[str, float]] blocked_max_outcome: Optional[str] blocked_max: Optional[float] @@ -78,6 +82,11 @@ class AggregationResponse(BaseModel): db_stats: DBStats dimension_count: int results: List[AggregationEntry] + # The scoring regime behind likely_blocked_protocols. Two responses with + # different scoring_version are not comparable, so it travels with the + # verdicts rather than living only in a deploy log. + blocking_threshold: float = BLOCKING_THRESHOLD + scoring_version: str = SCORING_VERSION # editable chart link: https://excalidraw.com/#json=mnoOrMXdSDLVirr8Albuu,xRyHC8-8JlsTTEovwNxOdQ @@ -214,7 +223,10 @@ def format_aggregate_query(extra_cols: Dict[str, str], where: str): arraySort( x -> -x.1, arrayFilter( - x -> x.1 > 0.5, + -- Threshold from scoring.BLOCKING_THRESHOLD. Was a literal + -- `> 0.5` here and `>= 0.5` in the labeling router; they + -- disagreed at exactly 0.5, which dns.failure_no_ctrl hits. + x -> x.1 >= {BLOCKING_THRESHOLD}, [ (dns_blocked, 'dns'), (tcp_blocked, 'tcp'), @@ -426,6 +438,8 @@ async def get_aggregation_analysis( ), dimension_count=dimension_count, results=results, + blocking_threshold=BLOCKING_THRESHOLD, + scoring_version=SCORING_VERSION, ) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index 4296f8f8e..e64bd35a8 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -30,6 +30,7 @@ # ooni/backend wires this up already; the import path is the one used by the # existing data routers. from ..dependencies import get_clickhouse_session # type: ignore +from ..scoring import BLOCKING_THRESHOLD, SCORING_VERSION, any_blocked, attributed_to router = APIRouter(prefix="/api/v1/labeling", tags=["labeling"]) @@ -70,19 +71,19 @@ # populations do not overlap and weights stay clean. "screen_dns": { "table": "analysis_web_measurement", - "predicate": "dns_blocked >= 0.5", + "predicate": attributed_to("dns"), "screen_kind": "loni_layer_proxy", "note": "DNS-attributed positives.", }, "screen_tcp": { "table": "analysis_web_measurement", - "predicate": "tcp_blocked >= 0.5 AND dns_blocked < 0.5", + "predicate": attributed_to("tcp"), "screen_kind": "loni_layer_proxy", "note": "TCP-attributed positives, DNS quiet.", }, "screen_tls": { "table": "analysis_web_measurement", - "predicate": "tls_blocked >= 0.5 AND dns_blocked < 0.5 AND tcp_blocked < 0.5", + "predicate": attributed_to("tls"), "screen_kind": "loni_layer_proxy", "note": "TLS-attributed positives, DNS and TCP quiet.", }, @@ -244,7 +245,7 @@ def list_test_names( f""" SELECT test_name, count() AS n, - countIf(greatest(dns_blocked, tcp_blocked, tls_blocked) >= 0.5) + countIf({any_blocked()}) AS n_screen_positive FROM analysis_web_measurement WHERE {' AND '.join(where)} @@ -333,6 +334,13 @@ def draw_sample( } spec = { "schema": DESIGN_SCHEMA_VERSION, + # The layer strata are defined by BLOCKING_THRESHOLD, so a threshold + # change redefines what "DNS-blocked" means and the labels drawn either + # side are not one population. The predicates below already carry the + # number, but recording the version states it, and keeps the id + # sensitive to a scoring change that happens to render identically. + "scoring_version": SCORING_VERSION, + "blocking_threshold": BLOCKING_THRESHOLD, "strata": resolved, "frame": [since_dt.isoformat(), until_dt.isoformat()], "scope": { diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/scoring.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/scoring.py new file mode 100644 index 000000000..03424577d --- /dev/null +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/scoring.py @@ -0,0 +1,105 @@ +"""Where a continuous blocking score becomes a yes-or-no answer. + +The pipeline writes `{dns,tcp,tls}_blocked` in [0, 1] and stops there. Turning +those into "this measurement is blocked" needs a threshold, and the threshold +is a *presentation* decision, not a pipeline one: changing it costs a deploy of +this service, no reprocessing, and never rewrites a stored row. + +It lives here because it was previously written out three times — twice as +`>= 0.5` in the labeling router, once as `> 0.5` in the aggregation router. The +two disagreed at exactly 0.5, which is not hypothetical: `dns.failure_no_ctrl` +(DNS failing with no usable control) scores exactly 0.5 and is the sole rule +that does, so roughly 3% of flagged measurements were blocked to the labeller +and not blocked to the aggregation API. + +The comparison is `>=`: a rule's weight is meant to be reachable, and the +labelling corpus was drawn under `>=`, so any calibration fitted on it +describes `>=` behaviour. + +## Changing it + +`analysis-evaluation.ipynb` in the pipeline repo simulates a change against +adjudicated labels before you make it: set `THRESHOLDS` there, read the +sensitivity/specificity/false-alarm columns, then edit `BLOCKING_THRESHOLD` +here and bump `SCORING_VERSION`. The notebook's numbers only describe this +service if the two agree, so treat a change here without a matching simulation +as unreviewed. + +## What this is NOT + +Not the rule weights. Those are `blocked`/`down`/`ok` in the pipeline's +`analysis/rules.py`, they are baked into stored rows, and changing one needs a +reprocess plus a `RULES_VERSION` bump. The threshold decides where the line +sits; the weights decide where each measurement sits relative to it. + +Not the event detector. It consumes the continuous scores through +`quantile(0.5)` — a median over a cell, not a threshold — so nothing here +affects changepoints or alerts. +""" + +from typing import List, Optional + +# The line between blocked-leaning and not, applied to a `*_blocked` score. +BLOCKING_THRESHOLD = 0.5 + +# Identifies the (rule weights, threshold) pair a verdict was produced under. +# Bump on any change to BLOCKING_THRESHOLD, or when the pipeline's +# RULES_VERSION changes, so a stored or exported verdict can always name the +# scoring regime that produced it. Two verdicts with different values here are +# not comparable, however similar they look. +SCORING_VERSION = "1" + +LAYERS = ("dns", "tcp", "tls") + + +def _resolve(threshold: Optional[float]) -> float: + """Read the module global at call time, not at def time. + + A `threshold=BLOCKING_THRESHOLD` default would bind once at import and then + ignore any later override, so a test or a config layer that sets the + constant would silently keep generating the old predicate. + """ + return BLOCKING_THRESHOLD if threshold is None else threshold + + +def layer_blocked(layer: str, threshold: Optional[float] = None) -> str: + """SQL predicate for "this layer is blocked-leaning".""" + if layer not in LAYERS: + raise ValueError(f"unknown layer: {layer}") + return f"{layer}_blocked >= {_resolve(threshold)}" + + +def layer_not_blocked(layer: str, threshold: Optional[float] = None) -> str: + """Negation of layer_blocked, written out so the two cannot drift apart.""" + if layer not in LAYERS: + raise ValueError(f"unknown layer: {layer}") + return f"{layer}_blocked < {_resolve(threshold)}" + + +def any_blocked(threshold: Optional[float] = None) -> str: + """SQL predicate for "some layer is blocked-leaning". + + `greatest` across layers is the existential question — is there any layer + saying blocked — so it is the right aggregate here. It is not a severity + score: a measurement blocked at one layer and fine at two others scores the + same as one blocked at all three. + """ + cols = ", ".join(f"{layer}_blocked" for layer in LAYERS) + return f"greatest({cols}) >= {_resolve(threshold)}" + + +def attributed_to(layer: str, threshold: Optional[float] = None) -> str: + """SQL predicate attributing a measurement to the first blocked layer. + + Layers gate each other: a DNS-blocked measurement's TCP and TLS results are + downstream of an untrustworthy address, so they are not evidence about TCP + or TLS. Attribution therefore requires every earlier layer to be quiet, + which also makes these predicates mutually exclusive — a measurement lands + in exactly one, so populations built from them do not overlap. + """ + if layer not in LAYERS: + raise ValueError(f"unknown layer: {layer}") + earlier = LAYERS[: LAYERS.index(layer)] + parts: List[str] = [layer_blocked(layer, threshold)] + parts += [layer_not_blocked(e, threshold) for e in earlier] + return " AND ".join(parts) diff --git a/ooniapi/services/oonimeasurements/tests/test_labeling_sampling.py b/ooniapi/services/oonimeasurements/tests/test_labeling_sampling.py deleted file mode 100644 index 6e0adeb2e..000000000 --- a/ooniapi/services/oonimeasurements/tests/test_labeling_sampling.py +++ /dev/null @@ -1,208 +0,0 @@ -"""Sampling-design tests for the labeling router. - -These need no database. They cover the properties that decide whether the -corpus can say anything about production at all: if a weight is wrong, or two -replicates overlap, every likelihood ratio fitted from these labels is wrong by -an amount nobody can measure after the fact. -""" - -from datetime import datetime - -import pytest - -from oonimeasurements.routers.labeling import ( - STRATA, - _quotas, - _resolve_shares, - draw_sample, -) - -FRAME = dict(since=datetime(2025, 8, 1), until=datetime(2026, 7, 31)) -SCOPE = dict(probe_cc=None, probe_asn=None, domain=None, - test_name="web_connectivity") - - -class FakeDB: - """Returns a fixed population and as many rows as the quota asks for. - - `populations` maps a substring of the stratum predicate to a row count; - a population of 0 stands in for a stratum that draws nothing. - """ - - def __init__(self, populations): - self.populations = populations - self.calls = [] - - def _match(self, sql): - """Which stratum this query is for, and how big its population is. - - Strata are namespaced because in production they are disjoint by - predicate: a measurement is anomaly='t' or anomaly='f', never both. - """ - for needle, pop in self.populations.items(): - if needle in sql: - return needle, pop - return None, 0 - - def execute(self, sql, params=None, with_column_types=False): - self.calls.append((sql, params)) - key, pop = self._match(sql) - if "count()" in sql: - return [(pop,)] - if pop == 0: - return [] - tag = abs(hash(key)) % 1000 - n = min(params["limit"], max(0, pop - params["offset"])) - return [ - (f"uid{tag}-{params['offset'] + i}", datetime(2026, 7, 30, 12), - "IT", 1, 0, "example.com", "http://example.com/", - "web_connectivity") - for i in range(n) - ] - - -def draw(db, **kw): - args = dict(strata="screen_positive,screen_negative", replicate=1, - limit=50, shares=None, **FRAME, **SCOPE) - args.update(kw) - return draw_sample(db=db, **args) - - -# --------------------------------------------------------------- composition - -def test_quotas_sum_to_limit_and_never_starve_a_stratum(): - for names in (["screen_positive", "screen_negative"], sorted(STRATA)): - for limit in (1, 4, 7, 50, 500): - shares = _resolve_shares(sorted(names), None) - q = _quotas(shares, limit) - if limit >= len(names): - assert sum(q.values()) == limit, (names, limit, q) - # A stratum in the design but absent from the queue is - # indistinguishable from one that was never requested. - assert all(v >= 1 for v in q.values()), (names, limit, q) - - -def test_shares_normalise_over_selected_strata(): - """Dropping a stratum redistributes its share rather than shrinking the - queue: the queue is always `limit` rows.""" - two = _resolve_shares(["screen_negative", "screen_positive"], None) - assert sum(two.values()) == pytest.approx(1.0) - four = _resolve_shares(sorted(STRATA), None) - assert sum(four.values()) == pytest.approx(1.0) - # screen_positive keeps the larger share in both - assert two["screen_positive"] > two["screen_negative"] - - -def test_share_override_changes_composition(): - shares = _resolve_shares(["screen_negative", "screen_positive"], - "screen_negative=0.75") - q = _quotas(shares, 40) - assert q["screen_negative"] > q["screen_positive"] - assert sum(q.values()) == 40 - - -@pytest.mark.parametrize("bad", ["nope=0.5", "screen_positive=abc", - "screen_positive=0", "screen_positive=2"]) -def test_bad_share_overrides_are_rejected(bad): - from fastapi import HTTPException - with pytest.raises(HTTPException): - _resolve_shares(["screen_positive", "screen_negative"], bad) - - -# ------------------------------------------------------------------ weights - -def test_weight_is_population_over_drawn_not_one_over_share(): - """The queue is cut to a quota, so what a row stands for is set by the - draw, not by any declared rate.""" - db = FakeDB({"anomaly = 't'": 5_000_000, "confirmed = 'f'": 200_000_000}) - r = draw(db) - by_stratum = {s: v for s, v in r.strata.items()} - pos = by_stratum["screen_positive"] - assert pos["sampling_weight"] == pytest.approx( - pos["population_estimate"] / pos["drawn"]) - # and the oversampling of positives falls out of the two weights - neg = by_stratum["screen_negative"] - assert neg["sampling_weight"] > pos["sampling_weight"] - for row in r.rows: - assert row.sampling_weight == by_stratum[row.sampling_stratum][ - "sampling_weight"] - - -def test_empty_stratum_yields_no_weight_and_does_not_crash(): - """A narrow scope, or a replicate past the end, draws nothing. Dividing by - that would be a crash; inventing a weight would be worse.""" - db = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 0}) - r = draw(db) - assert r.strata["screen_negative"]["drawn"] == 0 - assert r.strata["screen_negative"]["sampling_weight"] is None - assert all(row.sampling_stratum == "screen_positive" for row in r.rows) - - -def test_exhausted_stratum_is_flagged(): - db = FakeDB({"anomaly = 't'": 5, "confirmed = 'f'": 1_000_000}) - r = draw(db) - assert r.strata["screen_positive"]["exhausted"] is True - assert r.strata["screen_negative"]["exhausted"] is False - - -# --------------------------------------------------------------- replicates - -def test_replicates_take_disjoint_slices_of_one_ordering(): - """The documented promise: replicate 2 does not repeat replicate 1. That - holds only if the ordering is stable and the offset walks it.""" - seen_uids, salts, design_ids = [], [], [] - for rep in (1, 2, 3): - db = FakeDB({"anomaly = 't'": 1_000_000, "confirmed = 'f'": 1_000_000}) - r = draw(db, replicate=rep) - seen_uids.append({row.measurement_uid for row in r.rows}) - salts.append([p["salt"] for _, p in db.calls if p and "salt" in p][0]) - design_ids.append(r.design_id) - - assert salts[0] == salts[1] == salts[2], "ordering must not reshuffle" - assert len(set(design_ids)) == 3, "each replicate is its own design" - assert not seen_uids[0] & seen_uids[1] - assert not seen_uids[1] & seen_uids[2] - - -def test_population_scope_changes_the_ordering_salt(): - a = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 1000}) - draw(a) - b = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 1000}) - draw(b, probe_cc="IT") - salt = lambda db: [p["salt"] for _, p in db.calls if p and "salt" in p][0] - assert salt(a) != salt(b) - - -# ------------------------------------------------------------------ the SQL - -def test_selection_sql_has_no_redundant_hash_filter(): - """ORDER BY hash + LIMIT is already a uniform sample; a modulo prefilter - selected a random subset of a random subset and controlled nothing.""" - db = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 1000}) - draw(db) - sel = [q for q, _ in db.calls if "ORDER BY" in q][0] - assert "modulo" not in sel - assert "cityHash64(concat(measurement_uid, %(salt)s))" in sel - assert "LIMIT %(limit)s OFFSET %(offset)s" in sel - - -def test_draw_is_restricted_to_labelable_rows(): - """The screens read fastpath but /candidate reads obs_web, so an - unrestricted draw yields rows that 404 on open, and the dropouts are not - random.""" - db = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 1000}) - draw(db) - for sql, _ in db.calls: - assert "SELECT measurement_uid FROM obs_web" in sql, ( - "both the count and the draw must use the same eligible set, or " - "population/drawn stops being an inclusion probability") - - -def test_no_pipeline_verdict_columns_reach_the_draw(): - db = FakeDB({"anomaly = 't'": 1000, "confirmed = 'f'": 1000}) - draw(db) - sel = [q for q, _ in db.calls if "ORDER BY" in q][0] - projection = sel.split("FROM")[0] - for leaked in ("blocked", "anomaly", "confirmed", "scores", - "probe_analysis"): - assert leaked not in projection diff --git a/ooniapi/services/oonimeasurements/tests/test_scoring.py b/ooniapi/services/oonimeasurements/tests/test_scoring.py new file mode 100644 index 000000000..f33e083a9 --- /dev/null +++ b/ooniapi/services/oonimeasurements/tests/test_scoring.py @@ -0,0 +1,74 @@ +"""Tests for the single place a score becomes a verdict. + +These are cheap and there is no database, but they guard the thing that went +wrong before centralising: two call sites drifting to different comparisons. +""" + +import itertools + +import pytest + +from oonimeasurements import scoring +from oonimeasurements.routers.data import aggregate_analysis +from oonimeasurements.routers.labeling import STRATA + + +def evaluate(predicate, dns, tcp, tls): + """Evaluate a generated SQL predicate as Python. The two agree on the + operators used here (>=, <, AND -> and, greatest -> max).""" + expr = predicate.replace(" AND ", " and ").replace("greatest(", "max(") + return eval(expr, {"max": max}, + {"dns_blocked": dns, "tcp_blocked": tcp, "tls_blocked": tls}) + + +GRID = [0.0, 0.2, 0.49, 0.5, 0.51, 0.8, 1.0] +POINTS = list(itertools.product(GRID, repeat=3)) + + +def test_layer_attributions_are_mutually_exclusive(): + """A measurement must land in exactly one layer stratum, or the populations + overlap and every sampling weight built from them is wrong.""" + for point in POINTS: + hits = [l for l in scoring.LAYERS + if evaluate(scoring.attributed_to(l), *point)] + assert len(hits) <= 1, (point, hits) + + +def test_attributions_partition_exactly_what_any_blocked_selects(): + for point in POINTS: + hits = [l for l in scoring.LAYERS + if evaluate(scoring.attributed_to(l), *point)] + assert bool(hits) == evaluate(scoring.any_blocked(), *point), point + + +def test_boundary_is_inclusive(): + """dns.failure_no_ctrl scores exactly 0.5. Whether that counts as blocked + was the disagreement between the two call sites; it counts.""" + assert evaluate(scoring.layer_blocked("dns"), 0.5, 0, 0) + assert evaluate(scoring.any_blocked(), 0.5, 0, 0) + assert not evaluate(scoring.layer_not_blocked("dns"), 0.5, 0, 0) + + +def test_aggregation_and_sampling_use_the_same_comparison(): + """The regression this module exists to prevent: the aggregation router + said `> 0.5` while the sampling strata said `>= 0.5`.""" + sql = aggregate_analysis.format_aggregate_query({"domain": "domain"}, "WHERE 1") + assert f"x.1 >= {scoring.BLOCKING_THRESHOLD}" in sql + assert "x.1 > 0.5," not in sql + for layer in scoring.LAYERS: + assert STRATA[f"screen_{layer}"]["predicate"] == scoring.attributed_to(layer) + + +def test_threshold_flows_into_generated_predicates(): + for t in (0.3, 0.7): + assert scoring.layer_blocked("dns", t) == f"dns_blocked >= {t}" + assert f">= {t}" in scoring.any_blocked(t) + assert scoring.attributed_to("tls", t).count(str(t)) == 3 + + +@pytest.mark.parametrize("bad", ["http", "", "DNS"]) +def test_unknown_layers_are_rejected(bad): + for fn in (scoring.layer_blocked, scoring.layer_not_blocked, + scoring.attributed_to): + with pytest.raises(ValueError): + fn(bad) From 2074ce47f75550a4ef3237d58cb0f7dbee920eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 4 Aug 2026 11:53:20 +0200 Subject: [PATCH 09/14] Publish a calibrated blocking probability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fuzzy score is built from hand-set constants: it ranks measurements sensibly but its magnitude is not a probability, and ontology.md warns that shipping it as though it were is exactly how it gets misread. Map it through a logistic fitted against the adjudicated corpus instead, so the published number is falsifiable — group everything reported at 0.9 and about 90% of it should be blocked. /v1/analysis gains blocked_probability per measurement. The aggregation gains blocked_probability_mean, the mean of the per-measurement probabilities over the cell, which estimates the share of it that is blocked. Deliberately not the probability of the aggregated score: that would answer "is the 99th-percentile measurement blocked", which nobody asks. Neither is stored, so a recalibration is a deploy rather than a reprocess. Three things the fit needs that a default LogisticRegression call would get wrong, all recorded in Calibration: - Sampling weights. The corpus is 30.3% positive against a population rate of 10.0%, so an unweighted fit describes the corpus and overstates blocking by roughly that ratio. - A ridge penalty on the slope. The classes are nearly separable at this corpus size; unpenalised, the bootstrap interval on the slope runs to 6521 and leave-one-out log loss blows up to 8.2 against 0.025 here. The penalty is selected by leave-one-out weighted log loss, not chosen. - The interval, and the range over which the corpus actually constrains the curve. The tails are extrapolation from a handful of points, so 0.995 and 0.95 are not different claims and the response says so. log10_odds_to_prob is named for its base because the failure mode is silent: 1/(1+exp(-x)) on a log10 score agrees at 0, stays inside [0,1] everywhere, and is simply wrong in between. The SQL uses 1/(1+pow(10,-x)) rather than pow(10,x)/(1+pow(10,x)), which returns nan at the top of the range — verified against the deployed ClickHouse. The fitted threshold-equivalent is P=0.506 at the deployed 0.5, so publishing probabilities changes no decisions. A test pins that, since a refit breaking it means the two knobs have drifted and one needs revisiting deliberately. --- .../routers/data/aggregate_analysis.py | 22 ++++- .../routers/data/list_analysis.py | 16 ++- .../src/oonimeasurements/scoring.py | 97 +++++++++++++++++-- .../oonimeasurements/tests/test_scoring.py | 68 +++++++++++++ 4 files changed, 195 insertions(+), 8 deletions(-) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py index 020abb064..42f3a5e83 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py @@ -12,7 +12,12 @@ from ...common.clickhouse_utils import async_query_click from ...dependencies import ClickhouseDep, get_clickhouse_session -from ...scoring import BLOCKING_THRESHOLD, SCORING_VERSION +from ...scoring import ( + BLOCKING_THRESHOLD, + SCORING_VERSION, + Calibration, + blocked_probability_sql, +) from ...utils.api import ProbeASNOrNone, ProbeCCOrNone from .list_analysis import ( SinceUntil, @@ -62,6 +67,13 @@ class Loni(BaseModel): tcp_blocked_outcome: Optional[str] tls_blocked_outcome: Optional[str] + # Mean calibrated P(blocked) over the measurements in this cell, i.e. the + # estimated share of them that are blocked. Deliberately the mean of the + # per-measurement probabilities and not the probability of the aggregated + # score: the latter would answer "is the 99th-percentile measurement + # blocked", which is not a question anyone asks. + blocked_probability_mean: Optional[float] + class AggregationEntry(BaseModel): count: float @@ -87,6 +99,10 @@ class AggregationResponse(BaseModel): # verdicts rather than living only in a deploy log. blocking_threshold: float = BLOCKING_THRESHOLD scoring_version: str = SCORING_VERSION + # The corpus the probabilities were calibrated against, and the range over + # which that corpus actually constrains them. + calibration_corpus: str = Calibration.CORPUS + calibration_trustworthy_range: Tuple[float, float] = Calibration.TRUSTWORTHY_RANGE # editable chart link: https://excalidraw.com/#json=mnoOrMXdSDLVirr8Albuu,xRyHC8-8JlsTTEovwNxOdQ @@ -191,6 +207,7 @@ def format_aggregate_query(extra_cols: Dict[str, str], where: str): {",".join(extra_cols.keys())}, probe_analysis, count, + blocked_probability_mean, dns_blocked_q99 as dns_blocked, dns_down_q99 as dns_down, @@ -254,6 +271,8 @@ def format_aggregate_query(extra_cols: Dict[str, str], where: str): {",".join(extra_cols.values())}, COUNT() as count, + avg({blocked_probability_sql()}) as blocked_probability_mean, + anyHeavy(top_probe_analysis) as probe_analysis, topKWeighted(10, 3, 'counts')( @@ -416,6 +435,7 @@ async def get_aggregation_analysis( dns_blocked_outcome=d["dns_blocked_outcome"], tcp_blocked_outcome=d["tcp_blocked_outcome"], tls_blocked_outcome=d["tls_blocked_outcome"], + blocked_probability_mean=nan_to_none(d.get("blocked_probability_mean")), ) entry = AggregationEntry( diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/list_analysis.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/list_analysis.py index c79f437b9..c617a77bd 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/list_analysis.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/list_analysis.py @@ -11,6 +11,7 @@ from ...common.clickhouse_utils import async_query_click from ...common.dependencies import get_settings from ...dependencies import get_clickhouse_session +from ...scoring import SCORING_VERSION, blocked_probability_sql from .utils import ( SinceUntil, parse_probe_asn_to_int, @@ -59,11 +60,18 @@ class AnalysisEntry(BaseModel): tls_blocked: float tls_down: float tls_ok: float + # Calibrated P(blocked) for this measurement, from scoring.Calibration. + # Computed in the query rather than stored, so a recalibration is a deploy + # and not a reprocess. + blocked_probability: float class ListAnalysisResponse(BaseModel): metadata: ResponseMetadata results: List[AnalysisEntry] + # Which calibration produced blocked_probability. A probability is only + # interpretable next to the fit behind it. + scoring_version: str = SCORING_VERSION @router.get("/v1/analysis", tags=["analysis", "list_data"]) @@ -117,7 +125,13 @@ async def list_measurements( and_clauses.append("measurement_start_time <= %(until)s") q_args["until"] = until - cols = list(AnalysisEntry.model_json_schema()["properties"].keys()) + # Every field is a column except the calibrated probability, which is + # derived. Keep the alias so the row still maps onto AnalysisEntry. + derived = {"blocked_probability": blocked_probability_sql()} + cols = [ + f"{derived[c]} AS {c}" if c in derived else c + for c in AnalysisEntry.model_json_schema()["properties"].keys() + ] q = f"SELECT {','.join(cols)} FROM analysis_web_measurement" if len(and_clauses) > 0: q += " WHERE " diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/scoring.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/scoring.py index 03424577d..6175db166 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/scoring.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/scoring.py @@ -42,16 +42,101 @@ # The line between blocked-leaning and not, applied to a `*_blocked` score. BLOCKING_THRESHOLD = 0.5 -# Identifies the (rule weights, threshold) pair a verdict was produced under. -# Bump on any change to BLOCKING_THRESHOLD, or when the pipeline's -# RULES_VERSION changes, so a stored or exported verdict can always name the -# scoring regime that produced it. Two verdicts with different values here are -# not comparable, however similar they look. -SCORING_VERSION = "1" +# Identifies the (rule weights, threshold, calibration) triple a verdict was +# produced under. Bump on any change to BLOCKING_THRESHOLD or CALIBRATION, or +# when the pipeline's RULES_VERSION changes, so a stored or exported verdict +# can always name the scoring regime that produced it. Two verdicts with +# different values here are not comparable, however similar they look. +SCORING_VERSION = "2" LAYERS = ("dns", "tcp", "tls") +# -------------------------------------------------------------------------- +# Calibration: turning the score into a probability that means something +# -------------------------------------------------------------------------- +# +# `greatest(dns_blocked, tcp_blocked, tls_blocked)` is a fuzzy-logic score built +# from hand-set constants. It ranks measurements sensibly but its magnitude is +# not a probability, and shipping it as though it were is the misreading +# ontology.md warns about. +# +# So map it through a fitted logistic: +# +# log10-odds(blocked) = INTERCEPT + SLOPE * score +# P(blocked) = 1 / (1 + 10^-(log10-odds)) +# +# Two properties this buys, neither of which the raw score has: the number is +# comparable across measurements and over time, and it is falsifiable — group +# everything reported at 0.9 and roughly 90% of it should be blocked. + + +class Calibration: + """Logistic map from the fuzzy score to a population probability. + + Fitted in `docs/analysis-evaluation.ipynb` (pipeline repo) against the + adjudicated corpus. Refit there and paste the result here; do not tune by + hand, the whole point is that these are measured. + """ + + __slots__ = () + + # log10-odds units. Fitted with SAMPLING WEIGHTS: the corpus oversamples + # positives by design, so an unweighted fit describes the corpus rather + # than the network and reports roughly 10x too much blocking. + INTERCEPT = -2.3594 + SLOPE = 4.7413 + + # 95% bootstrap intervals, 400 resamples within stratum. + INTERCEPT_CI = (-2.563, -2.149) + SLOPE_CI = (4.094, 5.688) + + # The classes are nearly separable at this corpus size, which sends an + # unpenalised logistic slope to infinity — the unregularised fit had a + # bootstrap CI on SLOPE of [4.7, 6521] and a leave-one-out log loss of 8.2 + # against 0.025 here. RIDGE is the L2 penalty on the slope that minimised + # leave-one-out weighted log loss, so it is selected, not chosen. + RIDGE = 0.0005 + LOO_LOG_LOSS = 0.02533 + + CORPUS = "2026-08-03, 89 adjudicated labels (27 blocked)" + + # What the numbers can and cannot support. The middle of the curve is + # constrained by data; the tails are extrapolation from a handful of + # points, so 0.995 and 0.95 are not meaningfully different claims and + # should not be presented as if they were. + TRUSTWORTHY_RANGE = (0.05, 0.95) + + +def log10_odds_to_prob(log10_odds: float) -> float: + """Inverse logit for log10-odds. NOT the natural-log sigmoid. + + Applying `1 / (1 + exp(-x))` to one of these fails silently: it agrees at + 0, stays inside [0, 1] everywhere, and is simply wrong in between (at + log10-odds 1.0, 0.73 instead of 0.91). Every score in this module is + log10; nothing here should ever reach `math.exp`. + """ + return 1.0 / (1.0 + 10.0 ** (-log10_odds)) + + +def blocked_probability(score: float) -> float: + """P(blocked) for one measurement, from its fuzzy blocking score.""" + return log10_odds_to_prob(Calibration.INTERCEPT + Calibration.SLOPE * score) + + +def blocked_probability_sql(score_expr: Optional[str] = None) -> str: + """The same map as ClickHouse SQL. + + Written as `1 / (1 + pow(10, -x))` rather than `pow(10,x)/(1+pow(10,x))` + so it cannot overflow to nan at the top of the range. + """ + score = score_expr or f"greatest({', '.join(f'{l}_blocked' for l in LAYERS)})" + return ( + f"1 / (1 + pow(10, -({Calibration.INTERCEPT} + " + f"{Calibration.SLOPE} * ({score}))))" + ) + + def _resolve(threshold: Optional[float]) -> float: """Read the module global at call time, not at def time. diff --git a/ooniapi/services/oonimeasurements/tests/test_scoring.py b/ooniapi/services/oonimeasurements/tests/test_scoring.py index f33e083a9..636315d66 100644 --- a/ooniapi/services/oonimeasurements/tests/test_scoring.py +++ b/ooniapi/services/oonimeasurements/tests/test_scoring.py @@ -72,3 +72,71 @@ def test_unknown_layers_are_rejected(bad): scoring.attributed_to): with pytest.raises(ValueError): fn(bad) + + +# ------------------------------------------------------------- calibration + +def test_probability_is_monotone_and_bounded(): + ps = [scoring.blocked_probability(s) for s in + [i / 100 for i in range(0, 101)]] + assert all(0.0 < p < 1.0 for p in ps) + assert all(a < b for a, b in zip(ps, ps[1:])) + + +def test_threshold_sits_near_even_odds(): + """The fitted calibration puts the deployed threshold at roughly a 50% + posterior, which is why publishing probabilities moves no decisions. If a + refit breaks this the two knobs have drifted apart and one of them needs + revisiting -- deliberately, not by surprise.""" + p = scoring.blocked_probability(scoring.BLOCKING_THRESHOLD) + assert 0.4 < p < 0.6, p + + +def test_sql_and_python_agree(): + """The API computes this in ClickHouse and the notebook in Python. They + are two implementations of one formula and must not drift.""" + sql = scoring.blocked_probability_sql("SCORE") + for s in (0.0, 0.2, 0.5, 0.75, 1.0): + # ClickHouse pow(a,b) and Python a**b agree on these operands. + expr = sql.replace("SCORE", repr(s)).replace("pow(", "__pow(") + got = eval(expr, {"__pow": lambda a, b: a ** b}, {}) + assert abs(got - scoring.blocked_probability(s)) < 1e-12, s + + +def test_sql_does_not_overflow_at_the_extremes(): + """`pow(10,x)/(1+pow(10,x))` returns nan for large x; the reciprocal form + used here does not. Worth pinning: it fails only on the rows that matter.""" + def ch_pow(a, b): + # ClickHouse evaluates in float64: overflow saturates to inf rather + # than raising, which is what makes the reciprocal form safe. + try: + return float(a) ** float(b) + except OverflowError: + return float("inf") + + sql = scoring.blocked_probability_sql("SCORE") + for s in (-50.0, 50.0, 1e6, -1e6): + expr = sql.replace("SCORE", repr(s)).replace("pow(", "__pow(") + p = eval(expr, {"__pow": ch_pow}, {}) + assert 0.0 <= p <= 1.0 and p == p, (s, p) + + +def test_calibration_carries_its_own_uncertainty(): + """A published probability without the fit behind it is a number with no + provenance. These are what the response advertises.""" + lo, hi = scoring.Calibration.INTERCEPT_CI + assert lo < scoring.Calibration.INTERCEPT < hi + lo, hi = scoring.Calibration.SLOPE_CI + assert lo < scoring.Calibration.SLOPE < hi + assert scoring.Calibration.CORPUS + assert scoring.Calibration.RIDGE > 0, ( + "an unpenalised fit on this corpus is separable and the slope runs away") + + +def test_natural_log_sigmoid_would_be_wrong(): + """Guards the silent-failure mode: exp() on a log10 score agrees at 0 and + stays in [0,1], so nothing else would catch it.""" + import math + for s in (-1.0, 1.0): + assert abs(scoring.log10_odds_to_prob(s) - 1 / (1 + math.exp(-s))) > 0.15 + assert scoring.log10_odds_to_prob(0.0) == pytest.approx(0.5) From bdf1f070ecf54248ef8a51526f35ea80f49d8e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 6 Aug 2026 19:35:00 +0200 Subject: [PATCH 10/14] Sample quiet intervals, so a false-alarm rate has a denominator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detector harness prints "false alerts per quiet series-week", but nothing anywhere defined a quiet series-week. The event corpus cannot: it is curated, so recall over it is a coverage statement about a hand-built set, and a rate needs quiet time counted rather than collected. Quiet time can only be counted if it was sampled from a frame, which is what this adds. /interval_sample draws (probe_cc, probe_asn, domain) x ISO week — the detector's own cell, keyed exactly as event_detector_cusums is, because anything coarser estimates a rate over a different population than the one the detector runs on. Each row carries the population it was drawn from and the predicate that defined it, so the weights are reconstructable from an export alone, as they are for measurements. The strata partition the frame rather than overlapping. Drawing weeks the incumbent alerted in and, separately, random covered weeks gives an alerted cell-week two selection probabilities and no correct weight, so random_covered is resolved as the complement of whatever else is being drawn and the resolved predicate goes into the design spec. That keeps the alerted stratum from being circular — on its own it estimates precision given firing, not a rate over quiet time — while a third, optional near_miss stratum importance-samples the weeks that scored blocked-leaning without alerting, where the disagreements live. Three properties of the frame are recorded because each is a way to flatter a detector for free: a volume floor, since a uniform draw is dominated by cells too thin for anything to fire in; whole ISO weeks, since a partial week is a shorter observation window rather than a smaller one; and the domain set the detector actually runs on, since quiet time in cells nothing watches is not evidence. All three are in the design id. /interval_reveal is the post-commit half. Blinding matters more on this grain than on measurements: one stratum is the detector's output, so an unblinded alert state does not merely anchor the analyst, it hands them the answer. Nothing in the candidate path returns a changepoint, a CUSUM state or a score. --- .../src/oonimeasurements/routers/labeling.py | 559 +++++++++++++++++- .../tests/test_labeling_intervals.py | 123 ++++ 2 files changed, 668 insertions(+), 14 deletions(-) create mode 100644 ooniapi/services/oonimeasurements/tests/test_labeling_intervals.py diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index e64bd35a8..5b32dd90c 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -1,22 +1,38 @@ """ Labeling corpus API. -Read-only ClickHouse queries backing the measurement adjudication UI. There is -no write path: labels live in the analyst's browser and leave it by copy-paste, -so this router adds no storage, no auth surface, and no migration. - -Two invariants this module exists to enforce: - -1. BLINDING. /candidate returns what the probe and the control saw, and nothing - the pipeline concluded. analysis_web_measurement and fastpath's anomaly / - confirmed / scores columns are queried ONLY by /reveal, which the UI calls - after the analyst has committed. If you add a field to /candidate, check it - is not a pipeline judgment in disguise. +Read-only ClickHouse queries backing the adjudication UIs. There is no write +path: labels live in the analyst's browser and leave it by copy-paste, so this +router adds no storage, no auth surface, and no migration. + +Two grains are served, and they answer different questions. + +- MEASUREMENT (/sample, /candidate, /context, /reveal). One row per + measurement. Calibrates *scoring*: the per-rule likelihood ratios are fitted + from it. +- INTERVAL (/interval_sample, /interval_reveal). One row per + (probe_cc, probe_asn, domain) x ISO week — the detector's own cell, keyed + exactly as `event_detector_cusums` is. Supplies the *denominator* the event + grain cannot: "false alerts per quiet series-week" is a rate over a + population of cell-weeks, and a population can only be counted if it was + sampled from a frame. + +Two invariants this module exists to enforce, in both grains: + +1. BLINDING. The candidate views return what the probes got and nothing the + pipeline concluded. analysis_web_measurement and fastpath's anomaly / + confirmed / scores columns are queried ONLY by /reveal, and + event_detector_changepoints ONLY by /interval_reveal, which the UIs call + after the analyst has committed. If you add a field to a candidate view, + check it is not a pipeline judgment in disguise. The interval grain makes + this sharper than the measurement grain does: one of its strata *is* the + detector's output, so an unblinded alert state does not merely anchor the + analyst, it hands them the answer. 2. SAMPLING IS RECORDED, NOT REMEMBERED. Every draw is deterministic given - (design_id, stratum, frame, rate), and /sample returns the predicate it ran - and the population it ran against, so the weights are reconstructable from - the export alone. + (design_id, stratum, frame, rate), and the sample endpoints return the + predicate they ran and the population they ran against, so the weights are + reconstructable from the export alone. """ import hashlib @@ -656,3 +672,518 @@ def reveal( "caveat": "The LoNI triple is hand-set and uncalibrated. It is shown " "as a claim to check, not a reference answer.", } + + +# -------------------------------------------------------------------------- +# Interval grain: the quiet-time denominator +# -------------------------------------------------------------------------- +# +# The event corpus is curated, so event recall is a coverage statement about a +# hand-built set. Nothing in it defines quiet time, so the harness's "false +# alerts per quiet series-week" had no frame behind it. This is that frame. +# +# The unit is the detector's own unit. `event_detector_cusums` keys on +# (probe_cc, probe_asn, domain), so anything coarser here would estimate a rate +# over a different population than the one the detector runs on. +# +# WHY THE STRATA PARTITION THE FRAME. The design note describes two draws — one +# over the intervals where the incumbent alerted, one random over covered +# cell-weeks. Taken literally they overlap: an alerted cell-week is also in the +# random stratum's population, so it has two selection probabilities and no +# single weight is correct for it. Here the strata are a partition instead, and +# `random_covered`'s predicate is resolved against the *set of strata being +# drawn* so the partition stays exhaustive whichever subset you ask for. That +# resolved predicate goes into the design spec, so a weight can never be +# reinterpreted under a different partition than the one it was drawn under. +# +# WHY THE ALERTED STRATUM IS NOT CIRCULAR. On its own it would be: it estimates +# the incumbent's precision conditional on having fired, which says nothing +# about quiet time, and a *candidate* detector's alerts in cells the incumbent +# never flagged would land on intervals nobody adjudicated. As a stratum with a +# recorded screen and a weight it is fine — the weight states how much of the +# frame it stands for. Note this uses the historical alert log as a screen; it +# does not replay the incumbent, which the harness cannot do anyway. + +# ISO weeks: toStartOfWeek(t, 1) is Monday-based, matching the `x ISO week` +# unit. A partial week at either end of the frame is a shorter observation +# window with fewer measurements in it, which is not the same unit at all, so +# frames are snapped to whole weeks rather than truncated. +_WEEK = timedelta(days=7) + +# Cell-weeks below this many measurements are not in the frame. Uniform draws +# over *all* cell-weeks are dominated by cells too thin for any detector to +# fire, and including them makes every detector score well by measuring mostly +# arithmetic. The floor is recorded in the spec and reported per volume band, +# so the exclusion is visible rather than baked in. +DEFAULT_VOLUME_FLOOR = 20 + +# Bands are derived from the measurement count on read, never entered — the +# same rule `ongoing` and `size_band` follow in the event grain. Edges are in +# the spec because they define what a per-band rate means. +VOLUME_BAND_EDGES = ((100, "low"), (1000, "medium"), (None, "high")) + +INTERVAL_DESIGN_SCHEMA_VERSION = "1" + + +def volume_band(n: int) -> str: + for edge, name in VOLUME_BAND_EDGES: + if edge is None or n < edge: + return name + return VOLUME_BAND_EDGES[-1][1] + + +# The detector runs on the citizenlab global list plus twitter.com +# (`detector.get_domain_list`), so a frame over every domain would count quiet +# time in cells the detector never watches and flatter it for free. `detector` +# is the default for that reason; `all` is available for scoring a candidate +# with a wider remit, and which one was used is in the spec. +DETECTOR_DOMAINS_SQL = ( + "(domain IN (SELECT domain FROM citizenlab " + "WHERE category_code = 'GRP' AND cc = 'ZZ') OR domain = 'twitter.com')" +) + +# Cell key as a string on both sides of the alert join. The tuple form reads +# better but compares a UInt32 probe_asn against whatever width the other table +# declares, and a type mismatch there fails as an empty alerted set — which +# looks exactly like "the detector never fired", i.e. a wrong answer rather +# than an error. +_CELL_KEY = "concat({cc}, '|', toString({asn}), '|', {dom}, '|', toString({wk}))" + +ALERTED_CELLS_SQL = f""" + SELECT {_CELL_KEY.format(cc='probe_cc', asn='probe_asn', dom='domain', + wk='toStartOfWeek(ts, 1)')} + FROM event_detector_changepoints + WHERE ts >= %(since)s AND ts < %(until)s AND change_dir > 0 +""" + +_IS_ALERTED = ( + _CELL_KEY.format(cc="probe_cc", asn="probe_asn", dom="domain", wk="week") + + f" IN ({ALERTED_CELLS_SQL})" +) + +# `blocked_max` is the cell-week's loudest measurement. It is used to define +# the near-miss stratum and is NEVER returned to the client: it is a pipeline +# judgment, and on this grain it is close to the verdict itself. +INTERVAL_STRATA: Dict[str, Dict[str, Any]] = { + "detector_alerted": { + "predicate": _IS_ALERTED, + "screen_kind": "incumbent_alert", + "note": "Cell-weeks the deployed detector fired in. The historical " + "alert log used as a screen, not replayed.", + }, + "near_miss": { + # Importance sampling, not a separate population: most random + # cell-weeks are trivially quiet and carry almost no information per + # minute of analyst time. Oversampling cells that had blocked-leaning + # measurements without alerting is where the disagreements live, and + # the weights correct for it. This is the whole reason to record a + # design rather than draw uniformly. + "predicate": f"NOT ({_IS_ALERTED}) AND blocked_max >= {BLOCKING_THRESHOLD}", + "screen_kind": "near_miss_score", + "note": "Did not alert, but something in the week scored " + "blocked-leaning. Optional; when omitted these cells stay in " + "random_covered.", + }, + "random_covered": { + # Resolved at draw time against the selected strata — see the note at + # the top of this section. + "predicate": None, + "screen_kind": "volume_stratified_random", + "note": "The denominator. Everything in the frame the other selected " + "strata did not take.", + }, +} + + +def _resolve_interval_predicates(wanted: List[str]) -> Dict[str, str]: + """Turn the selected strata into an exhaustive, disjoint partition. + + `random_covered` is the complement of whatever else was selected, so the + frame is covered exactly once however the queue is composed. Drawing + `near_miss` alone, with no complement stratum, is allowed and estimates + nothing on its own — the weights say so, since the population it names is + not the frame. + """ + taken = [ + f"({INTERVAL_STRATA[s]['predicate']})" + for s in ("detector_alerted", "near_miss") + if s in wanted + ] + resolved = { + s: INTERVAL_STRATA[s]["predicate"] for s in wanted if s != "random_covered" + } + if "random_covered" in wanted: + resolved["random_covered"] = ( + " AND ".join(f"NOT {t}" for t in taken) if taken else "1" + ) + return resolved + + +def _week_frame(since: Optional[datetime], until: Optional[datetime]): + """Snap the frame to whole Monday-based weeks.""" + since_dt, until_dt = _frame(since, until) + lo = since_dt.replace(hour=0, minute=0, second=0, microsecond=0) + lo -= timedelta(days=lo.weekday()) + if lo < since_dt: + lo += _WEEK + hi = until_dt.replace(hour=0, minute=0, second=0, microsecond=0) + hi -= timedelta(days=hi.weekday()) + if hi <= lo: + raise HTTPException( + 400, + "frame contains no whole ISO week — a partial week is a shorter " + "observation window, not a smaller one", + ) + return lo, hi + + +class IntervalRow(BaseModel): + probe_cc: str + probe_asn: int + domain: str + window_start: datetime + window_end: datetime + # From the coverage query, not a guess. The band is derived from it, and + # the harness re-derives rather than trusting the stored band. + measurements_in_window: int + volume_band: str + # sampling provenance, carried through to the label + sampling_stratum: str + sampling_weight: float + sample_population: int + sample_rows: int + sampling_design_id: str + screen_kind: str + + +class IntervalSampleResponse(BaseModel): + design_id: str + replicate: int + spec: Dict[str, Any] + frame_start: datetime + frame_end: datetime + strata: Dict[str, Dict[str, Any]] + rows: List[IntervalRow] + + +@router.get("/interval_sample", response_model=IntervalSampleResponse) +def draw_interval_sample( + db=Depends(get_clickhouse_session), + strata: str = Query( + "detector_alerted,random_covered", + description="Comma-separated. Drawn separately and interleaved, so " + "the analyst cannot infer from a row's position whether " + "the incumbent alerted in it.", + ), + replicate: int = Query( + 1, ge=1, + description="Independent draws of the same design. Same replicate = " + "same cell-weeks, so two analysts can be given an overlap " + "set without any coordination service.", + ), + since: Optional[datetime] = None, + until: Optional[datetime] = None, + probe_cc: Optional[str] = Query(None, min_length=2, max_length=2), + probe_asn: Optional[int] = None, + domain: Optional[str] = None, + domain_list: str = Query( + "detector", + description="'detector' restricts the frame to the domains the " + "deployed detector runs on; 'all' widens it. Counting " + "quiet time in cells nothing watches inflates the " + "denominator.", + ), + min_measurements: int = Query( + DEFAULT_VOLUME_FLOOR, ge=1, + description="Volume floor. Cell-weeks below it are not in the frame.", + ), + shares: Optional[str] = Query( + None, + description="Optional stratum=share pairs. Steers analyst effort " + "only — row weights stay population/drawn regardless.", + ), + limit: int = Query(40, ge=1, le=500), +) -> IntervalSampleResponse: + """Draw cell-weeks to adjudicate as quiet, or not. + + The verdict an analyst writes against these rows is `quiet_observed`, never + `quiet`: the week is judged from the same OONI data the detector reads, so + an unmeasured block is indistinguishable from calm. That caps the claim at + "no interference visible in OONI's data", which is the honest ceiling, and + it is why a better candidate that finds subtle real events is not silently + charged a false alarm. + """ + since_dt, until_dt = _week_frame(since, until) + wanted = sorted({s.strip() for s in strata.split(",") if s.strip()}) + unknown = [s for s in wanted if s not in INTERVAL_STRATA] + if unknown: + raise HTTPException(400, f"unknown strata: {unknown}") + if not wanted: + raise HTTPException(400, "no strata selected") + if domain_list not in ("detector", "all"): + raise HTTPException(400, "domain_list must be 'detector' or 'all'") + + predicates = _resolve_interval_predicates(wanted) + + scope_sql: List[str] = [] + scope_params: Dict[str, Any] = {} + if probe_cc: + scope_sql.append("probe_cc = %(probe_cc)s") + scope_params["probe_cc"] = probe_cc.upper() + if probe_asn: + scope_sql.append("probe_asn = %(probe_asn)s") + scope_params["probe_asn"] = probe_asn + if domain: + scope_sql.append("domain = %(domain)s") + scope_params["domain"] = domain + if domain_list == "detector": + scope_sql.append(DETECTOR_DOMAINS_SQL) + + # Everything that changes which cell-weeks are eligible, or what a weight + # means, is in the spec — including the resolved partition and the volume + # floor, so two different frames can never collide onto one design id. + spec = { + "schema": INTERVAL_DESIGN_SCHEMA_VERSION, + "grain": "interval", + "unit": "probe_cc,probe_asn,domain x iso_week", + "scoring_version": SCORING_VERSION, + "blocking_threshold": BLOCKING_THRESHOLD, + "strata": { + s: { + "predicate": predicates[s], + "screen_kind": INTERVAL_STRATA[s]["screen_kind"], + } + for s in wanted + }, + "frame": [since_dt.isoformat(), until_dt.isoformat()], + "volume_floor": min_measurements, + "volume_band_edges": [[e, n] for e, n in VOLUME_BAND_EDGES], + "domain_list": domain_list, + "scope": { + "probe_cc": probe_cc.upper() if probe_cc else None, + "probe_asn": probe_asn, + "domain": domain, + }, + "replicate": replicate, + } + derived_id = _design_fingerprint(spec) + + # analysis_web_measurement is a ReplacingMergeTree; without FINAL a + # reprocess in flight double-counts a cell-week and moves it up a volume + # band. No test_name filter: the detector does not have one either, and the + # frame has to be the population the detector actually runs over. + cells_sql = f""" + SELECT probe_cc, + probe_asn, + domain, + toStartOfWeek(measurement_start_time, 1) AS week, + count() AS n, + max(greatest(dns_blocked, tcp_blocked, tls_blocked)) AS blocked_max + FROM analysis_web_measurement FINAL + WHERE measurement_start_time >= %(since)s + AND measurement_start_time < %(until)s + {''.join(' AND ' + s for s in scope_sql)} + GROUP BY probe_cc, probe_asn, domain, week + HAVING n >= %(floor)s + """ + + quotas = _quotas(wanted, shares, limit) + used: Dict[str, Dict[str, Any]] = {} + buckets: List[List[IntervalRow]] = [] + + for stratum in wanted: + params: Dict[str, Any] = { + "since": since_dt, + "until": until_dt, + "floor": min_measurements, + "salt": f"{derived_id}:{stratum}", + "limit": quotas[stratum], + **scope_params, + } + where = predicates[stratum] + + pop = db.execute( + f"SELECT count() FROM ({cells_sql}) AS cells WHERE {where}", params + ) + population = int(pop[0][0]) if pop else 0 + if not population: + used[stratum] = { + "predicate": where, + "table": "analysis_web_measurement", + "screen_kind": INTERVAL_STRATA[stratum]["screen_kind"], + "population_estimate": 0, + "drawn": 0, + "frame_start": since_dt.isoformat(), + "frame_end": until_dt.isoformat(), + "volume_floor": min_measurements, + "scope": spec["scope"], + } + continue + + # NOTE: no blocked_max, no alert state, no changepoints. The cell key, + # the window and how much data is in it — that is all an analyst gets + # before committing. + rows = db.execute( + f""" + SELECT probe_cc, probe_asn, domain, week, n + FROM ({cells_sql}) AS cells + WHERE {where} + ORDER BY cityHash64(concat( + {_CELL_KEY.format(cc='probe_cc', asn='probe_asn', + dom='domain', wk='week')}, %(salt)s)) + LIMIT %(limit)s + """, + params, + ) + + used[stratum] = { + "predicate": where, + "table": "analysis_web_measurement", + "screen_kind": INTERVAL_STRATA[stratum]["screen_kind"], + "population_estimate": population, + "drawn": len(rows), + "frame_start": since_dt.isoformat(), + "frame_end": until_dt.isoformat(), + "volume_floor": min_measurements, + "scope": spec["scope"], + } + buckets.append([ + IntervalRow( + probe_cc=r[0] or "", + probe_asn=int(r[1] or 0), + domain=r[2] or "", + window_start=datetime(r[3].year, r[3].month, r[3].day), + window_end=datetime(r[3].year, r[3].month, r[3].day) + _WEEK, + measurements_in_window=int(r[4]), + volume_band=volume_band(int(r[4])), + sampling_stratum=stratum, + sampling_weight=population / len(rows), + sample_population=population, + sample_rows=len(rows), + sampling_design_id=derived_id, + screen_kind=INTERVAL_STRATA[stratum]["screen_kind"], + ) + for r in rows + ]) + + interleaved: List[IntervalRow] = [] + for i in range(max((len(b) for b in buckets), default=0)): + for b in buckets: + if i < len(b): + interleaved.append(b[i]) + + return IntervalSampleResponse( + design_id=derived_id, + replicate=replicate, + spec=spec, + frame_start=since_dt, + frame_end=until_dt, + strata=used, + rows=interleaved[:limit], + ) + + +@router.get("/interval_reveal") +def interval_reveal( + probe_cc: str = Query(..., min_length=2, max_length=2), + probe_asn: int = Query(...), + domain: str = Query(...), + window_start: datetime = Query(...), + window_end: datetime = Query(...), + pad_days: int = Query(7, ge=0, le=28), + db=Depends(get_clickhouse_session), +) -> Dict[str, Any]: + """What the detector did in this cell-week. Shown after commit, never + before. + + Two things, because a bare alert flag is not diagnosable: the changepoints + themselves, and the hourly signal the detector consumed to produce them. + The signal is a median per cell-hour, exactly as `detector.get_observations` + computes it, so an analyst who disagrees with an alert can see whether the + detector saw something they did not or scored what they saw differently. + """ + lo = window_start - timedelta(days=pad_days) + hi = window_end + timedelta(days=pad_days) + params = { + "cc": probe_cc.upper(), + "asn": probe_asn, + "domain": domain, + "lo": lo, + "hi": hi, + "ws": window_start, + "we": window_end, + } + + cps = db.execute( + """ + SELECT ts, block_type, change_dir, s_pos, s_neg, current_state, h + FROM event_detector_changepoints + WHERE probe_cc = %(cc)s AND probe_asn = %(asn)s AND domain = %(domain)s + AND ts >= %(lo)s AND ts < %(hi)s + ORDER BY ts + """, + params, + ) + + signal = db.execute( + """ + WITH IF(resolver_asn = probe_asn, 1, 0) AS is_isp_resolver + SELECT toStartOfHour(measurement_start_time) AS ts, + count() AS n, + quantileIf(0.5)(dns_blocked, is_isp_resolver = 1) AS dns_isp_blocked, + quantileIf(0.5)(dns_blocked, is_isp_resolver = 0) AS dns_other_blocked, + quantile(0.5)(tcp_blocked) AS tcp_blocked, + quantile(0.5)(tls_blocked) AS tls_blocked + FROM analysis_web_measurement FINAL + WHERE probe_cc = %(cc)s AND probe_asn = %(asn)s AND domain = %(domain)s + AND measurement_start_time >= %(lo)s + AND measurement_start_time < %(hi)s + GROUP BY ts + ORDER BY ts + """, + params, + ) + + changepoints = [ + { + "ts": r[0], + "block_type": r[1], + "change_dir": int(r[2] or 0), + "s_pos": r[3], + "s_neg": r[4], + "current_state": r[5], + "h": r[6], + # Whether it lands in the adjudicated week is the whole question, + # so the client is not left to redo the comparison in local time. + "in_window": window_start <= r[0].replace(tzinfo=None) < window_end, + } + for r in cps + ] + + return { + "probe_cc": probe_cc.upper(), + "probe_asn": probe_asn, + "domain": domain, + "window_start": window_start, + "window_end": window_end, + "pad_days": pad_days, + "changepoints": changepoints, + "alerts_in_window": sum( + 1 for c in changepoints if c["in_window"] and c["change_dir"] > 0 + ), + "signal": [ + { + "ts": r[0], + "count": int(r[1]), + "dns_isp_blocked": r[2], + "dns_other_blocked": r[3], + "tcp_blocked": r[4], + "tls_blocked": r[5], + } + for r in signal + ], + "caveat": "The deployed detector is online: this is the alert log it " + "actually emitted, under whatever state it carried at the " + "time. It is not a replay and cannot be reproduced from " + "this window alone.", + } diff --git a/ooniapi/services/oonimeasurements/tests/test_labeling_intervals.py b/ooniapi/services/oonimeasurements/tests/test_labeling_intervals.py new file mode 100644 index 000000000..2a2d35572 --- /dev/null +++ b/ooniapi/services/oonimeasurements/tests/test_labeling_intervals.py @@ -0,0 +1,123 @@ +"""Tests for the interval sampler's frame arithmetic. + +No database. What is guarded here is the part that cannot be checked by looking +at the output: whether the strata cover the frame exactly once. A partition +that overlaps or leaks does not fail, it returns a plausible false-alarm rate +computed against the wrong denominator. +""" + +from datetime import datetime + +import pytest +from fastapi import HTTPException + +from oonimeasurements.routers.labeling import ( + DEFAULT_VOLUME_FLOOR, + INTERVAL_STRATA, + VOLUME_BAND_EDGES, + _design_fingerprint, + _resolve_interval_predicates, + _week_frame, + volume_band, +) + +ALL_STRATA = sorted(INTERVAL_STRATA) + + +def evaluate(predicate, alerted, blocked_max): + """Evaluate a generated SQL predicate as Python. + + The alerted subquery is opaque to this, so it is substituted for a boolean: + what is under test is how the strata compose, not how the join runs. + """ + expr = predicate.replace(INTERVAL_STRATA["detector_alerted"]["predicate"], "alerted") + expr = expr.replace(" AND ", " and ").replace("NOT ", "not ") + if expr == "1": + return True + return eval(expr, {}, {"alerted": alerted, "blocked_max": blocked_max}) + + +GRID = [(a, b) for a in (True, False) for b in (0.0, 0.49, 0.5, 1.0)] + + +@pytest.mark.parametrize( + "wanted", + [ + ["detector_alerted", "random_covered"], + ["detector_alerted", "near_miss", "random_covered"], + ["near_miss", "random_covered"], + ["random_covered"], + ], +) +def test_selected_strata_partition_the_frame(wanted): + """Every cell-week in the frame lands in exactly one selected stratum. + + Two would give it two selection probabilities and no correct weight; none + would drop it from the denominator silently, which biases the rate in the + unsafe direction. + """ + resolved = _resolve_interval_predicates(wanted) + for alerted, blocked_max in GRID: + hits = [s for s in wanted if evaluate(resolved[s], alerted, blocked_max)] + assert len(hits) == 1, (alerted, blocked_max, hits) + + +def test_near_miss_alone_does_not_claim_the_frame(): + """Drawing only the importance stratum is allowed and estimates nothing on + its own. It must not quietly widen to cover cells it did not screen for.""" + resolved = _resolve_interval_predicates(["near_miss"]) + assert not evaluate(resolved["near_miss"], False, 0.0) + assert not evaluate(resolved["near_miss"], True, 1.0) + + +def test_alerted_and_near_miss_are_disjoint(): + resolved = _resolve_interval_predicates(["detector_alerted", "near_miss"]) + for alerted, blocked_max in GRID: + hits = [s for s in resolved if evaluate(resolved[s], alerted, blocked_max)] + assert len(hits) <= 1, (alerted, blocked_max, hits) + + +def test_volume_bands_are_ordered_and_total(): + assert volume_band(0) == "low" + assert volume_band(DEFAULT_VOLUME_FLOOR) == "low" + assert volume_band(99) == "low" + assert volume_band(100) == "medium" + assert volume_band(999) == "medium" + assert volume_band(1000) == "high" + assert volume_band(10**9) == "high" + # Monotone, so a busier cell-week never lands in a quieter band. + seen = [volume_band(n) for n in (1, 100, 1000)] + assert seen == [name for _, name in VOLUME_BAND_EDGES] + + +def test_frame_snaps_to_whole_weeks(): + # A Thursday to a Thursday: both ends move to the Mondays inside the range. + lo, hi = _week_frame(datetime(2026, 3, 5), datetime(2026, 4, 2)) + assert lo == datetime(2026, 3, 9) + assert hi == datetime(2026, 3, 30) + assert lo.weekday() == 0 and hi.weekday() == 0 + assert (hi - lo).days % 7 == 0 + + +def test_frame_already_aligned_is_left_alone(): + lo, hi = _week_frame(datetime(2026, 3, 9), datetime(2026, 3, 30)) + assert (lo, hi) == (datetime(2026, 3, 9), datetime(2026, 3, 30)) + + +def test_frame_without_a_whole_week_is_rejected(): + """Rather than returning a shorter observation window as if it were a + smaller one.""" + with pytest.raises(HTTPException): + _week_frame(datetime(2026, 3, 10), datetime(2026, 3, 14)) + + +def test_design_id_tracks_the_partition(): + """The resolved predicate is in the spec, so the same nominal stratum drawn + under a different partition gets a different id — a weight can never be + reinterpreted under rules it was not drawn under.""" + two = _resolve_interval_predicates(["detector_alerted", "random_covered"]) + three = _resolve_interval_predicates( + ["detector_alerted", "near_miss", "random_covered"] + ) + assert two["random_covered"] != three["random_covered"] + assert _design_fingerprint({"strata": two}) != _design_fingerprint({"strata": three}) From 3b156307f4c2d607ce64e75caf0d75de6bab823a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 6 Aug 2026 20:59:40 +0200 Subject: [PATCH 11/14] Drop FINAL for performance reasons Must not be run while a backfill is in progress --- .../src/oonimeasurements/routers/labeling.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index 5b32dd90c..94ae28819 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -968,9 +968,8 @@ def draw_interval_sample( } derived_id = _design_fingerprint(spec) - # analysis_web_measurement is a ReplacingMergeTree; without FINAL a - # reprocess in flight double-counts a cell-week and moves it up a volume - # band. No test_name filter: the detector does not have one either, and the + # MUST not be run while a reprocess is in progress due to lack of FINAL + # No test_name filter: the detector does not have one either, and the # frame has to be the population the detector actually runs over. cells_sql = f""" SELECT probe_cc, @@ -979,7 +978,7 @@ def draw_interval_sample( toStartOfWeek(measurement_start_time, 1) AS week, count() AS n, max(greatest(dns_blocked, tcp_blocked, tls_blocked)) AS blocked_max - FROM analysis_web_measurement FINAL + FROM analysis_web_measurement WHERE measurement_start_time >= %(since)s AND measurement_start_time < %(until)s {''.join(' AND ' + s for s in scope_sql)} From 876a90d4e43bb244d60d9c21e1e70614f36af611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 6 Aug 2026 22:31:16 +0200 Subject: [PATCH 12/14] Say what the interval denominator is, now that it has two verdicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documentation and one comment; no behaviour change. The sampler's docstrings described a single silent verdict, but the labeller now distinguishes a week where OONI saw nothing wrong from a week inside a block that started earlier. Both mean no transition happened, so both are weeks the detector should have been silent through and both belong in the false-alarm denominator — "silent series-week" rather than "quiet series-week", which is the phrase the evaluation notebook now prints. Also records why the reveal's signal query does not use FINAL while the frame query does: it is read to draw a chart rather than to size a population, so an unmerged duplicate nudges a median instead of moving a cell-week into another volume band and corrupting a weight. --- .../src/oonimeasurements/routers/labeling.py | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index 94ae28819..5daf71ae7 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -13,9 +13,11 @@ - INTERVAL (/interval_sample, /interval_reveal). One row per (probe_cc, probe_asn, domain) x ISO week — the detector's own cell, keyed exactly as `event_detector_cusums` is. Supplies the *denominator* the event - grain cannot: "false alerts per quiet series-week" is a rate over a + grain cannot: "false alerts per silent series-week" is a rate over a population of cell-weeks, and a population can only be counted if it was - sampled from a frame. + sampled from a frame. Silent, not quiet: a week inside an ongoing block also + contains no transition for a changepoint detector to find, so it belongs in + the denominator too. Two invariants this module exists to enforce, in both grains: @@ -675,12 +677,13 @@ def reveal( # -------------------------------------------------------------------------- -# Interval grain: the quiet-time denominator +# Interval grain: the silent-time denominator # -------------------------------------------------------------------------- # # The event corpus is curated, so event recall is a coverage statement about a -# hand-built set. Nothing in it defines quiet time, so the harness's "false -# alerts per quiet series-week" had no frame behind it. This is that frame. +# hand-built set. Nothing in it defines a week the detector should have been +# silent through, so the harness's "false alerts per series-week" had no frame +# behind it. This is that frame. # # The unit is the detector's own unit. `event_detector_cusums` keys on # (probe_cc, probe_asn, domain), so anything coarser here would estimate a rate @@ -904,14 +907,20 @@ def draw_interval_sample( ), limit: int = Query(40, ge=1, le=500), ) -> IntervalSampleResponse: - """Draw cell-weeks to adjudicate as quiet, or not. - - The verdict an analyst writes against these rows is `quiet_observed`, never - `quiet`: the week is judged from the same OONI data the detector reads, so - an unmeasured block is indistinguishable from calm. That caps the claim at - "no interference visible in OONI's data", which is the honest ceiling, and - it is why a better candidate that finds subtle real events is not silently - charged a false alarm. + """Draw cell-weeks to adjudicate. + + What the analyst decides about each is whether the state *changed* inside + it, which is the only thing a changepoint detector can be right or wrong + about. Two of the verdicts mean it did not — `quiet_observed` for a week + where OONI saw nothing wrong, `blocked_throughout` for a week inside a + block that started earlier — and both belong in the false-alarm + denominator, because the detector should be silent in either. + + `quiet_observed`, never `quiet`: the week is judged from the same OONI data + the detector reads, so an unmeasured block is indistinguishable from calm. + That caps the claim at "no interference visible in OONI's data", which is + the honest ceiling, and it is why a better candidate that finds subtle real + events is not silently charged a false alarm. """ since_dt, until_dt = _week_frame(since, until) wanted = sorted({s.strip() for s in strata.split(",") if s.strip()}) @@ -1124,6 +1133,10 @@ def interval_reveal( params, ) + # No FINAL here, unlike the sampler's frame query. This one is read to draw + # a chart, not to size a population: an unmerged duplicate during a + # reprocess nudges a median, where in the frame it would move a cell-week + # into another volume band and corrupt a weight. signal = db.execute( """ WITH IF(resolver_asn = probe_asn, 1, 0) AS is_isp_resolver @@ -1133,7 +1146,7 @@ def interval_reveal( quantileIf(0.5)(dns_blocked, is_isp_resolver = 0) AS dns_other_blocked, quantile(0.5)(tcp_blocked) AS tcp_blocked, quantile(0.5)(tls_blocked) AS tls_blocked - FROM analysis_web_measurement FINAL + FROM analysis_web_measurement WHERE probe_cc = %(cc)s AND probe_asn = %(asn)s AND domain = %(domain)s AND measurement_start_time >= %(lo)s AND measurement_start_time < %(hi)s From c762801886e44a192269c4ee530e68e3f2951e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Aug 2026 15:18:53 +0200 Subject: [PATCH 13/14] Performance improvements to interval labeler --- .../src/oonimeasurements/routers/labeling.py | 159 ++++++++++++------ 1 file changed, 107 insertions(+), 52 deletions(-) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index 5daf71ae7..3a7916437 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -37,8 +37,10 @@ reconstructable from the export alone. """ +import time import hashlib import json +import logging from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional @@ -50,6 +52,8 @@ from ..dependencies import get_clickhouse_session # type: ignore from ..scoring import BLOCKING_THRESHOLD, SCORING_VERSION, any_blocked, attributed_to +log = logging.getLogger(__name__) + router = APIRouter(prefix="/api/v1/labeling", tags=["labeling"]) @@ -977,6 +981,13 @@ def draw_interval_sample( } derived_id = _design_fingerprint(spec) + quotas = _quotas(wanted, shares, limit) + + # `blocked_max` costs three float columns over the whole frame and only the + # near-miss stratum is defined in terms of it, so it is not read when that + # stratum was not asked for. + wants_blocked_max = "near_miss" in wanted + # MUST not be run while a reprocess is in progress due to lack of FINAL # No test_name filter: the detector does not have one either, and the # frame has to be the population the detector actually runs over. @@ -985,8 +996,9 @@ def draw_interval_sample( probe_asn, domain, toStartOfWeek(measurement_start_time, 1) AS week, - count() AS n, - max(greatest(dns_blocked, tcp_blocked, tls_blocked)) AS blocked_max + count() AS n + {", max(greatest(dns_blocked, tcp_blocked, tls_blocked)) AS blocked_max" + if wants_blocked_max else ""} FROM analysis_web_measurement WHERE measurement_start_time >= %(since)s AND measurement_start_time < %(until)s @@ -995,66 +1007,109 @@ def draw_interval_sample( HAVING n >= %(floor)s """ - quotas = _quotas(wanted, shares, limit) - used: Dict[str, Dict[str, Any]] = {} - buckets: List[List[IntervalRow]] = [] + # The strata are disjoint by construction (see + # `_resolve_interval_predicates`), which is what lets one expression label + # every cell-week in a single pass. Drawing each stratum under its own + # WHERE meant re-running this GROUP BY once to size the population and + # again to draw it — 2N aggregations of the same frame for N strata, and + # that aggregation is the whole cost of the endpoint. + # + # `random_covered` is the fallback rather than a branch of its own: its + # predicate is the negation of the others, so spelling it out would + # evaluate the alerted-set lookup a second time. The lookup is hoisted to + # an alias for the same reason — substituting the module constant into + # itself, so it cannot match anything else by accident. The predicate text + # recorded in the spec stays verbatim, since that text is what the design + # id hashes. + branches = ", ".join( + f"({predicates[s].replace(_IS_ALERTED, 'is_alerted')}), '{s}'" + for s in wanted + if s != "random_covered" + ) + fallback = "'random_covered'" if "random_covered" in wanted else "''" + stratum_sql = f"multiIf({branches}, {fallback})" if branches else fallback + # Drawing only the complement never asks about alerts, so in that case the + # changepoint set is not built at all. + alerted_sql = f"{_IS_ALERTED} AS is_alerted," if "is_alerted" in stratum_sql else "" + + # Per-stratum quotas, so an uneven `shares` split still takes exactly what + # it was promised out of one shared ordering. + quota_sql = "multiIf(" + ", ".join( + f"stratum = '{s}', {int(quotas[s])}" for s in wanted + ) + ", 0)" + + # Same salt the per-stratum draws used — `_design_fingerprint` plus the + # stratum name — so a replicate drawn before this rewrite and one drawn + # after select the same cell-weeks. + order_sql = ( + "cityHash64(concat(" + + _CELL_KEY.format(cc="probe_cc", asn="probe_asn", dom="domain", wk="week") + + ", %(design)s, ':', stratum))" + ) - for stratum in wanted: - params: Dict[str, Any] = { - "since": since_dt, - "until": until_dt, - "floor": min_measurements, - "salt": f"{derived_id}:{stratum}", - "limit": quotas[stratum], - **scope_params, - } - where = predicates[stratum] + params: Dict[str, Any] = { + "since": since_dt, + "until": until_dt, + "floor": min_measurements, + "design": derived_id, + **scope_params, + } - pop = db.execute( - f"SELECT count() FROM ({cells_sql}) AS cells WHERE {where}", params - ) - population = int(pop[0][0]) if pop else 0 - if not population: - used[stratum] = { - "predicate": where, - "table": "analysis_web_measurement", - "screen_kind": INTERVAL_STRATA[stratum]["screen_kind"], - "population_estimate": 0, - "drawn": 0, - "frame_start": since_dt.isoformat(), - "frame_end": until_dt.isoformat(), - "volume_floor": min_measurements, - "scope": spec["scope"], - } - continue + # NOTE: no blocked_max, no alert state, no changepoints leave this query. + # The cell key, the window and how much data is in it — that is all an + # analyst gets before committing. + t0 = time.monotonic() + rows = db.execute( + f""" + SELECT probe_cc, probe_asn, domain, week, n, stratum, population + FROM ( + SELECT probe_cc, probe_asn, domain, week, n, stratum, + count() OVER (PARTITION BY stratum) AS population, + row_number() OVER ( + PARTITION BY stratum ORDER BY {order_sql} + ) AS rn + FROM ( + SELECT probe_cc, probe_asn, domain, week, n, + {alerted_sql} + {stratum_sql} AS stratum + FROM ({cells_sql}) AS cells + ) AS labelled + WHERE stratum != '' + ) AS ranked + WHERE rn <= {quota_sql} + ORDER BY stratum, rn + """, + params, + ) + log.info("interval frame query: %.2fs", time.monotonic() - t0) - # NOTE: no blocked_max, no alert state, no changepoints. The cell key, - # the window and how much data is in it — that is all an analyst gets - # before committing. - rows = db.execute( - f""" - SELECT probe_cc, probe_asn, domain, week, n - FROM ({cells_sql}) AS cells - WHERE {where} - ORDER BY cityHash64(concat( - {_CELL_KEY.format(cc='probe_cc', asn='probe_asn', - dom='domain', wk='week')}, %(salt)s)) - LIMIT %(limit)s - """, - params, - ) + # A stratum with a population draws at least one row (quotas floor at 1), + # so an absent stratum here is an empty one and keeps its zero below. + drawn: Dict[str, List[Any]] = {s: [] for s in wanted} + populations: Dict[str, int] = {s: 0 for s in wanted} + for r in rows: + drawn[r[5]].append(r) + populations[r[5]] = int(r[6]) + + used: Dict[str, Dict[str, Any]] = {} + buckets: List[List[IntervalRow]] = [] + for stratum in wanted: + stratum_rows = drawn[stratum] + population = populations[stratum] used[stratum] = { - "predicate": where, + "predicate": predicates[stratum], "table": "analysis_web_measurement", "screen_kind": INTERVAL_STRATA[stratum]["screen_kind"], "population_estimate": population, - "drawn": len(rows), + "drawn": len(stratum_rows), "frame_start": since_dt.isoformat(), "frame_end": until_dt.isoformat(), "volume_floor": min_measurements, "scope": spec["scope"], } + if not stratum_rows: + continue buckets.append([ IntervalRow( probe_cc=r[0] or "", @@ -1065,13 +1120,13 @@ def draw_interval_sample( measurements_in_window=int(r[4]), volume_band=volume_band(int(r[4])), sampling_stratum=stratum, - sampling_weight=population / len(rows), + sampling_weight=population / len(stratum_rows), sample_population=population, - sample_rows=len(rows), + sample_rows=len(stratum_rows), sampling_design_id=derived_id, screen_kind=INTERVAL_STRATA[stratum]["screen_kind"], ) - for r in rows + for r in stratum_rows ]) interleaved: List[IntervalRow] = [] From c487dfd966632253036f2a88e9904f06c6f93fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 25 Sep 2026 14:33:41 +0200 Subject: [PATCH 14/14] WIP labeling --- .../src/oonimeasurements/routers/labeling.py | 1368 +++-------------- 1 file changed, 206 insertions(+), 1162 deletions(-) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py index 3a7916437..f163feb19 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/labeling.py @@ -1,46 +1,69 @@ """ Labeling corpus API. -Read-only ClickHouse queries backing the adjudication UIs. There is no write -path: labels live in the analyst's browser and leave it by copy-paste, so this -router adds no storage, no auth surface, and no migration. - -Two grains are served, and they answer different questions. - -- MEASUREMENT (/sample, /candidate, /context, /reveal). One row per - measurement. Calibrates *scoring*: the per-rule likelihood ratios are fitted - from it. -- INTERVAL (/interval_sample, /interval_reveal). One row per - (probe_cc, probe_asn, domain) x ISO week — the detector's own cell, keyed - exactly as `event_detector_cusums` is. Supplies the *denominator* the event - grain cannot: "false alerts per silent series-week" is a rate over a - population of cell-weeks, and a population can only be counted if it was - sampled from a frame. Silent, not quiet: a week inside an ongoing block also - contains no transition for a changepoint detector to find, so it belongs in - the denominator too. - -Two invariants this module exists to enforce, in both grains: - -1. BLINDING. The candidate views return what the probes got and nothing the - pipeline concluded. analysis_web_measurement and fastpath's anomaly / - confirmed / scores columns are queried ONLY by /reveal, and - event_detector_changepoints ONLY by /interval_reveal, which the UIs call - after the analyst has committed. If you add a field to a candidate view, - check it is not a pipeline judgment in disguise. The interval grain makes - this sharper than the measurement grain does: one of its strata *is* the - detector's output, so an unblinded alert state does not merely anchor the - analyst, it hands them the answer. - -2. SAMPLING IS RECORDED, NOT REMEMBERED. Every draw is deterministic given - (design_id, stratum, frame, rate), and the sample endpoints return the - predicate they ran and the population they ran against, so the weights are - reconstructable from the export alone. +/sample draws from `labeling_frames`, a materialised frame table built by the +analysis-evaluation notebook: one row per measurement_uid in the frame window, +carrying what each pipeline concluded (`blocked_fastpath`, `blocked_analysis`) +and the stratum that pair puts it in. + +* S11 both call it blocked +* S10 fastpath only +* S01_ok analysis only and msm_failure = 'f' +* S01_fail analysis only and msm_failure = 't' +* S00 neither + +The query used to populate this table is the following: + +INSERT INTO labeling_frames +SELECT + COALESCE(a.measurement_uid, b.measurement_uid) as measurement_uid, + a.blocked AS blocked_fastpath, + b.blocked AS blocked_analysis, + CASE + WHEN blocked_fastpath AND blocked_analysis THEN 'S11' + WHEN blocked_fastpath AND NOT blocked_analysis THEN 'S10' + WHEN NOT blocked_fastpath AND blocked_analysis + THEN IF(failed_fastpath, 'S01_fail', 'S01_ok') + ELSE 'S00' + END as stratum, + 'web_connectivity' as test_name, + COALESCE(a.day, b.day) as day, + 'anomaly = \'t\' OR confirmed = \'t\'' as fastpath_query, + 'dns_blocked > 0.5 OR tls_blocked > 0.5 OR tls_blocked > 0.5' as analysis_query, + COALESCE(a.failed_fastpath, FALSE) as failed_fastpath +FROM ( + SELECT + measurement_uid, + toStartOfDay(measurement_start_time) as day, + IF(anomaly = 't' OR confirmed = 't', TRUE, FALSE) as blocked, + IF(msm_failure = 't', TRUE, FALSE) as failed_fastpath + FROM fastpath + WHERE + measurement_start_time > '2026-07-03' + AND measurement_start_time < '2026-08-04' + AND test_name = 'web_connectivity' +) a +FULL OUTER JOIN ( + SELECT + measurement_uid, + toStartOfDay(measurement_start_time) as day, + IF(dns_blocked > 0.5 OR tls_blocked > 0.5 OR tls_blocked > 0.5, TRUE, FALSE) as blocked + FROM analysis_web_measurement + WHERE + measurement_start_time > '2026-07-03' + AND measurement_start_time < '2026-08-04' + AND test_name = 'web_connectivity' +) b +USING (measurement_uid) +WHERE measurement_uid != '' +SETTINGS join_algorithm = 'grace_hash'; """ import time import hashlib import json import logging +import math from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional @@ -56,180 +79,70 @@ router = APIRouter(prefix="/api/v1/labeling", tags=["labeling"]) +DESIGN_SCHEMA_VERSION = "3" -# -------------------------------------------------------------------------- -# Sampling design -# -------------------------------------------------------------------------- -# -# The design doc's screen is "any blocked-leaning RULE fired", which needs B1 -# (persisted fired-rule sets) to exist. It does not yet, so the screen below is -# a documented PROXY over the fastpath analysis which is derived from probe -# computed values. It's fine to use this as a PROXY, as it's recorded in the -# measurement itself and is stable, since we have not plans to change the -# fastpath scoring logic. -# -# This matters for the weights: the proxy screen has different, unmeasured -# coverage from the real one. Record `screen_kind` in the export so a later -# refit can tell proxy-screened rows from B1-screened ones and, if needed, -# drop the former. - +FRAME_TABLE = "labeling_frames" -STRATA: Dict[str, Dict[str, Any]] = { - "screen_positive": { - "table": "fastpath", - # confirmed rows are excluded: they are certain blocking with a known - # artefact, so a label on one adds almost nothing, and they were eating - # the positive quota. They keep their own census stratum below. - "predicate": f"anomaly = 't' AND confirmed = 'f' AND msm_failure = 'f'", - "screen_kind": "fastpath_proxy", - "note": "Proxy for B1's blocked-leaning-rule screen. LR numerator.", - }, - # Layer-attributed positives. A uniform draw from anomaly='t' oversamples - # whichever mechanism dominates globally (in practice TLS resets), so a - # corpus built from it calibrates one layer and starves the others. These - # read the pipeline's own layer scores, which means they oversample what - # the pipeline can already see — screen_negative stays the only stratum - # that can discover what it misses. Predicates are mutually exclusive so - # populations do not overlap and weights stay clean. - "screen_dns": { - "table": "analysis_web_measurement", - "predicate": attributed_to("dns"), - "screen_kind": "loni_layer_proxy", - "note": "DNS-attributed positives.", - }, - "screen_tcp": { - "table": "analysis_web_measurement", - "predicate": attributed_to("tcp"), - "screen_kind": "loni_layer_proxy", - "note": "TCP-attributed positives, DNS quiet.", - }, - "screen_tls": { - "table": "analysis_web_measurement", - "predicate": attributed_to("tls"), - "screen_kind": "loni_layer_proxy", - "note": "TLS-attributed positives, DNS and TCP quiet.", - }, - "screen_negative": { - "table": "fastpath", - "predicate": f"confirmed = 'f' AND anomaly = 'f' AND msm_failure = 'f'", - "screen_kind": "fastpath_proxy", - "note": "Bounds false negatives and carries the base rate. Small, " - "and the first thing cut under pressure. Do not cut it.", - }, - "fingerprint_match": { - "table": "fastpath", - "predicate": "confirmed = 't' AND msm_failure = 'f'", - "screen_kind": "fingerprint", - "note": "Census, not a sample. High-precision positives; tag the " - "labels so LRs can be refit without them as a circularity " - "check.", - }, - "incident_window": { - "table": "analysis_web_measurement", - "predicate": "1", # scoped entirely by the cc/domain/time params - "screen_kind": "incident_scope", - "note": "Draw inside a known event. Label the MEASUREMENT: rows here " - "that are genuinely ok are the most valuable in the corpus.", - }, -} - -# control_agreement (cheap negatives from probe/control agreement) needs an -# obs_web_ctrl join with per-layer agreement predicates. Left out on purpose -# rather than half-built: a negative stratum with a wrong predicate is worse -# than one that is absent, because it silently deflates every LR denominator. - -# Bump when STRATA definitions or the fingerprint's shape change: it forces new -# design ids, so old weights are never silently reinterpreted under new rules. -DESIGN_SCHEMA_VERSION = "2" - - -def _quotas(wanted: List[str], shares: Optional[str], limit: int) -> Dict[str, int]: - """How many rows each stratum contributes to the queue. - - Default is an equal split. `shares` reweights it — "screen_negative=0.5" - gives that stratum half the queue and splits the rest equally. Shares only - steer analyst effort; the weights on the rows are population/drawn either - way, so no share choice can bias an estimate, only its variance. - """ - fractions = {s: 1.0 for s in wanted} - if shares: - for part in shares.split(","): - name, _, value = part.partition("=") - name = name.strip() - if name not in wanted: - raise HTTPException(400, f"share for unknown stratum: {name}") - try: - fractions[name] = float(value) - except ValueError: - raise HTTPException(400, f"bad share: {part}") - if fractions[name] <= 0: - raise HTTPException(400, f"share must be positive: {part}") - total = sum(fractions.values()) - exact = {s: limit * f / total for s, f in fractions.items()} - quotas = {s: max(1, int(e)) for s, e in exact.items()} - # Hand out what rounding left over, largest remainder first. - leftover = limit - sum(quotas.values()) - for s in sorted(exact, key=lambda s: exact[s] - int(exact[s]), reverse=True): - if leftover <= 0: - break - quotas[s] += 1 - leftover -= 1 - # The min-1 floors can overshoot under an extreme share; trim the largest - # quotas back so the recorded drawn counts always match the returned queue. - while sum(quotas.values()) > limit: - biggest = max(quotas, key=lambda s: quotas[s]) - if quotas[biggest] == 1: - break # limit < number of strata; nothing sensible to trim - quotas[biggest] -= 1 - return quotas +# We divide +STRATA = ("S00", "S01_ok", "S01_fail", "S10", "S11") +# The stratums S10 and S01 are those in which the fastpath and data pipeline +# disagree, so they are the most valuable to sample. When they are in agreement +# is about blocking being present (S11) is of third important, while the rest is assigned +# to when they both think no blocking is happening (S00). +ALLOCATION: Dict[str, float] = {"S10": 0.3, "S01_ok": 0.15, "S01_fail": 0.15, "S11": 0.2} -def _design_fingerprint(spec: Dict[str, Any]) -> str: - """Content-address a sampling design. - The id is derived from the design so that there is never the same id used - for two different parameter sets, leaving two incompatible weightings - sharing a name. +def _allocate(sample_count: int) -> Dict[str, int]: + """Target n_h per stratum. Floors, with the remainder going to S00.""" + targets = {s: math.floor(sample_count * f) for s, f in ALLOCATION.items()} + targets["S00"] = sample_count - sum(targets.values()) + return {s: targets[s] for s in STRATA} - Redrawing with a higher limit returns a superset of the same rows, and two - analysts who enter the same parameters get the same queue, which is how - inter-rater agreement gets measured without any coordination service. - Want genuinely fresh rows from the same population? Increment `replicate`. - It is part of the spec, so it produces a different id and an independent - draw, on purpose and on the record. - """ - blob = json.dumps(spec, sort_keys=True, separators=(",", ":"), default=str) - return "d" + hashlib.sha256(blob.encode()).hexdigest()[:10] - - -class SampleRow(BaseModel): +class QueueRow(BaseModel): measurement_uid: str - measurement_start_time: datetime - probe_cc: str - probe_asn: int - resolver_asn: int - domain: str - input: Optional[str] test_name: str - # sampling provenance, carried through to the label - sampling_stratum: str - sampling_weight: float - sample_population: int - sample_rows: int - sampling_design_id: str - screen_kind: str + day: datetime + strata: str + draw_id: str + blocked_fastpath: bool + blocked_analysis: bool + + +class StratumDraw(BaseModel): + strata: str + N_h: int + n_h: int + n_h_target: int + +class FrameInfo(BaseModel): + table: str + frame_version: str + N_total: int + N_h: Dict[str, int] + day_start: datetime + day_end: datetime + test_names: List[str] + fastpath_query: List[str] + analysis_query: List[str] class SampleResponse(BaseModel): - design_id: str + draw_id: str + draw_timestamp: datetime + frame_version: str replicate: int + sample_count: int spec: Dict[str, Any] - frame_start: datetime - frame_end: datetime - strata: Dict[str, Dict[str, Any]] - rows: List[SampleRow] + frame: FrameInfo + strata: Dict[str, StratumDraw] + rows: List[QueueRow] + +def _design_fingerprint(spec: Dict[str, Any], d) -> str: + blob = json.dumps(spec, sort_keys=True, separators=(",", ":"), default=str) + return d + hashlib.sha256(blob.encode()).hexdigest()[:10] def _frame(since: Optional[datetime], until: Optional[datetime]): until = until or datetime.now(timezone.utc).replace(tzinfo=None) @@ -239,237 +152,124 @@ def _frame(since: Optional[datetime], until: Optional[datetime]): return since, until -@router.get("/test_names") -def list_test_names( - db=Depends(get_clickhouse_session), - since: Optional[datetime] = None, - until: Optional[datetime] = None, - probe_cc: Optional[str] = Query(None, min_length=2, max_length=2), -) -> Dict[str, Any]: - """What is labelable in this frame, and how much of it there is. - - Counted from analysis_web_measurement rather than fastpath on purpose: a - test with fastpath rows but no analysis rows will draw fine and then fail - to load, because the candidate view reads obs_web. This list is the set - that actually works end to end. +def _frame_census(db) -> FrameInfo: + """Snapshot the frame: its stratum counts and how it was defined. """ - since_dt, until_dt = _frame(since, until) - where = [ - "measurement_start_time >= %(since)s", - "measurement_start_time < %(until)s", - ] - params: Dict[str, Any] = {"since": since_dt, "until": until_dt} - if probe_cc: - where.append("probe_cc = %(probe_cc)s") - params["probe_cc"] = probe_cc.upper() - rows = db.execute( f""" - SELECT test_name, + SELECT stratum, count() AS n, - countIf({any_blocked()}) - AS n_screen_positive - FROM analysis_web_measurement - WHERE {' AND '.join(where)} - GROUP BY test_name - ORDER BY n DESC - """, - params, + min(day) AS day_start, + max(day) AS day_end, + groupUniqArray(toString(test_name)) AS test_names, + groupUniqArray(toString(fastpath_query)) AS fastpath_queries, + groupUniqArray(toString(analysis_query)) AS analysis_queries + FROM {FRAME_TABLE} + GROUP BY stratum + ORDER BY stratum + """ ) - return { - "frame_start": since_dt, - "frame_end": until_dt, - "test_names": [ - { - "test_name": r[0], - "measurements": int(r[1]), - "screen_positive": int(r[2]), - } - for r in rows - ], + if not rows: + raise HTTPException( + 503, + f"{FRAME_TABLE} is empty: build the frame before drawing from it", + ) + + descriptor = { + "table": FRAME_TABLE, + "N_total": sum(int(r[1]) for r in rows), + "N_h": {r[0]: int(r[1]) for r in rows}, + "day_start": min(r[2] for r in rows), + "day_end": max(r[3] for r in rows), + "test_names": sorted({t for r in rows for t in r[4]}), + "fastpath_query": sorted({q for r in rows for q in r[5]}), + "analysis_query": sorted({q for r in rows for q in r[6]}), } + missing = [s for s in STRATA if s not in descriptor["N_h"]] + if missing: + # Not fatal — an empty stratum is a real state of the world — but it + # means those quotas cannot be filled, so say so once, loudly. + log.warning("frame %s has no rows in strata %s", FRAME_TABLE, missing) + return FrameInfo(frame_version=_design_fingerprint(descriptor, "f"), **descriptor) @router.get("/sample", response_model=SampleResponse) def draw_sample( db=Depends(get_clickhouse_session), - strata: str = Query( - "screen_positive,screen_negative", - description="Comma-separated. Multiple strata are drawn separately " - "and interleaved, so the analyst cannot infer a row's " - "stratum from its position in the queue.", - ), replicate: int = Query( 1, ge=1, description="Independent draws of the same design. Same replicate = " "same rows (reproducible, extendable, comparable across " - "analysts). Increment it to sample rows the previous " - "replicate did not cover.", - ), - since: Optional[datetime] = None, - until: Optional[datetime] = None, - probe_cc: Optional[str] = Query(None, min_length=2, max_length=2), - probe_asn: Optional[int] = None, - domain: Optional[str] = None, - test_name: Optional[str] = Query( - "web_connectivity", - description="Comma-separated. Scoping a design to a test changes its " - "population, so weights are only valid within the same " - "test scope — change design_id when you change this. " - "Empty string means every test.", - ), - shares: Optional[str] = Query( - None, - description="Optional stratum=share pairs, e.g. " - "'screen_negative=0.5'. Reweights how the queue is split " - "across the selected strata; omitted strata share the " - "remainder equally. Steers effort only — row weights stay " - "population/drawn regardless.", + "analysts).", ), - limit: int = Query(50, ge=1, le=500), + sample_count: int = Query( + 100, ge=1, le=5000, + description="how many measurement_uids should be sampled", + ) ) -> SampleResponse: - since_dt, until_dt = _frame(since, until) - wanted = sorted({s.strip() for s in strata.split(",") if s.strip()}) - unknown = [s for s in wanted if s not in STRATA] - if unknown: - raise HTTPException(400, f"unknown strata: {unknown}") + started = time.monotonic() + frame = _frame_census(db) + targets = _allocate(sample_count) - tests = sorted({t.strip() for t in (test_name or "").split(",") if t.strip()}) - if "incident_window" in wanted and not (probe_cc and domain): - raise HTTPException( - 400, - "incident_window needs probe_cc and domain — an unscoped incident " - "draw is just a biased production sample", - ) + # The salt fixes each uid's rank within its stratum. + salt = f"{FRAME_TABLE}:{DESIGN_SCHEMA_VERSION}:r{replicate}" - # The spec is the design. Everything that changes which rows are eligible, - # or what a weight means, has to be in here — otherwise two different - # populations could collide onto one id, which is the failure this exists - # to make impossible. - resolved = { - s: { - "table": STRATA[s]["table"], - "predicate": STRATA[s]["predicate"], - "screen_kind": STRATA[s]["screen_kind"], - } - for s in wanted - } spec = { "schema": DESIGN_SCHEMA_VERSION, - # The layer strata are defined by BLOCKING_THRESHOLD, so a threshold - # change redefines what "DNS-blocked" means and the labels drawn either - # side are not one population. The predicates below already carry the - # number, but recording the version states it, and keeps the id - # sensitive to a scoring change that happens to render identically. - "scoring_version": SCORING_VERSION, - "blocking_threshold": BLOCKING_THRESHOLD, - "strata": resolved, - "frame": [since_dt.isoformat(), until_dt.isoformat()], - "scope": { - "probe_cc": probe_cc.upper() if probe_cc else None, - "probe_asn": probe_asn, - "domain": domain, - "test_names": tests or "all", - }, + "frame_table": FRAME_TABLE, + "frame_version": frame.frame_version, + "allocation": targets, + "sample_count": sample_count, "replicate": replicate, + "salt": salt, } - derived_id = _design_fingerprint(spec) - - # Shares, like limit, set how far down each stratum's fixed ordering the - # draw goes. They are deliberately not part of the spec: the same design - # with a bigger share returns a superset of the same stratum rows. - quotas = _quotas(wanted, shares, limit) - used: Dict[str, Dict[str, Any]] = {} - buckets: List[List[SampleRow]] = [] - - for stratum in wanted: - spec_s = STRATA[stratum] - table = spec_s["table"] - # analysis_web_measurement is a ReplacingMergeTree; during a reprocess - # the same uid exists in old and new versions until the merge runs, - # which would inflate the population and can draw a uid twice. - from_clause = f"{table} FINAL" if table == "analysis_web_measurement" else table - - where = [ - "measurement_start_time >= %(since)s", - "measurement_start_time < %(until)s", - f"({spec_s['predicate']})", - ] - params: Dict[str, Any] = { - "since": since_dt, - "until": until_dt, - "salt": f"{derived_id}:{stratum}", - "limit": quotas[stratum], - } - if probe_cc: - where.append("probe_cc = %(probe_cc)s") - params["probe_cc"] = probe_cc.upper() - if probe_asn: - where.append("probe_asn = %(probe_asn)s") - params["probe_asn"] = probe_asn - if domain: - where.append("domain = %(domain)s") - params["domain"] = domain - if tests: - where.append("test_name IN %(test_names)s") - params["test_names"] = tests - where_sql = " AND ".join(where) + draw_id = _design_fingerprint(spec, "dr") + + query = f""" + SELECT measurement_uid, + blocked_fastpath, + blocked_analysis, + stratum, + test_name, + day + FROM {FRAME_TABLE} + WHERE stratum = %(stratum)s + ORDER BY cityHash64(concat(measurement_uid, %(salt)s)), measurement_uid + LIMIT %(limit)s + """ - # Population first: the weight is 1/rate by construction, but the - # population is what lets anyone check that later. - pop = db.execute( - f"SELECT count() FROM {from_clause} WHERE {where_sql}", params - ) - population = int(pop[0][0]) if pop else 0 + used: Dict[str, StratumDraw] = {} + buckets: List[List[QueueRow]] = [] - resolver = ( - "resolver_asn" if table == "analysis_web_measurement" else "0" + for stratum in STRATA: + target = targets[stratum] + rows = ( + db.execute(query, {"stratum": stratum, "salt": salt, "limit": target}) + if target + else [] ) - # NOTE: no blocked/down/ok, no anomaly, no confirmed, no scores. - rows = db.execute( - f""" - SELECT measurement_uid, - measurement_start_time, - probe_cc, - probe_asn, - {resolver} AS resolver_asn, - domain, - input, - test_name - FROM {from_clause} - WHERE {where_sql} - ORDER BY cityHash64(concat(measurement_uid, %(salt)s)) - LIMIT %(limit)s - """, - params, + population = frame.N_h.get(stratum, 0) + n_h = len(rows) + if n_h < target: + log.warning( + "draw %s: stratum %s wanted %d, frame had %d", + draw_id, stratum, target, population, + ) + used[stratum] = StratumDraw( + strata=stratum, + N_h=population, + n_h=n_h, + n_h_target=target, ) - - used[stratum] = { - "predicate": spec_s["predicate"], - "table": table, - "screen_kind": spec_s["screen_kind"], - "population_estimate": population, - "drawn": len(rows), - "frame_start": since_dt.isoformat(), - "frame_end": until_dt.isoformat(), - "scope": spec["scope"], - } buckets.append([ - SampleRow( + QueueRow( measurement_uid=r[0], - measurement_start_time=r[1], - probe_cc=r[2] or "", - probe_asn=int(r[3] or 0), - resolver_asn=int(r[4] or 0), - domain=r[5] or "", - input=r[6], - test_name=r[7] or "", - sampling_stratum=stratum, - sampling_weight=population / len(rows), - sample_population=population, - sample_rows=len(rows), - sampling_design_id=derived_id, - screen_kind=spec_s["screen_kind"], + test_name=r[4] or "", + strata=r[3], + blocked_fastpath=bool(r[1]), + blocked_analysis=bool(r[2]), + day=r[5], + draw_id=draw_id, ) for r in rows ]) @@ -477,780 +277,24 @@ def draw_sample( # Interleave rather than concatenate. A queue that runs all the positives # first tells the analyst which stratum they are in, which is most of the # way to telling them the answer. - interleaved: List[SampleRow] = [] + interleaved: List[QueueRow] = [] for i in range(max((len(b) for b in buckets), default=0)): for b in buckets: if i < len(b): interleaved.append(b[i]) - return SampleResponse( - design_id=derived_id, - replicate=replicate, - spec=spec, - frame_start=since_dt, - frame_end=until_dt, - strata=used, - rows=interleaved[:limit], - ) - - -# -------------------------------------------------------------------------- -# The blinded candidate -# -------------------------------------------------------------------------- - - -def _rows_to_dicts(result, columns) -> List[Dict[str, Any]]: - return [dict(zip(columns, row)) for row in result] - - -@router.get("/candidate/{measurement_uid}") -def get_candidate( - measurement_uid: str, - db=Depends(get_clickhouse_session), -) -> Dict[str, Any]: - """Everything needed to judge one measurement, and nothing more. - - Deliberately absent: the LoNI triple, top_probe_analysis, anomaly, - confirmed, scores. Those are what the corpus exists to evaluate; an - analyst who sees them first is anchored, and every LR fit from those - labels is inflated by an amount nobody can measure. See /reveal. - """ - obs = db.execute( - """ - SELECT * FROM obs_web - WHERE measurement_uid = %(uid)s - ORDER BY observation_idx - """, - {"uid": measurement_uid}, - with_column_types=True, - ) - obs_rows, obs_types = obs - obs_cols = [c[0] for c in obs_types] - if not obs_rows: - raise HTTPException(404, "no observations for that measurement_uid") - - # obs_web_ctrl's exact columns vary by pipeline version, so select * and - # let the client field-match. Verify against your deployment before - # trusting the diff. - ctrl_rows, ctrl_types = db.execute( - "SELECT * FROM obs_web_ctrl WHERE measurement_uid = %(uid)s", - {"uid": measurement_uid}, - with_column_types=True, + log.info( + "draw %s: %d rows from frame %s in %.1fs", + draw_id, len(interleaved), frame.frame_version, time.monotonic() - started, ) - ctrl_cols = [c[0] for c in ctrl_types] - - return { - "measurement_uid": measurement_uid, - "observations": _rows_to_dicts(obs_rows, obs_cols), - "controls": _rows_to_dicts(ctrl_rows, ctrl_cols), - "blinded": True, - } - - -@router.get("/context") -def get_context( - hostname: str, - probe_cc: str = Query(..., min_length=2, max_length=2), - probe_asn: int = Query(...), - at: datetime = Query(..., description="Centre of the window"), - hours: int = Query(6, ge=1, le=72), - db=Depends(get_clickhouse_session), -) -> Dict[str, Any]: - """Failure-string counts per hour for this hostname on this network, - centred on the measurement. - - This is the panel that separates "one probe had a bad minute" from "this - network stopped resolving this name at 14:00". It is failure strings only — - still no verdicts. - """ - rows = db.execute( - """ - WITH multiIf( - dns_failure IS NOT NULL, concat('dns.', dns_failure), - tcp_failure IS NOT NULL, concat('tcp.', tcp_failure), - tls_failure IS NOT NULL, concat('tls.', tls_failure), - http_failure IS NOT NULL, concat('http.', http_failure), - 'ok' - ) AS failure_str - SELECT toStartOfHour(measurement_start_time) AS ts, - failure_str, - resolver_asn, - count() AS cnt - FROM obs_web - WHERE hostname = %(hostname)s - AND probe_cc = %(cc)s - AND probe_asn = %(asn)s - AND measurement_start_time >= %(since)s - AND measurement_start_time < %(until)s - GROUP BY ts, failure_str, resolver_asn - ORDER BY ts - """, - { - "hostname": hostname, - "cc": probe_cc.upper(), - "asn": probe_asn, - "since": at - timedelta(hours=hours), - "until": at + timedelta(hours=hours), - }, - ) - return { - "hostname": hostname, - "window_hours": hours, - "series": [ - { - "ts": r[0], - "failure_str": r[1], - "resolver_asn": int(r[2] or 0), - "count": int(r[3]), - } - for r in rows - ], - } - - -# -------------------------------------------------------------------------- -# The reveal — called only after the analyst commits -# -------------------------------------------------------------------------- - - -@router.get("/reveal/{measurement_uid}") -def reveal( - measurement_uid: str, - db=Depends(get_clickhouse_session), -) -> Dict[str, Any]: - """What the pipeline concluded. - - Shown after commit, never before. Two uses: analysts find rule bugs this - way, and the agreement rate between analyst and pipeline is a diagnostic - worth watching — as a signal that blinding is holding, not as a target to - improve. - """ - a = db.execute( - """ - SELECT top_probe_analysis, top_dns_failure, top_tcp_failure, - top_tls_failure, - dns_blocked, dns_down, dns_ok, - tcp_blocked, tcp_down, tcp_ok, - tls_blocked, tls_down, tls_ok - FROM analysis_web_measurement - WHERE measurement_uid = %(uid)s - LIMIT 1 - """, - {"uid": measurement_uid}, - ) - f = db.execute( - """ - SELECT anomaly, confirmed, msm_failure, scores - FROM fastpath WHERE measurement_uid = %(uid)s LIMIT 1 - """, - {"uid": measurement_uid}, - ) - - analysis = None - if a: - r = a[0] - analysis = { - "top_probe_analysis": r[0], - "top_dns_failure": r[1], - "top_tcp_failure": r[2], - "top_tls_failure": r[3], - "loni": { - "dns": {"blocked": r[4], "down": r[5], "ok": r[6]}, - "tcp": {"blocked": r[7], "down": r[8], "ok": r[9]}, - "tls": {"blocked": r[10], "down": r[11], "ok": r[12]}, - }, - } - - fastpath = None - if f: - r = f[0] - fastpath = { - "anomaly": r[0] == "t", - "confirmed": r[1] == "t", - "msm_failure": r[2] == "t", - "scores": r[3], - } - - return { - "measurement_uid": measurement_uid, - "analysis": analysis, - "fastpath": fastpath, - "caveat": "The LoNI triple is hand-set and uncalibrated. It is shown " - "as a claim to check, not a reference answer.", - } - - -# -------------------------------------------------------------------------- -# Interval grain: the silent-time denominator -# -------------------------------------------------------------------------- -# -# The event corpus is curated, so event recall is a coverage statement about a -# hand-built set. Nothing in it defines a week the detector should have been -# silent through, so the harness's "false alerts per series-week" had no frame -# behind it. This is that frame. -# -# The unit is the detector's own unit. `event_detector_cusums` keys on -# (probe_cc, probe_asn, domain), so anything coarser here would estimate a rate -# over a different population than the one the detector runs on. -# -# WHY THE STRATA PARTITION THE FRAME. The design note describes two draws — one -# over the intervals where the incumbent alerted, one random over covered -# cell-weeks. Taken literally they overlap: an alerted cell-week is also in the -# random stratum's population, so it has two selection probabilities and no -# single weight is correct for it. Here the strata are a partition instead, and -# `random_covered`'s predicate is resolved against the *set of strata being -# drawn* so the partition stays exhaustive whichever subset you ask for. That -# resolved predicate goes into the design spec, so a weight can never be -# reinterpreted under a different partition than the one it was drawn under. -# -# WHY THE ALERTED STRATUM IS NOT CIRCULAR. On its own it would be: it estimates -# the incumbent's precision conditional on having fired, which says nothing -# about quiet time, and a *candidate* detector's alerts in cells the incumbent -# never flagged would land on intervals nobody adjudicated. As a stratum with a -# recorded screen and a weight it is fine — the weight states how much of the -# frame it stands for. Note this uses the historical alert log as a screen; it -# does not replay the incumbent, which the harness cannot do anyway. - -# ISO weeks: toStartOfWeek(t, 1) is Monday-based, matching the `x ISO week` -# unit. A partial week at either end of the frame is a shorter observation -# window with fewer measurements in it, which is not the same unit at all, so -# frames are snapped to whole weeks rather than truncated. -_WEEK = timedelta(days=7) - -# Cell-weeks below this many measurements are not in the frame. Uniform draws -# over *all* cell-weeks are dominated by cells too thin for any detector to -# fire, and including them makes every detector score well by measuring mostly -# arithmetic. The floor is recorded in the spec and reported per volume band, -# so the exclusion is visible rather than baked in. -DEFAULT_VOLUME_FLOOR = 20 - -# Bands are derived from the measurement count on read, never entered — the -# same rule `ongoing` and `size_band` follow in the event grain. Edges are in -# the spec because they define what a per-band rate means. -VOLUME_BAND_EDGES = ((100, "low"), (1000, "medium"), (None, "high")) - -INTERVAL_DESIGN_SCHEMA_VERSION = "1" - - -def volume_band(n: int) -> str: - for edge, name in VOLUME_BAND_EDGES: - if edge is None or n < edge: - return name - return VOLUME_BAND_EDGES[-1][1] - - -# The detector runs on the citizenlab global list plus twitter.com -# (`detector.get_domain_list`), so a frame over every domain would count quiet -# time in cells the detector never watches and flatter it for free. `detector` -# is the default for that reason; `all` is available for scoring a candidate -# with a wider remit, and which one was used is in the spec. -DETECTOR_DOMAINS_SQL = ( - "(domain IN (SELECT domain FROM citizenlab " - "WHERE category_code = 'GRP' AND cc = 'ZZ') OR domain = 'twitter.com')" -) - -# Cell key as a string on both sides of the alert join. The tuple form reads -# better but compares a UInt32 probe_asn against whatever width the other table -# declares, and a type mismatch there fails as an empty alerted set — which -# looks exactly like "the detector never fired", i.e. a wrong answer rather -# than an error. -_CELL_KEY = "concat({cc}, '|', toString({asn}), '|', {dom}, '|', toString({wk}))" - -ALERTED_CELLS_SQL = f""" - SELECT {_CELL_KEY.format(cc='probe_cc', asn='probe_asn', dom='domain', - wk='toStartOfWeek(ts, 1)')} - FROM event_detector_changepoints - WHERE ts >= %(since)s AND ts < %(until)s AND change_dir > 0 -""" - -_IS_ALERTED = ( - _CELL_KEY.format(cc="probe_cc", asn="probe_asn", dom="domain", wk="week") - + f" IN ({ALERTED_CELLS_SQL})" -) - -# `blocked_max` is the cell-week's loudest measurement. It is used to define -# the near-miss stratum and is NEVER returned to the client: it is a pipeline -# judgment, and on this grain it is close to the verdict itself. -INTERVAL_STRATA: Dict[str, Dict[str, Any]] = { - "detector_alerted": { - "predicate": _IS_ALERTED, - "screen_kind": "incumbent_alert", - "note": "Cell-weeks the deployed detector fired in. The historical " - "alert log used as a screen, not replayed.", - }, - "near_miss": { - # Importance sampling, not a separate population: most random - # cell-weeks are trivially quiet and carry almost no information per - # minute of analyst time. Oversampling cells that had blocked-leaning - # measurements without alerting is where the disagreements live, and - # the weights correct for it. This is the whole reason to record a - # design rather than draw uniformly. - "predicate": f"NOT ({_IS_ALERTED}) AND blocked_max >= {BLOCKING_THRESHOLD}", - "screen_kind": "near_miss_score", - "note": "Did not alert, but something in the week scored " - "blocked-leaning. Optional; when omitted these cells stay in " - "random_covered.", - }, - "random_covered": { - # Resolved at draw time against the selected strata — see the note at - # the top of this section. - "predicate": None, - "screen_kind": "volume_stratified_random", - "note": "The denominator. Everything in the frame the other selected " - "strata did not take.", - }, -} - - -def _resolve_interval_predicates(wanted: List[str]) -> Dict[str, str]: - """Turn the selected strata into an exhaustive, disjoint partition. - - `random_covered` is the complement of whatever else was selected, so the - frame is covered exactly once however the queue is composed. Drawing - `near_miss` alone, with no complement stratum, is allowed and estimates - nothing on its own — the weights say so, since the population it names is - not the frame. - """ - taken = [ - f"({INTERVAL_STRATA[s]['predicate']})" - for s in ("detector_alerted", "near_miss") - if s in wanted - ] - resolved = { - s: INTERVAL_STRATA[s]["predicate"] for s in wanted if s != "random_covered" - } - if "random_covered" in wanted: - resolved["random_covered"] = ( - " AND ".join(f"NOT {t}" for t in taken) if taken else "1" - ) - return resolved - - -def _week_frame(since: Optional[datetime], until: Optional[datetime]): - """Snap the frame to whole Monday-based weeks.""" - since_dt, until_dt = _frame(since, until) - lo = since_dt.replace(hour=0, minute=0, second=0, microsecond=0) - lo -= timedelta(days=lo.weekday()) - if lo < since_dt: - lo += _WEEK - hi = until_dt.replace(hour=0, minute=0, second=0, microsecond=0) - hi -= timedelta(days=hi.weekday()) - if hi <= lo: - raise HTTPException( - 400, - "frame contains no whole ISO week — a partial week is a shorter " - "observation window, not a smaller one", - ) - return lo, hi - - -class IntervalRow(BaseModel): - probe_cc: str - probe_asn: int - domain: str - window_start: datetime - window_end: datetime - # From the coverage query, not a guess. The band is derived from it, and - # the harness re-derives rather than trusting the stored band. - measurements_in_window: int - volume_band: str - # sampling provenance, carried through to the label - sampling_stratum: str - sampling_weight: float - sample_population: int - sample_rows: int - sampling_design_id: str - screen_kind: str - - -class IntervalSampleResponse(BaseModel): - design_id: str - replicate: int - spec: Dict[str, Any] - frame_start: datetime - frame_end: datetime - strata: Dict[str, Dict[str, Any]] - rows: List[IntervalRow] - - -@router.get("/interval_sample", response_model=IntervalSampleResponse) -def draw_interval_sample( - db=Depends(get_clickhouse_session), - strata: str = Query( - "detector_alerted,random_covered", - description="Comma-separated. Drawn separately and interleaved, so " - "the analyst cannot infer from a row's position whether " - "the incumbent alerted in it.", - ), - replicate: int = Query( - 1, ge=1, - description="Independent draws of the same design. Same replicate = " - "same cell-weeks, so two analysts can be given an overlap " - "set without any coordination service.", - ), - since: Optional[datetime] = None, - until: Optional[datetime] = None, - probe_cc: Optional[str] = Query(None, min_length=2, max_length=2), - probe_asn: Optional[int] = None, - domain: Optional[str] = None, - domain_list: str = Query( - "detector", - description="'detector' restricts the frame to the domains the " - "deployed detector runs on; 'all' widens it. Counting " - "quiet time in cells nothing watches inflates the " - "denominator.", - ), - min_measurements: int = Query( - DEFAULT_VOLUME_FLOOR, ge=1, - description="Volume floor. Cell-weeks below it are not in the frame.", - ), - shares: Optional[str] = Query( - None, - description="Optional stratum=share pairs. Steers analyst effort " - "only — row weights stay population/drawn regardless.", - ), - limit: int = Query(40, ge=1, le=500), -) -> IntervalSampleResponse: - """Draw cell-weeks to adjudicate. - - What the analyst decides about each is whether the state *changed* inside - it, which is the only thing a changepoint detector can be right or wrong - about. Two of the verdicts mean it did not — `quiet_observed` for a week - where OONI saw nothing wrong, `blocked_throughout` for a week inside a - block that started earlier — and both belong in the false-alarm - denominator, because the detector should be silent in either. - - `quiet_observed`, never `quiet`: the week is judged from the same OONI data - the detector reads, so an unmeasured block is indistinguishable from calm. - That caps the claim at "no interference visible in OONI's data", which is - the honest ceiling, and it is why a better candidate that finds subtle real - events is not silently charged a false alarm. - """ - since_dt, until_dt = _week_frame(since, until) - wanted = sorted({s.strip() for s in strata.split(",") if s.strip()}) - unknown = [s for s in wanted if s not in INTERVAL_STRATA] - if unknown: - raise HTTPException(400, f"unknown strata: {unknown}") - if not wanted: - raise HTTPException(400, "no strata selected") - if domain_list not in ("detector", "all"): - raise HTTPException(400, "domain_list must be 'detector' or 'all'") - - predicates = _resolve_interval_predicates(wanted) - - scope_sql: List[str] = [] - scope_params: Dict[str, Any] = {} - if probe_cc: - scope_sql.append("probe_cc = %(probe_cc)s") - scope_params["probe_cc"] = probe_cc.upper() - if probe_asn: - scope_sql.append("probe_asn = %(probe_asn)s") - scope_params["probe_asn"] = probe_asn - if domain: - scope_sql.append("domain = %(domain)s") - scope_params["domain"] = domain - if domain_list == "detector": - scope_sql.append(DETECTOR_DOMAINS_SQL) - - # Everything that changes which cell-weeks are eligible, or what a weight - # means, is in the spec — including the resolved partition and the volume - # floor, so two different frames can never collide onto one design id. - spec = { - "schema": INTERVAL_DESIGN_SCHEMA_VERSION, - "grain": "interval", - "unit": "probe_cc,probe_asn,domain x iso_week", - "scoring_version": SCORING_VERSION, - "blocking_threshold": BLOCKING_THRESHOLD, - "strata": { - s: { - "predicate": predicates[s], - "screen_kind": INTERVAL_STRATA[s]["screen_kind"], - } - for s in wanted - }, - "frame": [since_dt.isoformat(), until_dt.isoformat()], - "volume_floor": min_measurements, - "volume_band_edges": [[e, n] for e, n in VOLUME_BAND_EDGES], - "domain_list": domain_list, - "scope": { - "probe_cc": probe_cc.upper() if probe_cc else None, - "probe_asn": probe_asn, - "domain": domain, - }, - "replicate": replicate, - } - derived_id = _design_fingerprint(spec) - - quotas = _quotas(wanted, shares, limit) - - # `blocked_max` costs three float columns over the whole frame and only the - # near-miss stratum is defined in terms of it, so it is not read when that - # stratum was not asked for. - wants_blocked_max = "near_miss" in wanted - - # MUST not be run while a reprocess is in progress due to lack of FINAL - # No test_name filter: the detector does not have one either, and the - # frame has to be the population the detector actually runs over. - cells_sql = f""" - SELECT probe_cc, - probe_asn, - domain, - toStartOfWeek(measurement_start_time, 1) AS week, - count() AS n - {", max(greatest(dns_blocked, tcp_blocked, tls_blocked)) AS blocked_max" - if wants_blocked_max else ""} - FROM analysis_web_measurement - WHERE measurement_start_time >= %(since)s - AND measurement_start_time < %(until)s - {''.join(' AND ' + s for s in scope_sql)} - GROUP BY probe_cc, probe_asn, domain, week - HAVING n >= %(floor)s - """ - - # The strata are disjoint by construction (see - # `_resolve_interval_predicates`), which is what lets one expression label - # every cell-week in a single pass. Drawing each stratum under its own - # WHERE meant re-running this GROUP BY once to size the population and - # again to draw it — 2N aggregations of the same frame for N strata, and - # that aggregation is the whole cost of the endpoint. - # - # `random_covered` is the fallback rather than a branch of its own: its - # predicate is the negation of the others, so spelling it out would - # evaluate the alerted-set lookup a second time. The lookup is hoisted to - # an alias for the same reason — substituting the module constant into - # itself, so it cannot match anything else by accident. The predicate text - # recorded in the spec stays verbatim, since that text is what the design - # id hashes. - branches = ", ".join( - f"({predicates[s].replace(_IS_ALERTED, 'is_alerted')}), '{s}'" - for s in wanted - if s != "random_covered" - ) - fallback = "'random_covered'" if "random_covered" in wanted else "''" - stratum_sql = f"multiIf({branches}, {fallback})" if branches else fallback - # Drawing only the complement never asks about alerts, so in that case the - # changepoint set is not built at all. - alerted_sql = f"{_IS_ALERTED} AS is_alerted," if "is_alerted" in stratum_sql else "" - - # Per-stratum quotas, so an uneven `shares` split still takes exactly what - # it was promised out of one shared ordering. - quota_sql = "multiIf(" + ", ".join( - f"stratum = '{s}', {int(quotas[s])}" for s in wanted - ) + ", 0)" - - # Same salt the per-stratum draws used — `_design_fingerprint` plus the - # stratum name — so a replicate drawn before this rewrite and one drawn - # after select the same cell-weeks. - order_sql = ( - "cityHash64(concat(" - + _CELL_KEY.format(cc="probe_cc", asn="probe_asn", dom="domain", wk="week") - + ", %(design)s, ':', stratum))" - ) - - params: Dict[str, Any] = { - "since": since_dt, - "until": until_dt, - "floor": min_measurements, - "design": derived_id, - **scope_params, - } - - # NOTE: no blocked_max, no alert state, no changepoints leave this query. - # The cell key, the window and how much data is in it — that is all an - # analyst gets before committing. - t0 = time.monotonic() - rows = db.execute( - f""" - SELECT probe_cc, probe_asn, domain, week, n, stratum, population - FROM ( - SELECT probe_cc, probe_asn, domain, week, n, stratum, - count() OVER (PARTITION BY stratum) AS population, - row_number() OVER ( - PARTITION BY stratum ORDER BY {order_sql} - ) AS rn - FROM ( - SELECT probe_cc, probe_asn, domain, week, n, - {alerted_sql} - {stratum_sql} AS stratum - FROM ({cells_sql}) AS cells - ) AS labelled - WHERE stratum != '' - ) AS ranked - WHERE rn <= {quota_sql} - ORDER BY stratum, rn - """, - params, - ) - log.info("interval frame query: %.2fs", time.monotonic() - t0) - - # A stratum with a population draws at least one row (quotas floor at 1), - # so an absent stratum here is an empty one and keeps its zero below. - drawn: Dict[str, List[Any]] = {s: [] for s in wanted} - populations: Dict[str, int] = {s: 0 for s in wanted} - for r in rows: - drawn[r[5]].append(r) - populations[r[5]] = int(r[6]) - - used: Dict[str, Dict[str, Any]] = {} - buckets: List[List[IntervalRow]] = [] - - for stratum in wanted: - stratum_rows = drawn[stratum] - population = populations[stratum] - used[stratum] = { - "predicate": predicates[stratum], - "table": "analysis_web_measurement", - "screen_kind": INTERVAL_STRATA[stratum]["screen_kind"], - "population_estimate": population, - "drawn": len(stratum_rows), - "frame_start": since_dt.isoformat(), - "frame_end": until_dt.isoformat(), - "volume_floor": min_measurements, - "scope": spec["scope"], - } - if not stratum_rows: - continue - buckets.append([ - IntervalRow( - probe_cc=r[0] or "", - probe_asn=int(r[1] or 0), - domain=r[2] or "", - window_start=datetime(r[3].year, r[3].month, r[3].day), - window_end=datetime(r[3].year, r[3].month, r[3].day) + _WEEK, - measurements_in_window=int(r[4]), - volume_band=volume_band(int(r[4])), - sampling_stratum=stratum, - sampling_weight=population / len(stratum_rows), - sample_population=population, - sample_rows=len(stratum_rows), - sampling_design_id=derived_id, - screen_kind=INTERVAL_STRATA[stratum]["screen_kind"], - ) - for r in stratum_rows - ]) - - interleaved: List[IntervalRow] = [] - for i in range(max((len(b) for b in buckets), default=0)): - for b in buckets: - if i < len(b): - interleaved.append(b[i]) - - return IntervalSampleResponse( - design_id=derived_id, + return SampleResponse( + draw_id=draw_id, + draw_timestamp=datetime.now(timezone.utc), + frame_version=frame.frame_version, replicate=replicate, + sample_count=sample_count, spec=spec, - frame_start=since_dt, - frame_end=until_dt, + frame=frame, strata=used, - rows=interleaved[:limit], - ) - - -@router.get("/interval_reveal") -def interval_reveal( - probe_cc: str = Query(..., min_length=2, max_length=2), - probe_asn: int = Query(...), - domain: str = Query(...), - window_start: datetime = Query(...), - window_end: datetime = Query(...), - pad_days: int = Query(7, ge=0, le=28), - db=Depends(get_clickhouse_session), -) -> Dict[str, Any]: - """What the detector did in this cell-week. Shown after commit, never - before. - - Two things, because a bare alert flag is not diagnosable: the changepoints - themselves, and the hourly signal the detector consumed to produce them. - The signal is a median per cell-hour, exactly as `detector.get_observations` - computes it, so an analyst who disagrees with an alert can see whether the - detector saw something they did not or scored what they saw differently. - """ - lo = window_start - timedelta(days=pad_days) - hi = window_end + timedelta(days=pad_days) - params = { - "cc": probe_cc.upper(), - "asn": probe_asn, - "domain": domain, - "lo": lo, - "hi": hi, - "ws": window_start, - "we": window_end, - } - - cps = db.execute( - """ - SELECT ts, block_type, change_dir, s_pos, s_neg, current_state, h - FROM event_detector_changepoints - WHERE probe_cc = %(cc)s AND probe_asn = %(asn)s AND domain = %(domain)s - AND ts >= %(lo)s AND ts < %(hi)s - ORDER BY ts - """, - params, + rows=interleaved, ) - - # No FINAL here, unlike the sampler's frame query. This one is read to draw - # a chart, not to size a population: an unmerged duplicate during a - # reprocess nudges a median, where in the frame it would move a cell-week - # into another volume band and corrupt a weight. - signal = db.execute( - """ - WITH IF(resolver_asn = probe_asn, 1, 0) AS is_isp_resolver - SELECT toStartOfHour(measurement_start_time) AS ts, - count() AS n, - quantileIf(0.5)(dns_blocked, is_isp_resolver = 1) AS dns_isp_blocked, - quantileIf(0.5)(dns_blocked, is_isp_resolver = 0) AS dns_other_blocked, - quantile(0.5)(tcp_blocked) AS tcp_blocked, - quantile(0.5)(tls_blocked) AS tls_blocked - FROM analysis_web_measurement - WHERE probe_cc = %(cc)s AND probe_asn = %(asn)s AND domain = %(domain)s - AND measurement_start_time >= %(lo)s - AND measurement_start_time < %(hi)s - GROUP BY ts - ORDER BY ts - """, - params, - ) - - changepoints = [ - { - "ts": r[0], - "block_type": r[1], - "change_dir": int(r[2] or 0), - "s_pos": r[3], - "s_neg": r[4], - "current_state": r[5], - "h": r[6], - # Whether it lands in the adjudicated week is the whole question, - # so the client is not left to redo the comparison in local time. - "in_window": window_start <= r[0].replace(tzinfo=None) < window_end, - } - for r in cps - ] - - return { - "probe_cc": probe_cc.upper(), - "probe_asn": probe_asn, - "domain": domain, - "window_start": window_start, - "window_end": window_end, - "pad_days": pad_days, - "changepoints": changepoints, - "alerts_in_window": sum( - 1 for c in changepoints if c["in_window"] and c["change_dir"] > 0 - ), - "signal": [ - { - "ts": r[0], - "count": int(r[1]), - "dns_isp_blocked": r[2], - "dns_other_blocked": r[3], - "tcp_blocked": r[4], - "tls_blocked": r[5], - } - for r in signal - ], - "caveat": "The deployed detector is online: this is the alert log it " - "actually emitted, under whatever state it carried at the " - "time. It is not a replay and cannot be reproduced from " - "this window alone.", - }