MW-1482: requisition-grain submission date, and Skipped as its own reporting outcome

dbt/models/marts/mart_logistics_summary.sql

@@ -28,7 +28,11 @@ with latest_month as (
 ),
 
 top_products as (
-  select program_name, product_name
+  -- keys aliased so they cannot collide with s.program_name / s.product_name:
+  -- with the same name on both sides of the join the ClickHouse analyser emits the
+  -- qualified name into the output, and the table ends up carrying a column
+  -- literally called "s.product_name"
+  select program_name as tp_program_name, product_name as tp_product_name
   from (
     select
       program_name,
@@ -49,8 +53,20 @@ top_products as (
   where consumption_rank <= 5
 )
 
-select s.*
+select
+  s.*,
+  -- Month completeness, so the report can hide the structurally partial newest
+  -- month the way every other chart on this dashboard already does. A missing
+  -- flag row degrades to "complete" (ClickHouse fills a LEFT JOIN miss with the
+  -- type default, not NULL), so a stale flags table can never blank the report.
+  if(mc.month = toDate(0), 1, mc.is_complete) as in_complete_month
 from {{ ref('mart_stock_status') }} s
 inner join top_products tp
-  on s.program_name = tp.program_name
- and s.product_name = tp.product_name
+  on s.program_name = tp.tp_program_name
+ and s.product_name = tp.tp_product_name
+left join (
+  select month, is_complete
+  from {{ ref('mart_month_completeness') }}
+  where family = 'stock'
+) mc
+  on mc.month = toStartOfMonth(s.period_end_date)

dbt/models/marts/mart_month_completeness.sql

@@ -24,6 +24,16 @@
 --                 obligations exist in advance, so they cannot be the yardstick)
 --   adjustments - distinct facilities with adjustments   (mart_adjustments)
 --
+-- A month must also carry a comparable BASE, which is the population the
+-- activity is measured against: obligations for reporting, and the activity
+-- itself for the other two families, where the two are the same thing. This
+-- catches a month whose cohort changed rather than whose data is late, and
+-- there is a live example: the weekly reporting schedule stops generating
+-- periods in April 2026, so May 2026 carries 6,072 obligations against 9,284
+-- the month before. Reported volume held, so the activity signal alone called
+-- May complete, and the pooled reporting rate jumped 51.4% to 75.8% on the
+-- final point of every trend - a composition change reading as improvement.
+--
 -- Rebuilt on every dbt run (one aggregation per family), so the flags track
 -- the data as months fill in. Charts consume the flags through the dataset
 -- layer; the incremental marts themselves stay untouched.
@@ -33,7 +43,8 @@ with monthly as (
   select
     'stock'                              as family,
     toStartOfMonth(period_end_date)      as month,
-    uniqExact(facility_id)               as units
+    uniqExact(facility_id)               as units,
+    uniqExact(facility_id)               as base
   from {{ ref('mart_stock_status') }}
   group by month
 
@@ -42,7 +53,12 @@ with monthly as (
   select
     'reporting'                          as family,
     toStartOfMonth(period_end_date)      as month,
-    countIf(reporting_status = 'Reported') as units
+    -- Actual reports, and deliberately NOT following the skip policy: this measures how
+    -- much data arrived in a month, so a skipped period contributes nothing whatever the
+    -- reporting rate is later decided to count. Changing this shifts in_complete_month,
+    -- which gates the months every reporting chart draws.
+    countIf(reporting_status = 'Reported') as units,
+    count()                                as base
   from {{ ref('mart_reporting_status') }}
   group by month
 
@@ -51,7 +67,8 @@ with monthly as (
   select
     'adjustments'                        as family,
     toStartOfMonth(period_end_date)      as month,
-    uniqExact(facility_id)               as units
+    uniqExact(facility_id)               as units,
+    uniqExact(facility_id)               as base
   from {{ ref('mart_adjustments') }}
   group by month
 
@@ -63,12 +80,19 @@ with_ratio as (
     family,
     month,
     units,
+    base,
     units / nullIf(
       max(units) over (
         partition by family
         order by month
         rows between 3 preceding and 1 preceding
-      ), 0)                              as coverage_ratio
+      ), 0)                              as coverage_ratio,
+    base / nullIf(
+      max(base) over (
+        partition by family
+        order by month
+        rows between 3 preceding and 1 preceding
+      ), 0)                              as base_ratio
   from monthly
 
 ),
@@ -79,9 +103,16 @@ flagged as (
     family,
     month,
     units,
+    base,
     coverage_ratio,
-    if(coverage_ratio is null
-       or coverage_ratio >= {{ var('month_completeness_threshold', 0.8) }},
+    base_ratio,
+    -- both signals must pass: late data fails the first, a changed cohort the
+    -- second. For stock and adjustments the two are the same measure, so the
+    -- base test is a no-op there by construction.
+    if((coverage_ratio is null
+        or coverage_ratio >= {{ var('month_completeness_threshold', 0.8) }})
+       and (base_ratio is null
+        or base_ratio >= {{ var('month_completeness_threshold', 0.8) }}),
        1, 0)                             as is_complete
   from with_ratio
 
@@ -91,7 +122,9 @@ select
   family,
   month,
   units,
+  base,
   round(coverage_ratio, 3)               as coverage_ratio,
+  round(base_ratio, 3)                   as base_ratio,
   is_complete,
   if(is_complete = 1
      and month = max(if(is_complete = 1, month, toDate(0)))

dbt/models/marts/mart_non_reporting_facilities.sql

@@ -15,6 +15,10 @@
 --     to determine the correct processing schedule per facility-program
 --   - Uses status_changes to check if a requisition was SUBMITTED
 --   - Only includes active, enabled facilities with active program support
+-- was_skipped marks the rows where the facility deliberately skipped the
+-- period rather than going silent. The rows stay in this table so no existing
+-- number moves; the flag is here so a reader can separate the two, matching
+-- the Skipped series on Reporting Rate by Program.
 -- Rolling 3-year window on requisition created_date.
 
 with facility_program_schedules as (
@@ -59,10 +63,27 @@ submitted as (
     and sc.status = 'SUBMITTED'
   where r.created_date >= now() - interval 3 year
     and r.emergency = false
+),
+
+-- Periods the facility deliberately skipped, detected the same way
+skipped as (
+  select distinct
+    r.facility_id                    as facility_id,
+    r.program_id                     as program_id,
+    r.processing_period_id           as processing_period_id
+  from {{ ref('stg_requisitions') }} r
+  inner join {{ ref('stg_status_changes') }} sc
+    on sc.requisition_id = r.id
+    and sc.status = 'SKIPPED'
+  where r.created_date >= now() - interval 3 year
+    and r.emergency = false
 )
 
 select
-  e.facility_id,
+  -- aliased explicitly: the ClickHouse analyser keeps a join key's output
+  -- column qualified when the name exists on both sides, which emits a column
+  -- literally called "e.facility_id" and breaks this table's ORDER BY
+  e.facility_id     as facility_id,
   f.code            as facility_code,
   f.name            as facility_name,
   f.active          as facility_active,
@@ -79,8 +100,13 @@ select
   {{ schedule_type('e.period_start_date', 'e.period_end_date') }}
                     as schedule_type,
 
-  'Did not report'  as reporting_status
+  'Did not report'  as reporting_status,
+  if(k.facility_id is null, 0, 1) as was_skipped
 from expected e
+left join skipped k
+  on e.facility_id = k.facility_id
+  and e.program_id = k.program_id
+  and e.period_id  = k.processing_period_id
 left join submitted s
   on e.facility_id = s.facility_id
   and e.program_id = s.program_id

dbt/models/marts/mart_reporting_status.sql

@@ -18,7 +18,12 @@
 --     determine the schedule each (facility, program) follows
 --   - status_changes (status='SUBMITTED') determines if a report happened
 --   - reporting_status: 'Reported' iff a SUBMITTED requisition exists for
---     the (facility, program, period) combination, else 'Did not report'
+--     the (facility, program, period) combination; 'Skipped' when no report
+--     exists but the facility deliberately skipped that period (a SKIPPED
+--     status change); 'Did not report' otherwise. Reported wins over Skipped
+--     if a key somehow carries both. Splitting Skipped out does NOT change
+--     the reporting_rate metric, which still counts only 'Reported' over all
+--     obligations - a skip stays a miss until the rate policy is decided.
 --   - submitted_date / submitted_week_of_month: from the SUBMITTED status
 --     change, used by the "Reporting Timeliness By Week" chart to bucket
 --     submissions into weeks 1–5 of the month
@@ -68,6 +73,24 @@ submitted as (
   where r.created_date >= now() - interval 3 year
     and r.emergency = false
   group by r.facility_id, r.program_id, r.processing_period_id
+),
+
+skipped as (
+  -- Periods the facility deliberately skipped. Detected through the SKIPPED
+  -- status change for symmetry with `submitted` above; the requisition's
+  -- current status agrees on every row in the Malawi data, so either route
+  -- gives the same set. Every program in that data has periods_skippable set,
+  -- so a skip is a legitimate outcome rather than a data error.
+  select distinct
+    r.facility_id                    as facility_id,
+    r.program_id                     as program_id,
+    r.processing_period_id           as processing_period_id
+  from {{ ref('stg_requisitions') }} r
+  inner join {{ ref('stg_status_changes') }} sc
+    on sc.requisition_id = r.id
+    and sc.status = 'SKIPPED'
+  where r.created_date >= now() - interval 3 year
+    and r.emergency = false
 )
 
 select
@@ -89,9 +112,9 @@ select
                                             as schedule_type,
 
   -- reporting outcome
-  case when s.facility_id is null
-       then 'Did not report'
-       else 'Reported'
+  case when s.facility_id is not null then 'Reported'
+       when k.facility_id is not null then 'Skipped'
+       else 'Did not report'
   end                                       as reporting_status,
   s.submitted_date,
 
@@ -120,6 +143,10 @@ left join submitted s
   on e.facility_id = s.facility_id
   and e.program_id = s.program_id
   and e.period_id  = s.processing_period_id
+left join skipped k
+  on e.facility_id = k.facility_id
+  and e.program_id = k.program_id
+  and e.period_id  = k.processing_period_id
 inner join {{ ref('stg_facilities') }} f
   on e.facility_id = f.id
 left join {{ ref('stg_facility_types') }} ft

dbt/models/marts/mart_requisition_summary.sql

@@ -25,7 +25,11 @@ select
   p.code            as program_code,
   p.name            as program_name,
   pp.name           as period_name,
-  pp.end_date       as period_end_date
+  pp.end_date       as period_end_date,
+
+  -- reporting cadence (shared macro - single source of truth across marts)
+  {{ schedule_type('pp.start_date', 'pp.end_date') }}
+                    as schedule_type
 from {{ ref('stg_requisitions') }} r
 left join {{ ref('stg_facilities') }} f
   on r.facility_id = f.id

dbt/models/marts/mart_stock_status.sql

@@ -44,7 +44,7 @@
 select
   -- line item identifiers
   li.id                         as line_item_id,
-  li.requisition_id,
+  li.requisition_id             as requisition_id,
 
   -- facility
   f.id                          as facility_id,
@@ -137,6 +137,12 @@ select
   r.status                      as requisition_status,
   r.modified_date               as requisition_modified_date,
 
+  -- the requisition's OWN submission timestamp: the earliest SUBMITTED status
+  -- change for this requisition. Kept at requisition grain because the
+  -- reporting mart can only offer it per facility-program-period and hands the
+  -- same date to every requisition sharing that tuple.
+  sub.submitted_date            as submitted_date,
+
   -- computed: order timeliness based on day-of-month of last requisition update
   -- (matches legacy 'Order Timeliness' computed column on stock_status_and_consumption)
   case
@@ -168,6 +174,18 @@ left join {{ ref('stg_processing_schedules') }} ps
   on pp.processing_schedule_id = ps.id
 left join {{ ref('stg_orderables') }} o
   on li.orderable_id = o.id
+left join (
+  -- key aliased so it cannot collide with li.requisition_id: the ClickHouse
+  -- analyser keeps a join key's output column qualified when the name exists on
+  -- both sides, which would break this table's ORDER BY
+  select
+    requisition_id    as sub_requisition_id,
+    min(created_date) as submitted_date
+  from {{ ref('stg_status_changes') }}
+  where status = 'SUBMITTED'
+  group by requisition_id
+) sub
+  on sub.sub_requisition_id = li.requisition_id
 where r.created_date >= now() - interval 3 year
   and pp.end_date >= now() - interval 3 year
 {% if is_incremental() %}

dbt/models/marts/schema.yml

@@ -177,6 +177,13 @@ models:
               arguments:
                 values:
                   - Did not report
+      - name: was_skipped
+        description: "1 when the facility deliberately skipped this period rather than going silent. The row stays in this table either way, so no existing count moves; the flag exists so the two can be told apart, matching the Skipped series on Reporting Rate by Program."
+        tests:
+          - not_null
+          - accepted_values:
+              arguments:
+                values: [0, 1]
       - name: schedule_type
         description: "Reporting frequency (Weekly / Monthly / Quarterly / BUQ), derived from the period length."
         tests:
@@ -244,10 +251,12 @@ models:
 
   - name: mart_reporting_status
     description: >
-      Reporting outcome per (facility × program × period). Superset of
-      mart_non_reporting_facilities — includes both 'Reported' and
-      'Did not report' rows, plus submission timing for the timeliness
-      chart on the Reporting Rate dashboard.
+      Reporting outcome per (facility × program × period): 'Reported',
+      'Skipped' where the facility deliberately skipped the period, or
+      'Did not report'. Carries submission timing for the timeliness chart on
+      the Reporting Rate dashboard. The reporting_rate metric counts only
+      'Reported' over all obligations, so a skip still reads as a miss there;
+      reporting_rate_excl_skipped is the alternative, defined and unused.
       BI contract: dashboards should query this mart, not raw tables.
     columns:
       - name: facility_name
@@ -267,6 +276,7 @@ models:
                 values:
                   - Did not report
                   - Reported
+                  - Skipped
       - name: schedule_type
         description: "Reporting frequency (Weekly / Monthly / Quarterly / BUQ), derived from the period length."
         tests: