diff --git a/dbt/models/marts/mart_logistics_summary.sql b/dbt/models/marts/mart_logistics_summary.sql index 49340ed..64b5d49 100644 --- a/dbt/models/marts/mart_logistics_summary.sql +++ b/dbt/models/marts/mart_logistics_summary.sql @@ -28,7 +28,11 @@ with latest_month as ( ), top_products as ( - select program_name, product_name + -- keys aliased so they cannot collide with s.program_name / s.product_name: + -- with the same name on both sides of the join the ClickHouse analyser emits the + -- qualified name into the output, and the table ends up carrying a column + -- literally called "s.product_name" + select program_name as tp_program_name, product_name as tp_product_name from ( select program_name, @@ -49,8 +53,20 @@ top_products as ( where consumption_rank <= 5 ) -select s.* +select + s.*, + -- Month completeness, so the report can hide the structurally partial newest + -- month the way every other chart on this dashboard already does. A missing + -- flag row degrades to "complete" (ClickHouse fills a LEFT JOIN miss with the + -- type default, not NULL), so a stale flags table can never blank the report. + if(mc.month = toDate(0), 1, mc.is_complete) as in_complete_month from {{ ref('mart_stock_status') }} s inner join top_products tp - on s.program_name = tp.program_name - and s.product_name = tp.product_name + on s.program_name = tp.tp_program_name + and s.product_name = tp.tp_product_name +left join ( + select month, is_complete + from {{ ref('mart_month_completeness') }} + where family = 'stock' +) mc + on mc.month = toStartOfMonth(s.period_end_date) diff --git a/dbt/models/marts/mart_month_completeness.sql b/dbt/models/marts/mart_month_completeness.sql index a4b5192..618dd54 100644 --- a/dbt/models/marts/mart_month_completeness.sql +++ b/dbt/models/marts/mart_month_completeness.sql @@ -24,6 +24,16 @@ -- obligations exist in advance, so they cannot be the yardstick) -- adjustments - distinct facilities with adjustments (mart_adjustments) -- +-- A month must also carry a comparable BASE, which is the population the +-- activity is measured against: obligations for reporting, and the activity +-- itself for the other two families, where the two are the same thing. This +-- catches a month whose cohort changed rather than whose data is late, and +-- there is a live example: the weekly reporting schedule stops generating +-- periods in April 2026, so May 2026 carries 6,072 obligations against 9,284 +-- the month before. Reported volume held, so the activity signal alone called +-- May complete, and the pooled reporting rate jumped 51.4% to 75.8% on the +-- final point of every trend - a composition change reading as improvement. +-- -- Rebuilt on every dbt run (one aggregation per family), so the flags track -- the data as months fill in. Charts consume the flags through the dataset -- layer; the incremental marts themselves stay untouched. @@ -33,7 +43,8 @@ with monthly as ( select 'stock' as family, toStartOfMonth(period_end_date) as month, - uniqExact(facility_id) as units + uniqExact(facility_id) as units, + uniqExact(facility_id) as base from {{ ref('mart_stock_status') }} group by month @@ -42,7 +53,12 @@ with monthly as ( select 'reporting' as family, toStartOfMonth(period_end_date) as month, - countIf(reporting_status = 'Reported') as units + -- Actual reports, and deliberately NOT following the skip policy: this measures how + -- much data arrived in a month, so a skipped period contributes nothing whatever the + -- reporting rate is later decided to count. Changing this shifts in_complete_month, + -- which gates the months every reporting chart draws. + countIf(reporting_status = 'Reported') as units, + count() as base from {{ ref('mart_reporting_status') }} group by month @@ -51,7 +67,8 @@ with monthly as ( select 'adjustments' as family, toStartOfMonth(period_end_date) as month, - uniqExact(facility_id) as units + uniqExact(facility_id) as units, + uniqExact(facility_id) as base from {{ ref('mart_adjustments') }} group by month @@ -63,12 +80,19 @@ with_ratio as ( family, month, units, + base, units / nullIf( max(units) over ( partition by family order by month rows between 3 preceding and 1 preceding - ), 0) as coverage_ratio + ), 0) as coverage_ratio, + base / nullIf( + max(base) over ( + partition by family + order by month + rows between 3 preceding and 1 preceding + ), 0) as base_ratio from monthly ), @@ -79,9 +103,16 @@ flagged as ( family, month, units, + base, coverage_ratio, - if(coverage_ratio is null - or coverage_ratio >= {{ var('month_completeness_threshold', 0.8) }}, + base_ratio, + -- both signals must pass: late data fails the first, a changed cohort the + -- second. For stock and adjustments the two are the same measure, so the + -- base test is a no-op there by construction. + if((coverage_ratio is null + or coverage_ratio >= {{ var('month_completeness_threshold', 0.8) }}) + and (base_ratio is null + or base_ratio >= {{ var('month_completeness_threshold', 0.8) }}), 1, 0) as is_complete from with_ratio @@ -91,7 +122,9 @@ select family, month, units, + base, round(coverage_ratio, 3) as coverage_ratio, + round(base_ratio, 3) as base_ratio, is_complete, if(is_complete = 1 and month = max(if(is_complete = 1, month, toDate(0))) diff --git a/dbt/models/marts/mart_non_reporting_facilities.sql b/dbt/models/marts/mart_non_reporting_facilities.sql index 3aca792..f3b1c3d 100644 --- a/dbt/models/marts/mart_non_reporting_facilities.sql +++ b/dbt/models/marts/mart_non_reporting_facilities.sql @@ -15,6 +15,10 @@ -- to determine the correct processing schedule per facility-program -- - Uses status_changes to check if a requisition was SUBMITTED -- - Only includes active, enabled facilities with active program support +-- was_skipped marks the rows where the facility deliberately skipped the +-- period rather than going silent. The rows stay in this table so no existing +-- number moves; the flag is here so a reader can separate the two, matching +-- the Skipped series on Reporting Rate by Program. -- Rolling 3-year window on requisition created_date. with facility_program_schedules as ( @@ -59,10 +63,27 @@ submitted as ( and sc.status = 'SUBMITTED' where r.created_date >= now() - interval 3 year and r.emergency = false +), + +-- Periods the facility deliberately skipped, detected the same way +skipped as ( + select distinct + r.facility_id as facility_id, + r.program_id as program_id, + r.processing_period_id as processing_period_id + from {{ ref('stg_requisitions') }} r + inner join {{ ref('stg_status_changes') }} sc + on sc.requisition_id = r.id + and sc.status = 'SKIPPED' + where r.created_date >= now() - interval 3 year + and r.emergency = false ) select - e.facility_id, + -- aliased explicitly: the ClickHouse analyser keeps a join key's output + -- column qualified when the name exists on both sides, which emits a column + -- literally called "e.facility_id" and breaks this table's ORDER BY + e.facility_id as facility_id, f.code as facility_code, f.name as facility_name, f.active as facility_active, @@ -79,8 +100,13 @@ select {{ schedule_type('e.period_start_date', 'e.period_end_date') }} as schedule_type, - 'Did not report' as reporting_status + 'Did not report' as reporting_status, + if(k.facility_id is null, 0, 1) as was_skipped from expected e +left join skipped k + on e.facility_id = k.facility_id + and e.program_id = k.program_id + and e.period_id = k.processing_period_id left join submitted s on e.facility_id = s.facility_id and e.program_id = s.program_id diff --git a/dbt/models/marts/mart_reporting_status.sql b/dbt/models/marts/mart_reporting_status.sql index 7fc7660..a511313 100644 --- a/dbt/models/marts/mart_reporting_status.sql +++ b/dbt/models/marts/mart_reporting_status.sql @@ -18,7 +18,12 @@ -- determine the schedule each (facility, program) follows -- - status_changes (status='SUBMITTED') determines if a report happened -- - reporting_status: 'Reported' iff a SUBMITTED requisition exists for --- the (facility, program, period) combination, else 'Did not report' +-- the (facility, program, period) combination; 'Skipped' when no report +-- exists but the facility deliberately skipped that period (a SKIPPED +-- status change); 'Did not report' otherwise. Reported wins over Skipped +-- if a key somehow carries both. Splitting Skipped out does NOT change +-- the reporting_rate metric, which still counts only 'Reported' over all +-- obligations - a skip stays a miss until the rate policy is decided. -- - submitted_date / submitted_week_of_month: from the SUBMITTED status -- change, used by the "Reporting Timeliness By Week" chart to bucket -- submissions into weeks 1–5 of the month @@ -68,6 +73,24 @@ submitted as ( where r.created_date >= now() - interval 3 year and r.emergency = false group by r.facility_id, r.program_id, r.processing_period_id +), + +skipped as ( + -- Periods the facility deliberately skipped. Detected through the SKIPPED + -- status change for symmetry with `submitted` above; the requisition's + -- current status agrees on every row in the Malawi data, so either route + -- gives the same set. Every program in that data has periods_skippable set, + -- so a skip is a legitimate outcome rather than a data error. + select distinct + r.facility_id as facility_id, + r.program_id as program_id, + r.processing_period_id as processing_period_id + from {{ ref('stg_requisitions') }} r + inner join {{ ref('stg_status_changes') }} sc + on sc.requisition_id = r.id + and sc.status = 'SKIPPED' + where r.created_date >= now() - interval 3 year + and r.emergency = false ) select @@ -89,9 +112,9 @@ select as schedule_type, -- reporting outcome - case when s.facility_id is null - then 'Did not report' - else 'Reported' + case when s.facility_id is not null then 'Reported' + when k.facility_id is not null then 'Skipped' + else 'Did not report' end as reporting_status, s.submitted_date, @@ -120,6 +143,10 @@ left join submitted s on e.facility_id = s.facility_id and e.program_id = s.program_id and e.period_id = s.processing_period_id +left join skipped k + on e.facility_id = k.facility_id + and e.program_id = k.program_id + and e.period_id = k.processing_period_id inner join {{ ref('stg_facilities') }} f on e.facility_id = f.id left join {{ ref('stg_facility_types') }} ft diff --git a/dbt/models/marts/mart_requisition_summary.sql b/dbt/models/marts/mart_requisition_summary.sql index 3ae4819..6746403 100644 --- a/dbt/models/marts/mart_requisition_summary.sql +++ b/dbt/models/marts/mart_requisition_summary.sql @@ -25,7 +25,11 @@ select p.code as program_code, p.name as program_name, pp.name as period_name, - pp.end_date as period_end_date + pp.end_date as period_end_date, + + -- reporting cadence (shared macro - single source of truth across marts) + {{ schedule_type('pp.start_date', 'pp.end_date') }} + as schedule_type from {{ ref('stg_requisitions') }} r left join {{ ref('stg_facilities') }} f on r.facility_id = f.id diff --git a/dbt/models/marts/mart_stock_status.sql b/dbt/models/marts/mart_stock_status.sql index 9a79a45..89549dc 100644 --- a/dbt/models/marts/mart_stock_status.sql +++ b/dbt/models/marts/mart_stock_status.sql @@ -44,7 +44,7 @@ select -- line item identifiers li.id as line_item_id, - li.requisition_id, + li.requisition_id as requisition_id, -- facility f.id as facility_id, @@ -137,6 +137,12 @@ select r.status as requisition_status, r.modified_date as requisition_modified_date, + -- the requisition's OWN submission timestamp: the earliest SUBMITTED status + -- change for this requisition. Kept at requisition grain because the + -- reporting mart can only offer it per facility-program-period and hands the + -- same date to every requisition sharing that tuple. + sub.submitted_date as submitted_date, + -- computed: order timeliness based on day-of-month of last requisition update -- (matches legacy 'Order Timeliness' computed column on stock_status_and_consumption) case @@ -168,6 +174,18 @@ left join {{ ref('stg_processing_schedules') }} ps on pp.processing_schedule_id = ps.id left join {{ ref('stg_orderables') }} o on li.orderable_id = o.id +left join ( + -- key aliased so it cannot collide with li.requisition_id: the ClickHouse + -- analyser keeps a join key's output column qualified when the name exists on + -- both sides, which would break this table's ORDER BY + select + requisition_id as sub_requisition_id, + min(created_date) as submitted_date + from {{ ref('stg_status_changes') }} + where status = 'SUBMITTED' + group by requisition_id +) sub + on sub.sub_requisition_id = li.requisition_id where r.created_date >= now() - interval 3 year and pp.end_date >= now() - interval 3 year {% if is_incremental() %} diff --git a/dbt/models/marts/schema.yml b/dbt/models/marts/schema.yml index 986aa3b..8fd7358 100644 --- a/dbt/models/marts/schema.yml +++ b/dbt/models/marts/schema.yml @@ -177,6 +177,13 @@ models: arguments: values: - Did not report + - name: was_skipped + description: "1 when the facility deliberately skipped this period rather than going silent. The row stays in this table either way, so no existing count moves; the flag exists so the two can be told apart, matching the Skipped series on Reporting Rate by Program." + tests: + - not_null + - accepted_values: + arguments: + values: [0, 1] - name: schedule_type description: "Reporting frequency (Weekly / Monthly / Quarterly / BUQ), derived from the period length." tests: @@ -244,10 +251,12 @@ models: - name: mart_reporting_status description: > - Reporting outcome per (facility × program × period). Superset of - mart_non_reporting_facilities — includes both 'Reported' and - 'Did not report' rows, plus submission timing for the timeliness - chart on the Reporting Rate dashboard. + Reporting outcome per (facility × program × period): 'Reported', + 'Skipped' where the facility deliberately skipped the period, or + 'Did not report'. Carries submission timing for the timeliness chart on + the Reporting Rate dashboard. The reporting_rate metric counts only + 'Reported' over all obligations, so a skip still reads as a miss there; + reporting_rate_excl_skipped is the alternative, defined and unused. BI contract: dashboards should query this mart, not raw tables. columns: - name: facility_name @@ -267,6 +276,7 @@ models: values: - Did not report - Reported + - Skipped - name: schedule_type description: "Reporting frequency (Weekly / Monthly / Quarterly / BUQ), derived from the period length." tests: