From 5832d66dcd622a1fa51eabaa47f2c8eeb1ad687b Mon Sep 17 00:00:00 2001 From: Kevin Marete Date: Fri, 9 Oct 2026 17:55:43 -0400 Subject: [PATCH 1/3] [DT-4263][2/4] Add DAR codes and DUOS-IDs to email sends Each send now carries the DAR code and the DUOS-IDs of the datasets it concerns, read from its entity reference according to what its email type stores there. EmailReference records that for every email type in an exhaustive switch, so a new type must be classified before it compiles, and the query receives it as one array indexed by type. Older rows that stored a different kind of reference (a DAR reference id on a DAR-code type, a DAR code on a reference-id type, a dataset name on a DUOS-ID type) are read as what they hold. Co-Authored-By: Claude Opus 5.5 --- .../consent/http/db/MailMessageDAO.java | 84 +++++-- .../http/db/mapper/MailSendMapper.java | 20 +- .../http/enumeration/EmailReference.java | 76 ++++++ .../consent/http/models/mail/MailSend.java | 4 +- .../resources/assets/schemas/MailSend.yaml | 12 +- .../consent/http/db/MailMessageDAOTest.java | 232 +++++++++++++++++- .../http/enumeration/EmailReferenceTest.java | 18 ++ .../http/resources/MailResourceTest.java | 8 +- .../http/service/EmailServiceTest.java | 4 +- 9 files changed, 434 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/broadinstitute/consent/http/enumeration/EmailReference.java create mode 100644 src/test/java/org/broadinstitute/consent/http/enumeration/EmailReferenceTest.java diff --git a/src/main/java/org/broadinstitute/consent/http/db/MailMessageDAO.java b/src/main/java/org/broadinstitute/consent/http/db/MailMessageDAO.java index e3333b64bd..a49e9624a2 100644 --- a/src/main/java/org/broadinstitute/consent/http/db/MailMessageDAO.java +++ b/src/main/java/org/broadinstitute/consent/http/db/MailMessageDAO.java @@ -5,6 +5,7 @@ import org.broadinstitute.consent.http.db.mapper.MailMessageMapper; import org.broadinstitute.consent.http.db.mapper.MailMessageSummaryMapper; import org.broadinstitute.consent.http.db.mapper.MailSendMapper; +import org.broadinstitute.consent.http.enumeration.EmailReference; import org.broadinstitute.consent.http.models.mail.MailMessage; import org.broadinstitute.consent.http.models.mail.MailMessageInsert; import org.broadinstitute.consent.http.models.mail.MailMessageSummary; @@ -98,6 +99,12 @@ List fetchMessageSummariesByCreateDate( @Bind("limit") Integer limit, @Bind("offset") Integer offset); + default List fetchSendsByCreateDate( + Date start, Date end, Integer limit, Integer offset, Integer recipientLimit) { + return fetchSendsByCreateDate( + start, end, limit, offset, recipientLimit, EmailReference.kindsByTypeInt()); + } + // One send per send_id, type and reference; older rows split on 10-minute gaps in sent time. @SqlQuery( """ @@ -156,27 +163,72 @@ PARTITION BY p.send_id ORDER BY u.display_name, MIN(n.email_entity_id) JOIN grouped n ON n.send_row_id = p.send_id LEFT JOIN users u ON u.user_id = n.user_id GROUP BY p.send_id, n.user_id, u.display_name + ), + sends AS ( + SELECT p.send_id, p.email_type, p.entity_reference_id, p.create_date, p.last_create_date, + p.recipient_count, + COALESCE( + json_agg( + json_build_object( + 'userId', nm.user_id, 'displayName', nm.display_name, 'sent', nm.sent) + ORDER BY nm.position + ) FILTER (WHERE nm.send_id IS NOT NULL), + '[]' + ) AS recipients + FROM page p + LEFT JOIN named nm ON nm.send_id = p.send_id AND nm.position <= :recipientLimit + GROUP BY p.send_id, p.email_type, p.entity_reference_id, p.create_date, + p.last_create_date, p.recipient_count ) - SELECT p.send_id, p.email_type, p.entity_reference_id, p.create_date, p.last_create_date, - p.recipient_count, - COALESCE( - json_agg( - json_build_object( - 'userId', nm.user_id, 'displayName', nm.display_name, 'sent', nm.sent) - ORDER BY nm.position - ) FILTER (WHERE nm.send_id IS NOT NULL), - '[]' - ) AS recipients - FROM page p - LEFT JOIN named nm ON nm.send_id = p.send_id AND nm.position <= :recipientLimit - GROUP BY p.send_id, p.email_type, p.entity_reference_id, p.create_date, p.last_create_date, - p.recipient_count - ORDER BY p.create_date DESC, p.send_id DESC + SELECT s.*, COALESCE(by_code.dar_code, by_dar.dar_code) AS dar_code, aliases.dataset_aliases + FROM sends s + CROSS JOIN LATERAL (SELECT (CAST(:kinds AS text[]))[s.email_type] AS kind) k + LEFT JOIN dar_collection by_code + ON k.kind IN ('DAR_CODE', 'DAR_REFERENCE_ID') AND by_code.dar_code = s.entity_reference_id + LEFT JOIN election el + ON k.kind = 'ELECTION_ID' AND el.election_id = CASE + WHEN s.entity_reference_id ~ '^[0-9]{1,9}$' + THEN CAST(s.entity_reference_id AS integer) END + LEFT JOIN data_access_request dar + ON dar.reference_id = CASE + WHEN k.kind = 'DAR_REFERENCE_ID' + OR (k.kind = 'DAR_CODE' AND s.entity_reference_id !~ '^DAR-') + THEN s.entity_reference_id + ELSE el.reference_id END + LEFT JOIN dar_collection by_dar ON by_dar.collection_id = dar.collection_id + LEFT JOIN LATERAL ( + SELECT array_agg(DISTINCT alias ORDER BY alias) AS dataset_aliases + FROM ( + SELECT ds.alias FROM dataset ds WHERE ds.dataset_id = el.dataset_id + UNION ALL + SELECT ds.alias FROM dar_dataset dd JOIN dataset ds ON ds.dataset_id = dd.dataset_id + WHERE el.dataset_id IS NULL AND dd.reference_id = dar.reference_id + UNION ALL + SELECT ds.alias FROM data_access_request d + JOIN dar_dataset dd ON dd.reference_id = d.reference_id + JOIN dataset ds ON ds.dataset_id = dd.dataset_id + WHERE d.collection_id = by_code.collection_id + AND d.parent_id IS NULL + AND d.submission_date IS NOT NULL + UNION ALL + SELECT ds.alias FROM dataset ds + WHERE k.kind = 'DUOS_ID' AND ds.alias = CASE + WHEN s.entity_reference_id ~ '^DUOS-[0-9]{1,18}$' + THEN CAST(substr(s.entity_reference_id, 6) AS bigint) END + UNION ALL + SELECT ds.alias FROM dataset ds + WHERE ds.name = s.entity_reference_id + AND (k.kind = 'DATASET_NAME' + OR (k.kind = 'DUOS_ID' AND s.entity_reference_id !~ '^DUOS-')) + ) related + ) aliases ON TRUE + ORDER BY s.create_date DESC, s.send_id DESC """) List fetchSendsByCreateDate( @Bind("start") Date start, @Bind("end") Date end, @Bind("limit") Integer limit, @Bind("offset") Integer offset, - @Bind("recipientLimit") Integer recipientLimit); + @Bind("recipientLimit") Integer recipientLimit, + @Bind("kinds") List kinds); } diff --git a/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java b/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java index 862a070eb0..20a011d63f 100644 --- a/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java +++ b/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java @@ -2,9 +2,12 @@ import com.google.gson.reflect.TypeToken; import java.lang.reflect.Type; +import java.sql.Array; import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; +import java.util.stream.Stream; +import org.broadinstitute.consent.http.models.Dataset; import org.broadinstitute.consent.http.models.mail.MailSend; import org.broadinstitute.consent.http.models.mail.MailSendRecipient; import org.broadinstitute.consent.http.util.gson.GsonUtil; @@ -26,6 +29,21 @@ public MailSend map(ResultSet r, StatementContext ctx) throws SQLException { r.getTimestamp("create_date"), r.getTimestamp("last_create_date"), r.getInt("recipient_count"), - recipients); + recipients, + r.getString("dar_code"), + datasetIdentifiers(r.getArray("dataset_aliases"))); + } + + private static List datasetIdentifiers(Array aliases) throws SQLException { + if (aliases == null) { + return List.of(); + } + try { + return Stream.of((Long[]) aliases.getArray()) + .map(alias -> Dataset.parseAliasToIdentifier(Math.toIntExact(alias))) + .toList(); + } finally { + aliases.free(); + } } } diff --git a/src/main/java/org/broadinstitute/consent/http/enumeration/EmailReference.java b/src/main/java/org/broadinstitute/consent/http/enumeration/EmailReference.java new file mode 100644 index 0000000000..e3d3401592 --- /dev/null +++ b/src/main/java/org/broadinstitute/consent/http/enumeration/EmailReference.java @@ -0,0 +1,76 @@ +package org.broadinstitute.consent.http.enumeration; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +/** What an email type stores as its {@code email_entity.entity_reference_id}. */ +public enum EmailReference { + DAR_CODE, + DAR_REFERENCE_ID, + ELECTION_ID, + DUOS_ID, + DATASET_NAME, + NONE; + + @SuppressWarnings("deprecation") + public static EmailReference of(EmailType type) { + return switch (type) { + case NEW_CASE, + NEW_DAR, + DATA_CUSTODIAN_APPROVAL, + RESEARCHER_DAR_APPROVED, + NEW_PROGRESS_REPORT_CASE, + RESEARCHER_PROGRESS_REPORT_APPROVED, + NEW_DAR_SO_NEEDS_TO_APPROVE -> + DAR_CODE; + case DAR_EXPIRED, + DAR_EXPIRATION_REMINDER, + NEW_PROGRESS_REPORT_REQUEST, + RESEARCHER_CLOSEOUT_COMPLETED, + SUBMITTED_CLOSEOUT, + SO_DAR_SUBMITTED, + SO_DAR_APPROVED, + SO_PROGRESS_REPORT_SUBMITTED, + SO_PROGRESS_REPORT_APPROVED, + DAC_RADAR_APPROVED -> + DAR_REFERENCE_ID; + case REMINDER -> ELECTION_ID; + case DATASET_DENIED, DATASET_APPROVED -> DUOS_ID; + case NEW_DATASET -> DATASET_NAME; + case COLLECT, + DISABLED_DATASET, + CLOSED_DATASET_ELECTION, + ADMIN_FLAGGED_DAR_APPROVED, + DAR_CANCEL, + DELEGATE_RESPONSIBILITIES, + NEW_RESEARCHER, + RESEARCHER_APPROVED, + NEW_DAA_REQUEST, + NEW_DAA_UPLOAD_RESEARCHER, + NEW_DAA_UPLOAD_SO, + NEW_LIBRARY_CARD_ISSUED, + DAC_VOTE_REMINDER_DIGEST, + NEW_STUDY_REGISTRATION_CONFIRMATION, + NEW_STUDY_DIGEST -> + NONE; + }; + } + + private static final List KINDS_BY_TYPE_INT = kindsByTypeInt(EmailType.values()); + + /** Each type's reference kind at the position Postgres reads as {@code kinds[type number]}. */ + public static List kindsByTypeInt() { + return KINDS_BY_TYPE_INT; + } + + private static List kindsByTypeInt(EmailType[] types) { + int max = Stream.of(types).mapToInt(EmailType::getTypeInt).max().orElse(0); + String[] kinds = new String[max]; + for (EmailType type : types) { + kinds[type.getTypeInt() - 1] = of(type).name(); + } + return Collections.unmodifiableList(Arrays.asList(kinds)); + } +} diff --git a/src/main/java/org/broadinstitute/consent/http/models/mail/MailSend.java b/src/main/java/org/broadinstitute/consent/http/models/mail/MailSend.java index 703482ec12..26a5952c36 100644 --- a/src/main/java/org/broadinstitute/consent/http/models/mail/MailSend.java +++ b/src/main/java/org/broadinstitute/consent/http/models/mail/MailSend.java @@ -13,4 +13,6 @@ public record MailSend( Date createDate, Date lastCreateDate, Integer recipientCount, - List recipients) {} + List recipients, + String darCode, + List datasetIdentifiers) {} diff --git a/src/main/resources/assets/schemas/MailSend.yaml b/src/main/resources/assets/schemas/MailSend.yaml index f6757b8ff6..5f4ffb40bc 100644 --- a/src/main/resources/assets/schemas/MailSend.yaml +++ b/src/main/resources/assets/schemas/MailSend.yaml @@ -35,7 +35,15 @@ properties: description: The user's display name sent: type: boolean - description: Whether SendGrid accepted any of this recipient's emails in the send. False when SendGrid rejected them, the user has turned email off, or email notifications are off for the whole environment, as on dev. Acceptance is not delivery; nothing records whether the email reached the inbox. + description: Whether SendGrid accepted any of this recipient's emails in the send. False when SendGrid rejected them, the user has turned email off, or email notifications are off for the whole environment, as on dev. + darCode: + type: [string, 'null'] + description: Code of the DAR the email concerns, such as DAR-123. Progress report emails carry their parent DAR's code. Null for email types unrelated to a DAR. + datasetIdentifiers: + type: array + description: DUOS-IDs of the datasets the email concerns, in alias order, as they are now rather than when the email was sent. For emails that identify a DAR only by its code, these are the submitted DAR's datasets, not its progress reports'. Empty for email types not tied to a DAR or dataset. + items: + type: string examples: - sendId: 1 emailType: 34 @@ -50,3 +58,5 @@ examples: - userId: 4 displayName: Grace Member sent: false + darCode: null + datasetIdentifiers: [] diff --git a/src/test/java/org/broadinstitute/consent/http/db/MailMessageDAOTest.java b/src/test/java/org/broadinstitute/consent/http/db/MailMessageDAOTest.java index 3bc1aea0f2..a0bd0f48aa 100644 --- a/src/test/java/org/broadinstitute/consent/http/db/MailMessageDAOTest.java +++ b/src/test/java/org/broadinstitute/consent/http/db/MailMessageDAOTest.java @@ -6,17 +6,25 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.sql.Timestamp; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; +import java.util.Arrays; import java.util.Comparator; import java.util.Date; import java.util.EnumSet; import java.util.List; import java.util.UUID; import java.util.stream.IntStream; +import org.broadinstitute.consent.http.enumeration.ElectionStatus; +import org.broadinstitute.consent.http.enumeration.ElectionType; import org.broadinstitute.consent.http.enumeration.EmailType; +import org.broadinstitute.consent.http.models.DataAccessRequest; +import org.broadinstitute.consent.http.models.DataAccessRequestData; +import org.broadinstitute.consent.http.models.DataUseBuilder; +import org.broadinstitute.consent.http.models.Dataset; import org.broadinstitute.consent.http.models.User; import org.broadinstitute.consent.http.models.mail.MailMessage; import org.broadinstitute.consent.http.models.mail.MailMessageInsert; @@ -692,7 +700,9 @@ void testFetchSendsByCreateDate_groups_the_recipients_of_one_send() { List.of( new MailSendRecipient(alice.getUserId(), "Alice", true), new MailSendRecipient(bob.getUserId(), "Bob", true), - new MailSendRecipient(carol.getUserId(), "Carol", true)))), + new MailSendRecipient(carol.getUserId(), "Carol", true)), + null, + List.of())), sends); } @@ -914,6 +924,226 @@ void testFetchSendsByCreateDate_excludes_the_end_instant_in_either_order() { assertEquals(forward, reversed); } + @Test + void testFetchSendsByCreateDate_reads_a_dar_code_as_its_submitted_dar() { + User user = createUser(); + String darCode = "DAR-" + randomInt(1000, 1_000_000); + Integer collectionId = + darCollectionDAO.insertDarCollection(darCode, user.getUserId(), new Date()); + DataAccessRequest dar = createDar(user, collectionId); + Dataset first = createDatasetFor(dar); + Dataset second = createDatasetFor(dar); + createDatasetFor(createProgressReport(user, dar)); + createDatasetFor(createDar(user, collectionId, null)); + generateSendRow(user, EmailType.NEW_DAR, darCode, Instant.now().minus(1, ChronoUnit.HOURS)); + + MailSend send = fetchSendsAroundNow().getFirst(); + + assertEquals(darCode, send.darCode()); + assertEquals( + List.of(first.getDatasetIdentifier(), second.getDatasetIdentifier()), + send.datasetIdentifiers()); + } + + @Test + void testFetchSendsByCreateDate_reads_a_progress_report_reference_as_its_parent_dar() { + User user = createUser(); + String darCode = "DAR-" + randomInt(1000, 1_000_000); + Integer collectionId = + darCollectionDAO.insertDarCollection(darCode, user.getUserId(), new Date()); + DataAccessRequest parent = createDar(user, collectionId); + createDatasetFor(parent); + DataAccessRequest progressReport = createProgressReport(user, parent); + Dataset reported = createDatasetFor(progressReport); + generateSendRow( + user, + EmailType.SO_PROGRESS_REPORT_SUBMITTED, + progressReport.getReferenceId(), + Instant.now().minus(1, ChronoUnit.HOURS)); + + MailSend send = fetchSendsAroundNow().getFirst(); + + assertEquals(darCode, send.darCode()); + assertEquals(List.of(reported.getDatasetIdentifier()), send.datasetIdentifiers()); + } + + @Test + void testFetchSendsByCreateDate_reads_an_election_id_as_its_dar_and_dataset() { + User user = createUser(); + String darCode = "DAR-" + randomInt(1000, 1_000_000); + Integer collectionId = + darCollectionDAO.insertDarCollection(darCode, user.getUserId(), new Date()); + DataAccessRequest dar = createDar(user, collectionId); + createDatasetFor(dar); + Dataset elected = createDatasetFor(dar); + Integer electionId = + electionDAO.insertElection( + ElectionType.DATA_ACCESS.getValue(), + ElectionStatus.OPEN.getValue(), + new Date(), + dar.getReferenceId(), + elected.getDatasetId()); + generateSendRow( + user, EmailType.REMINDER, electionId.toString(), Instant.now().minus(1, ChronoUnit.HOURS)); + + MailSend send = fetchSendsAroundNow().getFirst(); + + assertEquals(darCode, send.darCode()); + assertEquals(List.of(elected.getDatasetIdentifier()), send.datasetIdentifiers()); + } + + @Test + void testFetchSendsByCreateDate_reads_an_election_without_a_dataset_as_its_dars_datasets() { + User user = createUser(); + String darCode = "DAR-" + randomInt(1000, 1_000_000); + Integer collectionId = + darCollectionDAO.insertDarCollection(darCode, user.getUserId(), new Date()); + DataAccessRequest dar = createDar(user, collectionId); + Dataset requested = createDatasetFor(dar); + Integer electionId = + electionDAO.insertElection( + ElectionType.DATA_ACCESS.getValue(), + ElectionStatus.OPEN.getValue(), + new Date(), + dar.getReferenceId(), + null); + generateSendRow( + user, EmailType.REMINDER, electionId.toString(), Instant.now().minus(1, ChronoUnit.HOURS)); + + MailSend send = fetchSendsAroundNow().getFirst(); + + assertEquals(darCode, send.darCode()); + assertEquals(List.of(requested.getDatasetIdentifier()), send.datasetIdentifiers()); + } + + @Test + void testFetchSendsByCreateDate_reads_an_older_dar_reference_on_a_dar_code_type() { + User user = createUser(); + String darCode = "DAR-" + randomInt(1000, 1_000_000); + Integer collectionId = + darCollectionDAO.insertDarCollection(darCode, user.getUserId(), new Date()); + DataAccessRequest dar = createDar(user, collectionId); + Dataset requested = createDatasetFor(dar); + generateSendRow( + user, EmailType.NEW_DAR, dar.getReferenceId(), Instant.now().minus(1, ChronoUnit.HOURS)); + + MailSend send = fetchSendsAroundNow().getFirst(); + + assertEquals(darCode, send.darCode()); + assertEquals(List.of(requested.getDatasetIdentifier()), send.datasetIdentifiers()); + } + + @Test + void testFetchSendsByCreateDate_reads_a_duos_id_and_a_dataset_name() { + Dataset approved = createDataset(); + Dataset submitted = createDataset(); + Instant created = Instant.now().minus(1, ChronoUnit.HOURS); + generateSendRow( + createUser(), EmailType.DATASET_APPROVED, approved.getDatasetIdentifier(), created); + generateSendRow( + createUser(), EmailType.NEW_DATASET, submitted.getName(), created.plusSeconds(1)); + + List sends = fetchSendsAroundNow(); + + assertEquals( + List.of( + List.of(submitted.getDatasetIdentifier()), List.of(approved.getDatasetIdentifier())), + sends.stream().map(MailSend::datasetIdentifiers).toList()); + assertEquals(Arrays.asList(null, null), sends.stream().map(MailSend::darCode).toList()); + } + + @Test + void testFetchSendsByCreateDate_reads_an_older_dataset_name_on_a_duos_id_type() { + Dataset dataset = createDataset(); + generateSendRow( + createUser(), + EmailType.DATASET_DENIED, + dataset.getName(), + Instant.now().minus(1, ChronoUnit.HOURS)); + + assertEquals( + List.of(dataset.getDatasetIdentifier()), + fetchSendsAroundNow().getFirst().datasetIdentifiers()); + } + + @Test + void testFetchSendsByCreateDate_reads_an_older_dar_code_on_a_reference_id_type() { + User user = createUser(); + String darCode = "DAR-" + randomInt(1000, 1_000_000); + Integer collectionId = + darCollectionDAO.insertDarCollection(darCode, user.getUserId(), new Date()); + Dataset requested = createDatasetFor(createDar(user, collectionId)); + generateSendRow(user, EmailType.DAR_EXPIRED, darCode, Instant.now().minus(1, ChronoUnit.HOURS)); + + MailSend send = fetchSendsAroundNow().getFirst(); + + assertEquals(darCode, send.darCode()); + assertEquals(List.of(requested.getDatasetIdentifier()), send.datasetIdentifiers()); + } + + @Test + void testFetchSendsByCreateDate_leaves_unrelated_types_without_dar_or_datasets() { + generateSendRow( + createUser(), + EmailType.DAC_VOTE_REMINDER_DIGEST, + "2026-10-09", + Instant.now().minus(1, ChronoUnit.HOURS)); + + MailSend send = fetchSendsAroundNow().getFirst(); + + assertNull(send.darCode()); + assertEquals(List.of(), send.datasetIdentifiers()); + } + + private DataAccessRequest createDar(User user, Integer collectionId) { + return createDar(user, collectionId, new Date()); + } + + private DataAccessRequest createDar(User user, Integer collectionId, Date submissionDate) { + String referenceId = UUID.randomUUID().toString(); + Date now = new Date(); + dataAccessRequestDAO.insertDataAccessRequest( + collectionId, + referenceId, + user.getUserId(), + now, + submissionDate, + now, + new DataAccessRequestData(), + randomAlphabetic(10)); + return dataAccessRequestDAO.findByReferenceId(referenceId); + } + + private DataAccessRequest createProgressReport(User user, DataAccessRequest parent) { + String referenceId = UUID.randomUUID().toString(); + dataAccessRequestDAO.insertProgressReport( + parent.getId(), + parent.getCollectionId(), + referenceId, + user.getUserId(), + new DataAccessRequestData(), + randomAlphabetic(10)); + return dataAccessRequestDAO.findByReferenceId(referenceId); + } + + private Dataset createDatasetFor(DataAccessRequest dar) { + Dataset dataset = createDataset(); + dataAccessRequestDAO.insertDARDatasetRelation(dar.getReferenceId(), dataset.getDatasetId()); + return dataset; + } + + private Dataset createDataset() { + Integer id = + datasetDAO.insertDataset( + "Name_" + randomAlphabetic(20), + new Timestamp(System.currentTimeMillis()), + createUser().getUserId(), + null, + new DataUseBuilder().setGeneralUse(true).build().toString(), + null); + return datasetDAO.findDatasetById(id); + } + private User createUserNamed(String displayName) { User user = createUser(); userDAO.updateDisplayName(user.getUserId(), displayName); diff --git a/src/test/java/org/broadinstitute/consent/http/enumeration/EmailReferenceTest.java b/src/test/java/org/broadinstitute/consent/http/enumeration/EmailReferenceTest.java new file mode 100644 index 0000000000..181a9d0947 --- /dev/null +++ b/src/test/java/org/broadinstitute/consent/http/enumeration/EmailReferenceTest.java @@ -0,0 +1,18 @@ +package org.broadinstitute.consent.http.enumeration; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import org.junit.jupiter.api.Test; + +class EmailReferenceTest { + + @Test + void kinds_by_type_int_place_each_type_at_its_number() { + List kinds = EmailReference.kindsByTypeInt(); + + for (EmailType type : EmailType.values()) { + assertEquals(EmailReference.of(type).name(), kinds.get(type.getTypeInt() - 1), type.name()); + } + } +} diff --git a/src/test/java/org/broadinstitute/consent/http/resources/MailResourceTest.java b/src/test/java/org/broadinstitute/consent/http/resources/MailResourceTest.java index d96bd9072a..9d1ebcbd98 100644 --- a/src/test/java/org/broadinstitute/consent/http/resources/MailResourceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/resources/MailResourceTest.java @@ -227,12 +227,14 @@ void test_MailResource_sends_ListResponse() throws Exception { List.of( new MailSend( 1, - 34, - "2026-10-09", + 4, + "DAR-1", new Date(), new Date(), 1, - List.of(new MailSendRecipient(2, "A", true)))); + List.of(new MailSendRecipient(2, "A", true)), + "DAR-1", + List.of("DUOS-000001"))); when(emailService.fetchEmailSendsByCreateDate( df.parse("05/11/2021"), df.parse("05/11/2022"), 50, 10)) .thenReturn(sends); diff --git a/src/test/java/org/broadinstitute/consent/http/service/EmailServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/EmailServiceTest.java index 9225fc96e5..74dcb06bc1 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/EmailServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/EmailServiceTest.java @@ -356,7 +356,9 @@ void testFetchEmailSendsByCreateDate() { Date startDate = new Date(); Date endDate = new Date(); List sends = - List.of(new MailSend(1, 34, "2026-10-09", new Date(), new Date(), 0, List.of())); + List.of( + new MailSend( + 1, 34, "2026-10-09", new Date(), new Date(), 0, List.of(), null, List.of())); when(emailDAO.fetchSendsByCreateDate(startDate, endDate, 20, 0, 100)).thenReturn(sends); assertEquals(sends, service.fetchEmailSendsByCreateDate(startDate, endDate, 20, 0)); From 9c9fa28d90557bd154d48dba81b19df0cfcbe5ce Mon Sep 17 00:00:00 2001 From: Kevin Marete Date: Fri, 9 Oct 2026 22:14:24 -0400 Subject: [PATCH 2/3] Read dataset aliases whatever numeric type the column has dataset.alias is bigint where Liquibase created it but numeric on databases older than that changeset, such as the dev copy, which made the mapper's Long[] cast throw. Co-Authored-By: Claude Opus 5.5 --- .../http/db/mapper/MailSendMapper.java | 8 ++++-- .../http/db/mapper/MailSendMapperTest.java | 27 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/broadinstitute/consent/http/db/mapper/MailSendMapperTest.java diff --git a/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java b/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java index 20a011d63f..32fe448f40 100644 --- a/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java +++ b/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java @@ -39,8 +39,12 @@ private static List datasetIdentifiers(Array aliases) throws SQLExceptio return List.of(); } try { - return Stream.of((Long[]) aliases.getArray()) - .map(alias -> Dataset.parseAliasToIdentifier(Math.toIntExact(alias))) + // Number, since dataset.alias is bigint where Liquibase created it but numeric on older + // databases. + return Stream.of((Object[]) aliases.getArray()) + .map( + alias -> + Dataset.parseAliasToIdentifier(Math.toIntExact(((Number) alias).longValue()))) .toList(); } finally { aliases.free(); diff --git a/src/test/java/org/broadinstitute/consent/http/db/mapper/MailSendMapperTest.java b/src/test/java/org/broadinstitute/consent/http/db/mapper/MailSendMapperTest.java new file mode 100644 index 0000000000..b3ee29ddf9 --- /dev/null +++ b/src/test/java/org/broadinstitute/consent/http/db/mapper/MailSendMapperTest.java @@ -0,0 +1,27 @@ +package org.broadinstitute.consent.http.db.mapper; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.math.BigDecimal; +import java.sql.Array; +import java.sql.ResultSet; +import java.util.List; +import org.junit.jupiter.api.Test; + +class MailSendMapperTest { + + @Test + void reads_numeric_dataset_aliases_as_duos_ids() throws Exception { + ResultSet resultSet = mock(ResultSet.class); + Array aliases = mock(Array.class); + when(resultSet.getString("recipients")).thenReturn("[]"); + when(resultSet.getArray("dataset_aliases")).thenReturn(aliases); + when(aliases.getArray()).thenReturn(new BigDecimal[] {BigDecimal.valueOf(45), BigDecimal.ONE}); + + var send = new MailSendMapper().map(resultSet, null); + + assertEquals(List.of("DUOS-000045", "DUOS-000001"), send.datasetIdentifiers()); + } +} From 23610a3e31e77ea39cb0ee568f062e2afe7e658d Mon Sep 17 00:00:00 2001 From: Kevin Marete Date: Tue, 13 Oct 2026 11:16:55 -0400 Subject: [PATCH 3/3] Skip archived DARs, read long election ids, date the DUOS-ID switch Co-Authored-By: Claude Opus 5.5 --- .../consent/http/db/MailMessageDAO.java | 11 +++- .../http/db/mapper/MailSendMapper.java | 3 +- .../resources/assets/schemas/MailSend.yaml | 2 +- .../consent/http/db/MailMessageDAOTest.java | 60 +++++++++++++++++++ 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/broadinstitute/consent/http/db/MailMessageDAO.java b/src/main/java/org/broadinstitute/consent/http/db/MailMessageDAO.java index a49e9624a2..57ccb5cb8a 100644 --- a/src/main/java/org/broadinstitute/consent/http/db/MailMessageDAO.java +++ b/src/main/java/org/broadinstitute/consent/http/db/MailMessageDAO.java @@ -106,6 +106,7 @@ default List fetchSendsByCreateDate( } // One send per send_id, type and reference; older rows split on 10-minute gaps in sent time. + // DATASET_APPROVED/DENIED stored the dataset name until DUOS-IDs replaced it on 2026-07-30. @SqlQuery( """ WITH in_range AS ( @@ -187,8 +188,8 @@ CROSS JOIN LATERAL (SELECT (CAST(:kinds AS text[]))[s.email_type] AS kind) k ON k.kind IN ('DAR_CODE', 'DAR_REFERENCE_ID') AND by_code.dar_code = s.entity_reference_id LEFT JOIN election el ON k.kind = 'ELECTION_ID' AND el.election_id = CASE - WHEN s.entity_reference_id ~ '^[0-9]{1,9}$' - THEN CAST(s.entity_reference_id AS integer) END + WHEN s.entity_reference_id ~ '^[0-9]{1,18}$' + THEN CAST(s.entity_reference_id AS bigint) END LEFT JOIN data_access_request dar ON dar.reference_id = CASE WHEN k.kind = 'DAR_REFERENCE_ID' @@ -210,16 +211,20 @@ SELECT array_agg(DISTINCT alias ORDER BY alias) AS dataset_aliases WHERE d.collection_id = by_code.collection_id AND d.parent_id IS NULL AND d.submission_date IS NOT NULL + AND (LOWER(d.data ->> 'status') != 'archived' OR d.data ->> 'status' IS NULL) UNION ALL SELECT ds.alias FROM dataset ds WHERE k.kind = 'DUOS_ID' AND ds.alias = CASE WHEN s.entity_reference_id ~ '^DUOS-[0-9]{1,18}$' + AND s.create_date >= TIMESTAMPTZ '2026-07-30 17:02:38+00' THEN CAST(substr(s.entity_reference_id, 6) AS bigint) END UNION ALL SELECT ds.alias FROM dataset ds WHERE ds.name = s.entity_reference_id AND (k.kind = 'DATASET_NAME' - OR (k.kind = 'DUOS_ID' AND s.entity_reference_id !~ '^DUOS-')) + OR (k.kind = 'DUOS_ID' + AND (s.create_date < TIMESTAMPTZ '2026-07-30 17:02:38+00' + OR s.entity_reference_id !~ '^DUOS-[0-9]{1,18}$'))) ) related ) aliases ON TRUE ORDER BY s.create_date DESC, s.send_id DESC diff --git a/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java b/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java index 32fe448f40..ecc7e9852b 100644 --- a/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java +++ b/src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java @@ -39,8 +39,7 @@ private static List datasetIdentifiers(Array aliases) throws SQLExceptio return List.of(); } try { - // Number, since dataset.alias is bigint where Liquibase created it but numeric on older - // databases. + // Number: dataset.alias is bigint on newer databases and numeric on older ones. return Stream.of((Object[]) aliases.getArray()) .map( alias -> diff --git a/src/main/resources/assets/schemas/MailSend.yaml b/src/main/resources/assets/schemas/MailSend.yaml index 5f4ffb40bc..7f813d7e90 100644 --- a/src/main/resources/assets/schemas/MailSend.yaml +++ b/src/main/resources/assets/schemas/MailSend.yaml @@ -35,7 +35,7 @@ properties: description: The user's display name sent: type: boolean - description: Whether SendGrid accepted any of this recipient's emails in the send. False when SendGrid rejected them, the user has turned email off, or email notifications are off for the whole environment, as on dev. + description: Whether SendGrid accepted any of this recipient's emails in the send. False when SendGrid rejected them, the user has turned email off, or email notifications are off for the whole environment, as on dev. Acceptance is not delivery; nothing records whether the email reached the inbox. darCode: type: [string, 'null'] description: Code of the DAR the email concerns, such as DAR-123. Progress report emails carry their parent DAR's code. Null for email types unrelated to a DAR. diff --git a/src/test/java/org/broadinstitute/consent/http/db/MailMessageDAOTest.java b/src/test/java/org/broadinstitute/consent/http/db/MailMessageDAOTest.java index a0bd0f48aa..94d1b63b61 100644 --- a/src/test/java/org/broadinstitute/consent/http/db/MailMessageDAOTest.java +++ b/src/test/java/org/broadinstitute/consent/http/db/MailMessageDAOTest.java @@ -935,6 +935,9 @@ void testFetchSendsByCreateDate_reads_a_dar_code_as_its_submitted_dar() { Dataset second = createDatasetFor(dar); createDatasetFor(createProgressReport(user, dar)); createDatasetFor(createDar(user, collectionId, null)); + DataAccessRequest archived = createDar(user, collectionId); + createDatasetFor(archived); + dataAccessRequestDAO.archiveByReferenceIds(List.of(archived.getReferenceId())); generateSendRow(user, EmailType.NEW_DAR, darCode, Instant.now().minus(1, ChronoUnit.HOURS)); MailSend send = fetchSendsAroundNow().getFirst(); @@ -992,6 +995,38 @@ void testFetchSendsByCreateDate_reads_an_election_id_as_its_dar_and_dataset() { assertEquals(List.of(elected.getDatasetIdentifier()), send.datasetIdentifiers()); } + @Test + void testFetchSendsByCreateDate_reads_a_ten_digit_election_id() { + User user = createUser(); + String darCode = "DAR-" + randomInt(1000, 1_000_000); + Integer collectionId = + darCollectionDAO.insertDarCollection(darCode, user.getUserId(), new Date()); + DataAccessRequest dar = createDar(user, collectionId); + Dataset elected = createDatasetFor(dar); + Integer electionId = + electionDAO.insertElection( + ElectionType.DATA_ACCESS.getValue(), + ElectionStatus.OPEN.getValue(), + new Date(), + dar.getReferenceId(), + elected.getDatasetId()); + long largeId = 10_000_000_000L + electionId; + jdbi.useHandle( + handle -> + handle + .createUpdate("UPDATE election SET election_id = :largeId WHERE election_id = :id") + .bind("largeId", largeId) + .bind("id", electionId) + .execute()); + generateSendRow( + user, EmailType.REMINDER, Long.toString(largeId), Instant.now().minus(1, ChronoUnit.HOURS)); + + MailSend send = fetchSendsAroundNow().getFirst(); + + assertEquals(darCode, send.darCode()); + assertEquals(List.of(elected.getDatasetIdentifier()), send.datasetIdentifiers()); + } + @Test void testFetchSendsByCreateDate_reads_an_election_without_a_dataset_as_its_dars_datasets() { User user = createUser(); @@ -1066,6 +1101,31 @@ void testFetchSendsByCreateDate_reads_an_older_dataset_name_on_a_duos_id_type() fetchSendsAroundNow().getFirst().datasetIdentifiers()); } + @Test + void testFetchSendsByCreateDate_reads_an_older_name_shaped_like_a_duos_id_as_a_name() { + Dataset named = createDataset(); + String name = "DUOS-" + randomInt(100_000, 999_999); + jdbi.useHandle( + handle -> + handle + .createUpdate("UPDATE dataset SET name = :name WHERE dataset_id = :id") + .bind("name", name) + .bind("id", named.getDatasetId()) + .execute()); + Instant beforeDuosIds = Instant.parse("2026-07-01T12:00:00Z"); + generateSendRow(createUser(), EmailType.DATASET_DENIED, name, beforeDuosIds); + + List sends = + mailMessageDAO.fetchSendsByCreateDate( + Date.from(beforeDuosIds.minus(1, ChronoUnit.DAYS)), + Date.from(beforeDuosIds.plus(1, ChronoUnit.DAYS)), + 100, + 0, + 100); + + assertEquals(List.of(named.getDatasetIdentifier()), sends.getFirst().datasetIdentifiers()); + } + @Test void testFetchSendsByCreateDate_reads_an_older_dar_code_on_a_reference_id_type() { User user = createUser();