[DT-4263][3/7] Add DAR codes and DUOS-IDs to email sends

src/main/java/org/broadinstitute/consent/http/db/MailMessageDAO.java

@@ -5,6 +5,7 @@
 import org.broadinstitute.consent.http.db.mapper.MailMessageMapper;
 import org.broadinstitute.consent.http.db.mapper.MailMessageSummaryMapper;
 import org.broadinstitute.consent.http.db.mapper.MailSendMapper;
+import org.broadinstitute.consent.http.enumeration.EmailReference;
 import org.broadinstitute.consent.http.models.mail.MailMessage;
 import org.broadinstitute.consent.http.models.mail.MailMessageInsert;
 import org.broadinstitute.consent.http.models.mail.MailMessageSummary;
@@ -98,7 +99,14 @@ List<MailMessageSummary> fetchMessageSummariesByCreateDate(
       @Bind("limit") Integer limit,
       @Bind("offset") Integer offset);
 
+  default List<MailSend> fetchSendsByCreateDate(
+      Date start, Date end, Integer limit, Integer offset, Integer recipientLimit) {
+    return fetchSendsByCreateDate(
+        start, end, limit, offset, recipientLimit, EmailReference.kindsByTypeInt());
+  }
+
   // Rows sharing a send_id, type and reference are one send; older rows split on 10-minute gaps.
+  // :kinds holds, at each email type's number, what that type stores as its entity reference.
   @SqlQuery(
       """
       WITH in_range AS (
@@ -156,27 +164,72 @@ PARTITION BY p.send_id ORDER BY u.display_name, MIN(n.email_entity_id)
         JOIN grouped n ON n.send_row_id = p.send_id
         LEFT JOIN users u ON u.user_id = n.user_id
         GROUP BY p.send_id, n.user_id, u.display_name
+      ),
+      sends AS (
+        SELECT p.send_id, p.email_type, p.entity_reference_id, p.create_date, p.last_create_date,
+          p.recipient_count,
+          COALESCE(
+            json_agg(
+              json_build_object(
+                'userId', nm.user_id, 'displayName', nm.display_name, 'sent', nm.sent)
+              ORDER BY nm.position
+            ) FILTER (WHERE nm.send_id IS NOT NULL),
+            '[]'
+          ) AS recipients
+        FROM page p
+        LEFT JOIN named nm ON nm.send_id = p.send_id AND nm.position <= :recipientLimit
+        GROUP BY p.send_id, p.email_type, p.entity_reference_id, p.create_date,
+          p.last_create_date, p.recipient_count
       )
-      SELECT p.send_id, p.email_type, p.entity_reference_id, p.create_date, p.last_create_date,
-        p.recipient_count,
-        COALESCE(
-          json_agg(
-            json_build_object(
-              'userId', nm.user_id, 'displayName', nm.display_name, 'sent', nm.sent)
-            ORDER BY nm.position
-          ) FILTER (WHERE nm.send_id IS NOT NULL),
-          '[]'
-        ) AS recipients
-      FROM page p
-      LEFT JOIN named nm ON nm.send_id = p.send_id AND nm.position <= :recipientLimit
-      GROUP BY p.send_id, p.email_type, p.entity_reference_id, p.create_date, p.last_create_date,
-        p.recipient_count
-      ORDER BY p.create_date DESC, p.send_id DESC
+      SELECT s.*, COALESCE(by_code.dar_code, by_dar.dar_code) AS dar_code, aliases.dataset_aliases
+      FROM sends s
+      CROSS JOIN LATERAL (SELECT (CAST(:kinds AS text[]))[s.email_type] AS kind) k
+      LEFT JOIN dar_collection by_code
+        ON k.kind IN ('DAR_CODE', 'DAR_REFERENCE_ID') AND by_code.dar_code = s.entity_reference_id
+      LEFT JOIN election el
+        ON k.kind = 'ELECTION_ID' AND el.election_id = CASE
+          WHEN s.entity_reference_id ~ '^[0-9]{1,9}$'
+          THEN CAST(s.entity_reference_id AS integer) END
+      LEFT JOIN data_access_request dar
+        ON dar.reference_id = CASE
+          WHEN k.kind = 'DAR_REFERENCE_ID'
+            OR (k.kind = 'DAR_CODE' AND s.entity_reference_id !~ '^DAR-')
+          THEN s.entity_reference_id
+          ELSE el.reference_id END
+      LEFT JOIN dar_collection by_dar ON by_dar.collection_id = dar.collection_id
+      LEFT JOIN LATERAL (
+        SELECT array_agg(DISTINCT alias ORDER BY alias) AS dataset_aliases
+        FROM (
+          SELECT ds.alias FROM dataset ds WHERE ds.dataset_id = el.dataset_id
+          UNION ALL
+          SELECT ds.alias FROM dar_dataset dd JOIN dataset ds ON ds.dataset_id = dd.dataset_id
+          WHERE el.dataset_id IS NULL AND dd.reference_id = dar.reference_id
+          UNION ALL
+          SELECT ds.alias FROM data_access_request d
+          JOIN dar_dataset dd ON dd.reference_id = d.reference_id
+          JOIN dataset ds ON ds.dataset_id = dd.dataset_id
+          WHERE d.collection_id = by_code.collection_id
+            AND d.parent_id IS NULL
+            AND d.submission_date IS NOT NULL
+          UNION ALL
+          SELECT ds.alias FROM dataset ds
+          WHERE k.kind = 'DUOS_ID' AND ds.alias = CASE
+            WHEN s.entity_reference_id ~ '^DUOS-[0-9]{1,18}$'
+            THEN CAST(substr(s.entity_reference_id, 6) AS bigint) END
+          UNION ALL
+          SELECT ds.alias FROM dataset ds
+          WHERE ds.name = s.entity_reference_id
+            AND (k.kind = 'DATASET_NAME'
+              OR (k.kind = 'DUOS_ID' AND s.entity_reference_id !~ '^DUOS-'))
+        ) related
+      ) aliases ON TRUE
+      ORDER BY s.create_date DESC, s.send_id DESC
       """)
   List<MailSend> fetchSendsByCreateDate(
       @Bind("start") Date start,
       @Bind("end") Date end,
       @Bind("limit") Integer limit,
       @Bind("offset") Integer offset,
-      @Bind("recipientLimit") Integer recipientLimit);
+      @Bind("recipientLimit") Integer recipientLimit,
+      @Bind("kinds") List<String> kinds);
 }

src/main/java/org/broadinstitute/consent/http/db/mapper/MailSendMapper.java

@@ -2,9 +2,12 @@
 
 import com.google.gson.reflect.TypeToken;
 import java.lang.reflect.Type;
+import java.sql.Array;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.List;
+import java.util.stream.Stream;
+import org.broadinstitute.consent.http.models.Dataset;
 import org.broadinstitute.consent.http.models.mail.MailSend;
 import org.broadinstitute.consent.http.models.mail.MailSendRecipient;
 import org.broadinstitute.consent.http.util.gson.GsonUtil;
@@ -26,6 +29,25 @@ public MailSend map(ResultSet r, StatementContext ctx) throws SQLException {
         r.getTimestamp("create_date"),
         r.getTimestamp("last_create_date"),
         r.getInt("recipient_count"),
-        recipients);
+        recipients,
+        r.getString("dar_code"),
+        datasetIdentifiers(r.getArray("dataset_aliases")));
+  }
+
+  private static List<String> datasetIdentifiers(Array aliases) throws SQLException {
+    if (aliases == null) {
+      return List.of();
+    }
+    try {
+      // Number, since dataset.alias is bigint where Liquibase created it but numeric on older
+      // databases.
+      return Stream.of((Object[]) aliases.getArray())
+          .map(
+              alias ->
+                  Dataset.parseAliasToIdentifier(Math.toIntExact(((Number) alias).longValue())))
+          .toList();
+    } finally {
+      aliases.free();
+    }
   }
 }

src/main/java/org/broadinstitute/consent/http/enumeration/EmailReference.java

@@ -0,0 +1,76 @@
+package org.broadinstitute.consent.http.enumeration;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Stream;
+
+/** What an email type stores as its {@code email_entity.entity_reference_id}. */
+public enum EmailReference {
+  DAR_CODE,
+  DAR_REFERENCE_ID,
+  ELECTION_ID,
+  DUOS_ID,
+  DATASET_NAME,
+  NONE;
+
+  @SuppressWarnings("deprecation")
+  public static EmailReference of(EmailType type) {
+    return switch (type) {
+      case NEW_CASE,
+          NEW_DAR,
+          DATA_CUSTODIAN_APPROVAL,
+          RESEARCHER_DAR_APPROVED,
+          NEW_PROGRESS_REPORT_CASE,
+          RESEARCHER_PROGRESS_REPORT_APPROVED,
+          NEW_DAR_SO_NEEDS_TO_APPROVE ->
+          DAR_CODE;
+      case DAR_EXPIRED,
+          DAR_EXPIRATION_REMINDER,
+          NEW_PROGRESS_REPORT_REQUEST,
+          RESEARCHER_CLOSEOUT_COMPLETED,
+          SUBMITTED_CLOSEOUT,
+          SO_DAR_SUBMITTED,
+          SO_DAR_APPROVED,
+          SO_PROGRESS_REPORT_SUBMITTED,
+          SO_PROGRESS_REPORT_APPROVED,
+          DAC_RADAR_APPROVED ->
+          DAR_REFERENCE_ID;
+      case REMINDER -> ELECTION_ID;
+      case DATASET_DENIED, DATASET_APPROVED -> DUOS_ID;
+      case NEW_DATASET -> DATASET_NAME;
+      case COLLECT,
+          DISABLED_DATASET,
+          CLOSED_DATASET_ELECTION,
+          ADMIN_FLAGGED_DAR_APPROVED,
+          DAR_CANCEL,
+          DELEGATE_RESPONSIBILITIES,
+          NEW_RESEARCHER,
+          RESEARCHER_APPROVED,
+          NEW_DAA_REQUEST,
+          NEW_DAA_UPLOAD_RESEARCHER,
+          NEW_DAA_UPLOAD_SO,
+          NEW_LIBRARY_CARD_ISSUED,
+          DAC_VOTE_REMINDER_DIGEST,
+          NEW_STUDY_REGISTRATION_CONFIRMATION,
+          NEW_STUDY_DIGEST ->
+          NONE;
+    };
+  }
+
+  private static final List<String> KINDS_BY_TYPE_INT = kindsByTypeInt(EmailType.values());
+
+  /** Each type's reference kind at the position Postgres reads as {@code kinds[type number]}. */
+  public static List<String> kindsByTypeInt() {
+    return KINDS_BY_TYPE_INT;
+  }
+
+  private static List<String> kindsByTypeInt(EmailType[] types) {
+    int max = Stream.of(types).mapToInt(EmailType::getTypeInt).max().orElse(0);
+    String[] kinds = new String[max];
+    for (EmailType type : types) {
+      kinds[type.getTypeInt() - 1] = of(type).name();
+    }
+    return Collections.unmodifiableList(Arrays.asList(kinds));
+  }
+}

src/main/java/org/broadinstitute/consent/http/models/mail/MailSend.java

@@ -13,4 +13,6 @@ public record MailSend(
     Date createDate,
     Date lastCreateDate,
     Integer recipientCount,
-    List<MailSendRecipient> recipients) {}
+    List<MailSendRecipient> recipients,
+    String darCode,
+    List<String> datasetIdentifiers) {}

src/main/resources/assets/schemas/MailSend.yaml

@@ -35,7 +35,15 @@ properties:
           description: The user's display name
         sent:
           type: boolean
-          description: Whether SendGrid accepted any of this recipient's emails in the send. False when SendGrid rejected them, the user has turned email off, or email notifications are off for the whole environment, as on dev. Acceptance is not delivery; nothing records whether the email reached the inbox.
+          description: Whether SendGrid accepted any of this recipient's emails in the send. False when SendGrid rejected them, the user has turned email off, or email notifications are off for the whole environment, as on dev.
+  darCode:
+    type: [string, 'null']
+    description: Code of the DAR the email concerns, such as DAR-123. Progress report emails carry their parent DAR's code. Null for email types unrelated to a DAR.
+  datasetIdentifiers:
+    type: array
+    description: DUOS-IDs of the datasets the email concerns, in alias order, as they are now rather than when the email was sent. For emails that identify a DAR only by its code, these are the submitted DAR's datasets, not its progress reports'. Empty for email types not tied to a DAR or dataset.
+    items:
+      type: string
 examples:
   - sendId: 1
     emailType: 34
@@ -50,3 +58,5 @@ examples:
       - userId: 4
         displayName: Grace Member
         sent: false
+    darCode: null
+    datasetIdentifiers: []