TASKS-598: Docker image partnersinhealth/ces-etl

.github/workflows/build-and-deploy-to-sonatype.yml

@@ -7,24 +7,15 @@ on:
   repository_dispatch:
 
 jobs:
-  update-versions:
-    if: github.repository_owner == 'PIH'
-    permissions:
-      contents: write
-    concurrency:
-      group: update-versions-${{ github.ref }}
-      cancel-in-progress: true
-    uses: PIH/openmrs-contrib-distro-tools/.github/workflows/update-versions.yml@main
-    secrets: inherit
-
   build-and-publish:
     if: github.repository_owner == 'PIH'
-    needs: update-versions
     concurrency:
       group: build-and-deploy-${{ github.ref }}
       cancel-in-progress: true
     uses: PIH/openmrs-contrib-distro-tools/.github/workflows/build-and-deploy-to-sonatype.yml@main
     with:
+      image_name: partnersinhealth/ces-etl
+      docker_context: target/docker
       notify_ci_dashboard: true
     secrets: inherit
 

.github/workflows/release-to-sonatype.yml

@@ -9,4 +9,7 @@ permissions:
 jobs:
   release:
     uses: PIH/openmrs-contrib-distro-tools/.github/workflows/release-to-sonatype.yml@main
+    with:
+      image_name: partnersinhealth/ces-etl
+      docker_context: target/docker
     secrets: inherit

Dockerfile

@@ -0,0 +1,8 @@
+# Defaults to the published base image, so CI can build this. build-runtime-docker-image.sh
+# overrides it with partnersinhealth/petl:local for local builds.
+ARG PETL_BASE_IMAGE=partnersinhealth/petl:latest
+FROM ${PETL_BASE_IMAGE}
+
+COPY datasources /home/petl/configurations/datasources
+COPY jobs /home/petl/configurations/jobs
+COPY application-docker.yml /home/petl/bin/application.yml

README.md

@@ -22,3 +22,24 @@ because at the time of this writing (Oct 2021), CES ETL is colocated
 with the CES Laguna EMR  (a production system), which means that deploying
 `config-ces` causes EMR downtime. The config files in this directory
 can be "deployed" independently of `config-ces` and the EMR.
+
+# Docker image
+
+`partnersinhealth/ces-etl` layers this project's `datasources/`, `jobs/` and `application-docker.yml`
+(as its `application.yml`) on the [PETL](https://github.com/PIH/petl) base image,
+`partnersinhealth/petl`. CI builds and pushes it (`Dockerfile`, build context `target/docker/`,
+populated by `mvn package`) on every push to `master` and on every release, tagged `latest` and the
+version. To build it locally: `./build-runtime-docker-image.sh`, which layers on a locally built
+`partnersinhealth/petl:local`. It builds on `partnersinhealth/petl:latest` (the `Dockerfile`'s `PETL_BASE_IMAGE` default).
+
+It runs as [openmrs-contrib-distro-tools](https://github.com/PIH/openmrs-contrib-distro-tools)'
+`petl` service (see its `docs/services.md`), e.g. for ces-ci:
+
+    PETL_IMAGE_NAME=partnersinhealth/ces-etl
+    PETL_FULL_REFRESH_JOBS="create-partitions.yml refresh-cesci-data.yml"
+    PETL_SQLSERVER_DATABASE=openmrs_ces_ci
+
+`application-docker.yml` maps the `cesci` OpenMRS datasource and the `warehouse` SQL Server
+datasource onto distro-tools' `PETL_MYSQL_*` and `PETL_SQLSERVER_*` variables. The other sites'
+datasources aren't set; set one by its Spring environment-variable name where it's needed (e.g.
+`DATASOURCES_OPENMRS_CAPITAN_HOST`).

application-docker.yml

@@ -0,0 +1,29 @@
+# The image's application.yml. Per-server values come from the environment (distro-tools' petl
+# service sets PETL_MYSQL_* and PETL_SQLSERVER_*); any other property can be set by its Spring env
+# name, e.g. DATASOURCES_OPENMRS_CAPITAN_HOST. PETL's own job history is spring.datasource: H2 by
+# default, or SQL Server from SPRING_DATASOURCE_* (puppet's job_store => 'sqlserver').
+petl:
+  homeDir: "/home/petl"
+  datasourceDir: "${petl.homeDir}/configurations/datasources"
+  jobDir: "${petl.homeDir}/configurations/jobs"
+
+logging:
+  level:
+    root: "WARN"
+    org.pih: "INFO"
+
+datasources:
+  openmrs:
+    cesci:
+      host: "${PETL_MYSQL_HOST}"
+      port: "${PETL_MYSQL_PORT:3306}"
+      databaseName: "openmrs"
+      user: "${PETL_MYSQL_USER}"
+      password: "${PETL_MYSQL_PASSWORD}"
+      options: "${PETL_MYSQL_OPTIONS:}"
+  warehouse:
+    host: "${PETL_SQLSERVER_HOST}"
+    port: "${PETL_SQLSERVER_PORT:1433}"
+    databaseName: "${PETL_SQLSERVER_DATABASE}"
+    user: "${PETL_SQLSERVER_USER}"
+    password: "${PETL_SQLSERVER_PASSWORD}"

build-runtime-docker-image.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -eux
+
+# Populates target/docker/ (the same build context CI uses, never the source tree directly), then
+# builds on the locally built petl image; the Dockerfile's own default is what CI uses.
+mvn package
+
+docker build \
+  --build-arg PETL_BASE_IMAGE=partnersinhealth/petl:local \
+  -t partnersinhealth/ces-etl:local target/docker

datasources/openmrs-capitan.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.capitan.databaseName}
 user: ${datasources.openmrs.capitan.user}
 password: ${datasources.openmrs.capitan.password}
 options: ${datasources.openmrs.capitan.options}
-containerName: ${datasources.openmrs.capitan.containerName}
+containerName: ${datasources.openmrs.capitan.containerName:}

datasources/openmrs-cesci.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.cesci.databaseName}
 user: ${datasources.openmrs.cesci.user}
 password: ${datasources.openmrs.cesci.password}
 options: ${datasources.openmrs.cesci.options}
-containerName: ""
+containerName: ${datasources.openmrs.cesci.containerName:}

datasources/openmrs-honduras.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.honduras.databaseName}
 user: ${datasources.openmrs.honduras.user}
 password: ${datasources.openmrs.honduras.password}
 options: ${datasources.openmrs.honduras.options}
-containerName: ${datasources.openmrs.honduras.containerName}
+containerName: ${datasources.openmrs.honduras.containerName:}

datasources/openmrs-jaltenango.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.jaltenango.databaseName}
 user: ${datasources.openmrs.jaltenango.user}
 password: ${datasources.openmrs.jaltenango.password}
 options: ${datasources.openmrs.jaltenango.options}
-containerName: ${datasources.openmrs.jaltenango.containerName}
+containerName: ${datasources.openmrs.jaltenango.containerName:}

datasources/openmrs-laguna.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.laguna.databaseName}
 user: ${datasources.openmrs.laguna.user}
 password: ${datasources.openmrs.laguna.password}
 options: ${datasources.openmrs.laguna.options}
-containerName: ${datasources.openmrs.laguna.containerName}
+containerName: ${datasources.openmrs.laguna.containerName:}

datasources/openmrs-letrero.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.letrero.databaseName}
 user: ${datasources.openmrs.letrero.user}
 password: ${datasources.openmrs.letrero.password}
 options: ${datasources.openmrs.letrero.options}
-containerName: ${datasources.openmrs.letrero.containerName}
+containerName: ${datasources.openmrs.letrero.containerName:}

datasources/openmrs-matazano.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.matazano.databaseName}
 user: ${datasources.openmrs.matazano.user}
 password: ${datasources.openmrs.matazano.password}
 options: ${datasources.openmrs.matazano.options}
-containerName: ${datasources.openmrs.matazano.containerName}
+containerName: ${datasources.openmrs.matazano.containerName:}

datasources/openmrs-monterrey.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.monterrey.databaseName}
 user: ${datasources.openmrs.monterrey.user}
 password: ${datasources.openmrs.monterrey.password}
 options: ${datasources.openmrs.monterrey.options}
-containerName: ${datasources.openmrs.monterrey.containerName}
+containerName: ${datasources.openmrs.monterrey.containerName:}

datasources/openmrs-planAlta.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.planAlta.databaseName}
 user: ${datasources.openmrs.planAlta.user}
 password: ${datasources.openmrs.planAlta.password}
 options: ${datasources.openmrs.planAlta.options}
-containerName: ${datasources.openmrs.planAlta.containerName}
+containerName: ${datasources.openmrs.planAlta.containerName:}

datasources/openmrs-planBaja.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.planBaja.databaseName}
 user: ${datasources.openmrs.planBaja.user}
 password: ${datasources.openmrs.planBaja.password}
 options: ${datasources.openmrs.planBaja.options}
-containerName: ${datasources.openmrs.planBaja.containerName}
+containerName: ${datasources.openmrs.planBaja.containerName:}

datasources/openmrs-reforma.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.reforma.databaseName}
 user: ${datasources.openmrs.reforma.user}
 password: ${datasources.openmrs.reforma.password}
 options: ${datasources.openmrs.reforma.options}
-containerName: ${datasources.openmrs.reforma.containerName}
+containerName: ${datasources.openmrs.reforma.containerName:}

datasources/openmrs-salvador.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.salvador.databaseName}
 user: ${datasources.openmrs.salvador.user}
 password: ${datasources.openmrs.salvador.password}
 options: ${datasources.openmrs.salvador.options}
-containerName: ${datasources.openmrs.salvador.containerName}
+containerName: ${datasources.openmrs.salvador.containerName:}

datasources/openmrs-soledad.yml

@@ -5,4 +5,4 @@ databaseName: ${datasources.openmrs.soledad.databaseName}
 user: ${datasources.openmrs.soledad.user}
 password: ${datasources.openmrs.soledad.password}
 options: ${datasources.openmrs.soledad.options}
-containerName: ${datasources.openmrs.soledad.containerName}
+containerName: ${datasources.openmrs.soledad.containerName:}

pom.xml

@@ -91,6 +91,31 @@
                             <outputDirectory>${project.build.directory}/ces-etl/</outputDirectory>
                         </configuration>
                     </execution>
+                    <!-- The Docker build context (target/docker/), from the same files as the zip
+                         artifact; kept apart so Docker-only files stay out of the zip. -->
+                    <execution>
+                        <id>package-docker-context</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <encoding>UTF-8</encoding>
+                            <resources>
+                                <resource>
+                                    <directory>${basedir}</directory>
+                                    <filtering>false</filtering>
+                                    <includes>
+                                        <include>datasources/**</include>
+                                        <include>jobs/**</include>
+                                        <include>application-docker.yml</include>
+                                        <include>Dockerfile</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                            <outputDirectory>${project.build.directory}/docker/</outputDirectory>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>