From f020301a370cf150c274b0b65564ae541ba9fc40 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:55:53 +0000 Subject: [PATCH 1/6] [minor] Isolate ISLE database bootstrap and add recovery gate --- .../workflows/sitectl-create-smoke-test.yaml | 1 + README.md | 22 ++++++++++++++-- compose.yaml | 25 ++++++++++++++++++- scripts/test.sh | 1 + 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sitectl-create-smoke-test.yaml b/.github/workflows/sitectl-create-smoke-test.yaml index e51f9b6..a93eae4 100644 --- a/.github/workflows/sitectl-create-smoke-test.yaml +++ b/.github/workflows/sitectl-create-smoke-test.yaml @@ -34,3 +34,4 @@ jobs: packages: sitectl sitectl-isle sitectl-drupal package-versions: sitectl=1.6.3 sitectl-isle=1.3.0 sitectl-drupal=1.2.0 allow-unversioned-packages: false + run-verify: true diff --git a/README.md b/README.md index 1a9b9a0..a1f9fd1 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,8 @@ sitectl validate Update the application base tag or pin that base by digest with [`sitectl image`](https://sitectl.libops.io/commands/image): ```bash -sitectl image set --tag drupal=nginx-1.30.3-php84 --tag solr=9 --tag alpaca=2.4 -sitectl image set --build-arg drupal.BASE_IMAGE=libops/islandora:nginx-1.30.3-php84@sha256:... +sitectl image set --tag drupal=nginx-1.30.4-php84 --tag solr=9 --tag alpaca=2.4 +sitectl image set --build-arg drupal.BASE_IMAGE=libops/islandora:nginx-1.30.4-php84@sha256:0e20f5d10af9431c3140179ef6bf1e41299026043d1a98974180ab54a5330cef ``` Enable local development bind mounts with [`sitectl set`](https://sitectl.libops.io/commands/set): @@ -112,6 +112,24 @@ Only MariaDB and the one-shot `database-init` service receive `DB_ROOT_PASSWORD` ActiveMQ, Alpaca, and Drupal share the generated `ACTIVEMQ_PASSWORD` through service-specific secret targets. Drupal receives it as `DRUPAL_DEFAULT_BROKER_PASSWORD` together with the non-secret `admin` broker username, so the rendered Islandora settings use authenticated STOMP rather than relying on a broker default. +## Full-state recovery + +The ISLE plugin exposes the authoritative-versus-rebuildable recovery contract and creates a single checksummed bundle: + +```bash +sitectl isle recovery plan +sitectl isle recovery backup --output /var/backups/isle/site-$(date +%F).tar.gz +sitectl isle recovery validate --input /var/backups/isle/site-2026-08-07.tar.gz +``` + +The bundle contains the Drupal database and public/private files, plus the Fcrepo database and object data when Fcrepo is enabled. It deliberately excludes customer secrets, source-controlled project configuration, Solr and Blazegraph indexes, ActiveMQ queues, IIIF caches, and generated derivatives. Recreate the target from the matching site Git revision and template provenance lock, restore secrets from the organization's Vault backup, and rebuild the excluded derived state after restore. The target and bundle must agree on whether Fedora is enabled. + +Copy bundles to encrypted off-host storage with retention that meets the customer's documented RPO. At least quarterly, restore a selected bundle into a disposable context, run `sitectl healthcheck` and `sitectl verify --strict`, rebuild indexes and required derivatives, and record the achieved RPO/RTO. A destructive restore requires confirmation: + +```bash +sitectl isle recovery restore --input /var/backups/isle/site-2026-08-07.tar.gz +``` + ## License The Docker Compose template and LibOps-specific setup in this repository are licensed under the MIT License. The upstream Islandora starter site is licensed separately under the GNU General Public License v2; see `LICENSE.islandora-starter-site`. diff --git a/compose.yaml b/compose.yaml index 47a78ee..304b4e9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -112,6 +112,8 @@ services: start_period: ${DRUPAL_HEALTHCHECK_START_PERIOD:-5m} retries: ${DRUPAL_HEALTHCHECK_RETRIES:-20} depends_on: + database-init: + condition: service_completed_successfully mariadb: condition: service_healthy secrets: @@ -190,6 +192,27 @@ services: - ./secrets:/work/secrets:rw,z - ./compose.yaml:/work/compose.yaml:ro,z + database-init: + image: libops/base:3.2.2.0@sha256:851e17742b5fee57038855f46b1a46d2716c0012c5f3a4788b7e5c7bc12fed5e + restart: "no" + networks: + default: + environment: + DB_HOST: mariadb + DB_PORT: "3306" + DB_NAME: drupal_default + DB_USER: drupal_default + DB_CHARACTER_SET: utf8mb4 + DB_COLLATION: utf8mb4_unicode_ci + secrets: + - source: DB_ROOT_PASSWORD + - source: DRUPAL_DEFAULT_DB_PASSWORD + target: DB_PASSWORD + entrypoint: /usr/local/bin/init-database.sh + depends_on: + mariadb: + condition: service_healthy + mariadb: <<: *common image: libops/mariadb:11.8@sha256:1b0f53f277cedf95e1fa6df25580de01dd2d0722b09f39fe0d93cfb6263e26f3 @@ -200,7 +223,7 @@ services: mergepdf: <<: *common - image: islandora/mergepdf:6.3.19 + image: islandora/mergepdf:6.3.19@sha256:69f391d2058463f815c87b6703a39e38f5e61bf27b15b305ebf4bda64268110c secrets: - source: CERT_PUBLIC_KEY - source: CERT_AUTHORITY diff --git a/scripts/test.sh b/scripts/test.sh index 5d1e78e..2eead60 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -8,3 +8,4 @@ docker compose build --pull drupal docker compose run --rm -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" init docker compose up --remove-orphans --wait --wait-timeout "${COMPOSE_WAIT_TIMEOUT:-900}" -d sitectl healthcheck --persist --timeout "${SITECTL_HEALTHCHECK_TIMEOUT:-10m}" +sitectl verify --strict From 51a0eeb45ff112faae7c3ce49c8563081be1e25c Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 13:19:12 +0000 Subject: [PATCH 2/6] [patch] Make demo object workflow self-contained --- scripts/demo-objects.sh | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/scripts/demo-objects.sh b/scripts/demo-objects.sh index f0d51b0..e56a301 100755 --- a/scripts/demo-objects.sh +++ b/scripts/demo-objects.sh @@ -2,8 +2,37 @@ set -eou pipefail -# shellcheck disable=SC1091 -source "$(dirname "${BASH_SOURCE[0]}")/profile.sh" +site_url() { + sitectl stats --path . --format json | jq -er '.ingress.public_url' +} + +fcrepo_enabled() { + docker compose config --services 2>/dev/null | grep -qx 'fcrepo' +} + +container_url_for_url() { + local url="$1" + if fcrepo_enabled && [[ "${url}" =~ ^(https?)://(localhost|127\.0\.0\.1)(:[0-9]+)?(/.*)?$ ]]; then + printf '%s://drupal.internal%s\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[4]}" + return + fi + printf '%s\n' "${url}" +} + +container_network_for_url() { + local url="${1:?url is required}" + local compose_project_name + local traefik_container + if [[ "${url}" =~ ^https?://(localhost|127\.0\.0\.1)(:[0-9]+)?(/.*)?$ ]]; then + traefik_container="$(docker compose ps -q traefik)" + if [ -n "${traefik_container}" ]; then + printf 'container:%s\n' "${traefik_container}" + return + fi + fi + compose_project_name="$(docker compose config --format json | jq -er '.name')" + printf '%s_default\n' "${compose_project_name}" +} if [ ! -d "islandora_workbench" ]; then git clone https://github.com/mjordan/islandora_workbench @@ -13,7 +42,7 @@ if [ ! -d "islandora_demo_objects" ]; then git clone https://github.com/Islandora-Devops/islandora_demo_objects islandora_demo_objects fi -URL="$(site_url)" +URL="${SITECTL_DEMO_OBJECTS_URL:-$(site_url)}" WORKBENCH_URL="$(container_url_for_url "${URL}")" NETWORK="$(container_network_for_url "${WORKBENCH_URL}")" From 610a8809628a9eb709a19220992b53506e9f1c57 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 14:18:21 +0000 Subject: [PATCH 3/6] [patch] Surface Workbench import failures --- scripts/demo-objects.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/demo-objects.sh b/scripts/demo-objects.sh index e56a301..c775282 100755 --- a/scripts/demo-objects.sh +++ b/scripts/demo-objects.sh @@ -64,6 +64,7 @@ docker build \ tty_flag=( -i ) [ -t 0 ] && tty_flag=( -it ) +set +e docker run \ "${tty_flag[@]}" \ --rm \ @@ -74,3 +75,14 @@ docker run \ --name my-running-workbench \ workbench-docker:latest \ bash -lc "./workbench --config /islandora_demo_objects/create_islandora_objects.yml" +workbench_status=$? +set -e + +if [ "${workbench_status}" -ne 0 ]; then + workbench_log="islandora_workbench/workbench.log" + if [ -f "${workbench_log}" ]; then + printf 'Workbench failed; last 80 log lines:\n' >&2 + tail -n 80 -- "${workbench_log}" >&2 || true + fi + exit "${workbench_status}" +fi From b8882ac63c39eaa91ac82392be98748d8c426d61 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 14:44:24 +0000 Subject: [PATCH 4/6] [patch] Initialize private media storage --- Dockerfile | 3 ++- scripts/demo-objects.sh | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9d60029..da38805 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,5 +21,6 @@ COPY --link web/modules/custom/ /var/www/drupal/web/modules/custom/ COPY --link web/themes/custom/ /var/www/drupal/web/themes/custom/ COPY --link drupal/rootfs/opt/ /opt/ -RUN chown -R nginx:nginx /var/www/drupal && \ +RUN mkdir -p /var/www/drupal/private && \ + chown -R nginx:nginx /var/www/drupal && \ cleanup.sh diff --git a/scripts/demo-objects.sh b/scripts/demo-objects.sh index c775282..008ec9c 100755 --- a/scripts/demo-objects.sh +++ b/scripts/demo-objects.sh @@ -79,6 +79,8 @@ workbench_status=$? set -e if [ "${workbench_status}" -ne 0 ]; then + printf 'Drupal media storage state:\n' >&2 + docker compose exec -T drupal /var/www/drupal/vendor/bin/drush php:eval '$scheme = \Drupal::config("field.storage.media.field_media_image")->get("settings.uri_scheme"); $wrappers = \Drupal::service("stream_wrapper_manager")->getWrappers(); $flysystem = \Drupal\Core\Site\Settings::get("flysystem", []); $private = \Drupal\Core\Site\Settings::get("file_private_path", ""); print json_encode(["scheme" => $scheme, "registered" => isset($wrappers[$scheme]), "fedora_configured" => isset($flysystem["fedora"]), "private_path_exists" => is_string($private) && is_dir($private), "private_path_writable" => is_string($private) && is_writable($private)]);' >&2 || true workbench_log="islandora_workbench/workbench.log" if [ -f "${workbench_log}" ]; then printf 'Workbench failed; last 80 log lines:\n' >&2 From 54a1aae79a0cca2d23176fdc2e1b0f4f4935076b Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:48:33 +0000 Subject: [PATCH 5/6] [patch] Mount Drupal storage diagnostic --- compose.yaml | 4 ++++ scripts/demo-objects.sh | 2 +- scripts/drupal-media-storage-state.php | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 scripts/drupal-media-storage-state.php diff --git a/compose.yaml b/compose.yaml index 304b4e9..8365620 100644 --- a/compose.yaml +++ b/compose.yaml @@ -137,6 +137,10 @@ services: source: drupal-private-files target: /var/www/drupal/private type: volume + - read_only: true + source: ./scripts/drupal-media-storage-state.php + target: /var/www/drupal/drupal-media-storage-state.php + type: bind - drupal-solr-config:/opt/solr/server/solr/default:z,rw fits: diff --git a/scripts/demo-objects.sh b/scripts/demo-objects.sh index 008ec9c..1089cd0 100755 --- a/scripts/demo-objects.sh +++ b/scripts/demo-objects.sh @@ -80,7 +80,7 @@ set -e if [ "${workbench_status}" -ne 0 ]; then printf 'Drupal media storage state:\n' >&2 - docker compose exec -T drupal /var/www/drupal/vendor/bin/drush php:eval '$scheme = \Drupal::config("field.storage.media.field_media_image")->get("settings.uri_scheme"); $wrappers = \Drupal::service("stream_wrapper_manager")->getWrappers(); $flysystem = \Drupal\Core\Site\Settings::get("flysystem", []); $private = \Drupal\Core\Site\Settings::get("file_private_path", ""); print json_encode(["scheme" => $scheme, "registered" => isset($wrappers[$scheme]), "fedora_configured" => isset($flysystem["fedora"]), "private_path_exists" => is_string($private) && is_dir($private), "private_path_writable" => is_string($private) && is_writable($private)]);' >&2 || true + docker compose exec -T drupal /var/www/drupal/vendor/bin/drush php:script /var/www/drupal/drupal-media-storage-state.php >&2 || true workbench_log="islandora_workbench/workbench.log" if [ -f "${workbench_log}" ]; then printf 'Workbench failed; last 80 log lines:\n' >&2 diff --git a/scripts/drupal-media-storage-state.php b/scripts/drupal-media-storage-state.php new file mode 100644 index 0000000..fcd3402 --- /dev/null +++ b/scripts/drupal-media-storage-state.php @@ -0,0 +1,16 @@ +get('settings.uri_scheme'); +$wrappers = \Drupal::service('stream_wrapper_manager')->getWrappers(); +$flysystem = \Drupal\Core\Site\Settings::get('flysystem', []); +$private = \Drupal\Core\Site\Settings::get('file_private_path', ''); + +print json_encode([ + 'scheme' => $scheme, + 'registered' => isset($wrappers[$scheme]), + 'fedora_configured' => isset($flysystem['fedora']), + 'private_path_exists' => is_string($private) && is_dir($private), + 'private_path_writable' => is_string($private) && is_writable($private), +], JSON_THROW_ON_ERROR); From bf8843e0b40c149c26ff06503f0ee3ab6ba4293f Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 16:10:15 +0000 Subject: [PATCH 6/6] [patch] Test restored Islandora runtime contract --- .github/workflows/sitectl-create-smoke-test.yaml | 7 +++++-- Dockerfile | 2 +- README.md | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sitectl-create-smoke-test.yaml b/.github/workflows/sitectl-create-smoke-test.yaml index a93eae4..369dbc2 100644 --- a/.github/workflows/sitectl-create-smoke-test.yaml +++ b/.github/workflows/sitectl-create-smoke-test.yaml @@ -31,7 +31,10 @@ jobs: plugin: isle create-definition: ${{ matrix.create_definition }} create-args: ${{ matrix.create_args }} - packages: sitectl sitectl-isle sitectl-drupal - package-versions: sitectl=1.6.3 sitectl-isle=1.3.0 sitectl-drupal=1.2.0 + local-plugin-path: sitectl-isle + local-plugin-repository: libops/sitectl-isle + local-plugin-ref: c0f3cdf4a9532152af29c3fe032eccd39238d55b + packages: sitectl sitectl-drupal + package-versions: sitectl=1.8.1 sitectl-drupal=1.3.0 allow-unversioned-packages: false run-verify: true diff --git a/Dockerfile b/Dockerfile index da38805..a4c632a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=libops/islandora:nginx-1.30.4-php84@sha256:0e20f5d10af9431c3140179ef6bf1e41299026043d1a98974180ab54a5330cef +ARG BASE_IMAGE=libops/islandora:nginx-1.30.4-php84@sha256:0320df015cab9951ff0ba1e5f30c0a18641398706c3af6fe9d27c29f02b21d2e FROM ${BASE_IMAGE} ARG TARGETARCH diff --git a/README.md b/README.md index a1f9fd1..14cdca0 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Update the application base tag or pin that base by digest with [`sitectl image` ```bash sitectl image set --tag drupal=nginx-1.30.4-php84 --tag solr=9 --tag alpaca=2.4 -sitectl image set --build-arg drupal.BASE_IMAGE=libops/islandora:nginx-1.30.4-php84@sha256:0e20f5d10af9431c3140179ef6bf1e41299026043d1a98974180ab54a5330cef +sitectl image set --build-arg drupal.BASE_IMAGE=libops/islandora:nginx-1.30.4-php84@sha256:0320df015cab9951ff0ba1e5f30c0a18641398706c3af6fe9d27c29f02b21d2e ``` Enable local development bind mounts with [`sitectl set`](https://sitectl.libops.io/commands/set):