[minor] Isolate ISLE database bootstrap and verify storage

.github/workflows/sitectl-create-smoke-test.yaml

@@ -31,6 +31,10 @@ jobs:
       plugin: isle
       create-definition: ${{ matrix.create_definition }}
       create-args: ${{ matrix.create_args }}
-      packages: sitectl sitectl-isle sitectl-drupal
-      package-versions: sitectl=1.6.3 sitectl-isle=1.3.0 sitectl-drupal=1.2.0
+      local-plugin-path: sitectl-isle
+      local-plugin-repository: libops/sitectl-isle
+      local-plugin-ref: c0f3cdf4a9532152af29c3fe032eccd39238d55b
+      packages: sitectl sitectl-drupal
+      package-versions: sitectl=1.8.1 sitectl-drupal=1.3.0
       allow-unversioned-packages: false
+      run-verify: true

Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=libops/islandora:nginx-1.30.4-php84@sha256:0e20f5d10af9431c3140179ef6bf1e41299026043d1a98974180ab54a5330cef
+ARG BASE_IMAGE=libops/islandora:nginx-1.30.4-php84@sha256:0320df015cab9951ff0ba1e5f30c0a18641398706c3af6fe9d27c29f02b21d2e
 FROM ${BASE_IMAGE}
 
 ARG TARGETARCH
@@ -21,5 +21,6 @@ COPY --link web/modules/custom/ /var/www/drupal/web/modules/custom/
 COPY --link web/themes/custom/ /var/www/drupal/web/themes/custom/
 COPY --link drupal/rootfs/opt/ /opt/
 
-RUN chown -R nginx:nginx /var/www/drupal && \
+RUN mkdir -p /var/www/drupal/private && \
+    chown -R nginx:nginx /var/www/drupal && \
     cleanup.sh

README.md

@@ -55,8 +55,8 @@ sitectl validate
 Update the application base tag or pin that base by digest with [`sitectl image`](https://sitectl.libops.io/commands/image):
 
 ```bash
-sitectl image set --tag drupal=nginx-1.30.3-php84 --tag solr=9 --tag alpaca=2.4
-sitectl image set --build-arg drupal.BASE_IMAGE=libops/islandora:nginx-1.30.3-php84@sha256:...
+sitectl image set --tag drupal=nginx-1.30.4-php84 --tag solr=9 --tag alpaca=2.4
+sitectl image set --build-arg drupal.BASE_IMAGE=libops/islandora:nginx-1.30.4-php84@sha256:0320df015cab9951ff0ba1e5f30c0a18641398706c3af6fe9d27c29f02b21d2e
 ```
 
 Enable local development bind mounts with [`sitectl set`](https://sitectl.libops.io/commands/set):
@@ -112,6 +112,24 @@ Only MariaDB and the one-shot `database-init` service receive `DB_ROOT_PASSWORD`
 
 ActiveMQ, Alpaca, and Drupal share the generated `ACTIVEMQ_PASSWORD` through service-specific secret targets. Drupal receives it as `DRUPAL_DEFAULT_BROKER_PASSWORD` together with the non-secret `admin` broker username, so the rendered Islandora settings use authenticated STOMP rather than relying on a broker default.
 
+## Full-state recovery
+
+The ISLE plugin exposes the authoritative-versus-rebuildable recovery contract and creates a single checksummed bundle:
+
+```bash
+sitectl isle recovery plan
+sitectl isle recovery backup --output /var/backups/isle/site-$(date +%F).tar.gz
+sitectl isle recovery validate --input /var/backups/isle/site-2026-08-07.tar.gz
+```
+
+The bundle contains the Drupal database and public/private files, plus the Fcrepo database and object data when Fcrepo is enabled. It deliberately excludes customer secrets, source-controlled project configuration, Solr and Blazegraph indexes, ActiveMQ queues, IIIF caches, and generated derivatives. Recreate the target from the matching site Git revision and template provenance lock, restore secrets from the organization's Vault backup, and rebuild the excluded derived state after restore. The target and bundle must agree on whether Fedora is enabled.
+
+Copy bundles to encrypted off-host storage with retention that meets the customer's documented RPO. At least quarterly, restore a selected bundle into a disposable context, run `sitectl healthcheck` and `sitectl verify --strict`, rebuild indexes and required derivatives, and record the achieved RPO/RTO. A destructive restore requires confirmation:
+
+```bash
+sitectl isle recovery restore --input /var/backups/isle/site-2026-08-07.tar.gz
+```
+
 ## License
 
 The Docker Compose template and LibOps-specific setup in this repository are licensed under the MIT License. The upstream Islandora starter site is licensed separately under the GNU General Public License v2; see `LICENSE.islandora-starter-site`.

compose.yaml

@@ -112,6 +112,8 @@ services:
       start_period: ${DRUPAL_HEALTHCHECK_START_PERIOD:-5m}
       retries: ${DRUPAL_HEALTHCHECK_RETRIES:-20}
     depends_on:
+      database-init:
+        condition: service_completed_successfully
       mariadb:
         condition: service_healthy
     secrets:
@@ -135,6 +137,10 @@ services:
         source: drupal-private-files
         target: /var/www/drupal/private
         type: volume
+      - read_only: true
+        source: ./scripts/drupal-media-storage-state.php
+        target: /var/www/drupal/drupal-media-storage-state.php
+        type: bind
       - drupal-solr-config:/opt/solr/server/solr/default:z,rw
 
   fits:
@@ -190,6 +196,27 @@ services:
       - ./secrets:/work/secrets:rw,z
       - ./compose.yaml:/work/compose.yaml:ro,z
 
+  database-init:
+    image: libops/base:3.2.2.0@sha256:851e17742b5fee57038855f46b1a46d2716c0012c5f3a4788b7e5c7bc12fed5e
+    restart: "no"
+    networks:
+      default:
+    environment:
+      DB_HOST: mariadb
+      DB_PORT: "3306"
+      DB_NAME: drupal_default
+      DB_USER: drupal_default
+      DB_CHARACTER_SET: utf8mb4
+      DB_COLLATION: utf8mb4_unicode_ci
+    secrets:
+      - source: DB_ROOT_PASSWORD
+      - source: DRUPAL_DEFAULT_DB_PASSWORD
+        target: DB_PASSWORD
+    entrypoint: /usr/local/bin/init-database.sh
+    depends_on:
+      mariadb:
+        condition: service_healthy
+
   mariadb:
     <<: *common
     image: libops/mariadb:11.8@sha256:1b0f53f277cedf95e1fa6df25580de01dd2d0722b09f39fe0d93cfb6263e26f3
@@ -200,7 +227,7 @@ services:
 
   mergepdf:
     <<: *common
-    image: islandora/mergepdf:6.3.19
+    image: islandora/mergepdf:6.3.19@sha256:69f391d2058463f815c87b6703a39e38f5e61bf27b15b305ebf4bda64268110c
     secrets:
       - source: CERT_PUBLIC_KEY
       - source: CERT_AUTHORITY

scripts/demo-objects.sh

@@ -2,8 +2,37 @@
 
 set -eou pipefail
 
-# shellcheck disable=SC1091
-source "$(dirname "${BASH_SOURCE[0]}")/profile.sh"
+site_url() {
+  sitectl stats --path . --format json | jq -er '.ingress.public_url'
+}
+
+fcrepo_enabled() {
+  docker compose config --services 2>/dev/null | grep -qx 'fcrepo'
+}
+
+container_url_for_url() {
+  local url="$1"
+  if fcrepo_enabled && [[ "${url}" =~ ^(https?)://(localhost|127\.0\.0\.1)(:[0-9]+)?(/.*)?$ ]]; then
+    printf '%s://drupal.internal%s\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[4]}"
+    return
+  fi
+  printf '%s\n' "${url}"
+}
+
+container_network_for_url() {
+  local url="${1:?url is required}"
+  local compose_project_name
+  local traefik_container
+  if [[ "${url}" =~ ^https?://(localhost|127\.0\.0\.1)(:[0-9]+)?(/.*)?$ ]]; then
+    traefik_container="$(docker compose ps -q traefik)"
+    if [ -n "${traefik_container}" ]; then
+      printf 'container:%s\n' "${traefik_container}"
+      return
+    fi
+  fi
+  compose_project_name="$(docker compose config --format json | jq -er '.name')"
+  printf '%s_default\n' "${compose_project_name}"
+}
 
 if [ ! -d "islandora_workbench" ]; then
   git clone https://github.com/mjordan/islandora_workbench
@@ -13,7 +42,7 @@ if [ ! -d "islandora_demo_objects" ]; then
   git clone https://github.com/Islandora-Devops/islandora_demo_objects islandora_demo_objects
 fi
 
-URL="$(site_url)"
+URL="${SITECTL_DEMO_OBJECTS_URL:-$(site_url)}"
 WORKBENCH_URL="$(container_url_for_url "${URL}")"
 NETWORK="$(container_network_for_url "${WORKBENCH_URL}")"
 
@@ -35,6 +64,7 @@ docker build \
 tty_flag=( -i )
 [ -t 0 ] && tty_flag=( -it )
 
+set +e
 docker run \
   "${tty_flag[@]}" \
   --rm \
@@ -45,3 +75,16 @@ docker run \
   --name my-running-workbench \
   workbench-docker:latest \
   bash -lc "./workbench --config /islandora_demo_objects/create_islandora_objects.yml"
+workbench_status=$?
+set -e
+
+if [ "${workbench_status}" -ne 0 ]; then
+  printf 'Drupal media storage state:\n' >&2
+  docker compose exec -T drupal /var/www/drupal/vendor/bin/drush php:script /var/www/drupal/drupal-media-storage-state.php >&2 || true
+  workbench_log="islandora_workbench/workbench.log"
+  if [ -f "${workbench_log}" ]; then
+    printf 'Workbench failed; last 80 log lines:\n' >&2
+    tail -n 80 -- "${workbench_log}" >&2 || true
+  fi
+  exit "${workbench_status}"
+fi

scripts/drupal-media-storage-state.php

@@ -0,0 +1,16 @@
+<?php
+
+declare(strict_types=1);
+
+$scheme = \Drupal::config('field.storage.media.field_media_image')->get('settings.uri_scheme');
+$wrappers = \Drupal::service('stream_wrapper_manager')->getWrappers();
+$flysystem = \Drupal\Core\Site\Settings::get('flysystem', []);
+$private = \Drupal\Core\Site\Settings::get('file_private_path', '');
+
+print json_encode([
+    'scheme' => $scheme,
+    'registered' => isset($wrappers[$scheme]),
+    'fedora_configured' => isset($flysystem['fedora']),
+    'private_path_exists' => is_string($private) && is_dir($private),
+    'private_path_writable' => is_string($private) && is_writable($private),
+], JSON_THROW_ON_ERROR);

scripts/test.sh

@@ -8,3 +8,4 @@ docker compose build --pull drupal
 docker compose run --rm -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" init
 docker compose up --remove-orphans --wait --wait-timeout "${COMPOSE_WAIT_TIMEOUT:-900}" -d
 sitectl healthcheck --persist --timeout "${SITECTL_HEALTHCHECK_TIMEOUT:-10m}"
+sitectl verify --strict