From 198d5388dc989c09cb6de4ff891f203408e7840b Mon Sep 17 00:00:00 2001
From: Miliya-Ai <102935805+Miliya-Ai@users.noreply.github.com>
Date: Thu, 13 Aug 2026 18:02:25 -0400
Subject: [PATCH 1/9] Integrating SAM for fine-grained segmentation requires
 minimal changes to SAM itself (just adding a coordinate scaling parameter).
 The major involves creating a preprocessor that calls SAM and renaming the
 LLM object detection module before fixing all downstream dependencies

---
 .github/workflows/object-segmentation.yml     |  83 +++++++
 docker-compose.yml                            |  22 ++
 handlers/ocr-handler/Dockerfile               |   2 +
 handlers/ocr-handler/server.py                |  14 +-
 handlers/photo-audio-handler/src/server.ts    |   2 +-
 handlers/photo-audio-handler/src/utils.ts     |   7 +
 handlers/photo-tactile-svg/Dockerfile         |   1 +
 handlers/photo-tactile-svg/tactile_svg.py     |  13 +-
 handlers/svg-object-detection/Dockerfile      |   1 +
 handlers/svg-object-detection/od_svg.py       |   6 +-
 .../celebrity-detector/celebrity-detector.py  |   7 +-
 preprocessors/clothes-detector/clothes.py     |   7 +-
 preprocessors/grouping/grouping.py            |   7 +-
 .../object-depth-calculator.py                |   6 +-
 .../object-detection-llm.py                   |   2 +-
 preprocessors/object-segmentation/Dockerfile  |  52 +++++
 preprocessors/object-segmentation/README.md   |  44 ++++
 .../object-segmentation.py                    | 218 ++++++++++++++++++
 .../object-segmentation/requirements.txt      |   6 +
 preprocessors/ocr/ocr.py                      |  14 +-
 preprocessors/sorting/sorting.py              |   7 +-
 utils/object_detection/__init__.py            |  26 +++
 utils/segmentation/README.md                  |   5 +-
 utils/segmentation/sam_processor.py           |  23 +-
 24 files changed, 528 insertions(+), 47 deletions(-)
 create mode 100644 .github/workflows/object-segmentation.yml
 create mode 100644 preprocessors/object-segmentation/Dockerfile
 create mode 100644 preprocessors/object-segmentation/README.md
 create mode 100644 preprocessors/object-segmentation/object-segmentation.py
 create mode 100644 preprocessors/object-segmentation/requirements.txt
 create mode 100644 utils/object_detection/__init__.py

diff --git a/.github/workflows/object-segmentation.yml b/.github/workflows/object-segmentation.yml
new file mode 100644
index 000000000..790d348a0
--- /dev/null
+++ b/.github/workflows/object-segmentation.yml
@@ -0,0 +1,83 @@
+name: Object Segmentation
+on:
+  push:
+    branches: [ main, object-segmentation ]
+    tags: [ "preprocessor-object-segmentation-[0-9]+.[0-9]+.[0-9]+" ]
+    paths: [ "preprocessors/object-segmentation/**" ]
+  pull_request:
+    branches: [ main ]
+    paths: [ "preprocessors/object-segmentation/**" ]
+  workflow_run:
+    workflows: [ "Schemas (Trigger)" ]
+    types:
+      - completed
+  workflow_dispatch:
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: shared-reality-lab/image-preprocessor-object-segmentation
+jobs:
+  lint:
+    name: PEP 8 style check.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: Install flake8
+        run: pip install flake8
+      - name: Check with flake8
+        run: python -m flake8 ./preprocessors/object-segmentation --show-source
+  build-and-push-image:
+    name: Build and Push to Registry
+    needs: lint
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Log into GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Get Correct Tags
+        run: |
+          if [[ ${{ github.ref }} =~ ^refs/tags/preprocessor-object-segmentation-[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "TAGGED=true" >> $GITHUB_ENV
+          else
+            echo "TAGGED=false" >> $GITHUB_ENV
+          fi
+      - name: Get timestamp
+        run: echo "timestamp=$(date -u +'%Y-%m-%dT%H.%M')" >> $GITHUB_ENV
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            latest=${{ env.TAGGED }}
+          tags: |
+            type=match,enable=${{ env.TAGGED }},priority=300,pattern=preprocessor-object-segmentation-(\d+.\d+.\d+),group=1
+            type=raw,priority=200,value=unstable
+            type=raw,priority=100,value=${{ env.timestamp }}
+          labels: |
+            org.opencontainers.image.title=IMAGE Preprocessor Object Segmentation
+            org.opencontainers.image.description=Segments objects found by object-detection-llm using SAM, producing precise polygon outlines per object.
+            org.opencontainers.image.authors=IMAGE Project <image@cim.mcgill.ca>
+            org.opencontainers.image.documentation=https://github.com/Shared-Reality-Lab/IMAGE-server/tree/main/preprocessors/object-segmentation/README.md
+            org.opencontainers.image.licenses=AGPL-3.0-or-later
+            maintainer=IMAGE Project <image@cim.mcgill.ca>
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./preprocessors/object-segmentation/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/docker-compose.yml b/docker-compose.yml
index 4327e0c77..14ef72fc9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -193,6 +193,28 @@ services:
     env_file:
       ./config/llm.env
 
+  object-segmentation:
+    profiles: [test, production, default]
+    image: ghcr.io/shared-reality-lab/image-preprocessor-object-segmentation:${REGISTRY_TAG}
+    restart: "no"
+    labels:
+      ca.mcgill.a11y.image.preprocessor: 4
+      ca.mcgill.a11y.image.port: 5000
+      ca.mcgill.a11y.image.cacheTimeout: 3600
+      ca.mcgill.a11y.image.required_dependencies: "object-detection-llm"
+      ca.mcgill.a11y.image.optional_dependencies: ""
+    deploy:
+      resources:
+        reservations:
+          devices:
+          - driver: nvidia
+            capabilities: ["gpu", "utility", "compute"]
+    environment:
+      - PII_LOGGING_ENABLED=${PII_LOGGING_ENABLED}
+      - SAM_MODEL_PATH=/usr/src/app/models/sam2.1_l.pt
+      - MIN_CONTOUR_AREA=0.0001
+      - WARMUP_ENABLED=true
+
   multistage-diagram-segmentation:
     profiles: [production, test, default]
     image: ghcr.io/shared-reality-lab/image-preprocessor-multistage-diagram-segmentation:${REGISTRY_TAG}
diff --git a/handlers/ocr-handler/Dockerfile b/handlers/ocr-handler/Dockerfile
index e0cbc0a26..45d5ccc92 100644
--- a/handlers/ocr-handler/Dockerfile
+++ b/handlers/ocr-handler/Dockerfile
@@ -18,6 +18,8 @@ COPY /schemas /app/schemas
 
 COPY /config /app/config
 
+COPY /utils /app/utils
+
 COPY /handlers/ocr-handler/ /app
 
 EXPOSE 80
diff --git a/handlers/ocr-handler/server.py b/handlers/ocr-handler/server.py
index 3492fa68a..31b7e387e 100644
--- a/handlers/ocr-handler/server.py
+++ b/handlers/ocr-handler/server.py
@@ -22,6 +22,11 @@
 from flask import Flask, request, jsonify
 from datetime import datetime
 from config.logging_utils import configure_logging
+from utils.object_detection import (
+    GENERIC_OBJECT_DETECTION_NAME,
+    LLM_OBJECT_DETECTION_NAME,
+    get_object_detection_data,
+)
 
 configure_logging()
 
@@ -132,9 +137,12 @@ def render_ocr():
     text = ""
 
     # Object detection data is present
-    od = 'ca.mcgill.a11y.image.preprocessor.objectDetection'
-    if od in preprocessors and len(preprocessors[od]['objects']) > 0:
-        object_data = preprocessors[od]['objects']
+    od_data = get_object_detection_data(preprocessors)
+    if od_data is not None and len(od_data['objects']) > 0:
+        od = (GENERIC_OBJECT_DETECTION_NAME
+              if GENERIC_OBJECT_DETECTION_NAME in preprocessors
+              else LLM_OBJECT_DETECTION_NAME)
+        object_data = od_data['objects']
         text_lines = ocr_data['lines']
         text += "The following objects were detected: "
         done_once = False
diff --git a/handlers/photo-audio-handler/src/server.ts b/handlers/photo-audio-handler/src/server.ts
index 2d02d9208..649419d78 100644
--- a/handlers/photo-audio-handler/src/server.ts
+++ b/handlers/photo-audio-handler/src/server.ts
@@ -61,7 +61,7 @@ app.post("/handler", async (req, res) => {
     const preprocessors = req.body["preprocessors"];
     const secondCat = preprocessors["ca.mcgill.a11y.image.preprocessor.graphicTagger"];
     const semseg = preprocessors["ca.mcgill.a11y.image.preprocessor.semanticSegmentation"];
-    const objDet = preprocessors["ca.mcgill.a11y.image.preprocessor.objectDetection"];
+    const objDet = utils.getObjectDetectionData(preprocessors);
     const objGroup = preprocessors["ca.mcgill.a11y.image.preprocessor.grouping"];
     const action = preprocessors["ca.mcgill.a11y.image.preprocessor.actionRecognition"];
     //const collageDetector = preprocessors["ca.mcgill.a11y.image.preprocessor.collageDetector"];
diff --git a/handlers/photo-audio-handler/src/utils.ts b/handlers/photo-audio-handler/src/utils.ts
index 96a4995d8..dfae13857 100644
--- a/handlers/photo-audio-handler/src/utils.ts
+++ b/handlers/photo-audio-handler/src/utils.ts
@@ -39,6 +39,13 @@ type ObjDet = {
     objects: Obj[];
 };
 
+const GENERIC_OBJECT_DETECTION_NAME = "ca.mcgill.a11y.image.preprocessor.objectDetection";
+const LLM_OBJECT_DETECTION_NAME = "ca.mcgill.a11y.image.preprocessor.objectDetectionLLM";
+
+export function getObjectDetectionData(preprocessors: Record<string, ObjDet>): ObjDet | undefined {
+    return preprocessors[GENERIC_OBJECT_DETECTION_NAME] ?? preprocessors[LLM_OBJECT_DETECTION_NAME];
+}
+
 type ObjGroup = {
     grouped: { IDs: number[] }[];
     ungrouped: number[];
diff --git a/handlers/photo-tactile-svg/Dockerfile b/handlers/photo-tactile-svg/Dockerfile
index 031c928db..53b971070 100644
--- a/handlers/photo-tactile-svg/Dockerfile
+++ b/handlers/photo-tactile-svg/Dockerfile
@@ -14,6 +14,7 @@ RUN pip install -r requirements.txt
 
 COPY /schemas /usr/src/app/schemas
 COPY /config /usr/src/app/config
+COPY /utils /usr/src/app/utils
 COPY /handlers/photo-tactile-svg/ /usr/src/app
 
 EXPOSE 80
diff --git a/handlers/photo-tactile-svg/tactile_svg.py b/handlers/photo-tactile-svg/tactile_svg.py
index f8bb7adfb..51b0d7323 100644
--- a/handlers/photo-tactile-svg/tactile_svg.py
+++ b/handlers/photo-tactile-svg/tactile_svg.py
@@ -24,6 +24,7 @@
 import inflect
 from config.logging_utils import configure_logging
 from datetime import datetime
+from utils.object_detection import get_object_detection_data
 
 configure_logging()
 app = Flask(__name__)
@@ -99,9 +100,8 @@ def handle():
                   "and/ or semantic segmentation responses")
     if not (("ca.mcgill.a11y.image.preprocessor.semanticSegmentation"
              in preprocessors) or
-            all(x in preprocessors for x in
-                ["ca.mcgill.a11y.image.preprocessor.objectDetection",
-                 "ca.mcgill.a11y.image.preprocessor.grouping"])):
+            (get_object_detection_data(preprocessors) is not None and
+             "ca.mcgill.a11y.image.preprocessor.grouping" in preprocessors)):
         logging.debug("No Object Detector and Semantic Segmentation found")
         response = {
             "request_uuid": contents["request_uuid"],
@@ -152,17 +152,14 @@ def handle():
     form = inflect.engine()
     caption = ""
 
-    if "ca.mcgill.a11y.image.preprocessor.objectDetection"\
-        in preprocessors\
+    if get_object_detection_data(preprocessors) is not None\
             and "ca.mcgill.a11y.image.preprocessor.grouping" in preprocessors:
         logging.debug("Object detector and grouping preprocessor found. "
                       "Adding data to response...")
         caption = "This photo contains "
         obj_list = []
         preprocessor_names.append('Things and people')
-        o = preprocessors[
-            "ca.mcgill.a11y.image.preprocessor.objectDetection"
-            ]
+        o = get_object_detection_data(preprocessors)
         g = preprocessors["ca.mcgill.a11y.image.preprocessor.grouping"]
         objects = o["objects"]
         grouped = g["grouped"]
diff --git a/handlers/svg-object-detection/Dockerfile b/handlers/svg-object-detection/Dockerfile
index 9f47cae82..c0de03065 100644
--- a/handlers/svg-object-detection/Dockerfile
+++ b/handlers/svg-object-detection/Dockerfile
@@ -14,6 +14,7 @@ RUN pip install -r requirements.txt
 
 COPY /schemas /usr/src/app/schemas
 COPY /config /usr/src/app/config
+COPY /utils /usr/src/app/utils
 COPY /handlers/svg-object-detection/ /usr/src/app
 
 EXPOSE 80
diff --git a/handlers/svg-object-detection/od_svg.py b/handlers/svg-object-detection/od_svg.py
index 2128c0234..8770c937c 100644
--- a/handlers/svg-object-detection/od_svg.py
+++ b/handlers/svg-object-detection/od_svg.py
@@ -23,6 +23,7 @@
 import drawSvg as draw
 from datetime import datetime
 from config.logging_utils import configure_logging
+from utils.object_detection import get_object_detection_data
 
 configure_logging()
 
@@ -95,8 +96,7 @@ def handle():
         return response
 
     # No Object Detector found
-    if "ca.mcgill.a11y.image.preprocessor.objectDetection"\
-            not in preprocessors:
+    if get_object_detection_data(preprocessors) is None:
         logging.debug("No Object Detector found")
         response = {
             "request_uuid": contents["request_uuid"],
@@ -137,7 +137,7 @@ def handle():
         logging.debug("Sending response")
         return response
 
-    o = preprocessors["ca.mcgill.a11y.image.preprocessor.objectDetection"]
+    o = get_object_detection_data(preprocessors)
     g = preprocessors["ca.mcgill.a11y.image.preprocessor.grouping"]
     u = preprocessors["ca.mcgill.a11y.image.preprocessor.grouping"]
     objects = o["objects"]
diff --git a/preprocessors/celebrity-detector/celebrity-detector.py b/preprocessors/celebrity-detector/celebrity-detector.py
index 720fdd72b..0888b7037 100644
--- a/preprocessors/celebrity-detector/celebrity-detector.py
+++ b/preprocessors/celebrity-detector/celebrity-detector.py
@@ -27,6 +27,7 @@
 import numpy as np
 from datetime import datetime
 from utils.validation import Validator
+from utils.object_detection import get_object_detection_data
 
 app = Flask(__name__)
 
@@ -137,14 +138,12 @@ def categorise():
     # convert the uri to processable image
     if "graphic" not in content.keys():
         return "", 204
-    if "ca.mcgill.a11y.image.preprocessor.objectDetection" \
-            not in preprocessor:
+    oDpreprocessor = get_object_detection_data(preprocessor)
+    if oDpreprocessor is None:
         logging.info("Object detection output not "
                      "available. Skipping...")
         return "", 204
     else:
-        oDpreprocessor = \
-            preprocessor["ca.mcgill.a11y.image.preprocessor.objectDetection"]
         objects = oDpreprocessor["objects"]
         image_b64 = content["graphic"].split(",")[1]
         binary = base64.b64decode(image_b64)
diff --git a/preprocessors/clothes-detector/clothes.py b/preprocessors/clothes-detector/clothes.py
index b9071f7b0..d0825904f 100644
--- a/preprocessors/clothes-detector/clothes.py
+++ b/preprocessors/clothes-detector/clothes.py
@@ -34,6 +34,7 @@
 from predictors.YOLOv3 import YOLOv3Predictor
 from datetime import datetime
 from utils.validation import Validator
+from utils.object_detection import get_object_detection_data
 
 app = Flask(__name__)
 logging.basicConfig(level=logging.NOTSET)
@@ -120,14 +121,12 @@ def categorise():
     # convert the uri to processable image
     if "graphic" not in content.keys():
         return "", 204
-    if "ca.mcgill.a11y.image.preprocessor.objectDetection" \
-            not in preprocessor:
+    oDpreprocessor = get_object_detection_data(preprocessor)
+    if oDpreprocessor is None:
         logging.info("Object detection output not "
                      "available. Skipping...")
         return "", 204
     else:
-        oDpreprocessor = \
-            preprocessor["ca.mcgill.a11y.image.preprocessor.objectDetection"]
         objects = oDpreprocessor["objects"]
         image_b64 = content["graphic"].split(",")[1]
         binary = base64.b64decode(image_b64)
diff --git a/preprocessors/grouping/grouping.py b/preprocessors/grouping/grouping.py
index dd1f75e85..b0d2a1e80 100644
--- a/preprocessors/grouping/grouping.py
+++ b/preprocessors/grouping/grouping.py
@@ -23,6 +23,7 @@
 from config.logging_utils import configure_logging
 from datetime import datetime
 from utils.validation import Validator
+from utils.object_detection import get_object_detection_data
 
 configure_logging()
 
@@ -57,13 +58,11 @@ def readImage():
         return jsonify("Invalid Preprocessor JSON format"), 400
 
     preprocessor = content["preprocessors"]
-    if "ca.mcgill.a11y.image.preprocessor.objectDetection" \
-            not in preprocessor:
+    oDpreprocessor = get_object_detection_data(preprocessor)
+    if oDpreprocessor is None:
         logging.info("Object detection output not "
                      "available. Skipping...")
         return "", 204
-    oDpreprocessor = \
-        preprocessor["ca.mcgill.a11y.image.preprocessor.objectDetection"]
     objects = oDpreprocessor["objects"]
 
     for i in range(len(objects)):
diff --git a/preprocessors/object-depth-calculator/object-depth-calculator.py b/preprocessors/object-depth-calculator/object-depth-calculator.py
index 778ca5342..16457816d 100644
--- a/preprocessors/object-depth-calculator/object-depth-calculator.py
+++ b/preprocessors/object-depth-calculator/object-depth-calculator.py
@@ -23,6 +23,7 @@
 from datetime import datetime
 from config.logging_utils import configure_logging
 from utils.validation import Validator
+from utils.object_detection import get_object_detection_data
 
 configure_logging()
 
@@ -51,8 +52,7 @@ def objectdepth():
         logging.info("Request does not contain a depth-map. Skipping...")
         return "", 204  # No content
     logging.debug("passed depth-map check")
-    if ("ca.mcgill.a11y.image.preprocessor.objectDetection"
-            not in content["preprocessors"]):
+    if get_object_detection_data(content["preprocessors"]) is None:
         logging.info("Request does not contain objects. Skipping...")
         return "", 204  # No content
     logging.debug("passed objects check")
@@ -92,7 +92,7 @@ def objectdepth():
     image = np.asarray(bytearray(binary), dtype="uint8")
     img = cv2.imdecode(image, cv2.IMREAD_GRAYSCALE)/255
 
-    o = preprocessors["ca.mcgill.a11y.image.preprocessor.objectDetection"]
+    o = get_object_detection_data(preprocessors)
     objects = o["objects"]
     print(dimensions[0], dimensions[1])
     obj_depth = []
diff --git a/preprocessors/object-detection-llm/object-detection-llm.py b/preprocessors/object-detection-llm/object-detection-llm.py
index 85f783710..076f91704 100644
--- a/preprocessors/object-detection-llm/object-detection-llm.py
+++ b/preprocessors/object-detection-llm/object-detection-llm.py
@@ -68,7 +68,7 @@
 )
 
 PREPROCESSOR_NAME = \
-    "ca.mcgill.a11y.image.preprocessor.objectDetection"
+    "ca.mcgill.a11y.image.preprocessor.objectDetectionLLM"
 
 DATA_SCHEMA = './schemas/preprocessors/object-detection.schema.json'
 
diff --git a/preprocessors/object-segmentation/Dockerfile b/preprocessors/object-segmentation/Dockerfile
new file mode 100644
index 000000000..05f334247
--- /dev/null
+++ b/preprocessors/object-segmentation/Dockerfile
@@ -0,0 +1,52 @@
+FROM ultralytics/ultralytics:8.3.119-python
+
+# Set the working directory inside the container
+WORKDIR /usr/src/app
+
+# Create a non-root user to run the application
+RUN adduser --disabled-password python
+
+# Update PATH to include local bin for the python user
+ENV PATH="/usr/src/app/.local/bin:${PATH}"
+
+# Install system dependencies required for healthcheck (curl) and model downloads (wget)
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+    curl wget && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy the requirements file into the container
+COPY /preprocessors/object-segmentation/requirements.txt /usr/src/app/requirements.txt
+
+# Install Python dependencies
+RUN pip3 install --upgrade pip && \
+    pip3 install --no-cache-dir -r /usr/src/app/requirements.txt
+
+# Copy the schema, config, and shared utils files
+COPY /schemas /usr/src/app/schemas
+COPY /config /usr/src/app/config
+COPY /utils /usr/src/app/utils
+
+# Create model directory
+RUN mkdir -p /usr/src/app/models
+
+# Download SAM 2.1 Large model
+RUN wget -O /usr/src/app/models/sam2.1_l.pt https://image.a11y.mcgill.ca/models/semanticSegmentation/sam2.1_l.pt
+
+# Copy the preprocessor application code
+COPY /preprocessors/object-segmentation /usr/src/app
+
+# Set environment variables needed by the Flask app
+ENV FLASK_APP=object-segmentation.py
+
+# Expose the port the application runs on
+EXPOSE 5000
+
+# Switch to the non-root user
+USER python
+
+# Define the healthcheck command
+HEALTHCHECK --interval=60s --timeout=10s --start-period=120s --retries=5 CMD curl -f http://localhost:5000/health || exit 1
+
+# Define the command to run the application using gunicorn
+CMD [ "gunicorn", "object-segmentation:app", "-b", "0.0.0.0:5000", "--capture-output", "--timeout=120", "--log-level=debug" ]
diff --git a/preprocessors/object-segmentation/README.md b/preprocessors/object-segmentation/README.md
new file mode 100644
index 000000000..11032f068
--- /dev/null
+++ b/preprocessors/object-segmentation/README.md
@@ -0,0 +1,44 @@
+# Object Segmentation Preprocessor
+
+Alpha quality: not yet ready for use by end-users.
+
+This preprocessor segments objects already found by `object-detection-llm` using SAM (Segment Anything Model), producing precise per-object polygon outlines rather than just bounding boxes. It does not run its own LLM inference - it consumes `object-detection-llm`'s existing bounding boxes as SAM prompts.
+
+Output is shaped identically to the existing `semanticSegmentation` preprocessor's `segments` output (`schemas/preprocessors/segmentation.schema.json`), with an additional `objectID` field on each segment that ties it back to the detected object it came from.
+
+## Environment Variables
+
+```
+SAM_MODEL_PATH=[Path to SAM model file]
+MIN_CONTOUR_AREA=[Minimum normalized contour area to keep, default 0.0001]
+PII_LOGGING_ENABLED=[true or false]
+```
+
+**Note**: For production use, it's strongly recommended to set `PII_LOGGING_ENABLED=false` to prevent security risks. Logging personal information should only be done on test servers. The preprocessor uses a `logging.pii()` function that should be properly configured by the logging utilities module.
+
+## Libraries Used
+
+| Library | Link | Distribution License |
+| ------------- | ------------- | -------------|
+| Flask | [Link](https://pypi.org/project/Flask/) | BSD-3-Clause License |
+| jsonschema | [Link](https://pypi.org/project/jsonschema/) | MIT License |
+| gunicorn | [Link](https://github.com/benoitc/gunicorn) | MIT License |
+| pillow | [Link](https://pypi.org/project/Pillow/) | MIT-CMU |
+| opencv-python | [Link](https://pypi.org/project/opencv-python/) | Apache 2.0 |
+| ultralytics | [Link](https://pypi.org/project/ultralytics/) | AGPL-3.0 License |
+
+The versions for each of these libraries are specified in `requirements.txt`.
+
+## API Endpoints
+
+- `/preprocessor` (POST): Main endpoint for object segmentation
+- `/health` (GET): Health check endpoint
+- `/warmup` (GET): Warms up the SAM model with a dummy inference
+
+## Processing Pipeline
+
+1. Read `object-detection-llm`'s bounding boxes for the request (required; this preprocessor is a no-op without them).
+2. Decode the graphic from base64 to a PIL image. No additional resizing is performed here - the upstream `resize-graphic` pseudo-preprocessor already caps every incoming graphic to a fixed maximum dimension, so this preprocessor and `object-detection-llm` share the same pixel space.
+3. Segment each detected object's box with SAM, using the object's unique ID (not its type) as the SAM label to keep same-type objects (e.g. two people) from being merged into one contour set.
+4. Filter out contours below `MIN_CONTOUR_AREA`.
+5. Build a `segments` list matching `schemas/preprocessors/segmentation.schema.json`, with each segment tagged with the `objectID` it came from.
diff --git a/preprocessors/object-segmentation/object-segmentation.py b/preprocessors/object-segmentation/object-segmentation.py
new file mode 100644
index 000000000..dc7ca19c2
--- /dev/null
+++ b/preprocessors/object-segmentation/object-segmentation.py
@@ -0,0 +1,218 @@
+# Copyright (c) 2025 IMAGE Project, Shared Reality Lab, McGill University
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# and our Additional Terms along with this program.
+# If not, see
+# <https://github.com/Shared-Reality-Lab/IMAGE-server/blob/main/LICENSE>.
+
+import base64
+import logging
+import os
+import sys
+import time
+from io import BytesIO
+
+from flask import Flask, request, jsonify
+from datetime import datetime
+from PIL import Image
+
+from config.logging_utils import configure_logging
+from utils.object_detection import LLM_OBJECT_DETECTION_NAME
+from utils.segmentation import (
+    SAMClient,
+    create_segment_from_contours,
+    filter_contours_by_area,
+)
+from utils.validation import Validator
+
+configure_logging()
+
+logging.debug("Starting Object Segmentation Preprocessor...")
+
+app = Flask(__name__)
+
+MIN_CONTOUR_AREA = float(os.environ.get('MIN_CONTOUR_AREA', '0.0001'))
+
+PREPROCESSOR_NAME = \
+    "ca.mcgill.a11y.image.preprocessor.objectSegmentation"
+
+DATA_SCHEMA = './schemas/preprocessors/segmentation.schema.json'
+
+try:
+    sam_client = SAMClient()
+    validator = Validator(data_schema=DATA_SCHEMA)
+    logging.debug("SAM client and validator initialized")
+except Exception as e:
+    logging.error(f"Failed to initialize clients: {e}")
+    sys.exit(1)
+
+
+def decode_image(source):
+    """
+    Decode a base64 "graphic" data URI into a PIL Image, without any
+    LLM-specific resizing. The upstream resize-graphic pseudo-preprocessor
+    already caps every incoming graphic to a fixed maximum dimension
+    (aspect-ratio preserved), so every preprocessor - including this one
+    and object-detection-llm - sees the same image and shares a common
+    pixel space.
+    """
+    image_b64 = source.split(",")[1] if "," in source else source
+    binary = base64.b64decode(image_b64)
+    return Image.open(BytesIO(binary)).convert("RGB")
+
+
+@app.route("/preprocessor", methods=['POST'])
+def segment_objects():
+    """
+    Main endpoint to segment objects found by object-detection-llm using
+    SAM, producing precise per-object polygon outlines.
+    """
+    logging.debug("Received request for object segmentation.")
+
+    content = request.get_json()
+
+    if "graphic" not in content:
+        logging.info("No graphic content. Skipping...")
+        return jsonify({"error": "No graphic content"}), 204
+
+    ok, _ = validator.check_request(content)
+    if not ok:
+        return jsonify({"error": "Invalid Preprocessor JSON format"}), 400
+
+    preprocess_output = content["preprocessors"]
+    categoriser = "ca.mcgill.a11y.image.preprocessor.contentCategoriser"
+    if categoriser in preprocess_output:
+        categoriser_output = preprocess_output[categoriser]
+        categoriser_tags = categoriser_output["categories"]
+        if not categoriser_tags["photo"] and not categoriser_tags["collage"] \
+           and not categoriser_tags["illustration"]:
+            logging.info("Not a photo, collage, or illustration. Skipping...")
+            return "", 204
+
+    # Deliberately read the LLM detector's key directly rather than the
+    # generic objectDetection coalesce helper: this preprocessor only
+    # makes sense on objects that came from object-detection-llm (SAM
+    # is being applied specifically to LLM-detected boxes), not
+    # whichever object detector happened to also be running.
+    llm_detections = preprocess_output.get(LLM_OBJECT_DETECTION_NAME)
+    if not llm_detections or not llm_detections.get("objects"):
+        logging.info("No object-detection-llm output. Skipping...")
+        return "", 204
+
+    objects = llm_detections["objects"]
+
+    request_uuid = content["request_uuid"]
+    timestamp = time.time()
+
+    try:
+        pil_image = decode_image(content["graphic"])
+
+        # Use the unique object ID (not the type string) as the SAM label:
+        # SAMClient.segment_with_boxes(aggregate_by_label=True) merges
+        # contours from boxes sharing the same label, so labelling by
+        # type would silently merge contours from multiple same-type
+        # objects (e.g. two different people) into a single entry.
+        boxes = [
+            {"bbox_2d": obj["dimensions"], "label": str(obj["ID"])}
+            for obj in objects
+        ]
+        id_to_type = {str(obj["ID"]): obj["type"] for obj in objects}
+
+        contours_by_id = sam_client.segment_with_boxes(
+            pil_image,
+            boxes,
+            use_prompts=False,
+            aggregate_by_label=True,
+            return_structured=False,
+            coord_scale=1.0,
+        )
+
+        segments = []
+        for object_id, contours in contours_by_id.items():
+            filtered = filter_contours_by_area(
+                contours, min_area=MIN_CONTOUR_AREA
+            )
+            if not filtered:
+                continue
+            segment = create_segment_from_contours(
+                filtered, name=id_to_type[object_id]
+            )
+            segment["objectID"] = int(object_id)
+            segments.append(segment)
+
+        data = {"segments": segments}
+
+        ok, _ = validator.check_data(data)
+        if not ok:
+            return jsonify("Invalid Preprocessor JSON format"), 500
+
+        response = {
+            "request_uuid": request_uuid,
+            "timestamp": int(timestamp),
+            "name": PREPROCESSOR_NAME,
+            "data": data
+        }
+
+        ok, _ = validator.check_response(response)
+        if not ok:
+            return jsonify("Invalid Preprocessor JSON format"), 500
+
+        logging.info(
+            f"Successfully segmented {len(segments)} objects "
+            f"for request {request_uuid}."
+        )
+
+        return jsonify(response), 200
+
+    except Exception as e:
+        logging.error(
+            f"An unexpected error occurred during object segmentation "
+            f"for {request_uuid}: {e}", exc_info=True
+        )
+        return jsonify(
+            {"error": "An unexpected internal server error occurred"}
+        ), 500
+
+
+@app.route("/health", methods=["GET"])
+def health():
+    """
+    Health check endpoint to verify if the service is running
+    """
+    return jsonify({
+        "status": "healthy",
+        "timestamp": datetime.now().isoformat()
+    }), 200
+
+
+@app.route("/warmup", methods=["GET"])
+def warmup():
+    """
+    Warms up the SAM model by running a dummy inference.
+    """
+    try:
+        logging.info("Warming up SAM...")
+
+        sam_success = sam_client.warmup()
+
+        if not sam_success:
+            logging.error("SAM warmup failed.")
+
+        return jsonify({"status": "ok"}), 200
+
+    except Exception as e:
+        logging.error(f"Warmup failed: {str(e)}")
+        return jsonify({"status": "error", "message": str(e)}), 500
+
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0', port=5000, debug=True)
diff --git a/preprocessors/object-segmentation/requirements.txt b/preprocessors/object-segmentation/requirements.txt
new file mode 100644
index 000000000..de00a735e
--- /dev/null
+++ b/preprocessors/object-segmentation/requirements.txt
@@ -0,0 +1,6 @@
+Flask==3.1.3
+jsonschema==4.23.0
+gunicorn==23.0.0
+opencv-python==4.11.0.86
+ultralytics==8.3.99
+pillow==12.1.1
diff --git a/preprocessors/ocr/ocr.py b/preprocessors/ocr/ocr.py
index 4b4f9e55d..e3e2aed26 100644
--- a/preprocessors/ocr/ocr.py
+++ b/preprocessors/ocr/ocr.py
@@ -32,6 +32,11 @@
 )
 from config.logging_utils import configure_logging
 from utils.validation import Validator
+from utils.object_detection import (
+    GENERIC_OBJECT_DETECTION_NAME,
+    LLM_OBJECT_DETECTION_NAME,
+    get_object_detection_data,
+)
 
 configure_logging()
 
@@ -71,10 +76,13 @@ def get_ocr_text():
     if ocr_result is None:
         return jsonify("Could not retreive Azure results"), 500
 
-    od = 'ca.mcgill.a11y.image.preprocessor.objectDetection'
     preprocessors = content['preprocessors']
-    if od in preprocessors and len(preprocessors[od]['objects']) > 0:
-        ocr_result = find_obj_enclosing(od, preprocessors[od], ocr_result)
+    od_data = get_object_detection_data(preprocessors)
+    if od_data is not None and len(od_data['objects']) > 0:
+        od = (GENERIC_OBJECT_DETECTION_NAME
+              if GENERIC_OBJECT_DETECTION_NAME in preprocessors
+              else LLM_OBJECT_DETECTION_NAME)
+        ocr_result = find_obj_enclosing(od, od_data, ocr_result)
 
     name = 'ca.mcgill.a11y.image.preprocessor.ocrClouds'
     request_uuid = content['request_uuid']
diff --git a/preprocessors/sorting/sorting.py b/preprocessors/sorting/sorting.py
index 4b63509f3..a4972e172 100644
--- a/preprocessors/sorting/sorting.py
+++ b/preprocessors/sorting/sorting.py
@@ -21,6 +21,7 @@
 from datetime import datetime
 from config.logging_utils import configure_logging
 from utils.validation import Validator
+from utils.object_detection import get_object_detection_data
 
 configure_logging()
 
@@ -62,14 +63,12 @@ def readImage():
         return jsonify("Invalid Preprocessor JSON format"), 400
 
     preprocessor = content["preprocessors"]
-    if "ca.mcgill.a11y.image.preprocessor.objectDetection" \
-            not in preprocessor:
+    oDpreprocessor = get_object_detection_data(preprocessor)
+    if oDpreprocessor is None:
         logging.info("Object detection output not "
                      "available. Skipping...")
         return "", 204
 
-    oDpreprocessor = \
-        preprocessor["ca.mcgill.a11y.image.preprocessor.objectDetection"]
     objects = oDpreprocessor["objects"]
     for i in range(len(objects)):
         object_type.append(objects[i]["type"])
diff --git a/utils/object_detection/__init__.py b/utils/object_detection/__init__.py
new file mode 100644
index 000000000..36bdf25ba
--- /dev/null
+++ b/utils/object_detection/__init__.py
@@ -0,0 +1,26 @@
+# Copyright (c) 2025 IMAGE Project, Shared Reality Lab, McGill University
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+"""
+Shared helper for reading object-detection preprocessor output.
+"""
+
+GENERIC_OBJECT_DETECTION_NAME = \
+    "ca.mcgill.a11y.image.preprocessor.objectDetection"
+LLM_OBJECT_DETECTION_NAME = \
+    "ca.mcgill.a11y.image.preprocessor.objectDetectionLLM"
+
+
+def get_object_detection_data(preprocessors):
+    """Prefer the generic (YOLO/Azure) key; fall back to the LLM key."""
+    return preprocessors.get(GENERIC_OBJECT_DETECTION_NAME) \
+        or preprocessors.get(LLM_OBJECT_DETECTION_NAME)
diff --git a/utils/segmentation/README.md b/utils/segmentation/README.md
index 90e0a35d7..6aa27c01a 100644
--- a/utils/segmentation/README.md
+++ b/utils/segmentation/README.md
@@ -93,7 +93,7 @@ Initialize the SAM client using environment variables.
 - Uses `SAM_MODEL_PATH` environment variable for model path
 - Raises `ValueError` if environment variable is not set
 
-##### `segment_with_boxes(image, bounding_boxes, use_prompts=False, aggregate_by_label=True, return_structured=False, base_data=None)`
+##### `segment_with_boxes(image, bounding_boxes, use_prompts=False, aggregate_by_label=True, return_structured=False, base_data=None, coord_scale=1000.0)`
 Segment image using bounding boxes.
 - `image`: PIL Image object
 - `bounding_boxes`: List of dicts with 'bbox_2d' and 'label' keys
@@ -101,6 +101,7 @@ Segment image using bounding boxes.
 - `aggregate_by_label`: Group contours by label
 - `return_structured`: Return data in schema-compatible format
 - `base_data`: Base data structure to update (required if return_structured=True)
+- `coord_scale`: Divisor used to convert `bbox_2d` coordinates to pixel space. Defaults to `1000.0` for LLM-native 0-1000 grids (e.g. Qwen's raw output). Pass `1.0` if `bbox_2d` values are already normalized to 0-1 (e.g. the shared `object-detection.schema.json` format).
 
 Returns:
 - If `return_structured=False`: Dictionary mapping labels to lists of normalized contours
@@ -152,7 +153,7 @@ The segmentation utilities expect bounding boxes in the following format:
 ```
 
 Where:
-- `bbox_2d`: Bounding box coordinates in pixels [left, top, right, bottom]
+- `bbox_2d`: Bounding box coordinates `[left, top, right, bottom]`. By default these are interpreted as an LLM-native 0-1000 grid (e.g. Qwen's raw output) and divided by `coord_scale` (default `1000.0`) to get pixel coordinates. Pass `coord_scale=1.0` if your coordinates are already normalized to 0-1, as with the shared `object-detection.schema.json` format used by `object-detection-llm`/`yolo`/`azure`.
 - `label`: String identifier for the object (used as text prompt when `use_prompts=True`)
 
 ## Output Format
diff --git a/utils/segmentation/sam_processor.py b/utils/segmentation/sam_processor.py
index 4d2eca2dc..5f1554795 100644
--- a/utils/segmentation/sam_processor.py
+++ b/utils/segmentation/sam_processor.py
@@ -68,7 +68,8 @@ def segment_with_boxes(
         use_prompts: bool = False,
         aggregate_by_label: bool = True,
         return_structured: bool = False,
-        base_data: Optional[Dict[str, Any]] = None
+        base_data: Optional[Dict[str, Any]] = None,
+        coord_scale: float = 1000.0
     ) -> Dict[str, Any]:
         """
         Segment image regions using bounding boxes.
@@ -83,6 +84,13 @@ def segment_with_boxes(
                              update_data_with_contours (for schema validation)
             base_data: Base data structure to update
                     (required if return_structured=True)
+            coord_scale: Divisor to convert incoming bbox coordinates to
+                    pixel space. Use 1000.0 (default) for LLM-native
+                    0-1000 grids (e.g. Qwen's raw output, the existing
+                    multistage-diagram-segmentation caller). Use 1.0 if
+                    bounding_boxes are already normalized to 0-1 (e.g.
+                    the shared object-detection.schema.json format used
+                    by object-detection-llm/yolo/azure).
 
         Returns:
             If return_structured=False: Dictionary mapping labels to lists
@@ -141,13 +149,14 @@ def segment_with_boxes(
                 f"(normalized coords: {bbox})"
             )
 
-            # Convert normalized coordinates (0-1000) received from Qwen 3
-            # to pixel coordinates
+            # Convert normalized coordinates to pixel coordinates using
+            # coord_scale (1000.0 for LLM-native grids, 1.0 if already
+            # normalized to 0-1)
             bbox_pixels = [
-                (bbox[0] / 1000.0) * width,
-                (bbox[1] / 1000.0) * height,
-                (bbox[2] / 1000.0) * width,
-                (bbox[3] / 1000.0) * height
+                (bbox[0] / coord_scale) * width,
+                (bbox[1] / coord_scale) * height,
+                (bbox[2] / coord_scale) * width,
+                (bbox[3] / coord_scale) * height
             ]
 
             logging.pii(

From 5373de67e73ef354b99949d3b317b00ecc82143b Mon Sep 17 00:00:00 2001
From: Miliya-Ai <102935805+Miliya-Ai@users.noreply.github.com>
Date: Thu, 27 Aug 2026 18:52:47 -0400
Subject: [PATCH 2/9] Fix CI branch trigger and bump schemas for objectID

- object-segmentation.yml: push trigger listed the wrong branch name
  (object-segmentation instead of sam-segmentation), so pushes to this
  branch wouldn't build a test image.
- Bump schemas submodule to add the objectID field the new
  object-segmentation preprocessor already emits.
---
 .github/workflows/object-segmentation.yml | 2 +-
 schemas                                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/object-segmentation.yml b/.github/workflows/object-segmentation.yml
index 790d348a0..8ca8445d2 100644
--- a/.github/workflows/object-segmentation.yml
+++ b/.github/workflows/object-segmentation.yml
@@ -1,7 +1,7 @@
 name: Object Segmentation
 on:
   push:
-    branches: [ main, object-segmentation ]
+    branches: [ main, sam-segmentation ]
     tags: [ "preprocessor-object-segmentation-[0-9]+.[0-9]+.[0-9]+" ]
     paths: [ "preprocessors/object-segmentation/**" ]
   pull_request:
diff --git a/schemas b/schemas
index 2945b52da..ec0db2a79 160000
--- a/schemas
+++ b/schemas
@@ -1 +1 @@
-Subproject commit 2945b52da77bf74b1307e7e2286c6297ebef6157
+Subproject commit ec0db2a797fd9f059ac3cbd1d21ca3ef2bc57fed

From fdfc877f0d2bc9d281c6e90cbf4826f324b1f86d Mon Sep 17 00:00:00 2001
From: Miliya-Ai <102935805+Miliya-Ai@users.noreply.github.com>
Date: Thu, 27 Aug 2026 19:02:25 -0400
Subject: [PATCH 3/9] Make object-segmentation GPU opt-in per environment

Base docker-compose.yml now runs object-segmentation CPU-only, so
unicorn stays CPU-only by default. prod-docker-compose.yml adds the
nvidia device reservation for pegasus, which has GPU to spare.

Verified with `docker compose -f docker-compose.yml config` (no GPU
block) and `-f docker-compose.yml -f prod-docker-compose.yml config`
(GPU block present).
---
 docker-compose.yml      |  8 ++------
 prod-docker-compose.yml | 10 ++++++++++
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 14ef72fc9..48bb7771a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -203,12 +203,8 @@ services:
       ca.mcgill.a11y.image.cacheTimeout: 3600
       ca.mcgill.a11y.image.required_dependencies: "object-detection-llm"
       ca.mcgill.a11y.image.optional_dependencies: ""
-    deploy:
-      resources:
-        reservations:
-          devices:
-          - driver: nvidia
-            capabilities: ["gpu", "utility", "compute"]
+    # CPU-only by default (e.g. unicorn); pegasus opts into GPU via
+    # prod-docker-compose.yml.
     environment:
       - PII_LOGGING_ENABLED=${PII_LOGGING_ENABLED}
       - SAM_MODEL_PATH=/usr/src/app/models/sam2.1_l.pt
diff --git a/prod-docker-compose.yml b/prod-docker-compose.yml
index 716e9b185..de33fac01 100644
--- a/prod-docker-compose.yml
+++ b/prod-docker-compose.yml
@@ -70,3 +70,13 @@ services:
     environment:
       # temporary: backup server in place for NFB
       - NOMINATIM_FALLBACK_SERVER=https://nominatim.openstreetmap.org
+
+  # pegasus has GPU capacity to spare for this; unicorn does not, so it
+  # stays on the CPU-only default from docker-compose.yml.
+  object-segmentation:
+    deploy:
+      resources:
+        reservations:
+          devices:
+          - driver: nvidia
+            capabilities: ["gpu", "utility", "compute"]

From aa825909b5e73cee67280618bf2ca892c95736bf Mon Sep 17 00:00:00 2001
From: Miliya-Ai <102935805+Miliya-Ai@users.noreply.github.com>
Date: Tue, 1 Sep 2026 14:52:47 -0400
Subject: [PATCH 4/9] Switch object-segmentation from SAM 2.1 to SAM 3

sam3.pt is gated on Hugging Face (facebook/sam3), so the model can't be
mirrored to our own server and wget'd like sam2.1 was. Instead the CI
build now pulls it directly at build time using an HF_TOKEN build
secret (BuildKit --secret, not ARG, so the token never lands in the
image's layer history) from an account with approved access.

- ultralytics bumped to 8.4.137 (SAM 3 support landed in 8.3.237)
- Dockerfile downloads sam3.pt via huggingface-cli instead of wget
- object-segmentation.yml passes HF_TOKEN through to the build step
- SAM_MODEL_PATH default updated; multistage-diagram-segmentation is
  untouched and stays on SAM 2.1, out of scope for this change

Requires an HF_TOKEN repository secret to be added on GitHub before
this will build.
---
 .github/workflows/object-segmentation.yml        |  2 ++
 docker-compose.yml                               |  2 +-
 preprocessors/object-segmentation/Dockerfile     | 16 ++++++++++++----
 preprocessors/object-segmentation/README.md      |  4 +++-
 .../object-segmentation/requirements.txt         |  2 +-
 5 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/object-segmentation.yml b/.github/workflows/object-segmentation.yml
index 8ca8445d2..ab3ea88a5 100644
--- a/.github/workflows/object-segmentation.yml
+++ b/.github/workflows/object-segmentation.yml
@@ -81,3 +81,5 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          secrets: |
+            HF_TOKEN=${{ secrets.HF_TOKEN }}
diff --git a/docker-compose.yml b/docker-compose.yml
index 48bb7771a..a4cbab478 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -207,7 +207,7 @@ services:
     # prod-docker-compose.yml.
     environment:
       - PII_LOGGING_ENABLED=${PII_LOGGING_ENABLED}
-      - SAM_MODEL_PATH=/usr/src/app/models/sam2.1_l.pt
+      - SAM_MODEL_PATH=/usr/src/app/models/sam3.pt
       - MIN_CONTOUR_AREA=0.0001
       - WARMUP_ENABLED=true
 
diff --git a/preprocessors/object-segmentation/Dockerfile b/preprocessors/object-segmentation/Dockerfile
index 05f334247..bc4a1d5e4 100644
--- a/preprocessors/object-segmentation/Dockerfile
+++ b/preprocessors/object-segmentation/Dockerfile
@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
 FROM ultralytics/ultralytics:8.3.119-python
 
 # Set the working directory inside the container
@@ -9,10 +10,10 @@ RUN adduser --disabled-password python
 # Update PATH to include local bin for the python user
 ENV PATH="/usr/src/app/.local/bin:${PATH}"
 
-# Install system dependencies required for healthcheck (curl) and model downloads (wget)
+# Install system dependencies required for healthcheck
 RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-    curl wget && \
+    curl && \
     rm -rf /var/lib/apt/lists/*
 
 # Copy the requirements file into the container
@@ -30,8 +31,15 @@ COPY /utils /usr/src/app/utils
 # Create model directory
 RUN mkdir -p /usr/src/app/models
 
-# Download SAM 2.1 Large model
-RUN wget -O /usr/src/app/models/sam2.1_l.pt https://image.a11y.mcgill.ca/models/semanticSegmentation/sam2.1_l.pt
+# Download SAM 3 (gated on Hugging Face: facebook/sam3). Requires an
+# HF_TOKEN build secret from an account with approved access, passed via
+# BuildKit's --secret (never a plain ARG, which would bake the token into
+# the image's layer history).
+RUN --mount=type=secret,id=HF_TOKEN \
+    pip3 install --no-cache-dir huggingface_hub && \
+    huggingface-cli download facebook/sam3 sam3.pt \
+        --local-dir /usr/src/app/models \
+        --token "$(cat /run/secrets/HF_TOKEN)"
 
 # Copy the preprocessor application code
 COPY /preprocessors/object-segmentation /usr/src/app
diff --git a/preprocessors/object-segmentation/README.md b/preprocessors/object-segmentation/README.md
index 11032f068..71e832785 100644
--- a/preprocessors/object-segmentation/README.md
+++ b/preprocessors/object-segmentation/README.md
@@ -2,7 +2,9 @@
 
 Alpha quality: not yet ready for use by end-users.
 
-This preprocessor segments objects already found by `object-detection-llm` using SAM (Segment Anything Model), producing precise per-object polygon outlines rather than just bounding boxes. It does not run its own LLM inference - it consumes `object-detection-llm`'s existing bounding boxes as SAM prompts.
+This preprocessor segments objects already found by `object-detection-llm` using SAM 3 (Segment Anything Model), producing precise per-object polygon outlines rather than just bounding boxes. It does not run its own LLM inference - it consumes `object-detection-llm`'s existing bounding boxes as SAM prompts.
+
+SAM 3's weights (`facebook/sam3` on Hugging Face) are gated: building this image requires an `HF_TOKEN` build secret from an account with approved access, passed via BuildKit's `--secret` (see the `Dockerfile` and `.github/workflows/object-segmentation.yml`). Never bake the token into an `ARG` - that would persist it in the image's layer history.
 
 Output is shaped identically to the existing `semanticSegmentation` preprocessor's `segments` output (`schemas/preprocessors/segmentation.schema.json`), with an additional `objectID` field on each segment that ties it back to the detected object it came from.
 
diff --git a/preprocessors/object-segmentation/requirements.txt b/preprocessors/object-segmentation/requirements.txt
index de00a735e..21a7bde1b 100644
--- a/preprocessors/object-segmentation/requirements.txt
+++ b/preprocessors/object-segmentation/requirements.txt
@@ -2,5 +2,5 @@ Flask==3.1.3
 jsonschema==4.23.0
 gunicorn==23.0.0
 opencv-python==4.11.0.86
-ultralytics==8.3.99
+ultralytics==8.4.137
 pillow==12.1.1

From 8d5213d32abd37b7d60b36fc2771de830d07cce5 Mon Sep 17 00:00:00 2001
From: Miliya-Ai <102935805+Miliya-Ai@users.noreply.github.com>
Date: Tue, 1 Sep 2026 15:17:44 -0400
Subject: [PATCH 5/9] Fix Dockerfile: huggingface-cli is deprecated, use hf
 instead

Caught by a local docker build test: current huggingface_hub ships
`hf` as the CLI entry point and `huggingface-cli` now hard-fails
instead of just warning. Verified the `hf download` invocation reaches
HF's servers correctly with a throwaway token (got a clean "requires
approval" response, confirming syntax + secret mount + network path
all work).
---
 preprocessors/object-segmentation/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/preprocessors/object-segmentation/Dockerfile b/preprocessors/object-segmentation/Dockerfile
index bc4a1d5e4..e6fda1a4e 100644
--- a/preprocessors/object-segmentation/Dockerfile
+++ b/preprocessors/object-segmentation/Dockerfile
@@ -37,7 +37,7 @@ RUN mkdir -p /usr/src/app/models
 # the image's layer history).
 RUN --mount=type=secret,id=HF_TOKEN \
     pip3 install --no-cache-dir huggingface_hub && \
-    huggingface-cli download facebook/sam3 sam3.pt \
+    hf download facebook/sam3 sam3.pt \
         --local-dir /usr/src/app/models \
         --token "$(cat /run/secrets/HF_TOKEN)"
 

From 01c352060b4a6cae36352b0add8313dcb7a4476c Mon Sep 17 00:00:00 2001
From: Miliya-Ai <102935805+Miliya-Ai@users.noreply.github.com>
Date: Sat, 5 Sep 2026 20:56:07 -0400
Subject: [PATCH 6/9] SAM 3 requires the timm dependency in ultralytics

---
 preprocessors/object-segmentation/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/preprocessors/object-segmentation/requirements.txt b/preprocessors/object-segmentation/requirements.txt
index 21a7bde1b..dc1363388 100644
--- a/preprocessors/object-segmentation/requirements.txt
+++ b/preprocessors/object-segmentation/requirements.txt
@@ -4,3 +4,4 @@ gunicorn==23.0.0
 opencv-python==4.11.0.86
 ultralytics==8.4.137
 pillow==12.1.1
+timm==1.0.29

From 77e1f1eb2d6ee9c830b9f73a80ba6ec22776a487 Mon Sep 17 00:00:00 2001
From: Miliya-Ai <102935805+Miliya-Ai@users.noreply.github.com>
Date: Sat, 5 Sep 2026 22:44:24 -0400
Subject: [PATCH 7/9] mmsemseg only reports background elements

---
 docker-compose.yml                        |   2 +-
 handlers/photo-tactile-svg/README.md      |   4 +-
 handlers/photo-tactile-svg/tactile_svg.py | 114 +++++++++++++++-------
 preprocessors/mmsemseg/README.md          |   2 +-
 preprocessors/mmsemseg/segment.py         |  18 +++-
 5 files changed, 97 insertions(+), 43 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index a4cbab478..9b68a0faf 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -445,7 +445,7 @@ services:
     labels:
         ca.mcgill.a11y.image.handler: enable
         ca.mcgill.a11y.image.required_dependencies: "object-detection,object-grouping,semantic-segmentation,graphic-caption"
-        ca.mcgill.a11y.image.optional_dependencies: "object-detection-llm"
+        ca.mcgill.a11y.image.optional_dependencies: "object-detection-llm,object-segmentation"
     environment:
       - PII_LOGGING_ENABLED=${PII_LOGGING_ENABLED}
 
diff --git a/handlers/photo-tactile-svg/README.md b/handlers/photo-tactile-svg/README.md
index e8561fa1c..718e8654c 100644
--- a/handlers/photo-tactile-svg/README.md
+++ b/handlers/photo-tactile-svg/README.md
@@ -9,4 +9,6 @@ Alpha quality: Insufficiently refined to be tested by end-users.
 This is a [handler](https://github.com/Shared-Reality-Lab/IMAGE-server/wiki/2.-Handlers,-Preprocessors-and-Services#handlers=) component that creates a SVG that can be rendered as a tactile graphic to convey detected objects and detected semantic segments.
 
 Data from object detection and semantic segmentation are used to create a SVG.
-This SVG will be as per a specified [format](https://github.com/Shared-Reality-Lab/IMAGE-Monarch/tree/main#tactile-graphics) and might contain multiple layers with the number of layers equal to the number of object classes found within the photo. The semantic segments will be available independent of the layers.
\ No newline at end of file
+This SVG will be as per a specified [format](https://github.com/Shared-Reality-Lab/IMAGE-Monarch/tree/main#tactile-graphics) and might contain multiple layers with the number of layers equal to the number of object classes found within the photo. The semantic segments will be available independent of the layers.
+
+When `object-segmentation` (SAM 3) output is available for a detected object, its precise polygon outline is drawn in place of that object's plain bounding-box rectangle. Semantic segmentation is otherwise scoped to background/environmental elements (sky, walls, floors, etc.) that object detection doesn't cover.
\ No newline at end of file
diff --git a/handlers/photo-tactile-svg/tactile_svg.py b/handlers/photo-tactile-svg/tactile_svg.py
index 51b0d7323..4a9c3fb7c 100644
--- a/handlers/photo-tactile-svg/tactile_svg.py
+++ b/handlers/photo-tactile-svg/tactile_svg.py
@@ -31,6 +31,27 @@
 logging.basicConfig(level=logging.DEBUG)
 
 
+def object_contour_path(contours, dimensions, aria_label, **extra):
+    """Draw a SAM-precise polygon outline for a single object, in the
+    same style/coordinate space as the semantic segmentation contours
+    below, for use in place of a plain bounding-box rectangle."""
+    try:
+        p = draw.Path(stroke="#ff4477", stroke_width=2.5,
+                      fill='none', aria_label=aria_label, **extra)
+    except BaseException:
+        p = draw.Path(stroke="red", stroke_width=2.5,
+                      fill='none', aria_label=aria_label, **extra)
+    for c in contours:
+        coords = c["coordinates"]
+        for i in range(1, len(coords), 5):
+            if (i == 1):
+                p.M(coords[i][0] * dimensions[0],
+                    -coords[i][1] * dimensions[1])
+            p.L(coords[i][0] * dimensions[0],
+                -coords[i][1] * dimensions[1])
+    return p
+
+
 @app.route("/handler", methods=["POST"])
 def handle():
     logging.debug("Received request")
@@ -164,6 +185,15 @@ def handle():
         objects = o["objects"]
         grouped = g["grouped"]
         ungrouped = g["ungrouped"]
+        object_segmentation = preprocessors.get(
+            "ca.mcgill.a11y.image.preprocessor.objectSegmentation")
+        contours_by_object_id = {}
+        if object_segmentation:
+            contours_by_object_id = {
+                seg["objectID"]: seg["contours"]
+                for seg in object_segmentation["segments"]
+                if "objectID" in seg
+            }
         layer = 0
         # Loop through the object groups and generate a layer for each
         for group in grouped:
@@ -178,14 +208,54 @@ def handle():
             # Loop through the individual items
             # Draw a rectangle for each and tag objects
             for i, id in enumerate(ids):
-                x1 = objects[id]['dimensions'][0] * dimensions[0]
-                x2 = objects[id]['dimensions'][2] * dimensions[0]
-                y1 = objects[id]['dimensions'][1] * dimensions[1]
-                y2 = objects[id]['dimensions'][3] * dimensions[1]
+                label = obj_tag+" "+str(i+1)
+                if id in contours_by_object_id:
+                    g.append(object_contour_path(
+                        contours_by_object_id[id], dimensions, label))
+                else:
+                    x1 = objects[id]['dimensions'][0] * dimensions[0]
+                    x2 = objects[id]['dimensions'][2] * dimensions[0]
+                    y1 = objects[id]['dimensions'][1] * dimensions[1]
+                    y2 = objects[id]['dimensions'][3] * dimensions[1]
+                    width = abs(x2 - x1)
+                    height = abs(y2 - y1)
+                    start_y1 = -(y1 + height)
+                    g.append(
+                        draw.Rectangle(
+                            x1,
+                            start_y1,
+                            width,
+                            height,
+                            stroke="#ff4477",
+                            stroke_width=2.5,
+                            fill="none",
+                            aria_label=label))
+
+            svg.append(g)
+
+        # Loop through ungrouped objects and generate a layer for each
+        for val in ungrouped:
+            category = objects[val]["type"].strip()
+            # appending singular objects with appropriate article
+            obj_list.append(form.a(category))
+            layer += 1
+            if val in contours_by_object_id:
+                svg.append(object_contour_path(
+                    contours_by_object_id[val], dimensions, category,
+                    data_image_layer="Layer "+str(layer)))
+            else:
+                x1 = (objects[val]
+                      ['dimensions'][0] * dimensions[0])
+                x2 = (objects[val]
+                      ['dimensions'][2] * dimensions[0])
+                y1 = (objects[val]
+                      ['dimensions'][1] * dimensions[1])
+                y2 = (objects[val]
+                      ['dimensions'][3] * dimensions[1])
                 width = abs(x2 - x1)
                 height = abs(y2 - y1)
                 start_y1 = -(y1 + height)
-                g.append(
+                svg.append(
                     draw.Rectangle(
                         x1,
                         start_y1,
@@ -194,38 +264,8 @@ def handle():
                         stroke="#ff4477",
                         stroke_width=2.5,
                         fill="none",
-                        aria_label=obj_tag+" "+str(i+1)))
-
-            svg.append(g)
-
-        # Loop through ungrouped objects and generate a layer for each
-        for val in ungrouped:
-            category = objects[val]["type"].strip()
-            # appending singular objects with appropriate article
-            obj_list.append(form.a(category))
-            layer += 1
-            x1 = (objects[val]
-                  ['dimensions'][0] * dimensions[0])
-            x2 = (objects[val]
-                  ['dimensions'][2] * dimensions[0])
-            y1 = (objects[val]
-                  ['dimensions'][1] * dimensions[1])
-            y2 = (objects[val]
-                  ['dimensions'][3] * dimensions[1])
-            width = abs(x2 - x1)
-            height = abs(y2 - y1)
-            start_y1 = -(y1 + height)
-            svg.append(
-                draw.Rectangle(
-                    x1,
-                    start_y1,
-                    width,
-                    height,
-                    stroke="#ff4477",
-                    stroke_width=2.5,
-                    fill="none",
-                    aria_label=category,
-                    data_image_layer="Layer "+str(layer)))
+                        aria_label=category,
+                        data_image_layer="Layer "+str(layer)))
 
         if len(obj_list) > 1:
             obj_list[-1] = "and " + obj_list[-1] + "."
diff --git a/preprocessors/mmsemseg/README.md b/preprocessors/mmsemseg/README.md
index 1459356d3..b0dec79fd 100644
--- a/preprocessors/mmsemseg/README.md
+++ b/preprocessors/mmsemseg/README.md
@@ -2,7 +2,7 @@
 
 Beta quality: Useful enough for testing by end-users.
 
-This preprocessor is used for semantically segmenting images. The code for this preprocessor heavily relies on the [MMSegmentation](https://github.com/open-mmlab/mmsegmentation) framework.
+This preprocessor semantically segments images using the [MMSegmentation](https://github.com/open-mmlab/mmsegmentation) framework, but only returns "stuff"/background-type segments (e.g. sky, wall, floor, road) as classified by [ADE20K SceneParsing150](https://github.com/CSAILVision/sceneparsing/blob/master/objectInfo150.csv) - environmental elements that an object detector wouldn't catch. Foreground objects (people, furniture, vehicles, etc.) are segmented with far more precise outlines by `object-detection-llm` + `object-segmentation` (SAM 3) instead.
 
 The code to use this module as an API can be found in `segment.py`, additional functions are located in `utils.py`. This module is fully versionned, the versions of the libraries used can be found in `requirements.txt` and in the `Dockerfile`.
 
diff --git a/preprocessors/mmsemseg/segment.py b/preprocessors/mmsemseg/segment.py
index a089293a3..8c75f12e1 100644
--- a/preprocessors/mmsemseg/segment.py
+++ b/preprocessors/mmsemseg/segment.py
@@ -51,6 +51,13 @@
 COLORS = mmseg.core.evaluation.get_palette("ade20k")
 CLASS_NAMES = mmseg.core.evaluation.get_classes("ade20k")
 
+# this preprocessor is scoped down to only the
+# background elements they can't see, e.g. sky, walls, floors.
+STUFF_CLASS_IDS = frozenset({
+    0, 1, 2, 3, 5, 6, 9, 11, 13, 16, 21, 25, 26, 28, 29, 46, 48, 51, 52,
+    54, 59, 60, 61, 68, 79, 84, 91, 94, 96, 101, 105, 109, 113, 128, 140
+})
+
 app = Flask(__name__)
 
 
@@ -96,9 +103,14 @@ def run_segmentation(url, model, dictionary):
     # extracting contours
     pred = result[0].astype(np.int32)
     predicted_classes = np.bincount(pred.flatten()).argsort()[::-1]
-    logging.info("main classes detected : {}".format(predicted_classes[:5]))
-
-    for class_id in predicted_classes[:5]:
+    background_classes = [
+        c for c in predicted_classes if c in STUFF_CLASS_IDS
+    ][:5]
+    logging.info(
+        "main background classes detected : {}".format(background_classes)
+    )
+
+    for class_id in background_classes:
         logging.info("extracting contours for class: {}".format(str(class_id)))
 
         pred_color, class_name = visualize_result(pred, index=class_id)

From 6aedd3e482317452405bf494fb32cc0965cbf370 Mon Sep 17 00:00:00 2001
From: Miliya-Ai <102935805+Miliya-Ai@users.noreply.github.com>
Date: Sun, 6 Sep 2026 00:44:50 -0400
Subject: [PATCH 8/9] add object-segmentation to build.yml

---
 build.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/build.yml b/build.yml
index d5c6a4bbe..2ae1b0c8e 100644
--- a/build.yml
+++ b/build.yml
@@ -81,6 +81,11 @@ services:
             context: .
             dockerfile: ./preprocessors/object-detection-llm/Dockerfile
         image: "object-detection-llm:latest"
+    object-segmentation:
+        build:
+            context: .
+            dockerfile: ./preprocessors/object-segmentation/Dockerfile
+        image: "object-segmentation:latest"
     object-detection-azure:
         build:
             context: .

From 414ff4220fc355a1871e31e7864c222458ad3f4c Mon Sep 17 00:00:00 2001
From: Miliya-Ai <102935805+Miliya-Ai@users.noreply.github.com>
Date: Mon, 7 Sep 2026 20:14:08 -0400
Subject: [PATCH 9/9] add sam-segmentation to push-list for CI build

---
 .github/workflows/object-detection-llm.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/object-detection-llm.yml b/.github/workflows/object-detection-llm.yml
index a126c9d7d..36d52e557 100644
--- a/.github/workflows/object-detection-llm.yml
+++ b/.github/workflows/object-detection-llm.yml
@@ -1,7 +1,7 @@
 name: Object Detection with LLM
 on:
   push:
-    branches: [ main, object-detection-llm ]
+    branches: [ main, object-detection-llm, sam-segmentation ]
     tags: [ "preprocessor-object-detection-llm-[0-9]+.[0-9]+.[0-9]+" ]
     paths: [ "preprocessors/object-detection-llm/**" ]
   pull_request: