Replace mmsemseg with SAM 3 for object segmentation

.github/workflows/object-detection-llm.yml

@@ -1,7 +1,7 @@
 name: Object Detection with LLM
 on:
   push:
-    branches: [ main, object-detection-llm ]
+    branches: [ main, object-detection-llm, sam-segmentation ]
     tags: [ "preprocessor-object-detection-llm-[0-9]+.[0-9]+.[0-9]+" ]
     paths: [ "preprocessors/object-detection-llm/**" ]
   pull_request:

.github/workflows/object-segmentation.yml

@@ -0,0 +1,85 @@
+name: Object Segmentation
+on:
+  push:
+    branches: [ main, sam-segmentation ]
+    tags: [ "preprocessor-object-segmentation-[0-9]+.[0-9]+.[0-9]+" ]
+    paths: [ "preprocessors/object-segmentation/**" ]
+  pull_request:
+    branches: [ main ]
+    paths: [ "preprocessors/object-segmentation/**" ]
+  workflow_run:
+    workflows: [ "Schemas (Trigger)" ]
+    types:
+      - completed
+  workflow_dispatch:
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: shared-reality-lab/image-preprocessor-object-segmentation
+jobs:
+  lint:
+    name: PEP 8 style check.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: Install flake8
+        run: pip install flake8
+      - name: Check with flake8
+        run: python -m flake8 ./preprocessors/object-segmentation --show-source
+  build-and-push-image:
+    name: Build and Push to Registry
+    needs: lint
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Log into GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Get Correct Tags
+        run: |
+          if [[ ${{ github.ref }} =~ ^refs/tags/preprocessor-object-segmentation-[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "TAGGED=true" >> $GITHUB_ENV
+          else
+            echo "TAGGED=false" >> $GITHUB_ENV
+          fi
+      - name: Get timestamp
+        run: echo "timestamp=$(date -u +'%Y-%m-%dT%H.%M')" >> $GITHUB_ENV
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            latest=${{ env.TAGGED }}
+          tags: |
+            type=match,enable=${{ env.TAGGED }},priority=300,pattern=preprocessor-object-segmentation-(\d+.\d+.\d+),group=1
+            type=raw,priority=200,value=unstable
+            type=raw,priority=100,value=${{ env.timestamp }}
+          labels: |
+            org.opencontainers.image.title=IMAGE Preprocessor Object Segmentation
+            org.opencontainers.image.description=Segments objects found by object-detection-llm using SAM, producing precise polygon outlines per object.
+            org.opencontainers.image.authors=IMAGE Project <image@cim.mcgill.ca>
+            org.opencontainers.image.documentation=https://github.com/Shared-Reality-Lab/IMAGE-server/tree/main/preprocessors/object-segmentation/README.md
+            org.opencontainers.image.licenses=AGPL-3.0-or-later
+            maintainer=IMAGE Project <image@cim.mcgill.ca>
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./preprocessors/object-segmentation/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          secrets: |
+            HF_TOKEN=${{ secrets.HF_TOKEN }}

build.yml

@@ -81,6 +81,11 @@ services:
             context: .
             dockerfile: ./preprocessors/object-detection-llm/Dockerfile
         image: "object-detection-llm:latest"
+    object-segmentation:
+        build:
+            context: .
+            dockerfile: ./preprocessors/object-segmentation/Dockerfile
+        image: "object-segmentation:latest"
     object-detection-azure:
         build:
             context: .

docker-compose.yml

@@ -193,6 +193,24 @@ services:
     env_file:
       ./config/llm.env
 
+  object-segmentation:
+    profiles: [test, production, default]
+    image: ghcr.io/shared-reality-lab/image-preprocessor-object-segmentation:${REGISTRY_TAG}
+    restart: "no"
+    labels:
+      ca.mcgill.a11y.image.preprocessor: 4
+      ca.mcgill.a11y.image.port: 5000
+      ca.mcgill.a11y.image.cacheTimeout: 3600
+      ca.mcgill.a11y.image.required_dependencies: "object-detection-llm"
+      ca.mcgill.a11y.image.optional_dependencies: ""
+    # CPU-only by default (e.g. unicorn); pegasus opts into GPU via
+    # prod-docker-compose.yml.
+    environment:
+      - PII_LOGGING_ENABLED=${PII_LOGGING_ENABLED}
+      - SAM_MODEL_PATH=/usr/src/app/models/sam3.pt
+      - MIN_CONTOUR_AREA=0.0001
+      - WARMUP_ENABLED=true
+
   multistage-diagram-segmentation:
     profiles: [production, test, default]
     image: ghcr.io/shared-reality-lab/image-preprocessor-multistage-diagram-segmentation:${REGISTRY_TAG}
@@ -427,7 +445,7 @@ services:
     labels:
         ca.mcgill.a11y.image.handler: enable
         ca.mcgill.a11y.image.required_dependencies: "object-detection,object-grouping,semantic-segmentation,graphic-caption"
-        ca.mcgill.a11y.image.optional_dependencies: "object-detection-llm"
+        ca.mcgill.a11y.image.optional_dependencies: "object-detection-llm,object-segmentation"
     environment:
       - PII_LOGGING_ENABLED=${PII_LOGGING_ENABLED}
 

handlers/ocr-handler/Dockerfile

@@ -18,6 +18,8 @@ COPY /schemas /app/schemas
 
 COPY /config /app/config
 
+COPY /utils /app/utils
+
 COPY /handlers/ocr-handler/ /app
 
 EXPOSE 80

handlers/ocr-handler/server.py

@@ -22,6 +22,11 @@
 from flask import Flask, request, jsonify
 from datetime import datetime
 from config.logging_utils import configure_logging
+from utils.object_detection import (
+    GENERIC_OBJECT_DETECTION_NAME,
+    LLM_OBJECT_DETECTION_NAME,
+    get_object_detection_data,
+)
 
 configure_logging()
 
@@ -132,9 +137,12 @@ def render_ocr():
     text = ""
 
     # Object detection data is present
-    od = 'ca.mcgill.a11y.image.preprocessor.objectDetection'
-    if od in preprocessors and len(preprocessors[od]['objects']) > 0:
-        object_data = preprocessors[od]['objects']
+    od_data = get_object_detection_data(preprocessors)
+    if od_data is not None and len(od_data['objects']) > 0:
+        od = (GENERIC_OBJECT_DETECTION_NAME
+              if GENERIC_OBJECT_DETECTION_NAME in preprocessors
+              else LLM_OBJECT_DETECTION_NAME)
+        object_data = od_data['objects']
         text_lines = ocr_data['lines']
         text += "The following objects were detected: "
         done_once = False

handlers/photo-audio-handler/src/server.ts

@@ -61,7 +61,7 @@ app.post("/handler", async (req, res) => {
     const preprocessors = req.body["preprocessors"];
     const secondCat = preprocessors["ca.mcgill.a11y.image.preprocessor.graphicTagger"];
     const semseg = preprocessors["ca.mcgill.a11y.image.preprocessor.semanticSegmentation"];
-    const objDet = preprocessors["ca.mcgill.a11y.image.preprocessor.objectDetection"];
+    const objDet = utils.getObjectDetectionData(preprocessors);
     const objGroup = preprocessors["ca.mcgill.a11y.image.preprocessor.grouping"];
     const action = preprocessors["ca.mcgill.a11y.image.preprocessor.actionRecognition"];
     //const collageDetector = preprocessors["ca.mcgill.a11y.image.preprocessor.collageDetector"];

handlers/photo-audio-handler/src/utils.ts

@@ -39,6 +39,13 @@ type ObjDet = {
     objects: Obj[];
 };
 
+const GENERIC_OBJECT_DETECTION_NAME = "ca.mcgill.a11y.image.preprocessor.objectDetection";
+const LLM_OBJECT_DETECTION_NAME = "ca.mcgill.a11y.image.preprocessor.objectDetectionLLM";
+
+export function getObjectDetectionData(preprocessors: Record<string, ObjDet>): ObjDet | undefined {
+    return preprocessors[GENERIC_OBJECT_DETECTION_NAME] ?? preprocessors[LLM_OBJECT_DETECTION_NAME];
+}
+
 type ObjGroup = {
     grouped: { IDs: number[] }[];
     ungrouped: number[];

handlers/photo-tactile-svg/Dockerfile

@@ -14,6 +14,7 @@ RUN pip install -r requirements.txt
 
 COPY /schemas /usr/src/app/schemas
 COPY /config /usr/src/app/config
+COPY /utils /usr/src/app/utils
 COPY /handlers/photo-tactile-svg/ /usr/src/app
 
 EXPOSE 80

handlers/photo-tactile-svg/README.md

@@ -9,4 +9,6 @@ Alpha quality: Insufficiently refined to be tested by end-users.
 This is a [handler](https://github.com/Shared-Reality-Lab/IMAGE-server/wiki/2.-Handlers,-Preprocessors-and-Services#handlers=) component that creates a SVG that can be rendered as a tactile graphic to convey detected objects and detected semantic segments.
 
 Data from object detection and semantic segmentation are used to create a SVG.
-This SVG will be as per a specified [format](https://github.com/Shared-Reality-Lab/IMAGE-Monarch/tree/main#tactile-graphics) and might contain multiple layers with the number of layers equal to the number of object classes found within the photo. The semantic segments will be available independent of the layers.
+This SVG will be as per a specified [format](https://github.com/Shared-Reality-Lab/IMAGE-Monarch/tree/main#tactile-graphics) and might contain multiple layers with the number of layers equal to the number of object classes found within the photo. The semantic segments will be available independent of the layers.
+
+When `object-segmentation` (SAM 3) output is available for a detected object, its precise polygon outline is drawn in place of that object's plain bounding-box rectangle. Semantic segmentation is otherwise scoped to background/environmental elements (sky, walls, floors, etc.) that object detection doesn't cover.

handlers/photo-tactile-svg/tactile_svg.py

@@ -24,12 +24,34 @@
 import inflect
 from config.logging_utils import configure_logging
 from datetime import datetime
+from utils.object_detection import get_object_detection_data
 
 configure_logging()
 app = Flask(__name__)
 logging.basicConfig(level=logging.DEBUG)
 
 
+def object_contour_path(contours, dimensions, aria_label, **extra):
+    """Draw a SAM-precise polygon outline for a single object, in the
+    same style/coordinate space as the semantic segmentation contours
+    below, for use in place of a plain bounding-box rectangle."""
+    try:
+        p = draw.Path(stroke="#ff4477", stroke_width=2.5,
+                      fill='none', aria_label=aria_label, **extra)
+    except BaseException:
+        p = draw.Path(stroke="red", stroke_width=2.5,
+                      fill='none', aria_label=aria_label, **extra)
+    for c in contours:
+        coords = c["coordinates"]
+        for i in range(1, len(coords), 5):
+            if (i == 1):
+                p.M(coords[i][0] * dimensions[0],
+                    -coords[i][1] * dimensions[1])
+            p.L(coords[i][0] * dimensions[0],
+                -coords[i][1] * dimensions[1])
+    return p
+
+
 @app.route("/handler", methods=["POST"])
 def handle():
     logging.debug("Received request")
@@ -99,9 +121,8 @@ def handle():
                   "and/ or semantic segmentation responses")
     if not (("ca.mcgill.a11y.image.preprocessor.semanticSegmentation"
              in preprocessors) or
-            all(x in preprocessors for x in
-                ["ca.mcgill.a11y.image.preprocessor.objectDetection",
-                 "ca.mcgill.a11y.image.preprocessor.grouping"])):
+            (get_object_detection_data(preprocessors) is not None and
+             "ca.mcgill.a11y.image.preprocessor.grouping" in preprocessors)):
         logging.debug("No Object Detector and Semantic Segmentation found")
         response = {
             "request_uuid": contents["request_uuid"],
@@ -152,21 +173,27 @@ def handle():
     form = inflect.engine()
     caption = ""
 
-    if "ca.mcgill.a11y.image.preprocessor.objectDetection"\
-        in preprocessors\
+    if get_object_detection_data(preprocessors) is not None\
             and "ca.mcgill.a11y.image.preprocessor.grouping" in preprocessors:
         logging.debug("Object detector and grouping preprocessor found. "
                       "Adding data to response...")
         caption = "This photo contains "
         obj_list = []
         preprocessor_names.append('Things and people')
-        o = preprocessors[
-            "ca.mcgill.a11y.image.preprocessor.objectDetection"
-            ]
+        o = get_object_detection_data(preprocessors)
         g = preprocessors["ca.mcgill.a11y.image.preprocessor.grouping"]
         objects = o["objects"]
         grouped = g["grouped"]
         ungrouped = g["ungrouped"]
+        object_segmentation = preprocessors.get(
+            "ca.mcgill.a11y.image.preprocessor.objectSegmentation")
+        contours_by_object_id = {}
+        if object_segmentation:
+            contours_by_object_id = {
+                seg["objectID"]: seg["contours"]
+                for seg in object_segmentation["segments"]
+                if "objectID" in seg
+            }
         layer = 0
         # Loop through the object groups and generate a layer for each
         for group in grouped:
@@ -181,14 +208,54 @@ def handle():
             # Loop through the individual items
             # Draw a rectangle for each and tag objects
             for i, id in enumerate(ids):
-                x1 = objects[id]['dimensions'][0] * dimensions[0]
-                x2 = objects[id]['dimensions'][2] * dimensions[0]
-                y1 = objects[id]['dimensions'][1] * dimensions[1]
-                y2 = objects[id]['dimensions'][3] * dimensions[1]
+                label = obj_tag+" "+str(i+1)
+                if id in contours_by_object_id:
+                    g.append(object_contour_path(
+                        contours_by_object_id[id], dimensions, label))
+                else:
+                    x1 = objects[id]['dimensions'][0] * dimensions[0]
+                    x2 = objects[id]['dimensions'][2] * dimensions[0]
+                    y1 = objects[id]['dimensions'][1] * dimensions[1]
+                    y2 = objects[id]['dimensions'][3] * dimensions[1]
+                    width = abs(x2 - x1)
+                    height = abs(y2 - y1)
+                    start_y1 = -(y1 + height)
+                    g.append(
+                        draw.Rectangle(
+                            x1,
+                            start_y1,
+                            width,
+                            height,
+                            stroke="#ff4477",
+                            stroke_width=2.5,
+                            fill="none",
+                            aria_label=label))
+
+            svg.append(g)
+
+        # Loop through ungrouped objects and generate a layer for each
+        for val in ungrouped:
+            category = objects[val]["type"].strip()
+            # appending singular objects with appropriate article
+            obj_list.append(form.a(category))
+            layer += 1
+            if val in contours_by_object_id:
+                svg.append(object_contour_path(
+                    contours_by_object_id[val], dimensions, category,
+                    data_image_layer="Layer "+str(layer)))
+            else:
+                x1 = (objects[val]
+                      ['dimensions'][0] * dimensions[0])
+                x2 = (objects[val]
+                      ['dimensions'][2] * dimensions[0])
+                y1 = (objects[val]
+                      ['dimensions'][1] * dimensions[1])
+                y2 = (objects[val]
+                      ['dimensions'][3] * dimensions[1])
                 width = abs(x2 - x1)
                 height = abs(y2 - y1)
                 start_y1 = -(y1 + height)
-                g.append(
+                svg.append(
                     draw.Rectangle(
                         x1,
                         start_y1,
@@ -197,38 +264,8 @@ def handle():
                         stroke="#ff4477",
                         stroke_width=2.5,
                         fill="none",
-                        aria_label=obj_tag+" "+str(i+1)))
-
-            svg.append(g)
-
-        # Loop through ungrouped objects and generate a layer for each
-        for val in ungrouped:
-            category = objects[val]["type"].strip()
-            # appending singular objects with appropriate article
-            obj_list.append(form.a(category))
-            layer += 1
-            x1 = (objects[val]
-                  ['dimensions'][0] * dimensions[0])
-            x2 = (objects[val]
-                  ['dimensions'][2] * dimensions[0])
-            y1 = (objects[val]
-                  ['dimensions'][1] * dimensions[1])
-            y2 = (objects[val]
-                  ['dimensions'][3] * dimensions[1])
-            width = abs(x2 - x1)
-            height = abs(y2 - y1)
-            start_y1 = -(y1 + height)
-            svg.append(
-                draw.Rectangle(
-                    x1,
-                    start_y1,
-                    width,
-                    height,
-                    stroke="#ff4477",
-                    stroke_width=2.5,
-                    fill="none",
-                    aria_label=category,
-                    data_image_layer="Layer "+str(layer)))
+                        aria_label=category,
+                        data_image_layer="Layer "+str(layer)))
 
         if len(obj_list) > 1:
             obj_list[-1] = "and " + obj_list[-1] + "."