diff --git a/.github/workflows/object-detection-llm.yml b/.github/workflows/object-detection-llm.yml index a126c9d7..36d52e55 100644 --- a/.github/workflows/object-detection-llm.yml +++ b/.github/workflows/object-detection-llm.yml @@ -1,7 +1,7 @@ name: Object Detection with LLM on: push: - branches: [ main, object-detection-llm ] + branches: [ main, object-detection-llm, sam-segmentation ] tags: [ "preprocessor-object-detection-llm-[0-9]+.[0-9]+.[0-9]+" ] paths: [ "preprocessors/object-detection-llm/**" ] pull_request: diff --git a/.github/workflows/object-segmentation.yml b/.github/workflows/object-segmentation.yml new file mode 100644 index 00000000..ab3ea88a --- /dev/null +++ b/.github/workflows/object-segmentation.yml @@ -0,0 +1,85 @@ +name: Object Segmentation +on: + push: + branches: [ main, sam-segmentation ] + tags: [ "preprocessor-object-segmentation-[0-9]+.[0-9]+.[0-9]+" ] + paths: [ "preprocessors/object-segmentation/**" ] + pull_request: + branches: [ main ] + paths: [ "preprocessors/object-segmentation/**" ] + workflow_run: + workflows: [ "Schemas (Trigger)" ] + types: + - completed + workflow_dispatch: +env: + REGISTRY: ghcr.io + IMAGE_NAME: shared-reality-lab/image-preprocessor-object-segmentation +jobs: + lint: + name: PEP 8 style check. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install flake8 + run: pip install flake8 + - name: Check with flake8 + run: python -m flake8 ./preprocessors/object-segmentation --show-source + build-and-push-image: + name: Build and Push to Registry + needs: lint + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: true + - name: Log into GHCR + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Get Correct Tags + run: | + if [[ ${{ github.ref }} =~ ^refs/tags/preprocessor-object-segmentation-[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "TAGGED=true" >> $GITHUB_ENV + else + echo "TAGGED=false" >> $GITHUB_ENV + fi + - name: Get timestamp + run: echo "timestamp=$(date -u +'%Y-%m-%dT%H.%M')" >> $GITHUB_ENV + - name: Extract metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + flavor: | + latest=${{ env.TAGGED }} + tags: | + type=match,enable=${{ env.TAGGED }},priority=300,pattern=preprocessor-object-segmentation-(\d+.\d+.\d+),group=1 + type=raw,priority=200,value=unstable + type=raw,priority=100,value=${{ env.timestamp }} + labels: | + org.opencontainers.image.title=IMAGE Preprocessor Object Segmentation + org.opencontainers.image.description=Segments objects found by object-detection-llm using SAM, producing precise polygon outlines per object. + org.opencontainers.image.authors=IMAGE Project + org.opencontainers.image.documentation=https://github.com/Shared-Reality-Lab/IMAGE-server/tree/main/preprocessors/object-segmentation/README.md + org.opencontainers.image.licenses=AGPL-3.0-or-later + maintainer=IMAGE Project + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: . + file: ./preprocessors/object-segmentation/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + secrets: | + HF_TOKEN=${{ secrets.HF_TOKEN }} diff --git a/build.yml b/build.yml index d5c6a4bb..2ae1b0c8 100644 --- a/build.yml +++ b/build.yml @@ -81,6 +81,11 @@ services: context: . dockerfile: ./preprocessors/object-detection-llm/Dockerfile image: "object-detection-llm:latest" + object-segmentation: + build: + context: . + dockerfile: ./preprocessors/object-segmentation/Dockerfile + image: "object-segmentation:latest" object-detection-azure: build: context: . diff --git a/docker-compose.yml b/docker-compose.yml index 4327e0c7..9b68a0fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -193,6 +193,24 @@ services: env_file: ./config/llm.env + object-segmentation: + profiles: [test, production, default] + image: ghcr.io/shared-reality-lab/image-preprocessor-object-segmentation:${REGISTRY_TAG} + restart: "no" + labels: + ca.mcgill.a11y.image.preprocessor: 4 + ca.mcgill.a11y.image.port: 5000 + ca.mcgill.a11y.image.cacheTimeout: 3600 + ca.mcgill.a11y.image.required_dependencies: "object-detection-llm" + ca.mcgill.a11y.image.optional_dependencies: "" + # CPU-only by default (e.g. unicorn); pegasus opts into GPU via + # prod-docker-compose.yml. + environment: + - PII_LOGGING_ENABLED=${PII_LOGGING_ENABLED} + - SAM_MODEL_PATH=/usr/src/app/models/sam3.pt + - MIN_CONTOUR_AREA=0.0001 + - WARMUP_ENABLED=true + multistage-diagram-segmentation: profiles: [production, test, default] image: ghcr.io/shared-reality-lab/image-preprocessor-multistage-diagram-segmentation:${REGISTRY_TAG} @@ -427,7 +445,7 @@ services: labels: ca.mcgill.a11y.image.handler: enable ca.mcgill.a11y.image.required_dependencies: "object-detection,object-grouping,semantic-segmentation,graphic-caption" - ca.mcgill.a11y.image.optional_dependencies: "object-detection-llm" + ca.mcgill.a11y.image.optional_dependencies: "object-detection-llm,object-segmentation" environment: - PII_LOGGING_ENABLED=${PII_LOGGING_ENABLED} diff --git a/handlers/ocr-handler/Dockerfile b/handlers/ocr-handler/Dockerfile index e0cbc0a2..45d5ccc9 100644 --- a/handlers/ocr-handler/Dockerfile +++ b/handlers/ocr-handler/Dockerfile @@ -18,6 +18,8 @@ COPY /schemas /app/schemas COPY /config /app/config +COPY /utils /app/utils + COPY /handlers/ocr-handler/ /app EXPOSE 80 diff --git a/handlers/ocr-handler/server.py b/handlers/ocr-handler/server.py index 3492fa68..31b7e387 100644 --- a/handlers/ocr-handler/server.py +++ b/handlers/ocr-handler/server.py @@ -22,6 +22,11 @@ from flask import Flask, request, jsonify from datetime import datetime from config.logging_utils import configure_logging +from utils.object_detection import ( + GENERIC_OBJECT_DETECTION_NAME, + LLM_OBJECT_DETECTION_NAME, + get_object_detection_data, +) configure_logging() @@ -132,9 +137,12 @@ def render_ocr(): text = "" # Object detection data is present - od = 'ca.mcgill.a11y.image.preprocessor.objectDetection' - if od in preprocessors and len(preprocessors[od]['objects']) > 0: - object_data = preprocessors[od]['objects'] + od_data = get_object_detection_data(preprocessors) + if od_data is not None and len(od_data['objects']) > 0: + od = (GENERIC_OBJECT_DETECTION_NAME + if GENERIC_OBJECT_DETECTION_NAME in preprocessors + else LLM_OBJECT_DETECTION_NAME) + object_data = od_data['objects'] text_lines = ocr_data['lines'] text += "The following objects were detected: " done_once = False diff --git a/handlers/photo-audio-handler/src/server.ts b/handlers/photo-audio-handler/src/server.ts index 2d02d920..649419d7 100644 --- a/handlers/photo-audio-handler/src/server.ts +++ b/handlers/photo-audio-handler/src/server.ts @@ -61,7 +61,7 @@ app.post("/handler", async (req, res) => { const preprocessors = req.body["preprocessors"]; const secondCat = preprocessors["ca.mcgill.a11y.image.preprocessor.graphicTagger"]; const semseg = preprocessors["ca.mcgill.a11y.image.preprocessor.semanticSegmentation"]; - const objDet = preprocessors["ca.mcgill.a11y.image.preprocessor.objectDetection"]; + const objDet = utils.getObjectDetectionData(preprocessors); const objGroup = preprocessors["ca.mcgill.a11y.image.preprocessor.grouping"]; const action = preprocessors["ca.mcgill.a11y.image.preprocessor.actionRecognition"]; //const collageDetector = preprocessors["ca.mcgill.a11y.image.preprocessor.collageDetector"]; diff --git a/handlers/photo-audio-handler/src/utils.ts b/handlers/photo-audio-handler/src/utils.ts index 96a4995d..dfae1385 100644 --- a/handlers/photo-audio-handler/src/utils.ts +++ b/handlers/photo-audio-handler/src/utils.ts @@ -39,6 +39,13 @@ type ObjDet = { objects: Obj[]; }; +const GENERIC_OBJECT_DETECTION_NAME = "ca.mcgill.a11y.image.preprocessor.objectDetection"; +const LLM_OBJECT_DETECTION_NAME = "ca.mcgill.a11y.image.preprocessor.objectDetectionLLM"; + +export function getObjectDetectionData(preprocessors: Record): ObjDet | undefined { + return preprocessors[GENERIC_OBJECT_DETECTION_NAME] ?? preprocessors[LLM_OBJECT_DETECTION_NAME]; +} + type ObjGroup = { grouped: { IDs: number[] }[]; ungrouped: number[]; diff --git a/handlers/photo-tactile-svg/Dockerfile b/handlers/photo-tactile-svg/Dockerfile index 031c928d..53b97107 100644 --- a/handlers/photo-tactile-svg/Dockerfile +++ b/handlers/photo-tactile-svg/Dockerfile @@ -14,6 +14,7 @@ RUN pip install -r requirements.txt COPY /schemas /usr/src/app/schemas COPY /config /usr/src/app/config +COPY /utils /usr/src/app/utils COPY /handlers/photo-tactile-svg/ /usr/src/app EXPOSE 80 diff --git a/handlers/photo-tactile-svg/README.md b/handlers/photo-tactile-svg/README.md index e8561fa1..718e8654 100644 --- a/handlers/photo-tactile-svg/README.md +++ b/handlers/photo-tactile-svg/README.md @@ -9,4 +9,6 @@ Alpha quality: Insufficiently refined to be tested by end-users. This is a [handler](https://github.com/Shared-Reality-Lab/IMAGE-server/wiki/2.-Handlers,-Preprocessors-and-Services#handlers=) component that creates a SVG that can be rendered as a tactile graphic to convey detected objects and detected semantic segments. Data from object detection and semantic segmentation are used to create a SVG. -This SVG will be as per a specified [format](https://github.com/Shared-Reality-Lab/IMAGE-Monarch/tree/main#tactile-graphics) and might contain multiple layers with the number of layers equal to the number of object classes found within the photo. The semantic segments will be available independent of the layers. \ No newline at end of file +This SVG will be as per a specified [format](https://github.com/Shared-Reality-Lab/IMAGE-Monarch/tree/main#tactile-graphics) and might contain multiple layers with the number of layers equal to the number of object classes found within the photo. The semantic segments will be available independent of the layers. + +When `object-segmentation` (SAM 3) output is available for a detected object, its precise polygon outline is drawn in place of that object's plain bounding-box rectangle. Semantic segmentation is otherwise scoped to background/environmental elements (sky, walls, floors, etc.) that object detection doesn't cover. \ No newline at end of file diff --git a/handlers/photo-tactile-svg/tactile_svg.py b/handlers/photo-tactile-svg/tactile_svg.py index f8bb7adf..4a9c3fb7 100644 --- a/handlers/photo-tactile-svg/tactile_svg.py +++ b/handlers/photo-tactile-svg/tactile_svg.py @@ -24,12 +24,34 @@ import inflect from config.logging_utils import configure_logging from datetime import datetime +from utils.object_detection import get_object_detection_data configure_logging() app = Flask(__name__) logging.basicConfig(level=logging.DEBUG) +def object_contour_path(contours, dimensions, aria_label, **extra): + """Draw a SAM-precise polygon outline for a single object, in the + same style/coordinate space as the semantic segmentation contours + below, for use in place of a plain bounding-box rectangle.""" + try: + p = draw.Path(stroke="#ff4477", stroke_width=2.5, + fill='none', aria_label=aria_label, **extra) + except BaseException: + p = draw.Path(stroke="red", stroke_width=2.5, + fill='none', aria_label=aria_label, **extra) + for c in contours: + coords = c["coordinates"] + for i in range(1, len(coords), 5): + if (i == 1): + p.M(coords[i][0] * dimensions[0], + -coords[i][1] * dimensions[1]) + p.L(coords[i][0] * dimensions[0], + -coords[i][1] * dimensions[1]) + return p + + @app.route("/handler", methods=["POST"]) def handle(): logging.debug("Received request") @@ -99,9 +121,8 @@ def handle(): "and/ or semantic segmentation responses") if not (("ca.mcgill.a11y.image.preprocessor.semanticSegmentation" in preprocessors) or - all(x in preprocessors for x in - ["ca.mcgill.a11y.image.preprocessor.objectDetection", - "ca.mcgill.a11y.image.preprocessor.grouping"])): + (get_object_detection_data(preprocessors) is not None and + "ca.mcgill.a11y.image.preprocessor.grouping" in preprocessors)): logging.debug("No Object Detector and Semantic Segmentation found") response = { "request_uuid": contents["request_uuid"], @@ -152,21 +173,27 @@ def handle(): form = inflect.engine() caption = "" - if "ca.mcgill.a11y.image.preprocessor.objectDetection"\ - in preprocessors\ + if get_object_detection_data(preprocessors) is not None\ and "ca.mcgill.a11y.image.preprocessor.grouping" in preprocessors: logging.debug("Object detector and grouping preprocessor found. " "Adding data to response...") caption = "This photo contains " obj_list = [] preprocessor_names.append('Things and people') - o = preprocessors[ - "ca.mcgill.a11y.image.preprocessor.objectDetection" - ] + o = get_object_detection_data(preprocessors) g = preprocessors["ca.mcgill.a11y.image.preprocessor.grouping"] objects = o["objects"] grouped = g["grouped"] ungrouped = g["ungrouped"] + object_segmentation = preprocessors.get( + "ca.mcgill.a11y.image.preprocessor.objectSegmentation") + contours_by_object_id = {} + if object_segmentation: + contours_by_object_id = { + seg["objectID"]: seg["contours"] + for seg in object_segmentation["segments"] + if "objectID" in seg + } layer = 0 # Loop through the object groups and generate a layer for each for group in grouped: @@ -181,14 +208,54 @@ def handle(): # Loop through the individual items # Draw a rectangle for each and tag objects for i, id in enumerate(ids): - x1 = objects[id]['dimensions'][0] * dimensions[0] - x2 = objects[id]['dimensions'][2] * dimensions[0] - y1 = objects[id]['dimensions'][1] * dimensions[1] - y2 = objects[id]['dimensions'][3] * dimensions[1] + label = obj_tag+" "+str(i+1) + if id in contours_by_object_id: + g.append(object_contour_path( + contours_by_object_id[id], dimensions, label)) + else: + x1 = objects[id]['dimensions'][0] * dimensions[0] + x2 = objects[id]['dimensions'][2] * dimensions[0] + y1 = objects[id]['dimensions'][1] * dimensions[1] + y2 = objects[id]['dimensions'][3] * dimensions[1] + width = abs(x2 - x1) + height = abs(y2 - y1) + start_y1 = -(y1 + height) + g.append( + draw.Rectangle( + x1, + start_y1, + width, + height, + stroke="#ff4477", + stroke_width=2.5, + fill="none", + aria_label=label)) + + svg.append(g) + + # Loop through ungrouped objects and generate a layer for each + for val in ungrouped: + category = objects[val]["type"].strip() + # appending singular objects with appropriate article + obj_list.append(form.a(category)) + layer += 1 + if val in contours_by_object_id: + svg.append(object_contour_path( + contours_by_object_id[val], dimensions, category, + data_image_layer="Layer "+str(layer))) + else: + x1 = (objects[val] + ['dimensions'][0] * dimensions[0]) + x2 = (objects[val] + ['dimensions'][2] * dimensions[0]) + y1 = (objects[val] + ['dimensions'][1] * dimensions[1]) + y2 = (objects[val] + ['dimensions'][3] * dimensions[1]) width = abs(x2 - x1) height = abs(y2 - y1) start_y1 = -(y1 + height) - g.append( + svg.append( draw.Rectangle( x1, start_y1, @@ -197,38 +264,8 @@ def handle(): stroke="#ff4477", stroke_width=2.5, fill="none", - aria_label=obj_tag+" "+str(i+1))) - - svg.append(g) - - # Loop through ungrouped objects and generate a layer for each - for val in ungrouped: - category = objects[val]["type"].strip() - # appending singular objects with appropriate article - obj_list.append(form.a(category)) - layer += 1 - x1 = (objects[val] - ['dimensions'][0] * dimensions[0]) - x2 = (objects[val] - ['dimensions'][2] * dimensions[0]) - y1 = (objects[val] - ['dimensions'][1] * dimensions[1]) - y2 = (objects[val] - ['dimensions'][3] * dimensions[1]) - width = abs(x2 - x1) - height = abs(y2 - y1) - start_y1 = -(y1 + height) - svg.append( - draw.Rectangle( - x1, - start_y1, - width, - height, - stroke="#ff4477", - stroke_width=2.5, - fill="none", - aria_label=category, - data_image_layer="Layer "+str(layer))) + aria_label=category, + data_image_layer="Layer "+str(layer))) if len(obj_list) > 1: obj_list[-1] = "and " + obj_list[-1] + "." diff --git a/handlers/svg-object-detection/Dockerfile b/handlers/svg-object-detection/Dockerfile index 9f47cae8..c0de0306 100644 --- a/handlers/svg-object-detection/Dockerfile +++ b/handlers/svg-object-detection/Dockerfile @@ -14,6 +14,7 @@ RUN pip install -r requirements.txt COPY /schemas /usr/src/app/schemas COPY /config /usr/src/app/config +COPY /utils /usr/src/app/utils COPY /handlers/svg-object-detection/ /usr/src/app EXPOSE 80 diff --git a/handlers/svg-object-detection/od_svg.py b/handlers/svg-object-detection/od_svg.py index 2128c023..8770c937 100644 --- a/handlers/svg-object-detection/od_svg.py +++ b/handlers/svg-object-detection/od_svg.py @@ -23,6 +23,7 @@ import drawSvg as draw from datetime import datetime from config.logging_utils import configure_logging +from utils.object_detection import get_object_detection_data configure_logging() @@ -95,8 +96,7 @@ def handle(): return response # No Object Detector found - if "ca.mcgill.a11y.image.preprocessor.objectDetection"\ - not in preprocessors: + if get_object_detection_data(preprocessors) is None: logging.debug("No Object Detector found") response = { "request_uuid": contents["request_uuid"], @@ -137,7 +137,7 @@ def handle(): logging.debug("Sending response") return response - o = preprocessors["ca.mcgill.a11y.image.preprocessor.objectDetection"] + o = get_object_detection_data(preprocessors) g = preprocessors["ca.mcgill.a11y.image.preprocessor.grouping"] u = preprocessors["ca.mcgill.a11y.image.preprocessor.grouping"] objects = o["objects"] diff --git a/preprocessors/celebrity-detector/celebrity-detector.py b/preprocessors/celebrity-detector/celebrity-detector.py index 720fdd72..0888b703 100644 --- a/preprocessors/celebrity-detector/celebrity-detector.py +++ b/preprocessors/celebrity-detector/celebrity-detector.py @@ -27,6 +27,7 @@ import numpy as np from datetime import datetime from utils.validation import Validator +from utils.object_detection import get_object_detection_data app = Flask(__name__) @@ -137,14 +138,12 @@ def categorise(): # convert the uri to processable image if "graphic" not in content.keys(): return "", 204 - if "ca.mcgill.a11y.image.preprocessor.objectDetection" \ - not in preprocessor: + oDpreprocessor = get_object_detection_data(preprocessor) + if oDpreprocessor is None: logging.info("Object detection output not " "available. Skipping...") return "", 204 else: - oDpreprocessor = \ - preprocessor["ca.mcgill.a11y.image.preprocessor.objectDetection"] objects = oDpreprocessor["objects"] image_b64 = content["graphic"].split(",")[1] binary = base64.b64decode(image_b64) diff --git a/preprocessors/clothes-detector/clothes.py b/preprocessors/clothes-detector/clothes.py index b9071f7b..d0825904 100644 --- a/preprocessors/clothes-detector/clothes.py +++ b/preprocessors/clothes-detector/clothes.py @@ -34,6 +34,7 @@ from predictors.YOLOv3 import YOLOv3Predictor from datetime import datetime from utils.validation import Validator +from utils.object_detection import get_object_detection_data app = Flask(__name__) logging.basicConfig(level=logging.NOTSET) @@ -120,14 +121,12 @@ def categorise(): # convert the uri to processable image if "graphic" not in content.keys(): return "", 204 - if "ca.mcgill.a11y.image.preprocessor.objectDetection" \ - not in preprocessor: + oDpreprocessor = get_object_detection_data(preprocessor) + if oDpreprocessor is None: logging.info("Object detection output not " "available. Skipping...") return "", 204 else: - oDpreprocessor = \ - preprocessor["ca.mcgill.a11y.image.preprocessor.objectDetection"] objects = oDpreprocessor["objects"] image_b64 = content["graphic"].split(",")[1] binary = base64.b64decode(image_b64) diff --git a/preprocessors/grouping/grouping.py b/preprocessors/grouping/grouping.py index dd1f75e8..b0d2a1e8 100644 --- a/preprocessors/grouping/grouping.py +++ b/preprocessors/grouping/grouping.py @@ -23,6 +23,7 @@ from config.logging_utils import configure_logging from datetime import datetime from utils.validation import Validator +from utils.object_detection import get_object_detection_data configure_logging() @@ -57,13 +58,11 @@ def readImage(): return jsonify("Invalid Preprocessor JSON format"), 400 preprocessor = content["preprocessors"] - if "ca.mcgill.a11y.image.preprocessor.objectDetection" \ - not in preprocessor: + oDpreprocessor = get_object_detection_data(preprocessor) + if oDpreprocessor is None: logging.info("Object detection output not " "available. Skipping...") return "", 204 - oDpreprocessor = \ - preprocessor["ca.mcgill.a11y.image.preprocessor.objectDetection"] objects = oDpreprocessor["objects"] for i in range(len(objects)): diff --git a/preprocessors/mmsemseg/README.md b/preprocessors/mmsemseg/README.md index 1459356d..b0dec79f 100644 --- a/preprocessors/mmsemseg/README.md +++ b/preprocessors/mmsemseg/README.md @@ -2,7 +2,7 @@ Beta quality: Useful enough for testing by end-users. -This preprocessor is used for semantically segmenting images. The code for this preprocessor heavily relies on the [MMSegmentation](https://github.com/open-mmlab/mmsegmentation) framework. +This preprocessor semantically segments images using the [MMSegmentation](https://github.com/open-mmlab/mmsegmentation) framework, but only returns "stuff"/background-type segments (e.g. sky, wall, floor, road) as classified by [ADE20K SceneParsing150](https://github.com/CSAILVision/sceneparsing/blob/master/objectInfo150.csv) - environmental elements that an object detector wouldn't catch. Foreground objects (people, furniture, vehicles, etc.) are segmented with far more precise outlines by `object-detection-llm` + `object-segmentation` (SAM 3) instead. The code to use this module as an API can be found in `segment.py`, additional functions are located in `utils.py`. This module is fully versionned, the versions of the libraries used can be found in `requirements.txt` and in the `Dockerfile`. diff --git a/preprocessors/mmsemseg/segment.py b/preprocessors/mmsemseg/segment.py index a089293a..8c75f12e 100644 --- a/preprocessors/mmsemseg/segment.py +++ b/preprocessors/mmsemseg/segment.py @@ -51,6 +51,13 @@ COLORS = mmseg.core.evaluation.get_palette("ade20k") CLASS_NAMES = mmseg.core.evaluation.get_classes("ade20k") +# this preprocessor is scoped down to only the +# background elements they can't see, e.g. sky, walls, floors. +STUFF_CLASS_IDS = frozenset({ + 0, 1, 2, 3, 5, 6, 9, 11, 13, 16, 21, 25, 26, 28, 29, 46, 48, 51, 52, + 54, 59, 60, 61, 68, 79, 84, 91, 94, 96, 101, 105, 109, 113, 128, 140 +}) + app = Flask(__name__) @@ -96,9 +103,14 @@ def run_segmentation(url, model, dictionary): # extracting contours pred = result[0].astype(np.int32) predicted_classes = np.bincount(pred.flatten()).argsort()[::-1] - logging.info("main classes detected : {}".format(predicted_classes[:5])) - - for class_id in predicted_classes[:5]: + background_classes = [ + c for c in predicted_classes if c in STUFF_CLASS_IDS + ][:5] + logging.info( + "main background classes detected : {}".format(background_classes) + ) + + for class_id in background_classes: logging.info("extracting contours for class: {}".format(str(class_id))) pred_color, class_name = visualize_result(pred, index=class_id) diff --git a/preprocessors/object-depth-calculator/object-depth-calculator.py b/preprocessors/object-depth-calculator/object-depth-calculator.py index 778ca534..16457816 100644 --- a/preprocessors/object-depth-calculator/object-depth-calculator.py +++ b/preprocessors/object-depth-calculator/object-depth-calculator.py @@ -23,6 +23,7 @@ from datetime import datetime from config.logging_utils import configure_logging from utils.validation import Validator +from utils.object_detection import get_object_detection_data configure_logging() @@ -51,8 +52,7 @@ def objectdepth(): logging.info("Request does not contain a depth-map. Skipping...") return "", 204 # No content logging.debug("passed depth-map check") - if ("ca.mcgill.a11y.image.preprocessor.objectDetection" - not in content["preprocessors"]): + if get_object_detection_data(content["preprocessors"]) is None: logging.info("Request does not contain objects. Skipping...") return "", 204 # No content logging.debug("passed objects check") @@ -92,7 +92,7 @@ def objectdepth(): image = np.asarray(bytearray(binary), dtype="uint8") img = cv2.imdecode(image, cv2.IMREAD_GRAYSCALE)/255 - o = preprocessors["ca.mcgill.a11y.image.preprocessor.objectDetection"] + o = get_object_detection_data(preprocessors) objects = o["objects"] print(dimensions[0], dimensions[1]) obj_depth = [] diff --git a/preprocessors/object-detection-llm/object-detection-llm.py b/preprocessors/object-detection-llm/object-detection-llm.py index 85f78371..076f9170 100644 --- a/preprocessors/object-detection-llm/object-detection-llm.py +++ b/preprocessors/object-detection-llm/object-detection-llm.py @@ -68,7 +68,7 @@ ) PREPROCESSOR_NAME = \ - "ca.mcgill.a11y.image.preprocessor.objectDetection" + "ca.mcgill.a11y.image.preprocessor.objectDetectionLLM" DATA_SCHEMA = './schemas/preprocessors/object-detection.schema.json' diff --git a/preprocessors/object-segmentation/Dockerfile b/preprocessors/object-segmentation/Dockerfile new file mode 100644 index 00000000..e6fda1a4 --- /dev/null +++ b/preprocessors/object-segmentation/Dockerfile @@ -0,0 +1,60 @@ +# syntax=docker/dockerfile:1 +FROM ultralytics/ultralytics:8.3.119-python + +# Set the working directory inside the container +WORKDIR /usr/src/app + +# Create a non-root user to run the application +RUN adduser --disabled-password python + +# Update PATH to include local bin for the python user +ENV PATH="/usr/src/app/.local/bin:${PATH}" + +# Install system dependencies required for healthcheck +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + curl && \ + rm -rf /var/lib/apt/lists/* + +# Copy the requirements file into the container +COPY /preprocessors/object-segmentation/requirements.txt /usr/src/app/requirements.txt + +# Install Python dependencies +RUN pip3 install --upgrade pip && \ + pip3 install --no-cache-dir -r /usr/src/app/requirements.txt + +# Copy the schema, config, and shared utils files +COPY /schemas /usr/src/app/schemas +COPY /config /usr/src/app/config +COPY /utils /usr/src/app/utils + +# Create model directory +RUN mkdir -p /usr/src/app/models + +# Download SAM 3 (gated on Hugging Face: facebook/sam3). Requires an +# HF_TOKEN build secret from an account with approved access, passed via +# BuildKit's --secret (never a plain ARG, which would bake the token into +# the image's layer history). +RUN --mount=type=secret,id=HF_TOKEN \ + pip3 install --no-cache-dir huggingface_hub && \ + hf download facebook/sam3 sam3.pt \ + --local-dir /usr/src/app/models \ + --token "$(cat /run/secrets/HF_TOKEN)" + +# Copy the preprocessor application code +COPY /preprocessors/object-segmentation /usr/src/app + +# Set environment variables needed by the Flask app +ENV FLASK_APP=object-segmentation.py + +# Expose the port the application runs on +EXPOSE 5000 + +# Switch to the non-root user +USER python + +# Define the healthcheck command +HEALTHCHECK --interval=60s --timeout=10s --start-period=120s --retries=5 CMD curl -f http://localhost:5000/health || exit 1 + +# Define the command to run the application using gunicorn +CMD [ "gunicorn", "object-segmentation:app", "-b", "0.0.0.0:5000", "--capture-output", "--timeout=120", "--log-level=debug" ] diff --git a/preprocessors/object-segmentation/README.md b/preprocessors/object-segmentation/README.md new file mode 100644 index 00000000..71e83278 --- /dev/null +++ b/preprocessors/object-segmentation/README.md @@ -0,0 +1,46 @@ +# Object Segmentation Preprocessor + +Alpha quality: not yet ready for use by end-users. + +This preprocessor segments objects already found by `object-detection-llm` using SAM 3 (Segment Anything Model), producing precise per-object polygon outlines rather than just bounding boxes. It does not run its own LLM inference - it consumes `object-detection-llm`'s existing bounding boxes as SAM prompts. + +SAM 3's weights (`facebook/sam3` on Hugging Face) are gated: building this image requires an `HF_TOKEN` build secret from an account with approved access, passed via BuildKit's `--secret` (see the `Dockerfile` and `.github/workflows/object-segmentation.yml`). Never bake the token into an `ARG` - that would persist it in the image's layer history. + +Output is shaped identically to the existing `semanticSegmentation` preprocessor's `segments` output (`schemas/preprocessors/segmentation.schema.json`), with an additional `objectID` field on each segment that ties it back to the detected object it came from. + +## Environment Variables + +``` +SAM_MODEL_PATH=[Path to SAM model file] +MIN_CONTOUR_AREA=[Minimum normalized contour area to keep, default 0.0001] +PII_LOGGING_ENABLED=[true or false] +``` + +**Note**: For production use, it's strongly recommended to set `PII_LOGGING_ENABLED=false` to prevent security risks. Logging personal information should only be done on test servers. The preprocessor uses a `logging.pii()` function that should be properly configured by the logging utilities module. + +## Libraries Used + +| Library | Link | Distribution License | +| ------------- | ------------- | -------------| +| Flask | [Link](https://pypi.org/project/Flask/) | BSD-3-Clause License | +| jsonschema | [Link](https://pypi.org/project/jsonschema/) | MIT License | +| gunicorn | [Link](https://github.com/benoitc/gunicorn) | MIT License | +| pillow | [Link](https://pypi.org/project/Pillow/) | MIT-CMU | +| opencv-python | [Link](https://pypi.org/project/opencv-python/) | Apache 2.0 | +| ultralytics | [Link](https://pypi.org/project/ultralytics/) | AGPL-3.0 License | + +The versions for each of these libraries are specified in `requirements.txt`. + +## API Endpoints + +- `/preprocessor` (POST): Main endpoint for object segmentation +- `/health` (GET): Health check endpoint +- `/warmup` (GET): Warms up the SAM model with a dummy inference + +## Processing Pipeline + +1. Read `object-detection-llm`'s bounding boxes for the request (required; this preprocessor is a no-op without them). +2. Decode the graphic from base64 to a PIL image. No additional resizing is performed here - the upstream `resize-graphic` pseudo-preprocessor already caps every incoming graphic to a fixed maximum dimension, so this preprocessor and `object-detection-llm` share the same pixel space. +3. Segment each detected object's box with SAM, using the object's unique ID (not its type) as the SAM label to keep same-type objects (e.g. two people) from being merged into one contour set. +4. Filter out contours below `MIN_CONTOUR_AREA`. +5. Build a `segments` list matching `schemas/preprocessors/segmentation.schema.json`, with each segment tagged with the `objectID` it came from. diff --git a/preprocessors/object-segmentation/object-segmentation.py b/preprocessors/object-segmentation/object-segmentation.py new file mode 100644 index 00000000..dc7ca19c --- /dev/null +++ b/preprocessors/object-segmentation/object-segmentation.py @@ -0,0 +1,218 @@ +# Copyright (c) 2025 IMAGE Project, Shared Reality Lab, McGill University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# You should have received a copy of the GNU Affero General Public License +# and our Additional Terms along with this program. +# If not, see +# . + +import base64 +import logging +import os +import sys +import time +from io import BytesIO + +from flask import Flask, request, jsonify +from datetime import datetime +from PIL import Image + +from config.logging_utils import configure_logging +from utils.object_detection import LLM_OBJECT_DETECTION_NAME +from utils.segmentation import ( + SAMClient, + create_segment_from_contours, + filter_contours_by_area, +) +from utils.validation import Validator + +configure_logging() + +logging.debug("Starting Object Segmentation Preprocessor...") + +app = Flask(__name__) + +MIN_CONTOUR_AREA = float(os.environ.get('MIN_CONTOUR_AREA', '0.0001')) + +PREPROCESSOR_NAME = \ + "ca.mcgill.a11y.image.preprocessor.objectSegmentation" + +DATA_SCHEMA = './schemas/preprocessors/segmentation.schema.json' + +try: + sam_client = SAMClient() + validator = Validator(data_schema=DATA_SCHEMA) + logging.debug("SAM client and validator initialized") +except Exception as e: + logging.error(f"Failed to initialize clients: {e}") + sys.exit(1) + + +def decode_image(source): + """ + Decode a base64 "graphic" data URI into a PIL Image, without any + LLM-specific resizing. The upstream resize-graphic pseudo-preprocessor + already caps every incoming graphic to a fixed maximum dimension + (aspect-ratio preserved), so every preprocessor - including this one + and object-detection-llm - sees the same image and shares a common + pixel space. + """ + image_b64 = source.split(",")[1] if "," in source else source + binary = base64.b64decode(image_b64) + return Image.open(BytesIO(binary)).convert("RGB") + + +@app.route("/preprocessor", methods=['POST']) +def segment_objects(): + """ + Main endpoint to segment objects found by object-detection-llm using + SAM, producing precise per-object polygon outlines. + """ + logging.debug("Received request for object segmentation.") + + content = request.get_json() + + if "graphic" not in content: + logging.info("No graphic content. Skipping...") + return jsonify({"error": "No graphic content"}), 204 + + ok, _ = validator.check_request(content) + if not ok: + return jsonify({"error": "Invalid Preprocessor JSON format"}), 400 + + preprocess_output = content["preprocessors"] + categoriser = "ca.mcgill.a11y.image.preprocessor.contentCategoriser" + if categoriser in preprocess_output: + categoriser_output = preprocess_output[categoriser] + categoriser_tags = categoriser_output["categories"] + if not categoriser_tags["photo"] and not categoriser_tags["collage"] \ + and not categoriser_tags["illustration"]: + logging.info("Not a photo, collage, or illustration. Skipping...") + return "", 204 + + # Deliberately read the LLM detector's key directly rather than the + # generic objectDetection coalesce helper: this preprocessor only + # makes sense on objects that came from object-detection-llm (SAM + # is being applied specifically to LLM-detected boxes), not + # whichever object detector happened to also be running. + llm_detections = preprocess_output.get(LLM_OBJECT_DETECTION_NAME) + if not llm_detections or not llm_detections.get("objects"): + logging.info("No object-detection-llm output. Skipping...") + return "", 204 + + objects = llm_detections["objects"] + + request_uuid = content["request_uuid"] + timestamp = time.time() + + try: + pil_image = decode_image(content["graphic"]) + + # Use the unique object ID (not the type string) as the SAM label: + # SAMClient.segment_with_boxes(aggregate_by_label=True) merges + # contours from boxes sharing the same label, so labelling by + # type would silently merge contours from multiple same-type + # objects (e.g. two different people) into a single entry. + boxes = [ + {"bbox_2d": obj["dimensions"], "label": str(obj["ID"])} + for obj in objects + ] + id_to_type = {str(obj["ID"]): obj["type"] for obj in objects} + + contours_by_id = sam_client.segment_with_boxes( + pil_image, + boxes, + use_prompts=False, + aggregate_by_label=True, + return_structured=False, + coord_scale=1.0, + ) + + segments = [] + for object_id, contours in contours_by_id.items(): + filtered = filter_contours_by_area( + contours, min_area=MIN_CONTOUR_AREA + ) + if not filtered: + continue + segment = create_segment_from_contours( + filtered, name=id_to_type[object_id] + ) + segment["objectID"] = int(object_id) + segments.append(segment) + + data = {"segments": segments} + + ok, _ = validator.check_data(data) + if not ok: + return jsonify("Invalid Preprocessor JSON format"), 500 + + response = { + "request_uuid": request_uuid, + "timestamp": int(timestamp), + "name": PREPROCESSOR_NAME, + "data": data + } + + ok, _ = validator.check_response(response) + if not ok: + return jsonify("Invalid Preprocessor JSON format"), 500 + + logging.info( + f"Successfully segmented {len(segments)} objects " + f"for request {request_uuid}." + ) + + return jsonify(response), 200 + + except Exception as e: + logging.error( + f"An unexpected error occurred during object segmentation " + f"for {request_uuid}: {e}", exc_info=True + ) + return jsonify( + {"error": "An unexpected internal server error occurred"} + ), 500 + + +@app.route("/health", methods=["GET"]) +def health(): + """ + Health check endpoint to verify if the service is running + """ + return jsonify({ + "status": "healthy", + "timestamp": datetime.now().isoformat() + }), 200 + + +@app.route("/warmup", methods=["GET"]) +def warmup(): + """ + Warms up the SAM model by running a dummy inference. + """ + try: + logging.info("Warming up SAM...") + + sam_success = sam_client.warmup() + + if not sam_success: + logging.error("SAM warmup failed.") + + return jsonify({"status": "ok"}), 200 + + except Exception as e: + logging.error(f"Warmup failed: {str(e)}") + return jsonify({"status": "error", "message": str(e)}), 500 + + +if __name__ == "__main__": + app.run(host='0.0.0.0', port=5000, debug=True) diff --git a/preprocessors/object-segmentation/requirements.txt b/preprocessors/object-segmentation/requirements.txt new file mode 100644 index 00000000..dc136338 --- /dev/null +++ b/preprocessors/object-segmentation/requirements.txt @@ -0,0 +1,7 @@ +Flask==3.1.3 +jsonschema==4.23.0 +gunicorn==23.0.0 +opencv-python==4.11.0.86 +ultralytics==8.4.137 +pillow==12.1.1 +timm==1.0.29 diff --git a/preprocessors/ocr/ocr.py b/preprocessors/ocr/ocr.py index 4b4f9e55..e3e2aed2 100644 --- a/preprocessors/ocr/ocr.py +++ b/preprocessors/ocr/ocr.py @@ -32,6 +32,11 @@ ) from config.logging_utils import configure_logging from utils.validation import Validator +from utils.object_detection import ( + GENERIC_OBJECT_DETECTION_NAME, + LLM_OBJECT_DETECTION_NAME, + get_object_detection_data, +) configure_logging() @@ -71,10 +76,13 @@ def get_ocr_text(): if ocr_result is None: return jsonify("Could not retreive Azure results"), 500 - od = 'ca.mcgill.a11y.image.preprocessor.objectDetection' preprocessors = content['preprocessors'] - if od in preprocessors and len(preprocessors[od]['objects']) > 0: - ocr_result = find_obj_enclosing(od, preprocessors[od], ocr_result) + od_data = get_object_detection_data(preprocessors) + if od_data is not None and len(od_data['objects']) > 0: + od = (GENERIC_OBJECT_DETECTION_NAME + if GENERIC_OBJECT_DETECTION_NAME in preprocessors + else LLM_OBJECT_DETECTION_NAME) + ocr_result = find_obj_enclosing(od, od_data, ocr_result) name = 'ca.mcgill.a11y.image.preprocessor.ocrClouds' request_uuid = content['request_uuid'] diff --git a/preprocessors/sorting/sorting.py b/preprocessors/sorting/sorting.py index 4b63509f..a4972e17 100644 --- a/preprocessors/sorting/sorting.py +++ b/preprocessors/sorting/sorting.py @@ -21,6 +21,7 @@ from datetime import datetime from config.logging_utils import configure_logging from utils.validation import Validator +from utils.object_detection import get_object_detection_data configure_logging() @@ -62,14 +63,12 @@ def readImage(): return jsonify("Invalid Preprocessor JSON format"), 400 preprocessor = content["preprocessors"] - if "ca.mcgill.a11y.image.preprocessor.objectDetection" \ - not in preprocessor: + oDpreprocessor = get_object_detection_data(preprocessor) + if oDpreprocessor is None: logging.info("Object detection output not " "available. Skipping...") return "", 204 - oDpreprocessor = \ - preprocessor["ca.mcgill.a11y.image.preprocessor.objectDetection"] objects = oDpreprocessor["objects"] for i in range(len(objects)): object_type.append(objects[i]["type"]) diff --git a/prod-docker-compose.yml b/prod-docker-compose.yml index 716e9b18..de33fac0 100644 --- a/prod-docker-compose.yml +++ b/prod-docker-compose.yml @@ -70,3 +70,13 @@ services: environment: # temporary: backup server in place for NFB - NOMINATIM_FALLBACK_SERVER=https://nominatim.openstreetmap.org + + # pegasus has GPU capacity to spare for this; unicorn does not, so it + # stays on the CPU-only default from docker-compose.yml. + object-segmentation: + deploy: + resources: + reservations: + devices: + - driver: nvidia + capabilities: ["gpu", "utility", "compute"] diff --git a/schemas b/schemas index 2945b52d..ec0db2a7 160000 --- a/schemas +++ b/schemas @@ -1 +1 @@ -Subproject commit 2945b52da77bf74b1307e7e2286c6297ebef6157 +Subproject commit ec0db2a797fd9f059ac3cbd1d21ca3ef2bc57fed diff --git a/utils/object_detection/__init__.py b/utils/object_detection/__init__.py new file mode 100644 index 00000000..36bdf25b --- /dev/null +++ b/utils/object_detection/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) 2025 IMAGE Project, Shared Reality Lab, McGill University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +""" +Shared helper for reading object-detection preprocessor output. +""" + +GENERIC_OBJECT_DETECTION_NAME = \ + "ca.mcgill.a11y.image.preprocessor.objectDetection" +LLM_OBJECT_DETECTION_NAME = \ + "ca.mcgill.a11y.image.preprocessor.objectDetectionLLM" + + +def get_object_detection_data(preprocessors): + """Prefer the generic (YOLO/Azure) key; fall back to the LLM key.""" + return preprocessors.get(GENERIC_OBJECT_DETECTION_NAME) \ + or preprocessors.get(LLM_OBJECT_DETECTION_NAME) diff --git a/utils/segmentation/README.md b/utils/segmentation/README.md index 90e0a35d..6aa27c01 100644 --- a/utils/segmentation/README.md +++ b/utils/segmentation/README.md @@ -93,7 +93,7 @@ Initialize the SAM client using environment variables. - Uses `SAM_MODEL_PATH` environment variable for model path - Raises `ValueError` if environment variable is not set -##### `segment_with_boxes(image, bounding_boxes, use_prompts=False, aggregate_by_label=True, return_structured=False, base_data=None)` +##### `segment_with_boxes(image, bounding_boxes, use_prompts=False, aggregate_by_label=True, return_structured=False, base_data=None, coord_scale=1000.0)` Segment image using bounding boxes. - `image`: PIL Image object - `bounding_boxes`: List of dicts with 'bbox_2d' and 'label' keys @@ -101,6 +101,7 @@ Segment image using bounding boxes. - `aggregate_by_label`: Group contours by label - `return_structured`: Return data in schema-compatible format - `base_data`: Base data structure to update (required if return_structured=True) +- `coord_scale`: Divisor used to convert `bbox_2d` coordinates to pixel space. Defaults to `1000.0` for LLM-native 0-1000 grids (e.g. Qwen's raw output). Pass `1.0` if `bbox_2d` values are already normalized to 0-1 (e.g. the shared `object-detection.schema.json` format). Returns: - If `return_structured=False`: Dictionary mapping labels to lists of normalized contours @@ -152,7 +153,7 @@ The segmentation utilities expect bounding boxes in the following format: ``` Where: -- `bbox_2d`: Bounding box coordinates in pixels [left, top, right, bottom] +- `bbox_2d`: Bounding box coordinates `[left, top, right, bottom]`. By default these are interpreted as an LLM-native 0-1000 grid (e.g. Qwen's raw output) and divided by `coord_scale` (default `1000.0`) to get pixel coordinates. Pass `coord_scale=1.0` if your coordinates are already normalized to 0-1, as with the shared `object-detection.schema.json` format used by `object-detection-llm`/`yolo`/`azure`. - `label`: String identifier for the object (used as text prompt when `use_prompts=True`) ## Output Format diff --git a/utils/segmentation/sam_processor.py b/utils/segmentation/sam_processor.py index 4d2eca2d..5f155479 100644 --- a/utils/segmentation/sam_processor.py +++ b/utils/segmentation/sam_processor.py @@ -68,7 +68,8 @@ def segment_with_boxes( use_prompts: bool = False, aggregate_by_label: bool = True, return_structured: bool = False, - base_data: Optional[Dict[str, Any]] = None + base_data: Optional[Dict[str, Any]] = None, + coord_scale: float = 1000.0 ) -> Dict[str, Any]: """ Segment image regions using bounding boxes. @@ -83,6 +84,13 @@ def segment_with_boxes( update_data_with_contours (for schema validation) base_data: Base data structure to update (required if return_structured=True) + coord_scale: Divisor to convert incoming bbox coordinates to + pixel space. Use 1000.0 (default) for LLM-native + 0-1000 grids (e.g. Qwen's raw output, the existing + multistage-diagram-segmentation caller). Use 1.0 if + bounding_boxes are already normalized to 0-1 (e.g. + the shared object-detection.schema.json format used + by object-detection-llm/yolo/azure). Returns: If return_structured=False: Dictionary mapping labels to lists @@ -141,13 +149,14 @@ def segment_with_boxes( f"(normalized coords: {bbox})" ) - # Convert normalized coordinates (0-1000) received from Qwen 3 - # to pixel coordinates + # Convert normalized coordinates to pixel coordinates using + # coord_scale (1000.0 for LLM-native grids, 1.0 if already + # normalized to 0-1) bbox_pixels = [ - (bbox[0] / 1000.0) * width, - (bbox[1] / 1000.0) * height, - (bbox[2] / 1000.0) * width, - (bbox[3] / 1000.0) * height + (bbox[0] / coord_scale) * width, + (bbox[1] / coord_scale) * height, + (bbox[2] / coord_scale) * width, + (bbox[3] / coord_scale) * height ] logging.pii(