Add new text-followup handler to use the output of text-followup preprocessor

.github/workflows/text-followup-handler.yml

@@ -0,0 +1,82 @@
+name: Text Followup Handler
+on:
+  push:
+    branches: [ main ]
+    tags: [ "handler-text-followup-[0-9]+.[0-9]+.[0-9]+" ]
+    paths: [ "handlers/text-followup/**" ]
+  pull_request:
+    branches: [ main ]
+    paths: [ "handlers/text-followup/**" ]
+  workflow_run:
+    workflows: [ "Schemas (Trigger)" ]
+    types:
+      - completed
+  workflow_dispatch:
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: shared-reality-lab/image-handler-text-followup
+jobs:
+  lint:
+    name: PEP 8 style check.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: Install flake8
+        run: pip install flake8
+      - name: Check with flake8
+        run: python -m flake8 ./handlers/text-followup --show-source
+  build-and-push-image:
+    name: Build and Push to Registry
+    needs: lint
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Log into GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Get Correct Tags
+        run: |
+          if [[ ${{ github.ref }} =~ ^refs/tags/handler-text-followup-[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "TAGGED=true" >> $GITHUB_ENV
+          else
+            echo "TAGGED=false" >> $GITHUB_ENV
+          fi
+      - name: Get timestamp
+        run: echo "timestamp=$(date -u +'%Y-%m-%dT%H.%M')" >> $GITHUB_ENV
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            latest=${{ env.TAGGED }}
+          tags: |
+            type=match,enable=${{ env.TAGGED }},priority=300,pattern=handler-text-followup-(\d+.\d+.\d+),group=1
+            type=raw,priority=200,value=unstable
+            type=raw,priority=100,value=${{ env.timestamp }}
+          labels: |
+            org.opencontainers.image.title=IMAGE Handler Text Followup
+            org.opencontainers.image.description=Handler to create responses for followup queries.
+            org.opencontainers.image.authors=IMAGE Project <[email protected]>
+            org.opencontainers.image.licenses=AGPL-3.0-or-later
+            maintainer=IMAGE Project <[email protected]>
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./handlers/text-followup/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

build.yml

@@ -263,6 +263,11 @@ services:
             context: .
             dockerfile: ./handlers/svg-action-recognition/Dockerfile
         image: "svg-action-recognition:latest"
+    text-followup-handler:
+        build:
+            context: .
+            dockerfile: ./handlers/text-followup-handler/Dockerfile
+        image: "text-followup-handler:latest"
     # Supercollider on Fedora 34
     supercollider-base:
         build:

docker-compose.yml

@@ -291,6 +291,13 @@ services:
     labels:
         ca.mcgill.a11y.image.handler: enable
 
+  text-followup-handler:
+    profiles: [test, default]
+    image: ghcr.io/shared-reality-lab/image-handler-text-followup:${REGISTRY_TAG}
+    restart: unless-stopped
+    labels:
+        ca.mcgill.a11y.image.handler: enable
+
 # end - common services
 
 # start - unicorn exclusive services

handlers/text-followup-handler/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.13
+
+RUN apt-get install libcairo2
+
+RUN adduser --disabled-password python
+WORKDIR /usr/src/app
+ENV PATH="/home/python/.local/bin:${PATH}"
+
+RUN pip install --upgrade pip
+COPY /handlers/text-followup-handler/requirements.txt /usr/src/app/requirements.txt
+RUN pip install -r requirements.txt
+
+COPY /schemas /usr/src/app/schemas
+COPY /handlers/text-followup-handler/ /usr/src/app
+
+EXPOSE 80
+USER python
+CMD ["gunicorn", "followup:app", "-b", "0.0.0.0:80", "--capture-output", "--log-level=debug" ]

handlers/text-followup-handler/README.md

@@ -0,0 +1,12 @@
+# Text Followup Handler
+
+Alpha quality: Insufficiently refined to be tested by end-users.
+
+![license: AGPL](https://camo.githubusercontent.com/b53b1136762ea55ee6a2d641c9f8283b8335a79b3cb95cbab5a988e678e269b8/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d4147504c2d73756363657373) [GitHub Container Registry Package](https://github.com/Shared-Reality-Lab/IMAGE-server/pkgs/container/image-handler-text-followup)
+
+## What is this?
+
+This is a [handler](https://github.com/Shared-Reality-Lab/IMAGE-server/wiki/2.-Handlers,-Preprocessors-and-Services#handlers=) component that returns a text-only response to a followup query posed by the user.
+
+Data from text-followup preprocessor are used to generate the response.
+Only the brief response from the preprocessor is included in the handler response.

handlers/text-followup-handler/followup.py

@@ -0,0 +1,185 @@
+# Copyright (c) 2025 IMAGE Project, Shared Reality Lab, McGill University
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# and our Additional Terms along with this program.
+# If not, see
+# <https://github.com/Shared-Reality-Lab/IMAGE-server/blob/main/LICENSE>.
+
+from flask import Flask, jsonify, request
+import json
+import jsonschema
+from jsonschema.exceptions import ValidationError
+import logging
+import time
+
+app = Flask(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+@app.route("/handler", methods=["POST"])
+def handle():
+    logging.debug("Received request")
+    # Load necessary schema files
+    with open("./schemas/definitions.json") as f:
+        definitions_schema = json.load(f)
+    with open("./schemas/request.schema.json") as f:
+        request_schema = json.load(f)
+    with open("./schemas/handler-response.schema.json") as f:
+        response_schema = json.load(f)
+    with open("./schemas/renderers/text.schema.json") as f:
+        renderer_schema = json.load(f)
+
+    store = {
+        definitions_schema["$id"]: definitions_schema,
+        request_schema["$id"]: request_schema,
+        response_schema["$id"]: response_schema,
+        renderer_schema["$id"]: renderer_schema
+    }
+    resolver = jsonschema.RefResolver.from_schema(
+        request_schema, store=store
+    )
+    # Get and validate request contents
+    contents = request.get_json()
+    try:
+        logging.debug("Validating request schema")
+        validator = jsonschema.Draft7Validator(
+            request_schema, resolver=resolver
+        )
+        validator.validate(contents)
+    except ValidationError as e:
+        logging.error(e)
+        return jsonify("Invalid request received!"), 400
+
+    preprocessors = contents['preprocessors']
+
+    logging.debug("Checking whether renderer is supported")
+    if ("ca.mcgill.a11y.image.renderer.Text" not in contents["renderers"]):
+        logging.debug("Text Renderer is not supported")
+        response = {
+            "request_uuid": contents["request_uuid"],
+            "timestamp": int(time.time()),
+            "renderings": []
+        }
+        try:
+            validator = jsonschema.Draft7Validator(
+                response_schema, resolver=resolver)
+            validator.validate(response)
+        except jsonschema.exceptions.ValidationError as error:
+            logging.error(error)
+            return jsonify("Invalid Preprocessor JSON format"), 500
+        logging.debug("Sending response")
+        return response
+
+    # Throws error when text-followup preprocessor is not found
+    logging.debug("Checking for text-followup "
+                  "preprocessor responses")
+    if not ("ca.mcgill.a11y.image.preprocessor.text-followup"
+            in preprocessors):
+        logging.debug("Text-followup preprocessor not found")
+        response = {
+            "request_uuid": contents["request_uuid"],
+            "timestamp": int(time.time()),
+            "renderings": []
+        }
+        try:
+            validator = jsonschema.Draft7Validator(
+                response_schema, resolver=resolver)
+            validator.validate(response)
+        except jsonschema.exceptions.ValidationError as error:
+            logging.error(error)
+            return jsonify("Invalid Preprocessor JSON format"), 500
+        logging.debug("Sending response")
+        return response
+
+    # Checking for graphic and dimensions
+    logging.debug("Checking whether graphic and"
+                  " dimensions are available")
+    if "graphic" in contents and "dimensions" in contents:
+        # If an existing graphic exists, often it is
+        # best to use that for convenience.
+        # see the following for SVG coordinate info:
+        # developer.mozilla.org/en-US/docs/Web/SVG/Tutorial/Positions
+        logging.debug("Graphic has dimensions defined")
+    else:
+        logging.debug("Graphic and/or dimensions are not defined")
+        response = {
+            "request_uuid": contents["request_uuid"],
+            "timestamp": int(time.time()),
+            "renderings": []
+        }
+        try:
+            validator = jsonschema.Draft7Validator(
+                response_schema, resolver=resolver)
+            validator.validate(response)
+        except jsonschema.exceptions.ValidationError as error:
+            logging.error(error)
+            return jsonify("Invalid Preprocessor JSON format"), 500
+        logging.debug("Sending response")
+        return response
+    # Checking whether this is an actual follow up query
+    logging.debug("Checking whether this is "
+                  "an actual follow up query")
+    if "followup" not in contents:
+        logging.debug("Follow-up query is not defined")
+        response = {
+            "request_uuid": contents["request_uuid"],
+            "timestamp": int(time.time()),
+            "renderings": []
+        }
+        try:
+            validator = jsonschema.Draft7Validator(
+                response_schema, resolver=resolver)
+            validator.validate(response)
+        except jsonschema.exceptions.ValidationError as error:
+            logging.error(error)
+            return jsonify("Invalid Preprocessor JSON format"), 500
+        logging.debug("Sending response")
+        return response
+
+    data = {"text": (preprocessors["ca.mcgill.a11y.image.preprocessor.text"
+                                   "-followup"]["response_brief"])}
+
+    rendering = {
+        "type_id": "ca.mcgill.a11y.image.renderer.Text",
+        "description": "Response to a follow-up query",
+        "data": data
+    }
+
+    try:
+        validator = jsonschema.Draft7Validator(
+            renderer_schema, resolver=resolver
+        )
+        validator.validate(data)
+    except ValidationError as e:
+        logging.error(e)
+        logging.debug("Failed to validate the response renderer!")
+        return jsonify("Failed to validate the response renderer"), 500
+    response = {
+        "request_uuid": contents["request_uuid"],
+        "timestamp": int(time.time()),
+        "renderings": [rendering]
+    }
+    try:
+        validator = jsonschema.Draft7Validator(
+            response_schema, resolver=resolver
+        )
+        validator.validate(response)
+    except ValidationError as e:
+        logging.debug("Failed to generate a valid response")
+        logging.error(e)
+        return jsonify("Failed to generate a valid response"), 500
+    logging.debug("Sending response")
+    return response
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=80, debug=True)

handlers/text-followup-handler/requirements.txt

@@ -0,0 +1,3 @@
+jsonschema==4.23.0
+Flask==3.0.3
+gunicorn