feat: updated image-size and render-text examples with file api (#68)

examples/echo/README.md

@@ -0,0 +1,11 @@
+## Overview
+
+An example of a simple text-to-text DIAL application.
+
+It returns back the content and attachments from the last user message.
+
+Upon start the Docker image exposes `openai/deployments/echo/chat/completions` endpoint at port `5000`.
+
+## Usage
+
+Find how to integrate the application into the DIAL Core and call it using DIAL API in the [cookbook](https://github.com/epam/ai-dial/blob/main/dial-cookbook/examples/how_to_call_text_to_text_applications.ipynb).

examples/echo/app.py

@@ -1,6 +1,6 @@
 """
 A DIAL application which returns back the content and attachments
-of the last user message.
+from the last user message.
 """
 
 import uvicorn

examples/image_size/README.md

@@ -0,0 +1,24 @@
+## Overview
+
+An example of a simple image-to-text DIAL application.
+
+It takes an image from the last user message attachments and returns back the image dimensions.
+
+Upon start the Docker image exposes `openai/deployments/image-size/chat/completions` endpoint at port `5000`.
+
+## Configuration
+
+The application supports image attachments provided in one of the following format:
+
+1. Base64 encoded image
+2. URL to the image, which might be either
+   * public URL or
+   * URL pointing to a file in the DIAL file storage. `DIAL_URL` environment variable should be set to support image stored in the storage.
+
+|Variable|Default|Description|
+|---|---|---|
+|DIAL_URL||URL of the core DIAL server. Optional. Used to access images stored in the DIAL file storage|
+
+## Usage
+
+Find how to integrate the application into the DIAL Core and call it using DIAL API in the [cookbook](https://github.com/epam/ai-dial/blob/main/dial-cookbook/examples/how_to_call_image_to_text_applications.ipynb).

examples/image_size/app/image.py

@@ -2,10 +2,26 @@
 from io import BytesIO
 from typing import Tuple
 
+import aiohttp
 from PIL import Image
 
 
-def get_image_base64_size(image_base64) -> Tuple[int, int]:
+def get_image_base64_size(image_base64: str) -> Tuple[int, int]:
     image_binary = base64.b64decode(image_base64)
     img = Image.open(BytesIO(image_binary))
     return img.size
+
+
+def bytes_to_base64(data: bytes) -> str:
+    return base64.b64encode(data).decode()
+
+
+async def download_image_as_bytes(url: str) -> bytes:
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url) as response:
+            response.raise_for_status()
+            return await response.content.read()
+
+
+async def download_image_as_base64(url: str) -> str:
+    return bytes_to_base64(await download_image_as_bytes(url))

examples/image_size/app/main.py

@@ -4,39 +4,109 @@
 returns its width and height as text.
 """
 
+import os
+from urllib.parse import urlparse
+
 import uvicorn
 
 from aidial_sdk import DIALApp
 from aidial_sdk import HTTPException as DIALException
 from aidial_sdk.chat_completion import ChatCompletion, Request, Response
 
-from .image import get_image_base64_size
+from .image import download_image_as_base64, get_image_base64_size
+
+DIAL_URL = os.environ.get("DIAL_URL")
+
+
+# A helper to distinguish relative URLs from absolute ones
+# Relative URLs are treated as URLs to the DIAL File storage
+# Absolute URLs are treated as publicly accessible URLs to external resources
+def is_relative_url(url) -> bool:
+    parsed_url = urlparse(url)
+    return (
+        not parsed_url.scheme
+        and not parsed_url.netloc
+        and not url.startswith("/")
+    )
 
 
 # ChatCompletion is an abstract class for applications and model adapters
 class ImageSizeApplication(ChatCompletion):
     async def chat_completion(self, request: Request, response: Response):
         # Create a single choice
         with response.create_single_choice() as choice:
-            # Get the image from the last message attachments
-            try:
-                image = request.messages[-1].custom_content.attachments[0].data  # type: ignore
-            except Exception:
-                # Raise an exception if no image was found
+            # Validate the request:
+            # - the request must contain at least one message, and
+            # - the last message must contain one and only one image attachment.
+
+            if len(request.messages) == 0:
+                raise DIALException(
+                    message="The request must contain at least one message",
+                    status_code=422,
+                )
+
+            message = request.messages[-1]
+
+            if (
+                message.custom_content is None
+                or message.custom_content.attachments is None
+            ):
                 raise DIALException(
                     message="No image attachment was found in the last message",
                     status_code=422,
                 )
+
+            attachments = message.custom_content.attachments
+
+            if len(attachments) != 1:
+                raise DIALException(
+                    message="Only one attachment is expected in the last message",
+                    status_code=422,
+                )
+
+            # Get the image from the last message attachments
+            attachment = attachments[0]
+
+            # The attachment contains either the image content as a base64 string or an image URL
+            if attachment.data is not None:
+                image_data = attachment.data
+            elif attachment.url is not None:
+                image_url = attachment.url
+
+                # Download the image from the URL
+                if is_relative_url(image_url):
+                    if DIAL_URL is None:
+                        # DIAL SDK automatically converts standard Python exceptions to 500 Internal Server Error
+                        raise ValueError(
+                            "DIAL_URL environment variable is unset"
+                        )
+
+                    image_abs_url = f"{DIAL_URL}/v1/{image_url}"
+                else:
+                    image_abs_url = image_url
+
+                image_data = await download_image_as_base64(image_abs_url)
+            else:
+                raise DIALException(
+                    message="Either 'data' or 'url' field must be provided in the attachment",
+                    status_code=422,
+                )
+
             # Compute the image size
-            (w, h) = get_image_base64_size(image)
-            # Display the image size
+            (w, h) = get_image_base64_size(image_data)
+
+            # Return the image size
             choice.append_content(f"Size: {w}x{h}px")
 
 
 # DIALApp extends FastAPI to provide a user-friendly interface for routing requests to your applications
-app = DIALApp()
+app = DIALApp(
+    dial_url=DIAL_URL,
+    propagation_auth_headers=DIAL_URL is not None,
+    add_healthcheck=True,
+)
+
 app.add_chat_completion("image-size", ImageSizeApplication())
 
-# Run built app
 if __name__ == "__main__":
     uvicorn.run(app, port=5000)

examples/image_size/requirements.txt

@@ -1,2 +1,3 @@
 aidial-sdk>=0.2
-pillow==10.2.0
+pillow==10.2.0
+aiohttp==3.9.0

examples/render_text/README.md

@@ -0,0 +1,25 @@
+## Overview
+
+An example of a simple text-to-image DIAL application.
+
+It takes a text from the last user message attachments and returns back the image with the rasterized text.
+
+The generated image is added as an image attachment to the response message and also as a Markdown image in the response text.
+
+Upon start the Docker image exposes `openai/deployments/render-text/chat/completions` endpoint at port `5000`.
+
+## Configuration
+
+The application returns the image in one of the following formats:
+1. Base64 encoded image
+2. URL to the image stored in the DIAL file storage. `DIAL_URL` environment variable should be set to support image uploading to the storage.
+
+The format of the image attachment is controlled by the user message, which is expected to have the following format: `(base64|url),<text to render>`.
+
+|Variable|Default|Description|
+|---|---|---|
+|DIAL_URL||URL of the core DIAL server. Optional. Used to upload generated images the DIAL file storage|
+
+## Usage
+
+Find how to integrate the application into the DIAL Core and call it using DIAL API in the [cookbook](https://github.com/epam/ai-dial/blob/main/dial-cookbook/examples/how_to_call_text_to_image_applications.ipynb).

examples/render_text/app/image.py

@@ -1,11 +1,13 @@
 import base64
+import os
 import textwrap
 from io import BytesIO
 
+import aiohttp
 from PIL import Image, ImageDraw, ImageFont
 
 
-def text_to_image_base64(text, img_size=(200, 100), font_size=20) -> str:
+def text_to_image_base64(text: str, img_size=(200, 100), font_size=20) -> str:
     img = Image.new("RGB", img_size, color="yellow")
     d = ImageDraw.Draw(img)
 
@@ -25,3 +27,30 @@ def text_to_image_base64(text, img_size=(200, 100), font_size=20) -> str:
     img_base64 = base64.b64encode(img_buffer.getvalue()).decode()
 
     return img_base64
+
+
+async def upload_png_image(
+    dial_url: str, filepath: str, image_base64: str
+) -> str:
+    async with aiohttp.ClientSession() as session:
+        async with session.get(f"{dial_url}/v1/bucket") as response:
+            response.raise_for_status()
+            appdata = (await response.json())["appdata"]
+
+        image_bytes = base64.b64decode(image_base64)
+
+        data = aiohttp.FormData()
+        data.add_field(
+            name="file",
+            content_type="image/png",
+            value=BytesIO(image_bytes),
+            filename=os.path.basename(filepath),
+        )
+
+        async with session.put(
+            f"{dial_url}/v1/files/{appdata}/{filepath}", data=data
+        ) as response:
+            response.raise_for_status()
+            metadata = await response.json()
+
+    return metadata["url"]

examples/render_text/app/main.py

@@ -4,12 +4,17 @@
 sends the image back to the user in an attachment.
 """
 
+import os
+
 import uvicorn
 
 from aidial_sdk import DIALApp
+from aidial_sdk import HTTPException as DIALException
 from aidial_sdk.chat_completion import ChatCompletion, Request, Response
 
-from .image import text_to_image_base64
+from .image import text_to_image_base64, upload_png_image
+
+DIAL_URL = os.environ.get("DIAL_URL")
 
 
 # ChatCompletion is an abstract class for applications and model adapters
@@ -20,14 +25,46 @@ async def chat_completion(self, request: Request, response: Response):
             # Get the last message content
             content = request.messages[-1].content or ""
 
+            # The image may be returned either as base64 string or as URL
+            # The content specifies the mode of return: 'base64' or 'url'
+            try:
+                command, text = content.split(",", 1)
+                if command not in ["base64", "url"]:
+                    raise DIALException(
+                        message="The command must be either 'base64' or 'url'",
+                        status_code=422,
+                    )
+            except ValueError:
+                raise DIALException(
+                    message="The content must be in the format '(base64|url),<text>'",
+                    status_code=422,
+                )
+
             # Rasterize the user message to an image
-            image_base64 = text_to_image_base64(content)
+            image_base64 = text_to_image_base64(text)
             image_type = "image/png"
 
             # Add the image as an attachment
-            choice.add_attachment(
-                type=image_type, title="Image", data=image_base64
-            )
+            if command == "base64":
+                # As base64 string
+                choice.add_attachment(
+                    type=image_type, title="Image", data=image_base64
+                )
+            else:
+                # As URL to DIAL File storage
+                if DIAL_URL is None:
+                    # DIAL SDK automatically converts standard Python exceptions to 500 Internal Server Error
+                    raise ValueError("DIAL_URL environment variable is unset")
+
+                # Upload the image to DIAL File storage
+                image_url = await upload_png_image(
+                    DIAL_URL, "images/picture.png", image_base64
+                )
+
+                # And return as an attachment
+                choice.add_attachment(
+                    type=image_type, title="Image", url=image_url
+                )
 
             # Return the image in Markdown format
             choice.append_content(
@@ -36,9 +73,13 @@ async def chat_completion(self, request: Request, response: Response):
 
 
 # DIALApp extends FastAPI to provide a user-friendly interface for routing requests to your applications
-app = DIALApp()
+app = DIALApp(
+    dial_url=DIAL_URL,
+    propagation_auth_headers=DIAL_URL is not None,
+    add_healthcheck=True,
+)
+
 app.add_chat_completion("render-text", RenderTextApplication())
 
-# Run built app
 if __name__ == "__main__":
     uvicorn.run(app, port=5000)

examples/render_text/requirements.txt

@@ -1,2 +1,3 @@
 aidial-sdk>=0.2
-pillow==10.2.0
+pillow==10.2.0
+aiohttp==3.9.0

pyproject.toml

@@ -54,6 +54,7 @@ pytest = "^7.4.2"
 httpx = "^0.25.0"
 nox = "^2023.4.22"
 pillow = "^10.2.0"
+aiohttp = "^3.9.0"
 
 [tool.poetry.group.lint.dependencies]
 flake8 = "^6.0.0"

tests/examples/test_render_text.py

@@ -18,7 +18,7 @@ def test_app():
             "messages": [
                 {
                     "role": "user",
-                    "content": "Hello world!",
+                    "content": "base64,Hello world!",
                 }
             ]
         },