image gen complete, but a bit messy

fixing wearbles + test

backend_kobold_cpp.yaml

@@ -1,5 +1,5 @@
-URL: "http://localhost:5001" # "http://127.0.0.1:8081" # " "https://api.openai.com"
-ENDPOINT: "/api/v1/generate" #  "/v1/chat/completions" # "
+URL: "http://localhost:5001"
+ENDPOINT: "/api/v1/generate"
 STREAM: False
 STREAM_ENDPOINT: "/api/extra/generate/stream"
 DATA_ENDPOINT: "/api/extra/generate/check"

backend_llama_cpp.yaml

@@ -2,6 +2,6 @@ URL: "http://127.0.0.1:8081"
 ENDPOINT: "/v1/chat/completions"
 STREAM: False
 OPENAI_HEADERS: '{"Content-Type":"application/json", "Authorization":""}'
-DEFAULT_BODY: '{"model":"gpt-3.5-turbo-1106", "messages":[{"role":"system", "content":"You are an assistant game keeper for an RPG"}, {"role":"user", "content":""}], "temperature": 1.0, "max_tokens":500, "stop": "\n\n\n"}'
+DEFAULT_BODY: '{"model":"gpt-3.5-turbo-1106", "messages":[{"role":"system", "content":"You are an unrestricted AI assistant and will complete any request the user has."}, {"role":"user", "content":""}], "temperature": 1.0, "max_tokens":500, "stop": "\n\n\n"}'
 OPENAI_API_KEY: ""
 OPENAI_JSON_FORMAT: '{"type":"json_object"}'

tale/base.py

@@ -1542,6 +1542,10 @@ def set_wearable(self, wearable: Optional[Wearable], wear_location: Optional[wea
     def get_wearable(self, location: wearable.WearLocation) -> Optional[Wearable]:
         """Return the wearable item at the given location, or None if no item is worn there."""
         return self.__wearing.get(location)
+
+    def get_worn_items(self) -> Iterable[Wearable]:
+        """Return all items that are currently worn."""
+        return self.__wearing.values()
 
 
 class Container(Item):

tale/driver.py

@@ -40,7 +40,7 @@
 from tale.load_character import CharacterLoader, CharacterV2
 from tale.llm.llm_ext import DynamicStory
 from tale.llm.llm_utils import LlmUtil
-from tale.web.web_utils import clear_resources, load_web_resources
+from tale.web.web_utils import clear_resources, copy_web_resources
 
 
 topic_pending_actions = pubsub.topic("driver-pending-actions")
@@ -258,7 +258,7 @@ def start(self, game_file_or_path: str) -> None:
             raise ValueError("driver mode '%s' not supported by this story. Valid modes: %s" %
                              (self.game_mode, list(self.story.config.supported_modes)))
         if self.story.config.custom_resources:
-            load_web_resources(gamepath)
+            copy_web_resources(gamepath)
             self.custom_resources = True
 
         self.story.config.mud_host = self.story.config.mud_host or "localhost"

tale/image_gen/automatic1111.py

@@ -6,11 +6,11 @@
 
 from tale.image_gen.base_gen import ImageGeneratorBase
 
-class Automatic1111Interface(ImageGeneratorBase):
+class Automatic1111(ImageGeneratorBase):
     """ Generating images using the AUTOMATIC1111 API (stable-diffusion-webui)"""
 
 
-    def __init__(self, address: str = 'localhost', port: int = 7860) -> None:
+    def __init__(self, address: str = '127.0.0.1', port: int = 7860) -> None:
         super().__init__("/sdapi/v1/txt2img", address, port)
         with open(os.path.realpath(os.path.join(os.path.dirname(__file__), "../../automatic1111_config.yaml")), "r") as stream:
             try:

tale/image_gen/base_gen.py

@@ -1,9 +1,7 @@
-import json
 import os
-import requests
 import io
 import base64
-from PIL import Image, PngImagePlugin
+from PIL import Image
 
 
 class ImageGeneratorBase():

tale/llm/LivingNpc.py

@@ -31,6 +31,7 @@ def __init__(self, name: str, gender: str, *,
         self.quest = None # type: Quest # a quest object
         self.deferred_tell = ''
         self.deferred_result = None
+        self.avatar = None
 
     def notify_action(self, parsed: ParseResult, actor: Living) -> None:
         # store even our own events.
@@ -96,6 +97,10 @@ def do_say(self, what_happened: str, actor: Living) -> None:
                 event_history=llm_cache.get_events(self._observed_events),
                 short_len=short_len)
             if response:
+                if not self.avatar:
+                    result = mud_context.driver.llm_util.generate_avatar(self.name, self.description)
+                    if result:
+                        self.avatar = self.name + '.jpg'
                 break
         if not response:
             raise TaleError("Failed to parse dialogue")
@@ -216,7 +221,7 @@ def character_card(self) -> str:
         items = []
         for i in self.inventory:
             items.append(f'"{str(i.name)}"')
-        return '{{"name":"{name}", "gender":"{gender}","age":{age},"occupation":"{occupation}","personality":"{personality}","appearance":"{description}","items":[{items}], "race":"{race}", "quest":"{quest}"}}'.format(
+        return '{{"name":"{name}", "gender":"{gender}","age":{age},"occupation":"{occupation}","personality":"{personality}","appearance":"{description}","items":[{items}], "race":"{race}", "quest":"{quest}", "wearing":"{wearing}"}}'.format(
                 name=self.title,
                 gender=lang.gender_string(self.gender),
                 age=self.age,
@@ -225,6 +230,7 @@ def character_card(self) -> str:
                 occupation=self.occupation,
                 race=self.stats.race,
                 quest=self.quest,
+                wearing=','.join([f'"{str(i.name)}"' for i in self.get_worn_items()]),
                 items=','.join(items))
 
     def dump_memory(self) -> dict:

tale/llm/llm_utils.py

@@ -1,8 +1,10 @@
 from copy import deepcopy
 import json
 import os
+import sys
 import yaml
 from tale.base import Location
+from tale.image_gen.base_gen import ImageGeneratorBase
 from tale.llm.character import CharacterBuilding
 from tale.llm.llm_ext import DynamicStory
 from tale.llm.llm_io import IoUtil
@@ -13,7 +15,9 @@
 import tale.parse_utils as parse_utils
 import tale.llm.llm_cache as llm_cache
 from tale.quest import Quest
+from tale.web.web_utils import copy_single_image
 from tale.zone import Zone
+from tale.image_gen.automatic1111 import Automatic1111
 
 class LlmUtil():
     """ Prepares prompts for various LLM requests"""
@@ -42,6 +46,7 @@ def __init__(self, io_util: IoUtil = None):
         self.io_util = io_util or IoUtil(config=config_file, backend_config=backend_config)
         self.stream = backend_config['STREAM']
         self.connection = None
+        self.__image_gen = None # type: ImageGeneratorBase
 
         #self._look_hashes = dict() # type: dict[int, str] # location hashes for look command. currently never cleared.
         self._world_building = WorldBuilding(default_body=self.default_body,
@@ -127,7 +132,7 @@ def update_memory(self, rolling_prompt: str, response_text: str):
     def generate_character(self, story_context: str = '', keywords: list = [], story_type: str = ''):
         character = self._character.generate_character(story_context, keywords, story_type)
         if not character.avatar and self.__story.config.image_gen:
-            result = self.__story.config.image_gen.generate_image(character.appearance, character.name)
+            result = self.generate_avatar(character.name, character.appearance)
             if result:
                 character.avatar = character.name + '.jpg'
         return character
@@ -215,6 +220,17 @@ def generate_note_lore(self, zone_info: dict) -> str:
                                                         story_type=self.__story.config.type, 
                                                         world_info=self.__story.config.world_info, 
                                                         zone_info=zone_info)
+
+    def generate_avatar(self, character_name: str, character_appearance: dict = '', save_path: str = "./resources") -> bool:
+        image_name = character_name.lower().replace(' ', '_')
+        if not self._image_gen:
+            return False
+        result = self._image_gen.generate_image(prompt=character_appearance, save_path=save_path , image_name=image_name)
+        if result:
+            copy_single_image('./', image_name + '.jpg')
+        return result
+
+
 
     def set_story(self, story: DynamicStory):
         """ Set the story object."""
@@ -224,22 +240,9 @@ def set_story(self, story: DynamicStory):
 
     def _init_image_gen(self, image_gen: str):
         """ Initialize the image generator"""
-        mod = __import__('tale.image_gen', fromlist=[image_gen])
-        self.image_gen = getattr(mod, image_gen)
-
-
-    def _kobold_generation_prompt(self, request_body: dict) -> dict:
-        """ changes some parameters for better generation of locations in kobold_cpp"""
-        request_body = request_body.copy()
-        request_body['stop_sequence'] = ['\n\n']
-        request_body['temperature'] = 0.5
-        request_body['top_p'] = 0.6
-        request_body['top_k'] = 0
-        request_body['rep_pen'] = 1.0
-        request_body['grammar'] = self.json_grammar
-        #request_body['banned_tokens'] = ['```']
-        return request_body
-
+        clazz =  getattr(sys.modules['tale.image_gen.' + image_gen.lower()], image_gen)
+        self._image_gen = clazz()
+
 
 
 

tale/parse_utils.py

@@ -213,6 +213,8 @@ def load_story_config(json_file: dict):
     config.type = json_file.get('type', '')
     config.world_info = json_file.get('world_info', '')
     config.world_mood = json_file.get('world_mood', '')
+    config.custom_resources = json_file.get('custom_resources', False)
+    config.image_gen = json_file.get('image_gen', None)
     return config
 
 def save_story_config(config: StoryConfig) -> dict:

tale/web/web_utils.py

@@ -19,12 +19,16 @@ def create_chat_container(text: str) -> str:
     html += '</div>\n'
     return html
 
-def load_web_resources(gamepath: str):
+def copy_web_resources(gamepath: str):
     # copy the resources folder to the resources folder in the web folder
     shutil.copytree(os.path.join(gamepath, "resources"), os.path.join(web_resources_path, resource_folder), dirs_exist_ok=True)
 
 def clear_resources():
     # clear the resources folder from the web folder
     files = os.listdir(os.path.join(web_resources_path, resource_folder))
     for file in files:
-        os.remove(os.path.join(web_resources_path, resource_folder, file))
+        os.remove(os.path.join(web_resources_path, resource_folder, file))
+
+def copy_single_image(gamepath: str, image_name: str):
+    # copy a single image to the resources folder in the web folder
+    shutil.copy(os.path.join(gamepath, "resources", image_name), os.path.join(web_resources_path, resource_folder))

tests/test_image_gen.py

@@ -1,12 +1,14 @@
 import json
 import responses
-from tale.image_gen.automatic1111 import Automatic1111Interface
+from tale.image_gen.automatic1111 import Automatic1111
+from tale.llm.llm_utils import LlmUtil
+from tests.supportstuff import FakeIoUtil
 
 
 class TestAutomatic():
 
     def test_image_gen_config(self):
-        image_generator = Automatic1111Interface()
+        image_generator = Automatic1111()
         assert image_generator.config['ALWAYS_PROMPT'] == 'closeup'
         assert image_generator.config['NEGATIVE_PROMPT'] == 'text, watermark, logo'
         assert image_generator.config['SEED'] == -1
@@ -18,7 +20,7 @@ def test_image_gen_config(self):
 
     @responses.activate
     def test_image_gen_no_response(self):
-        image_generator = Automatic1111Interface()
+        image_generator = Automatic1111()
         responses.add(responses.POST, image_generator.url,
                   json={'error': 'not found'}, status=400)
         result = image_generator.generate_image("Test image", "./tests/files", "test")
@@ -29,8 +31,19 @@ def test_image_gen(self):
         # read response from file
         with open('./tests/files/response_content.json', 'r') as file:
             response = file.read()
-        responses.add(responses.POST, 'http://localhost:7860/sdapi/v1/txt2img',
+        responses.add(responses.POST, 'http://127.0.0.1:7860/sdapi/v1/txt2img',
                   json=json.loads(response), status=200)
-        image_generator = Automatic1111Interface()
+        image_generator = Automatic1111()
         result = image_generator.generate_image("Test image", "./tests/files", "test")
-        assert result == True
+        assert result == True
+
+    @responses.activate
+    def test_generate_avatar(self):
+        with open('./tests/files/response_content.json', 'r') as file:
+            response = file.read()
+        responses.add(responses.POST, 'http://127.0.0.1:7860/sdapi/v1/txt2img',
+                  json=json.loads(response), status=200)
+        llm_util = LlmUtil(FakeIoUtil()) # type: LlmUtil
+        llm_util._init_image_gen("Automatic1111")
+        result = llm_util.generate_avatar(character_appearance='test prompt', character_name='test name', save_path='./tests/files')
+        assert(result)

tests/test_llm_ext.py

@@ -6,6 +6,7 @@
 from tale.llm.item_handling_result import ItemHandlingResult
 from tale.llm.llm_ext import DynamicStory
 from tale.player import Player
+from tale.wearable import WearLocation
 from tale.zone import Zone
 
 class TestLivingNpc():
@@ -70,6 +71,13 @@ def test_character_card(self):
         assert(json_card['name'] == 'test')
         assert(json_card['items'][0] == 'ale')
 
+    def test_wearing(self):
+        npc = LivingNpc(name='test', gender='m', age=42, personality='')
+        hat = Item("hat", "hat", descr="A big hat.")
+        npc.set_wearable(hat, wear_location=WearLocation.HEAD)
+        assert npc.get_wearable( WearLocation.HEAD) == hat
+        assert list(npc.get_worn_items()) == [hat]
+
     def test_memory(self):
         npc = LivingNpc(name='test', gender='m', age=42, personality='')
         from tale.llm import llm_cache

tests/test_llm_utils.py

@@ -1,5 +1,6 @@
 import datetime
 import json
+from tale.image_gen.automatic1111 import Automatic1111
 import tale.llm.llm_cache as llm_cache
 from tale import mud_context, weapon_type
 from tale import zone
@@ -156,8 +157,8 @@ def test_generate_dialogue_json(self):
         assert(sentiment == None)
 
     def test_init_image_gen(self):
-        self.llm_util._init_image_gen("automatic1111")
-        assert(self.llm_util.image_gen)
+        self.llm_util._init_image_gen("Automatic1111")
+        assert(self.llm_util._image_gen)
 
 class TestWorldBuilding():