Merge pull request #985 from Capsize-Games/devastator

Fix custom path during installation

README.md

@@ -158,49 +158,4 @@ Run a single unit tests `python -m unittest src/airunner/tests/<file_name>`
 Example
 ```bash
 python -m unittest src/airunner/tests/test_prompt_weight_convert.py
-```
-
----
-
-## Privacy and Security
-
-Although AI Runner v3.0 is built with Huggingface libraries, we have taken
-care to strip the application of any telemetry or tracking features.
-
----
-
-### Internet access
-
-Only the setup wizard needs access to the internet in order to download the required models.
-
-For more information see the [Darklock](https://github.com/capsize-games/darklock)  and
-[Facehuggershield](https://github.com/capsize-games/facehuggershield) libraries.
-
----
-
-### Disc access
-
-Write access for the transformers library has been disabled, preventing it from creating a huggingface 
-cache directory at runtime.
-
-The application itself may still access the disc for reading and writing, however we have restricted
-reads and writes to the user provided `airunner` directory (by default this is located at `~/.local/share/airunner`).
-
-All other attempts to access the disc are blocked and logged for your review.
-
-For more information see `src/security/restrict_os_access.py`.
-
----
-
-### Huggingface Hub
-
-**_Huggingface Hub  contains telemetry and tracking features that have been completely disabled in AI Runner._**
-
-The security measures taken for this library are as follows
-
-- Prevented from accessing the internet
-- Prevented from accessing the disc
-- All environment variables set for maximum security
-- All telemetry disabled
-
-See [Facehuggershield](https://github.com/capsize-games/facehuggershield) for more information.
+```

setup.py

@@ -54,9 +54,6 @@
         "inflect==7.3.1",
         "pycountry==24.6.1",
 
-        # Security
-        "facehuggershield==0.1.11",
-
         # Llama index
         "llama-index==0.11.7",
         "llama-index-readers-file==0.2.0",

src/airunner/alembic/env.py

@@ -10,7 +10,6 @@
 config.set_main_option("sqlalchemy.url", f"sqlite:///{db_path}")
 
 # check if db file exists
-print("*"*100)
 if not os.path.exists(db_path):
     print(f"Database file not found at {db_path}")
 else:

src/airunner/app.py

@@ -66,44 +66,10 @@ def __init__(
 
         self.register(SignalCode.LOG_LOGGED_SIGNAL, self.on_log_logged_signal)
 
-        self.create_paths()
         self.start()
         self.run_setup_wizard()
         self.run()
 
-    def create_paths(self):
-        self.logger.debug("Creating paths")
-        art_path = os.path.expanduser((
-            os.path.join(
-                self.path_settings.base_path,
-                "art",
-            )
-        ))
-        models_path = os.path.expanduser((
-            os.path.join(
-                art_path,
-                "models",
-            )
-        ))
-        images_path = os.path.expanduser((
-            os.path.join(
-                art_path,
-                "other",
-                "images"
-            )
-        ))
-        versions = self.session.query(distinct(AIModels.version)).filter(AIModels.category == 'stablediffusion').all()
-        for version in versions:
-            os.makedirs(
-                os.path.join(models_path, version[0], "embeddings"),
-                exist_ok=True
-            )
-            os.makedirs(
-                os.path.join(models_path, version[0], "lora"),
-                exist_ok=True
-            )
-        os.makedirs(images_path, exist_ok=True)
-
     def run_setup_wizard(self):
         application_settings = self.session.query(ApplicationSettings).first()
         if application_settings.run_setup_wizard:

src/airunner/enums.py

@@ -162,6 +162,7 @@ class SignalCode(Enum):
     DRAW_GRID_SIGNAL = "draw_grid_signal"
     SET_SCENE_RECT_SIGNAL = "set_scene_rect_signal"
     DOWNLOAD_COMPLETE = "scan_for_models"
+    PATH_SET = "path_set"
     SD_IMAGE_GENERATE_REQUEST_SIGNAL = "image_generate_request_signal"
     SD_PROGRESS_SIGNAL = "progress_signal"
     SD_REQUEST_SIGNAL = "sd_request_signal"

src/airunner/handlers/stablediffusion/download_worker.py

@@ -5,7 +5,7 @@
 from PySide6.QtCore import QObject, Signal
 from airunner.enums import SignalCode
 from airunner.mediator_mixin import MediatorMixin
-from facehuggershield.huggingface.settings import DEFAULT_HF_ENDPOINT
+DEFAULT_HF_ENDPOINT = "https://huggingface.co"
 from airunner.windows.main.settings_mixin import SettingsMixin
 
 

src/airunner/installer.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 
-import facehuggershield
-facehuggershield.huggingface.activate(
-    show_stdout=True,
-    darklock_os_whitelisted_operations=[
-        "makedirs"
-    ]
-)
+# import facehuggershield
+# facehuggershield.huggingface.activate(
+#     show_stdout=True,
+#     darklock_os_whitelisted_operations=[
+#         "makedirs"
+#     ]
+# )
 from airunner.app_installer import AppInstaller
 
 

src/airunner/main.py

@@ -10,18 +10,18 @@
 # Importing this module sets the Hugging Face environment
 # variables for the application.
 ################################################################
-import facehuggershield
+# import facehuggershield
 from airunner.settings import NLTK_DOWNLOAD_DIR
 import os
 base_path = os.path.join(os.path.expanduser("~"), ".local", "share", "airunner")
-facehuggershield.huggingface.activate(
-    show_stdout=True,
-    darklock_os_whitelisted_directories=[
-        base_path,
-        NLTK_DOWNLOAD_DIR,
-        "/tmp"
-    ]
-)
+# facehuggershield.huggingface.activate(
+#     show_stdout=True,
+#     darklock_os_whitelisted_directories=[
+#         base_path,
+#         NLTK_DOWNLOAD_DIR,
+#         "/tmp"
+#     ]
+# )
 
 ################################################################
 # Set the environment variable for PyTorch to use expandable
@@ -64,7 +64,7 @@ def main():
     engine = create_engine("sqlite:///" + os.path.join(base_dir, "airunner.db"))
     session = scoped_session(sessionmaker(bind=engine))
     application_settings = session.query(ApplicationSettings).first()
-    if application_settings.run_setup_wizard:
+    if application_settings.run_setup_wizard or True:
         from airunner.app_installer import AppInstaller
         AppInstaller()
     else:

src/airunner/settings.py

@@ -114,42 +114,48 @@
 )
 BASE_PATH = "~/.local/share/airunner"
 DEFAULT_PATH_SETTINGS = {
-    "base_path": BASE_PATH,
     "documents_path": os.path.expanduser(
         os.path.join(
-            BASE_PATH,
             "text/other",
             "documents"
         )
     ),
     "ebook_path": os.path.expanduser(
         os.path.join(
-            BASE_PATH,
             "text/other",
             "ebooks"
         )
     ),
     "image_path": os.path.expanduser(
         os.path.join(
-            BASE_PATH,
             "art/other",
             "images"
         )
     ),
     "llama_index_path": os.path.expanduser(
         os.path.join(
-            BASE_PATH,
             "text/rag",
             "db"
         )
     ),
     "webpages_path": os.path.expanduser(
         os.path.join(
-            BASE_PATH,
             "text/other",
             "webpages"
         )
     ),
+    "stt_model_path": os.path.expanduser(
+        os.path.join(
+            "text/models/stt",
+            "models"
+        )
+    ),
+    "tts_model_path": os.path.expanduser(
+        os.path.join(
+            "text/models/tts",
+            "models"
+        )
+    ),
 }
 MALE = Gender.MALE
 FEMALE = Gender.FEMALE

src/airunner/utils/os/create_airunner_directory.py

@@ -26,6 +26,8 @@ def create_airunner_paths(path_settings: PathSettings):
         "image_path",
         "llama_index_path",
         "webpages_path",
+        "stt_model_path",
+        "tts_model_path",
     ):
         path = getattr(path_settings, attr)
         # Path sanitization

src/airunner/windows/download_wizard/download_wizard_window.py

@@ -29,6 +29,8 @@ def __init__(self):
             QWizard.WizardButton.FinishButton
         ).clicked.connect(self.save_settings)
 
+        self.button(QWizard.WizardButton.NextButton).clicked.connect(self.next_button_clicked)
+
         self.init_pages()
 
     def save_settings(self):
@@ -40,6 +42,16 @@ def save_settings(self):
         self.update_application_settings("run_setup_wizard", False)
         self.update_application_settings("download_wizard_completed", True)
 
+    def next_button_clicked(self):
+        current_page = self.currentPage()
+        print("NEXT BUTTON CLICKED", current_page)
+        if hasattr(current_page, 'next'):
+            print("PAGE HAS NEXT")
+            current_page.next()
+        if hasattr(current_page, "start"):
+            print("PAGE HAS RUN")
+            current_page.start()
+
     def init_pages(self):
         """
         Initialize the wizard pages based on setup settings.
@@ -51,8 +63,6 @@ def init_pages(self):
             self.application_settings.stable_diffusion_agreement_checked and
             self.application_settings.airunner_agreement_checked
         ):
-            create_airunner_paths(self.path_settings)
-
             self.setPage(0, PathSettings(self))
             self.setPage(1, InstallPage(self))
             self.setPage(2, InstallSuccessPage(self))

src/airunner/windows/setup_wizard/base_wizard.py

@@ -13,8 +13,8 @@ class BaseWizard(
 
     def __init__(self, parent: QWizard):
         MediatorMixin.__init__(self)
-
         super(BaseWizard, self).__init__()
+
         if self.class_name_:
             self.ui = self.class_name_()
             self.ui.setupUi(self)

src/airunner/windows/setup_wizard/installation_settings/install_page.py

@@ -17,6 +17,8 @@
 from airunner.windows.main.settings_mixin import SettingsMixin
 from airunner.windows.setup_wizard.base_wizard import BaseWizard
 from airunner.windows.setup_wizard.installation_settings.templates.install_page_ui import Ui_install_page
+from airunner.settings import DEFAULT_PATH_SETTINGS
+from airunner.utils.os.create_airunner_directory import create_airunner_paths
 
 nltk.data.path.append(NLTK_DOWNLOAD_DIR)
 
@@ -72,6 +74,7 @@ def __init__(self, parent):
             lambda: self.file_download_finished.emit()
         )
         self.register(SignalCode.DOWNLOAD_COMPLETE, self.download_finished)
+        self.register(SignalCode.PATH_SET, self.path_set)
 
     def download_stable_diffusion(self):
         self.parent.on_set_downloading_status_label({
@@ -328,6 +331,10 @@ def download_finished(self):
         self.total_models_in_current_step -= 1
         if self.total_models_in_current_step <= 0:
             self.set_page()
+
+    @Slot()
+    def path_set(self):
+        self.set_page()
 
     def run(self):
         if (
@@ -343,48 +350,50 @@ def set_page(self):
             self.application_settings.stable_diffusion_agreement_checked and
             self.current_step == -1
         ):
+            """
+            Create the airunner paths
+            """
+            create_airunner_paths(self.path_settings)
+            self.update_application_settings("paths_initialized", True)
             self.parent.on_set_downloading_status_label({
                 "label": f"Downloading Stable Diffusion"
             })
-            self.current_step = 0
+            self.current_step = 1
             self.download_stable_diffusion()
         elif (
             self.application_settings.stable_diffusion_agreement_checked and
-            self.current_step == 0
+            self.current_step == 1
         ):
             self.parent.on_set_downloading_status_label({
                 "label": f"Downloading Controlnet"
             })
-            self.current_step = 1
+            self.current_step = 2
             self.download_controlnet()
         elif (
             self.application_settings.stable_diffusion_agreement_checked and
-            self.current_step < 2
+            self.current_step == 2
         ):
-            self.current_step = 2
+            self.current_step = 3
             self.download_controlnet_processors()
-        elif self.current_step < 3:
+        elif self.current_step == 3:
             self.parent.on_set_downloading_status_label({
                 "label": f"Downloading LLM"
             })
-            self.current_step = 3
+            self.current_step = 4
             self.download_llms()
-        elif self.current_step < 4:
+        elif self.current_step == 4:
             self.parent.on_set_downloading_status_label({
                 "label": f"Downloading Text-to-Speech"
             })
-            self.current_step = 4
+            self.current_step = 5
             self.download_tts()
-        elif self.current_step < 5:
+        elif self.current_step == 5:
             self.parent.on_set_downloading_status_label({
                 "label": f"Downloading Speech-to-Text"
             })
-            self.current_step = 5
-            self.download_stt()
-        elif self.current_step < 6:
             self.current_step = 6
             self.download_nltk_files()
-        else:
+        elif self.current_step == 6:
              self.hf_downloader.download_model(
                  requested_path="",
                  requested_file_name="",
@@ -430,6 +439,8 @@ def __init__(self, parent):
         self.thread = QThread()
         self.worker = InstallWorker(self)
         self.worker.moveToThread(self.thread)
+
+    def start(self):
         self.thread.started.connect(self.worker.run)
         self.thread.start()