From 7d146f254ff2aac9e0bdf40afa81d6cd6cc2e9c1 Mon Sep 17 00:00:00 2001 From: chrisoro Date: Tue, 22 Oct 2024 16:46:22 +0200 Subject: [PATCH] add missing exc handling --- README.md | 75 ++++++--- src/__init__.py | 2 +- src/config/__init__.py | 2 + src/config/models.py | 36 +++-- src/dataloader.py | 4 +- src/item/data/item_type.py | 28 +++- src/item/descr/find_aspect.py | 4 +- src/item/descr/read_descr_tts.py | 255 ++++++++++++++++++++++--------- src/main.py | 4 +- src/overlay.py | 10 +- src/scripts/common.py | 37 +++++ src/{ => scripts}/loot_filter.py | 47 ++---- src/scripts/loot_filter_tts.py | 139 +++++++++++++++++ src/scripts/vision_mode.py | 59 ++++--- src/tts.py | 80 +++++----- 15 files changed, 563 insertions(+), 219 deletions(-) create mode 100644 src/scripts/common.py rename src/{ => scripts}/loot_filter.py (87%) create mode 100644 src/scripts/loot_filter_tts.py diff --git a/README.md b/README.md index 379ffc75..fe0f21eb 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,37 @@ feature request or issue reports join the [discord](https://discord.gg/YyzaPhAN6 and it should work. If you don't want to run it as admin, you can disable the mouse control in the params.ini by setting `vision_mode_only` to `true`. +### TTS + +D4 uses a third-party TTS engine called Tolk. Tolk has a feature that allows custom third-party TTS DLLs to be loaded. +D4 automatically loads the DLL, which actually just sends the text to another application rather than reading it aloud. +This is similar to having a Braille TTS application for D4. + +To use TTS, you need to: + +- Copy `saapi64.dll` to your D4 directory +- Enable `Use Screen Reader` and `3rd Party Screen Reader` in D4 Accesibility settings +- Set `use_tts` in your `params.ini` to either `full` or `mixed` (or via the [GUI](#GUI)) + +#### Restrictions + +Currently, `use_tts` enables either a mixed mode where image processing is still used for item and affix position detection, +but TTS is used for everything text-related. This results in a small improvement in performance and a major improvement +in accuracy. Or a full mode where only TTS is used. + +**The following is currently supported using use_tts=mixed:** + +- Full item detection for all wearable items, e.g. armor, weapons, and accessories. Both in `vision_mode` and +`loot_filter`. +- Basic item detection for all? other items, e.g. only type + rarity + +**The following is currently supported using use_tts=mixed:** + +- Full item detection for all wearable items, e.g. armor, weapons, and accessories. Only in `loot_filter`. +- For everything else, mixed mode is used + +We might also discontinue the pure image processing mode in the future, as TTS is easier to maintain. + ### Configs The config folder in `C:/Users//.d4lf` contains: @@ -75,17 +106,18 @@ The config folder in `C:/Users//.d4lf` contains: | [general] | Description | |---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | profiles | A set of profiles separated by comma. d4lf will look for these yaml files in config/profiles and in C:/Users/WINDOWS_USER/.d4lf/profiles | -| keep_aspects | - `all`: Keep all legendary items
- `upgrade`: Keep all legendary items that upgrade your codex of power
- `none`: Keep no legendary items based on aspect (they are still filtered!) | +| browser | Which browser to use to get builds, please make sure you pick an installed browser: chrome, edge or firefox are currently supported | +| check_chest_tabs | Which chest tabs will be checked and filtered for items in case chest is open when starting the filter. You need to buy all slots. Counting is done left to right. E.g. 1,2,4 will check tab 1, tab 2, tab 4 | +| full_dump | When using the import build feature, whether to use the full dump (e.g. contains all filter items) or not | | handle_rares | - `filter`: Filter them based on your profiles
- `ignore`: Ignores all rares, vision mode shows them as blue and auto mode never junks or favorites them
- `junk`: Vision mode shows them always as red, auto mode always junks rares | | handle_uniques | How to handle uniques that do not match any filter. This property does not apply to filtered uniques. All mythics are favorited regardless of filter.
- `favorite`: Mark the unique as favorite and vision mode will show it as green (default)
- `ignore`: Do nothing with the unique and vision mode will show it as green
- `junk`: Mark any uniques that don't match any filters as junk and show as red in vision mode | -| run_vision_mode_on_startup | If the vision mode should automatically start when starting d4lf. Otherwise has to be started manually with the vision button or the hotkey | -| check_chest_tabs | Which chest tabs will be checked and filtered for items in case chest is open when starting the filter. You need to buy all slots. Counting is done left to right. E.g. 1,2,4 will check tab 1, tab 2, tab 4 | -| move_to_inv_item_type
move_to_stash_item_type | Which types of items to move when using fast move functionality. Will only affect tabs defined in check_chest_tabs. You can select more than one option.
- `favorites`: Move favorites only
- `junk`: Move junk only
- `unmarked`: Only items not marked as favorite or junk
- `everything`: Move everything | +| hidden_transparency | The overlay will become transparent after not hovering it for a while. This can be changed by specifying any value between [0, 1] with 0 being completely invisible and 1 completely visible | +| keep_aspects | - `all`: Keep all legendary items
- `upgrade`: Keep all legendary items that upgrade your codex of power
- `none`: Keep no legendary items based on aspect (they are still filtered!) | | mark_as_favorite | Whether to favorite matched items or not. Defaults to true | | minimum_overlay_font_size | The minimum font size for the vision overlay, specifically the green text that shows which filter(s) are matching. Note: For small profile names, the font may actually be larger than this size but will never go below this size. | -| hidden_transparency | The overlay will become transparent after not hovering it for a while. This can be changed by specifying any value between [0, 1] with 0 being completely invisible and 1 completely visible | -| browser | Which browser to use to get builds, please make sure you pick an installed browser: chrome, edge or firefox are currently supported | -| full_dump | When using the import build feature, whether to use the full dump (e.g. contains all filter items) or not | +| move_to_inv_item_type
move_to_stash_item_type | Which types of items to move when using fast move functionality. Will only affect tabs defined in check_chest_tabs. You can select more than one option.
- `favorites`: Move favorites only
- `junk`: Move junk only
- `unmarked`: Only items not marked as favorite or junk
- `everything`: Move everything | +| run_vision_mode_on_startup | If the vision mode should automatically start when starting d4lf. Otherwise has to be started manually with the vision button or the hotkey | +| use_tts | use TTS instead of OCR, see [TTS](#TTS) | | [char] | Description | |-----------|-----------------------------------| @@ -423,45 +455,46 @@ in [assets/lang/enUS/uniques.json](assets/lang/enUS/uniques.json). ### Python Setup -- You can use [miniconda](https://docs.conda.io/projects/miniconda/en/latest/) or just plain python. +- You can use plain python or something like [miniconda](https://docs.conda.io/projects/miniconda/en/latest/). -Conda setup: +Python setup (windows, linux venv activation differs): ```bash git clone https://github.com/aeon0/d4lf cd d4lf -conda env create -f environment.yml -conda activate d4lf +python -m venv venv +venv\Scripts\activate +python -m pip install -r requirements.txt python -m src.main ``` -Python setup (windows, linux venv activation differs): +Conda setup: ```bash git clone https://github.com/aeon0/d4lf cd d4lf -python -m venv venv -venv\Scripts\activate -python -m pip install -r requirements.txt +conda env create -f environment.yml +conda activate d4lf python -m src.main ``` ### Formatting & Linting -Ruff is used for linting and auto formatting. You can run it with: +Just use pre-commit. ```bash -ruff format +pre-commit install ``` +or directly via + ```bash -ruff check +pre-commit run -a ``` -Setup VS Code by using the ruff extension. Also turn on "trim trailing whitespaces" is VS Code settings. - ## Credits - Icon based of: [CarbotAnimations](https://www.youtube.com/carbotanimations/about) -- Some of the OCR code is originally from [@gleed](https://github.com/aliig). Good guy. +- Some of the OCR code is originally from [@gleed](https://github.com/aliig). Good guy - Names and textures for matching from [Blizzard](https://www.blizzard.com) +- Thanks to NekrosStratia for the initial idea and help with TTS mode diff --git a/src/__init__.py b/src/__init__.py index 2fe88cb4..85ff7ded 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -2,4 +2,4 @@ TP = concurrent.futures.ThreadPoolExecutor() -__version__ = "5.9.0alpha1" +__version__ = "5.9.0alpha2" diff --git a/src/config/__init__.py b/src/config/__init__.py index 2fa80ba9..936a1505 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -8,4 +8,6 @@ def get_base_dir(bundled: bool = False) -> Path: return Path(__file__).parent.parent.parent +AFFIX_COMPARISON_CHARS = 60 + BASE_DIR = get_base_dir(False) diff --git a/src/config/models.py b/src/config/models.py index a9367bd0..26f99814 100644 --- a/src/config/models.py +++ b/src/config/models.py @@ -29,17 +29,21 @@ class AspectFilterType(enum.StrEnum): upgrade = enum.auto() +class ComparisonType(enum.StrEnum): + larger = enum.auto() + smaller = enum.auto() + + class HandleRaresType(enum.StrEnum): filter = enum.auto() ignore = enum.auto() junk = enum.auto() -class MoveItemsType(enum.StrEnum): - everything = enum.auto() - favorites = enum.auto() - junk = enum.auto() - unmarked = enum.auto() +class ItemRefreshType(enum.StrEnum): + force_with_filter = enum.auto() + force_without_filter = enum.auto() + no_refresh = enum.auto() class LogLevels(enum.StrEnum): @@ -50,21 +54,23 @@ class LogLevels(enum.StrEnum): critical = enum.auto() +class MoveItemsType(enum.StrEnum): + everything = enum.auto() + favorites = enum.auto() + junk = enum.auto() + unmarked = enum.auto() + + class UnfilteredUniquesType(enum.StrEnum): favorite = enum.auto() ignore = enum.auto() junk = enum.auto() -class ComparisonType(enum.StrEnum): - larger = enum.auto() - smaller = enum.auto() - - -class ItemRefreshType(enum.StrEnum): - force_with_filter = enum.auto() - force_without_filter = enum.auto() - no_refresh = enum.auto() +class UseTTSType(enum.StrEnum): + full = enum.auto() + mixed = enum.auto() + off = enum.auto() class _IniBaseModel(BaseModel): @@ -279,7 +285,7 @@ class GeneralModel(_IniBaseModel): "C:/Users/USERNAME/.d4lf/profiles/*.yaml", ) run_vision_mode_on_startup: bool = Field(default=True, description="Whether to run vision mode on startup or not") - use_tts: bool = Field(default=False, description="Whether to use tts or not") + use_tts: UseTTSType = Field(default=UseTTSType.off, description="Whether to use tts or not") @field_validator("check_chest_tabs", mode="before") def check_chest_tabs_index(cls, v: str) -> list[int]: diff --git a/src/dataloader.py b/src/dataloader.py index df1aeb75..873e6223 100644 --- a/src/dataloader.py +++ b/src/dataloader.py @@ -2,7 +2,7 @@ import logging import threading -from src.config import BASE_DIR +from src.config import AFFIX_COMPARISON_CHARS, BASE_DIR from src.config.loader import IniConfigLoader from src.item.data.item_type import ItemType @@ -67,5 +67,5 @@ def load_data(self): with open(BASE_DIR / f"assets/lang/{IniConfigLoader().general.language}/uniques.json", encoding="utf-8") as f: data = json.load(f) for key, d in data.items(): - self.aspect_unique_dict[key] = d["desc"][:60] + self.aspect_unique_dict[key] = d["desc"][:AFFIX_COMPARISON_CHARS] self.aspect_unique_num_idx[key] = d["num_idx"] diff --git a/src/item/data/item_type.py b/src/item/data/item_type.py index 292ffa7b..8aa4777d 100644 --- a/src/item/data/item_type.py +++ b/src/item/data/item_type.py @@ -33,11 +33,14 @@ class ItemType(Enum): Wand = "wand" # Custom Types Compass = "compass" + Consumable = "consumable" + Gem = "gem" Incense = "incense" Material = "material" + Rune = "rune" Sigil = "nightmare sigil" - Tribute = "Tribute" TemperManual = "temper manual" + Tribute = "tribute" def is_armor(item_type: ItemType) -> bool: @@ -50,6 +53,22 @@ def is_armor(item_type: ItemType) -> bool: ] +def is_consumable(item_type: ItemType) -> bool: + return item_type in [ + ItemType.Consumable, + ItemType.Elixir, + ItemType.Incense, + ItemType.TemperManual, + ] + + +def is_mapping(item_type: ItemType) -> bool: + return item_type in [ + ItemType.Compass, + ItemType.Sigil, + ] + + def is_jewelry(item_type: ItemType) -> bool: return item_type in [ ItemType.Amulet, @@ -57,6 +76,13 @@ def is_jewelry(item_type: ItemType) -> bool: ] +def is_socketable(item_type: ItemType) -> bool: + return item_type in [ + ItemType.Gem, + ItemType.Rune, + ] + + def is_weapon(item_type: ItemType) -> bool: return item_type in [ ItemType.Axe, diff --git a/src/item/descr/find_aspect.py b/src/item/descr/find_aspect.py index 7b49e8db..7e1fbe86 100644 --- a/src/item/descr/find_aspect.py +++ b/src/item/descr/find_aspect.py @@ -2,6 +2,7 @@ import numpy as np +from src.config import AFFIX_COMPARISON_CHARS from src.dataloader import Dataloader from src.item.data.aspect import Aspect from src.item.descr.text import clean_str, closest_match, find_number @@ -22,8 +23,7 @@ def find_aspect(img_item_descr: np.ndarray, aspect_bullet: TemplateMatch, do_pre concatenated_str = image_to_text(img_full_aspect, do_pre_proc=do_pre_proc).text.lower().replace("\n", " ") cleaned_str = clean_str(concatenated_str) - # Note: If you adjust the [:45] it also needs to be adapted in the dataloader - cleaned_str = cleaned_str[:60] + cleaned_str = cleaned_str[:AFFIX_COMPARISON_CHARS] found_key = closest_match(cleaned_str, Dataloader().aspect_unique_dict) num_idx = Dataloader().aspect_unique_num_idx diff --git a/src/item/descr/read_descr_tts.py b/src/item/descr/read_descr_tts.py index a88b0e67..c528158f 100644 --- a/src/item/descr/read_descr_tts.py +++ b/src/item/descr/read_descr_tts.py @@ -6,87 +6,30 @@ import src.tts from src import TP +from src.config import AFFIX_COMPARISON_CHARS from src.dataloader import Dataloader from src.item.data.affix import Affix, AffixType from src.item.data.aspect import Aspect -from src.item.data.item_type import ItemType, is_armor, is_jewelry, is_weapon +from src.item.data.item_type import ItemType, is_armor, is_consumable, is_jewelry, is_mapping, is_socketable, is_weapon from src.item.data.rarity import ItemRarity from src.item.descr.text import clean_str, closest_match, find_number from src.item.descr.texture import find_affix_bullets, find_seperator_short, find_seperators_long from src.item.models import Item +from src.template_finder import TemplateMatch from src.utils.window import screenshot LOGGER = logging.getLogger(__name__) -def create_item_from_tts(tts_item: list[str]) -> Item | None: - if tts_item[0] == "Compass": - return Item(rarity=ItemRarity.Common, item_type=ItemType.Compass) - if tts_item[0] == "Nightmare Sigil": - return Item(rarity=ItemRarity.Common, item_type=ItemType.Sigil) - - if any(word in tts_item[0] for word in ["Tribute of"]): - search_string = tts_item[0].lower() - else: - search_string = tts_item[1].lower().replace("ancestral", "").strip() - item = Item() - search_string_split = search_string.split(" ") - res = rapidfuzz.process.extractOne( - search_string_split[0], [rar.value for rar in ItemRarity], scorer=rapidfuzz.distance.Levenshtein.distance - ) - try: - item.rarity = ItemRarity(res[0]) if res else None - except ValueError: - return None - res = rapidfuzz.process.extractOne( - " ".join(search_string_split[1:]), [it.value for it in ItemType], scorer=rapidfuzz.distance.Levenshtein.distance - ) - try: - item.item_type = ItemType(res[0]) if res else None - except ValueError: - return None - for _i, line in enumerate(tts_item): - if "item power" in line.lower(): - item.power = find_number(line) - break - return item - - -def get_affixes_from_tts_section(tts_section: list[str], item: Item, length: int): - if item.rarity in [ItemRarity.Mythic, ItemRarity.Unique]: - length += 1 - dps = None - item_power = None - masterwork = None - start = 0 - for i, line in enumerate(tts_section): - if "masterwork" in line.lower(): - masterwork = i - if "item power" in line.lower(): - item_power = i - if "damage per second" in line.lower(): - dps = i - base_value = masterwork if masterwork else item_power - if is_weapon(item.item_type): - start = dps + 2 - elif is_jewelry(item.item_type): - start = base_value - elif is_armor(item.item_type): - start = base_value + 1 - elif item.item_type == ItemType.Shield: - start = base_value + 3 - start += 1 - return tts_section[start : start + length] - - -def add_affixes_from_tts(tts_section, item, inherent_affix_bullets, affix_bullets): - affixes = get_affixes_from_tts_section( +def _add_affixes_from_tts_mixed( + tts_section: list[str], item: Item, inherent_affix_bullets: list[TemplateMatch], affix_bullets: list[TemplateMatch] +) -> Item: + affixes = _get_affixes_from_tts_section( tts_section, item, len(inherent_affix_bullets) + len([x for x in affix_bullets if any(x.name.startswith(s) for s in ["affix", "greater_affix", "rerolled"])]), ) - # the first len(inherent_affix_bullets) are inherent affixes for i, affix in enumerate(affixes): if i < len(inherent_affix_bullets): name = rapidfuzz.process.extractOne( @@ -121,7 +64,7 @@ def add_affixes_from_tts(tts_section, item, inherent_affix_bullets, affix_bullet ) ) else: - name = closest_match(clean_str(affix), Dataloader().aspect_unique_dict) + name = closest_match(clean_str(affix)[:AFFIX_COMPARISON_CHARS], Dataloader().aspect_unique_dict) item.aspect = Aspect( name=name, loc=affix_bullets[i - len(inherent_affix_bullets) - len(affix_bullets)].center, @@ -131,23 +74,165 @@ def add_affixes_from_tts(tts_section, item, inherent_affix_bullets, affix_bullet return item -def read_descr(img_item_descr: np.ndarray) -> Item | None: - tts_section = copy.copy(src.tts.LAST_ITEM_SECTION) +def _add_affixes_from_tts(tts_section: list[str], item: Item) -> Item: + inherent_num = 0 + affixes_num = 3 if item.rarity == ItemRarity.Legendary else 4 + if is_weapon(item.item_type) or item.item_type in [ItemType.Amulet, ItemType.Boots]: + inherent_num = 1 + elif item.item_type in [ItemType.Ring]: + inherent_num = 2 + elif item.item_type in [ItemType.Shield]: + inherent_num = 4 + affixes = _get_affixes_from_tts_section(tts_section, item, inherent_num + affixes_num) + for i, affix in enumerate(affixes): + if i < inherent_num: + name = rapidfuzz.process.extractOne( + clean_str(affix), list(Dataloader().affix_dict), scorer=rapidfuzz.distance.Levenshtein.distance + ) + item.inherent.append( + Affix( + name=name[0], + text=affix, + type=AffixType.inherent, + value=find_number(affix), + ) + ) + elif i < inherent_num + affixes_num: + name = rapidfuzz.process.extractOne( + clean_str(affix), list(Dataloader().affix_dict), scorer=rapidfuzz.distance.Levenshtein.distance + ) + item.affixes.append( + Affix( + name=name[0], + text=affix, + type=AffixType.normal, + value=find_number(affix), + ) + ) + else: + name = closest_match(clean_str(affix)[:AFFIX_COMPARISON_CHARS], Dataloader().aspect_unique_dict) + item.aspect = Aspect( + name=name, + text=affix, + value=find_number(affix), + ) + return item + + +def _create_base_item_from_tts(tts_item: list[str]) -> Item | None: + if tts_item[0].startswith(src.tts.ItemIdentifiers.COMPASS.value): + return Item(rarity=ItemRarity.Common, item_type=ItemType.Compass) + if tts_item[0].startswith(src.tts.ItemIdentifiers.NIGHTMARE_SIGIL.value): + return Item(rarity=ItemRarity.Common, item_type=ItemType.Sigil) + if tts_item[0].startswith(src.tts.ItemIdentifiers.TRIBUTE.value): + item = Item(item_type=ItemType.Tribute) + search_string_split = tts_item[1].split(" ") + item.rarity = _get_item_rarity(search_string_split[0]) + return item + if tts_item[0].startswith(src.tts.ItemIdentifiers.WHISPERING_KEY.value): + return Item(item_type=ItemType.Consumable) + if any(tts_item[1].lower().endswith(x) for x in ["summoning"]): + return Item(item_type=ItemType.Material) + if any(tts_item[1].lower().endswith(x) for x in ["gem"]): + return Item(item_type=ItemType.Gem) + if "rune of" in tts_item[1].lower(): + item = Item(item_type=ItemType.Rune) + search_string_split = tts_item[1].lower().split(" rune of ") + item.rarity = _get_item_rarity(search_string_split[0]) + return item + item = Item() + if tts_item[1].lower().endswith("elixir"): + item.item_type = ItemType.Elixir + elif tts_item[1].lower().endswith("incense"): + item.item_type = ItemType.Incense + elif any(tts_item[1].lower().endswith(x) for x in ["consumable", "scroll"]): + item.item_type = ItemType.Consumable + if is_consumable(item.item_type): + search_string_split = tts_item[1].split(" ") + item.rarity = _get_item_rarity(search_string_split[0]) + return item + + search_string = tts_item[1].lower().replace("ancestral", "").strip() + search_string_split = search_string.split(" ") + item.rarity = _get_item_rarity(search_string_split[0]) + item.item_type = _get_item_type(" ".join(search_string_split[1:])) + for _i, line in enumerate(tts_item): + if "item power" in line.lower(): + item.power = int(find_number(line)) + break + return item + + +def _get_affixes_from_tts_section(tts_section: list[str], item: Item, length: int): + if item.rarity in [ItemRarity.Mythic, ItemRarity.Unique]: + length += 1 + dps = None + item_power = None + masterwork = None + start = 0 + for i, line in enumerate(tts_section): + if "masterwork" in line.lower(): + masterwork = i + if "item power" in line.lower(): + item_power = i + if "damage per second" in line.lower(): + dps = i + base_value = masterwork if masterwork else item_power + if is_weapon(item.item_type): + start = dps + 2 + elif is_jewelry(item.item_type): + start = base_value + elif is_armor(item.item_type): + start = base_value + 1 + elif item.item_type == ItemType.Shield: + start = base_value + 3 + start += 1 + return tts_section[start : start + length] + + +def _get_item_rarity(data: str) -> ItemRarity | None: + res = rapidfuzz.process.extractOne(data, [rar.value for rar in ItemRarity], scorer=rapidfuzz.distance.Levenshtein.distance) + try: + return ItemRarity(res[0]) if res else None + except ValueError: + return None + + +def _get_item_type(data: str): + res = rapidfuzz.process.extractOne(data, [it.value for it in ItemType], scorer=rapidfuzz.distance.Levenshtein.distance) + try: + return ItemType(res[0]) if res else None + except ValueError: + return None + + +def _is_codex_upgrade(tts_section: list[str], item: Item) -> bool: + return any("upgrades an aspect in the codex of power on salvage" in line.lower() for line in tts_section) + + +def read_descr_mixed(img_item_descr: np.ndarray) -> Item | None: + tts_section = copy.copy(src.tts.LAST_ITEM) if not tts_section: return None - if (item := create_item_from_tts(tts_section)) is None: + if (item := _create_base_item_from_tts(tts_section)) is None: return None - if item.item_type in [ItemType.Material, ItemType.TemperManual, ItemType.Elixir, ItemType.Incense] or ( - item.rarity in [ItemRarity.Magic, ItemRarity.Common] and item.item_type != ItemType.Sigil + if any( + [ + is_consumable(item.item_type), + is_mapping(item.item_type), + is_socketable(item.item_type), + item.item_type in [ItemType.Material, ItemType.Tribute], + ] ): return item + if all([not is_armor(item.item_type), not is_jewelry(item.item_type), not is_weapon(item.item_type)]): + return None if (sep_short_match := find_seperator_short(img_item_descr)) is None: LOGGER.warning("Could not detect item_seperator_short.") screenshot("failed_seperator_short", img=img_item_descr) return None - futures = {} - futures["sep_long"] = TP.submit(find_seperators_long, img_item_descr, sep_short_match) + futures = {"sep_long": TP.submit(find_seperators_long, img_item_descr, sep_short_match)} affix_bullets = find_affix_bullets(img_item_descr, sep_short_match) sep_long_match = futures["sep_long"].result() if futures["sep_long"] is not None else None @@ -187,4 +272,28 @@ def read_descr(img_item_descr: np.ndarray) -> Item | None: inherent_affix_bullets = affix_bullets[:number_inherents] affix_bullets = affix_bullets[number_inherents:] - return add_affixes_from_tts(tts_section, item, inherent_affix_bullets, affix_bullets) + item.codex_upgrade = _is_codex_upgrade(tts_section, item) + + return _add_affixes_from_tts_mixed(tts_section, item, inherent_affix_bullets, affix_bullets) + + +def read_description() -> Item | None: + tts_section = copy.copy(src.tts.LAST_ITEM) + if not tts_section: + return None + if (item := _create_base_item_from_tts(tts_section)) is None: + return None + if any( + [ + is_consumable(item.item_type), + is_mapping(item.item_type), + is_socketable(item.item_type), + item.item_type in [ItemType.Material, ItemType.Tribute], + ] + ): + return item + if all([not is_armor(item.item_type), not is_jewelry(item.item_type), not is_weapon(item.item_type)]): + return None + + item.codex_upgrade = _is_codex_upgrade(tts_section, item) + return _add_affixes_from_tts(tts_section, item) diff --git a/src/main.py b/src/main.py index a15a80f4..24d0e342 100644 --- a/src/main.py +++ b/src/main.py @@ -12,7 +12,7 @@ from src import __version__, tts from src.cam import Cam from src.config.loader import IniConfigLoader -from src.config.models import ItemRefreshType +from src.config.models import ItemRefreshType, UseTTSType from src.gui.qt_gui import start_gui from src.item.filter import Filter from src.logger import LOG_DIR @@ -71,7 +71,7 @@ def main(): ) keyboard.add_hotkey(IniConfigLoader().advanced_options.move_to_inv, lambda: overlay.move_items_to_inventory()) keyboard.add_hotkey(IniConfigLoader().advanced_options.move_to_chest, lambda: overlay.move_items_to_stash()) - if IniConfigLoader().general.use_tts: + if IniConfigLoader().general.use_tts in [UseTTSType.full, UseTTSType.mixed]: tts.start_connection() overlay = Overlay() overlay.run() diff --git a/src/overlay.py b/src/overlay.py index 6eb3deff..e84184d3 100644 --- a/src/overlay.py +++ b/src/overlay.py @@ -4,11 +4,12 @@ import tkinter as tk import typing +import src.scripts.loot_filter +import src.scripts.loot_filter_tts from src.cam import Cam from src.config.loader import IniConfigLoader -from src.config.models import ItemRefreshType +from src.config.models import ItemRefreshType, UseTTSType from src.config.ui import ResManager -from src.loot_filter import run_loot_filter from src.loot_mover import move_items_to_inventory, move_items_to_stash from src.scripts.vision_mode import vision_mode from src.utils.process_handler import kill_thread @@ -146,7 +147,10 @@ def toggle_size(self): move_window_to_foreground(win_spec) def filter_items(self, force_refresh=ItemRefreshType.no_refresh): - self._start_or_stop_loot_interaction_thread(run_loot_filter, (force_refresh,)) + if IniConfigLoader().general.use_tts == UseTTSType.full: + self._start_or_stop_loot_interaction_thread(src.scripts.loot_filter_tts.run_loot_filter, (force_refresh,)) + else: + self._start_or_stop_loot_interaction_thread(src.scripts.loot_filter.run_loot_filter, (force_refresh,)) def move_items_to_inventory(self): self._start_or_stop_loot_interaction_thread(move_items_to_inventory) diff --git a/src/scripts/common.py b/src/scripts/common.py new file mode 100644 index 00000000..e86a44c0 --- /dev/null +++ b/src/scripts/common.py @@ -0,0 +1,37 @@ +import logging +import time + +import keyboard + +from src.cam import Cam +from src.utils.custom_mouse import mouse + +LOGGER = logging.getLogger(__name__) + + +def mark_as_junk(): + keyboard.send("space") + time.sleep(0.13) + + +def mark_as_favorite(): + LOGGER.info("Mark as favorite") + keyboard.send("space") + time.sleep(0.17) + keyboard.send("space") + time.sleep(0.13) + + +def reset_item_status(occupied, inv): + for item_slot in occupied: + if item_slot.is_fav: + inv.hover_item(item_slot) + keyboard.send("space") + if item_slot.is_junk: + inv.hover_item(item_slot) + keyboard.send("space") + time.sleep(0.13) + keyboard.send("space") + + if occupied: + mouse.move(*Cam().abs_window_to_monitor((0, 0))) diff --git a/src/loot_filter.py b/src/scripts/loot_filter.py similarity index 87% rename from src/loot_filter.py rename to src/scripts/loot_filter.py index 2397bba2..2a8961fb 100644 --- a/src/loot_filter.py +++ b/src/scripts/loot_filter.py @@ -11,6 +11,7 @@ from src.item.descr.read_descr import read_descr from src.item.filter import Filter from src.item.find_descr import find_descr +from src.scripts.common import mark_as_favorite, mark_as_junk, reset_item_status from src.ui.char_inventory import CharInventory from src.ui.chest import Chest from src.ui.inventory_base import InventoryBase @@ -52,7 +53,7 @@ def check_items(inv: InventoryBase, force_refresh: ItemRefreshType): screenshot("failed_descr_detection", img=img) break inv.hover_item(item) - time.sleep(0.18) + time.sleep(0.15) img = Cam().grab() start_detect = time.time() found, rarity, cropped_descr, _ = find_descr(img, item.center) @@ -73,7 +74,7 @@ def check_items(inv: InventoryBase, force_refresh: ItemRefreshType): item_descr = read_descr(rarity, cropped_descr, False) if item_descr is None: LOGGER.info("Retry item detection") - time.sleep(0.3) + time.sleep(0.2) found, rarity, cropped_descr, _ = find_descr(Cam().grab(), item.center) if found: item_descr = read_descr(rarity, cropped_descr) @@ -120,50 +121,22 @@ def check_items(inv: InventoryBase, force_refresh: ItemRefreshType): # property if item_descr.rarity == ItemRarity.Unique: if not res.keep: - _mark_as_junk() + mark_as_junk() elif res.keep: if res.all_unique_filters_are_aspects and not res.unique_aspect_in_profile: if IniConfigLoader().general.handle_uniques == UnfilteredUniquesType.favorite: - _mark_as_favorite() + mark_as_favorite() elif IniConfigLoader().general.mark_as_favorite: - _mark_as_favorite() + mark_as_favorite() else: if not res.keep: - _mark_as_junk() + mark_as_junk() elif ( res.keep and (matched_any_affixes or item_descr.rarity == ItemRarity.Mythic or item_descr.item_type == ItemType.Sigil) and IniConfigLoader().general.mark_as_favorite ): - _mark_as_favorite() - - -def _mark_as_junk(): - keyboard.send("space") - time.sleep(0.13) - - -def _mark_as_favorite(): - LOGGER.info("Mark as favorite") - keyboard.send("space") - time.sleep(0.17) - keyboard.send("space") - time.sleep(0.13) - - -def reset_item_status(occupied, inv): - for item_slot in occupied: - if item_slot.is_fav: - inv.hover_item(item_slot) - keyboard.send("space") - if item_slot.is_junk: - inv.hover_item(item_slot) - keyboard.send("space") - time.sleep(0.13) - keyboard.send("space") - - if occupied: - mouse.move(*Cam().abs_window_to_monitor((0, 0))) + mark_as_favorite() def run_loot_filter(force_refresh: ItemRefreshType = ItemRefreshType.no_refresh): @@ -175,10 +148,10 @@ def run_loot_filter(force_refresh: ItemRefreshType = ItemRefreshType.no_refresh) if chest.is_open(): for i in IniConfigLoader().general.check_chest_tabs: chest.switch_to_tab(i) - time.sleep(0.5) + time.sleep(0.4) check_items(chest, force_refresh) mouse.move(*Cam().abs_window_to_monitor((0, 0))) - time.sleep(0.5) + time.sleep(0.4) check_items(inv, force_refresh) else: if not inv.open(): diff --git a/src/scripts/loot_filter_tts.py b/src/scripts/loot_filter_tts.py new file mode 100644 index 00000000..196874d8 --- /dev/null +++ b/src/scripts/loot_filter_tts.py @@ -0,0 +1,139 @@ +import logging +import time + +import keyboard + +import src.item.descr.read_descr_tts +from src.cam import Cam +from src.config.loader import IniConfigLoader +from src.config.models import HandleRaresType, ItemRefreshType, UnfilteredUniquesType +from src.item.data.item_type import ItemType +from src.item.data.rarity import ItemRarity +from src.item.filter import Filter +from src.scripts.common import mark_as_favorite, mark_as_junk, reset_item_status +from src.ui.char_inventory import CharInventory +from src.ui.chest import Chest +from src.ui.inventory_base import InventoryBase +from src.utils.custom_mouse import mouse +from src.utils.window import screenshot + +LOGGER = logging.getLogger(__name__) + + +def check_items(inv: InventoryBase, force_refresh: ItemRefreshType): + occupied, _ = inv.get_item_slots() + + if force_refresh == ItemRefreshType.force_with_filter or force_refresh == ItemRefreshType.force_without_filter: + reset_item_status(occupied, inv) + occupied, _ = inv.get_item_slots() + + if force_refresh == ItemRefreshType.force_without_filter: + return + + num_fav = sum(1 for slot in occupied if slot.is_fav) + num_junk = sum(1 for slot in occupied if slot.is_junk) + LOGGER.info(f"Items: {len(occupied)} (favorite: {num_fav}, junk: {num_junk}) in {inv.menu_name}") + for item in occupied: + if item.is_junk or item.is_fav: + continue + inv.hover_item(item) + time.sleep(0.15) + img = Cam().grab() + item_descr = None + try: + item_descr = src.item.descr.read_descr_tts.read_description() + LOGGER.debug(f"Parsed item based on TTS: {item_descr}") + except Exception: + screenshot("tts_error", img=img) + LOGGER.exception(f"Error in TTS read_descr. {src.tts.LAST_ITEM=}") + if item_descr is None: + LOGGER.info("Retry item detection") + time.sleep(0.3) + try: + item_descr = src.item.descr.read_descr_tts.read_description() + LOGGER.debug(f"Parsed item based on TTS: {item_descr}") + except Exception: + screenshot("tts_error", img=img) + LOGGER.exception(f"Error in TTS read_descr. {src.tts.LAST_ITEM=}") + if item_descr is None: + continue + + # Hardcoded filters + if item_descr.rarity == ItemRarity.Common and item_descr.item_type == ItemType.Material: + LOGGER.info("Matched: Material") + continue + if item_descr.rarity == ItemRarity.Legendary and item_descr.item_type == ItemType.Material: + LOGGER.info("Matched: Extracted Aspect") + continue + if item_descr.item_type == ItemType.Elixir: + LOGGER.info("Matched: Elixir") + continue + if item_descr.item_type == ItemType.Incense: + LOGGER.info("Matched: Incense") + continue + if item_descr.item_type == ItemType.TemperManual: + LOGGER.info("Matched: Temper Manual") + continue + if item_descr.rarity in [ItemRarity.Magic, ItemRarity.Common] and item_descr.item_type != ItemType.Sigil: + keyboard.send("space") + time.sleep(0.13) + continue + if item_descr.rarity == ItemRarity.Rare and IniConfigLoader().general.handle_rares == HandleRaresType.ignore: + LOGGER.info("Matched: Rare, ignore Item") + continue + if item_descr.rarity == ItemRarity.Rare and IniConfigLoader().general.handle_rares == HandleRaresType.junk: + keyboard.send("space") + time.sleep(0.13) + continue + + # Check if we want to keep the item + start_filter = time.time() + res = Filter().should_keep(item_descr) + matched_any_affixes = len(res.matched) > 0 and len(res.matched[0].matched_affixes) > 0 + LOGGER.debug(f" Runtime (Filter): {time.time() - start_filter:.2f}s") + + # Uniques have special handling. If they have an aspect specifically called out by a profile they are treated + # like any other item. If not, and there are no non-aspect filters, then they are handled by the handle_uniques + # property + if item_descr.rarity == ItemRarity.Unique: + if not res.keep: + mark_as_junk() + elif res.keep: + if res.all_unique_filters_are_aspects and not res.unique_aspect_in_profile: + if IniConfigLoader().general.handle_uniques == UnfilteredUniquesType.favorite: + mark_as_favorite() + elif IniConfigLoader().general.mark_as_favorite: + mark_as_favorite() + else: + if not res.keep: + mark_as_junk() + elif ( + res.keep + and (matched_any_affixes or item_descr.rarity == ItemRarity.Mythic or item_descr.item_type == ItemType.Sigil) + and IniConfigLoader().general.mark_as_favorite + ): + mark_as_favorite() + + +def run_loot_filter(force_refresh: ItemRefreshType = ItemRefreshType.no_refresh): + mouse.move(*Cam().abs_window_to_monitor((0, 0))) + LOGGER.info("Run Loot filter") + inv = CharInventory() + chest = Chest() + + if chest.is_open(): + for i in IniConfigLoader().general.check_chest_tabs: + chest.switch_to_tab(i) + time.sleep(0.4) + check_items(chest, force_refresh) + mouse.move(*Cam().abs_window_to_monitor((0, 0))) + time.sleep(0.4) + check_items(inv, force_refresh) + else: + if not inv.open(): + screenshot("inventory_not_open", img=Cam().grab()) + LOGGER.error("Inventory did not open up") + return + check_items(inv, force_refresh) + mouse.move(*Cam().abs_window_to_monitor((0, 0))) + LOGGER.info("Loot Filter done") diff --git a/src/scripts/vision_mode.py b/src/scripts/vision_mode.py index a7547f9c..50df1e02 100644 --- a/src/scripts/vision_mode.py +++ b/src/scripts/vision_mode.py @@ -12,9 +12,9 @@ import src.tts from src.cam import Cam from src.config.loader import IniConfigLoader -from src.config.models import HandleRaresType +from src.config.models import HandleRaresType, UseTTSType from src.config.ui import ResManager -from src.item.data.item_type import ItemType +from src.item.data.item_type import ItemType, is_armor, is_consumable, is_jewelry, is_mapping, is_socketable, is_weapon from src.item.data.rarity import ItemRarity from src.item.descr.read_descr import read_descr from src.item.filter import Filter @@ -24,6 +24,7 @@ from src.utils.custom_mouse import mouse from src.utils.image_operations import compare_histograms, crop from src.utils.ocr.read import image_to_text +from src.utils.window import screenshot LOGGER = logging.getLogger(__name__) @@ -205,15 +206,16 @@ def vision_mode(): root.update() # Check if the item is a match based on our filters - match = True last_top_left_corner = top_left_corner last_center = item_center - if IniConfigLoader().general.use_tts: + item_descr = None + if IniConfigLoader().general.use_tts in [UseTTSType.full, UseTTSType.mixed]: try: - item_descr = src.item.descr.read_descr_tts.read_descr(cropped_descr) + item_descr = src.item.descr.read_descr_tts.read_descr_mixed(cropped_descr) + LOGGER.debug(f"Parsed item based on TTS: {item_descr}") except Exception: screenshot("tts_error", img=cropped_descr) - LOGGER.exception(f"Error in TTS read_descr. {src.tts.LAST_ITEM_SECTION=}") + LOGGER.exception(f"Error in TTS read_descr. {src.tts.LAST_ITEM=}") else: item_descr = read_descr(rarity, cropped_descr, False) if item_descr is None: @@ -223,27 +225,28 @@ def vision_mode(): continue ignored_item = False - if item_descr.item_type == ItemType.Material: - LOGGER.info("Matched: Material") + if is_consumable(item_descr.item_type): + LOGGER.info("Matched: Consumable") + ignored_item = True + if is_mapping(item_descr.item_type): + LOGGER.info("Matched: Mapping") ignored_item = True - elif item_descr.item_type == ItemType.Elixir: - LOGGER.info("Matched: Elixir") + if is_socketable(item_descr.item_type): + LOGGER.info("Matched: Socketable") ignored_item = True - elif item_descr.item_type == ItemType.Incense: - LOGGER.info("Matched: Incense") + elif item_descr.item_type == ItemType.Tribute: + LOGGER.info("Matched: Tribute") ignored_item = True - elif item_descr.item_type == ItemType.TemperManual: - LOGGER.info("Matched: Temper Manual") + elif item_descr.item_type == ItemType.Material: + LOGGER.info("Matched: Material") ignored_item = True - elif rarity in [ItemRarity.Magic, ItemRarity.Common] and item_descr.item_type != ItemType.Sigil: - match = False - item_descr = None - elif rarity == ItemRarity.Rare and IniConfigLoader().general.handle_rares == HandleRaresType.ignore: + if ( + rarity == ItemRarity.Rare + and (is_armor(item_descr.item_type) or is_weapon(item_descr.item_type) or is_jewelry(item_descr.item_type)) + and IniConfigLoader().general.handle_rares in [HandleRaresType.ignore, HandleRaresType.junk] + ): LOGGER.info("Matched: Rare, ignore Item") ignored_item = True - elif rarity == ItemRarity.Rare and IniConfigLoader().general.handle_rares == HandleRaresType.junk: - match = False - item_descr = None if ignored_item: create_signal_rect(canvas, w, thick, "#00b3b3") @@ -251,9 +254,15 @@ def vision_mode(): root.update() continue - if item_descr is not None: - res = Filter().should_keep(item_descr) - match = res.keep + if item_descr is None: + LOGGER.info("Unknown Item") + create_signal_rect(canvas, w, thick, "#ce7e00") + root.update_idletasks() + root.update() + continue + + res = Filter().should_keep(item_descr) + match = res.keep # Adapt colors based on config if match: @@ -290,7 +299,7 @@ def vision_mode(): if __name__ == "__main__": try: - from src.utils.window import WindowSpec, screenshot, start_detecting_window + from src.utils.window import WindowSpec, start_detecting_window src.logger.setup() win_spec = WindowSpec(IniConfigLoader().advanced_options.process_name) diff --git a/src/tts.py b/src/tts.py index 527cf9f0..9fd0e90b 100644 --- a/src/tts.py +++ b/src/tts.py @@ -1,29 +1,44 @@ +import enum import logging import queue +import re import threading import win32file import win32pipe -LAST_ITEM_SECTION = [] +LAST_ITEM = [] LOGGER = logging.getLogger(__name__) _DATA_QUEUE = queue.Queue(maxsize=100) -_PIPE_NAME = r"\\.\pipe\d4lf" + + +class ItemIdentifiers(enum.Enum): + COMPASS = "Compass" + NIGHTMARE_SIGIL = "Nightmare Sigil" + TRIBUTE = "TRIBUTE OF" + WHISPERING_KEY = "WHISPERING KEY" def create_pipe() -> int: return win32pipe.CreateNamedPipe( - _PIPE_NAME, win32pipe.PIPE_ACCESS_INBOUND, win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_WAIT, 1, 2048, 10 * 2 ** (10 * 2), 0, None + r"\\.\pipe\d4lf", + win32pipe.PIPE_ACCESS_INBOUND, + win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_WAIT, + 1, + 2048, + 10 * 2 ** (10 * 2), + 0, + None, ) def read_pipe() -> None: while True: handle = create_pipe() - LOGGER.debug(f"Named pipe '{_PIPE_NAME}' created. Waiting for client to connect...") + LOGGER.debug("Waiting for TTS client to connect") win32pipe.ConnectNamedPipe(handle, None) - LOGGER.debug("Client connected. Waiting for data...") + LOGGER.debug("TTS client connected") while True: try: @@ -35,10 +50,9 @@ def read_pipe() -> None: _DATA_QUEUE.put(s) except Exception as e: print(f"Error while reading data: {e}") - break win32file.CloseHandle(handle) - print("Client disconnected. Waiting for a new client...") + print("TTS client disconnected") def detector() -> None: @@ -46,48 +60,40 @@ def detector() -> None: while True: data = fix_data(_DATA_QUEUE.get()) local_cache.append(data) - if any(word in data.lower() for word in ["markasjunk", "markasfavorite", "unmarkitem"]): - start = find_item_start(local_cache) - global LAST_ITEM_SECTION - LAST_ITEM_SECTION = local_cache[start:] + if any(word in data.lower() for word in ["mouse button"]) and (start := find_item_start(local_cache)) is not None: + global LAST_ITEM + LAST_ITEM = local_cache[start:] + LOGGER.debug(f"TTS Found: {LAST_ITEM}") local_cache = [] -def has_more_than_consecutive_capitals(s: str, n: int = 5) -> bool: - capital_count = 0 - for char in s: - if char.isupper(): - capital_count += 1 - if capital_count > n: - return True - else: - capital_count = 0 - return False +def find_item_start(data: list[str]) -> int | None: + ignored_words = ["COMPASS AFFIXES", "DUNGEON AFFIXES"] + for index, item in reversed(list(enumerate(data))): + if any(ignored in item for ignored in ignored_words): + continue -def find_item_start(data: list[str]) -> int: - item_starts = ["Compass", "Nightmare Sigil", "Tribute of"] - for index in range(len(data) - 1, -1, -1): - if any(word in data[index] for word in item_starts): + if any(item.startswith(x) for x in [y.value for y in ItemIdentifiers]): return index - for index in range(len(data) - 1, -1, -1): - if has_more_than_consecutive_capitals(data[index]): + + cleaned_str = re.sub(r"[^A-Za-z]", "", item) + if len(cleaned_str) >= 3 and item.isupper(): return index - return -1 + + return None def fix_data(data: str) -> str: - return ( - data.replace("'", "") - .replace(""", "") - .replace("[FAVORITED ITEM]. ", "") - .replace("ￂﾠ", "") - .replace("(Spiritborn Only", "") - .strip() - ) + to_remove = ["'", """, "[FAVORITED ITEM]. ", "ￂﾠ", "(Spiritborn Only)", "[MARKED AS JUNK]. "] + + for item in to_remove: + data = data.replace(item, "") + + return data.strip() def start_connection() -> None: - LOGGER.info("Starting tts connection...") + LOGGER.info("Starting tts listener") threading.Thread(target=detector, daemon=True).start() threading.Thread(target=read_pipe, daemon=True).start()