Enhancement: Replace pather.adjust_abs_range_to_screen() with ui_mana…

…ger.get_closest_non_hud_pixel() (#908)

src/char/bone_necro.py

@@ -14,6 +14,7 @@
 import time
 import os
 from ui_manager import ScreenObjects
+from ui_manager import get_closest_non_hud_pixel
 
 class Bone_Necro(IChar):
     def __init__(self, skill_hotkeys: dict, pather: Pather):
@@ -106,8 +107,8 @@ def _cast_circle(self, cast_dir: tuple[float,float],cast_start_angle: float=0.0,
             angle = self._lerp(cast_start_angle,cast_end_angle,float(i)/cast_div)
             target = unit_vector(rotate_vec(cast_dir, angle))
             Logger.debug(f"Circle cast - current angle: {angle}º")
-            circle_pos_screen = self._pather._adjust_abs_range_to_screen(target*radius)
-            circle_pos_monitor = convert_abs_to_monitor(circle_pos_screen)
+            circle_pos_abs = get_closest_non_hud_pixel(pos = target*radius, pos_type="abs")
+            circle_pos_monitor = convert_abs_to_monitor(circle_pos_abs)
             start = time.time()
             mouse.move(*circle_pos_monitor,delay_factor=[0.95*delay, 1.05*delay])
             duration = time.time() - start

src/char/i_char.py

@@ -15,7 +15,7 @@
 from config import Config
 from screen import grab, convert_monitor_to_screen, convert_screen_to_abs, convert_abs_to_monitor, convert_screen_to_monitor
 import template_finder
-from ui_manager import detect_screen_object, ScreenObjects
+from ui_manager import detect_screen_object, ScreenObjects, get_closest_non_hud_pixel
 
 class IChar:
     _CrossGameCapabilities: None | CharacterCapabilities = None
@@ -309,8 +309,8 @@ def _pre_buff_cta(self):
 
 
     def vec_to_monitor(self, target):
-        circle_pos_screen = self._pather._adjust_abs_range_to_screen(target)
-        return convert_abs_to_monitor(circle_pos_screen)
+        circle_pos_abs = get_closest_non_hud_pixel(pos = target, pos_type="abs")
+        return convert_abs_to_monitor(circle_pos_abs)
 
     def _lerp(self,a: float,b: float, f:float):
         return a + f * (b - a)

src/char/necro.py

@@ -13,7 +13,7 @@
 import numpy as np
 import time
 import os
-from ui_manager import wait_until_visible, ScreenObjects
+from ui_manager import wait_until_visible, ScreenObjects, get_closest_non_hud_pixel
 
 class Necro(IChar):
     def __init__(self, skill_hotkeys: dict, pather: Pather):
@@ -352,11 +352,10 @@ def _cast_circle(self, cast_dir: tuple[float,float],cast_start_angle: float=0.0,
             target = unit_vector(rotate_vec(cast_dir, angle))
             #Logger.info("current angle ~> "+str(angle))
             for j in range(cast_v_div):
-                circle_pos_screen = self._pather._adjust_abs_range_to_screen((target*120.0*float(j+1.0))*offset)
-                circle_pos_monitor = convert_abs_to_monitor(circle_pos_screen)
+                circle_pos_abs = get_closest_non_hud_pixel(pos = (target*120.0*float(j+1.0))*offset, pos_type="abs")
+                circle_pos_monitor = convert_abs_to_monitor(circle_pos_abs)
                 mouse.move(*circle_pos_monitor,delay_factor=[0.3*delay, .6*delay])
 
-
                 #Logger.info("circle move")
         mouse.release(button="right")
         keyboard.send(Config().char["stand_still"], do_press=False)

src/char/poison_necro.py

@@ -13,6 +13,7 @@
 import numpy as np
 import time
 import os
+from ui_manager import get_closest_non_hud_pixel
 
 class Poison_Necro(IChar):
     def __init__(self, skill_hotkeys: dict, pather: Pather):
@@ -372,8 +373,8 @@ def _cast_circle(self, cast_dir: tuple[float,float],cast_start_angle: float=0.0,
             target = unit_vector(rotate_vec(cast_dir, angle))
             #Logger.info("current angle ~> "+str(angle))
             for j in range(cast_v_div):
-                circle_pos_screen = self._pather._adjust_abs_range_to_screen((target*120.0*float(j+1.0))*offset)
-                circle_pos_monitor = screen.convert_abs_to_monitor(circle_pos_screen)
+                circle_pos_abs = get_closest_non_hud_pixel(pos = ((target*120.0*float(j+1.0))*offset), pos_type="abs")
+                circle_pos_monitor = screen.convert_abs_to_monitor(circle_pos_abs)
                 mouse.move(*circle_pos_monitor,delay_factor=[0.3*delay, .6*delay])
 
 

src/d2r_image/processing_data.py

@@ -2,9 +2,6 @@
 from d2r_image.data_models import ItemQuality
 import cv2
 
-HUD_MASK = cv2.imread(f"assets/hud_mask.png", cv2.IMREAD_GRAYSCALE)
-HUD_MASK = cv2.threshold(HUD_MASK, 1, 255, cv2.THRESH_BINARY)[1]
-
 ITEM_COLORS = ['white', 'gray', 'blue', 'green', 'yellow', 'gold', 'orange']
 GAUS_FILTER = (21, 1)
 EXPECTED_HEIGHT_RANGE = [round(num) for num in [x / 1.5 for x in [14, 40]]]

src/d2r_image/processing_helpers.py

@@ -12,10 +12,11 @@
 from d2r_image.processing_data import Runeword
 import d2r_image.d2data_lookup as d2data_lookup
 from d2r_image.d2data_lookup import fuzzy_base_item_match
-from d2r_image.processing_data import EXPECTED_HEIGHT_RANGE, EXPECTED_WIDTH_RANGE, GAUS_FILTER, ITEM_COLORS, QUALITY_COLOR_MAP, Runeword, HUD_MASK, BOX_EXPECTED_HEIGHT_RANGE, BOX_EXPECTED_WIDTH_RANGE
+from d2r_image.processing_data import EXPECTED_HEIGHT_RANGE, EXPECTED_WIDTH_RANGE, GAUS_FILTER, ITEM_COLORS, QUALITY_COLOR_MAP, Runeword, BOX_EXPECTED_HEIGHT_RANGE, BOX_EXPECTED_WIDTH_RANGE
 from d2r_image.strings_store import base_items
 from utils.misc import color_filter, erode_to_black, slugify
 from d2r_image.ocr import image_to_text
+from ui_manager import get_hud_mask
 
 from screen import convert_screen_to_monitor
 from utils.misc import color_filter, cut_roi, roi_center
@@ -163,8 +164,8 @@ def _contains_color(img: np.ndarray, color: str) -> bool:
 
 def clean_img(inp_img: np.ndarray, black_thresh: int = 14) -> np.ndarray:
     img = inp_img[:, :, :]
-    if img.shape[0] == HUD_MASK.shape[0] and img.shape[1] == HUD_MASK.shape[1]:
-        img = cv2.bitwise_and(img, img, mask=HUD_MASK)
+    if img.shape[0] == get_hud_mask().shape[0] and img.shape[1] == get_hud_mask().shape[1]:
+        img = cv2.bitwise_and(img, img, mask=get_hud_mask())
     # In order to not filter out highlighted items, change their color to black
     highlight_mask = color_filter(img, Config().colors["item_highlight"])[0]
     img[highlight_mask > 0] = (0, 0, 0)

src/pather.py

@@ -13,7 +13,7 @@
 from screen import convert_screen_to_monitor, convert_abs_to_screen, convert_abs_to_monitor, convert_screen_to_abs, grab, stop_detecting_window
 import template_finder
 from char import IChar
-from ui_manager import detect_screen_object, ScreenObjects, is_visible, select_screen_object_match
+from ui_manager import detect_screen_object, ScreenObjects, is_visible, select_screen_object_match, get_closest_non_hud_pixel
 
 class Location:
     # A5 Town
@@ -535,44 +535,6 @@ def traverse_nodes_fixed(self, key: str | list[tuple[float, float]], char: IChar
         #     cv2.imwrite(f"./log/screenshots/info/nil_path_{key}_" + time.strftime("%Y%m%d_%H%M%S") + ".png", grab())
         return True
 
-    def _adjust_abs_range_to_screen(self, abs_pos: tuple[float, float]) -> tuple[float, float]:
-        """
-        Adjust an absolute coordinate so it will not go out of screen or click on any ui which will not move the char
-        :param abs_pos: Absolute position of the desired position to move to
-        :return: Absolute position of a valid position that can be clicked on
-        """
-        f = 1.0
-        # Check for x-range
-        if abs_pos[0] > self._range_x[1]:
-            f = min(f, abs(self._range_x[1] / float(abs_pos[0])))
-        elif abs_pos[0] < self._range_x[0]:
-            f = min(f, abs(self._range_x[0] / float(abs_pos[0])))
-        # Check y-range
-        if abs_pos[1] > self._range_y[1]:
-            f = min(f, abs(self._range_y[1] / float(abs_pos[1])))
-        if abs_pos[1] < self._range_y[0]:
-            f = min(f, abs(self._range_y[0] / float(abs_pos[1])))
-        # Scale the position by the factor f
-        if f < 1.0:
-            abs_pos = (int(abs_pos[0] * f), int(abs_pos[1] * f))
-        # Check if adjusted position is "inside globe"
-        screen_pos = convert_abs_to_screen(abs_pos)
-        if is_in_roi(Config().ui_roi["mana_globe"], screen_pos) or is_in_roi(Config().ui_roi["health_globe"], screen_pos):
-            # convert any of health or mana roi top coordinate to abs (x-coordinate is just a dummy 0 value)
-            new_range_y_bottom = convert_screen_to_abs((0, Config().ui_roi["mana_globe"][1]))[1]
-            f = abs(new_range_y_bottom / float(abs_pos[1]))
-            abs_pos = (int(abs_pos[0] * f), int(abs_pos[1] * f))
-        # Check if clicking on merc img
-        screen_pos = convert_abs_to_screen(abs_pos)
-        if is_in_roi(Config().ui_roi["merc_icon"], screen_pos):
-            width = Config().ui_roi["merc_icon"][2]
-            height = Config().ui_roi["merc_icon"][3]
-            w_abs, h_abs = convert_screen_to_abs((width, height))
-            fw = abs(w_abs / float(abs_pos[0]))
-            fh = abs(h_abs / float(abs_pos[1]))
-            f = max(fw, fh)
-            abs_pos = (int(abs_pos[0] * f), int(abs_pos[1] * f))
-        return abs_pos
 
     def find_abs_node_pos(self, node_idx: int, img: np.ndarray, threshold: float = 0.68) -> tuple[float, float]:
         node = self._nodes[node_idx]
@@ -590,7 +552,7 @@ def find_abs_node_pos(self, node_idx: int, img: np.ndarray, threshold: float = 0
             # Calc the abs node position with the relative coordinates (relative to ref)
             node_pos_rel = self._get_node(node_idx, template_match.name)
             node_pos_abs = self._convert_rel_to_abs(node_pos_rel, ref_pos_abs)
-            node_pos_abs = self._adjust_abs_range_to_screen(node_pos_abs)
+            node_pos_abs = get_closest_non_hud_pixel(pos = node_pos_abs, pos_type="abs")
             return node_pos_abs
         return None
 
@@ -668,7 +630,7 @@ def traverse_nodes(
                     else:
                         angle = random.random() * math.pi * 2
                         pos_abs = (math.cos(angle) * 150, math.sin(angle) * 150)
-                    pos_abs = self._adjust_abs_range_to_screen(pos_abs)
+                    pos_abs = get_closest_non_hud_pixel(pos = node_pos_abs, pos_type="abs")
                     Logger.debug(f"Pather: taking a random guess towards " + str(pos_abs))
                     x_m, y_m = convert_abs_to_monitor(pos_abs)
                     char.move((x_m, y_m), force_move=True)
@@ -733,7 +695,7 @@ def display_all_nodes(pather: Pather, filter: str = None):
                         # Calc the abs node position with the relative coordinates (relative to ref)
                         node_pos_rel = pather._get_node(node_idx, template_type)
                         node_pos_abs = pather._convert_rel_to_abs(node_pos_rel, ref_pos_abs)
-                        node_pos_abs = pather._adjust_abs_range_to_screen(node_pos_abs)
+                        node_pos_abs = get_closest_non_hud_pixel(pos = node_pos_abs, pos_type="abs")
                         x, y = convert_abs_to_screen(node_pos_abs)
                         cv2.circle(display_img, (x, y), 5, (255, 0, 0), 3)
                         cv2.putText(display_img, str(node_idx), (x, y), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)

src/target_detect.py

@@ -7,6 +7,7 @@
 from utils.misc import color_filter, is_in_roi
 import json
 from dataclasses import dataclass
+from ui_manager import get_hud_mask
 
 FILTER_RANGES=[
     {"erode": 1, "blur": 3, "lh": 38, "ls": 169, "lv": 50, "uh": 70, "us": 255, "uv": 255}, # poison
@@ -120,8 +121,7 @@ def _process_image(img, mask_char:bool=False, mask_hud:bool=True, erode:int=None
     """
     img = img
     if mask_hud:
-        hud_mask = cv2.imread(f"./assets/hud_mask.png", cv2.IMREAD_GRAYSCALE)
-        img = cv2.bitwise_and(img, img, mask=hud_mask)
+        img = cv2.bitwise_and(img, img, mask=get_hud_mask())
     if mask_char: img = cv2.rectangle(img, (600,250), (700,400), (0,0,0), -1) # black out character by drawing a black box above him (e.g. ignore set glow)
     if erode:
         kernel = np.ones((erode, erode), 'uint8')
@@ -180,8 +180,6 @@ class LiveViewer:
     def __init__(self):
         with open("./src/utils/live-view-last-settings.json") as f:
             self.settings = json.loads(f.read())
-        self.hud_mask = cv2.imread(f"./assets/hud_mask.png", cv2.IMREAD_GRAYSCALE)
-        self.hud_mask = cv2.threshold(self.hud_mask, 1, 255, cv2.THRESH_BINARY)[1]
         self.use_existing_image = False
         self.existing_image_path = "test/assets/mobs.png"
         self.setup()
@@ -209,7 +207,7 @@ def value_update(self, ignore: bool = False):
             self.image = grab()
         else:
             self.image = cv2.imread(self.existing_image_path)
-        self.image = cv2.bitwise_and(self.image, self.image, mask=self.hud_mask)
+        self.image = cv2.bitwise_and(self.image, self.image, mask=get_hud_mask())
         # black out character
         self.image = cv2.rectangle(self.image, (550,250), (700,400), (0,0,0), -1)
         try:

src/ui_manager.py

@@ -3,12 +3,14 @@
 import numpy as np
 import time
 import cv2
+from functools import cache
+
 from typing import TypeVar, Callable
 from utils.custom_mouse import mouse
 from utils.misc import wait, cut_roi, image_is_equal
 from logger import Logger
 from config import Config
-from screen import grab, convert_abs_to_monitor
+from screen import convert_abs_to_screen, convert_monitor_to_screen, convert_screen_to_abs, convert_screen_to_monitor, grab, convert_abs_to_monitor
 import template_finder
 from template_finder import TemplateMatch
 from dataclasses import dataclass
@@ -331,6 +333,65 @@ def center_mouse(delay_factor: list = None):
     else:
         mouse.move(*center_m, randomize=20, delay_factor = [0.1, 0.2])
 
+@cache
+def get_hud_mask() -> np.ndarray:
+    mask = cv2.imread(f"assets/hud_mask.png", cv2.IMREAD_GRAYSCALE)
+    return cv2.threshold(mask, 1, 255, cv2.THRESH_BINARY)[1]
+
+def _find_nearest_nonzero(img: np.ndarray, pos: tuple[int, int]) -> tuple[int, int]:
+    """
+    Finds the nearest nonzero pixel to the target pixel.
+    :param img: The image to search in.
+    :param pos: The target pixel
+    :return: The nearest nonzero pixel
+    """
+
+    x, y = pos
+    if x < 0:
+        x = 0
+    elif x >= img.shape[1]:
+        x = img.shape[1] - 1
+    if y < 0:
+        y = 0
+    elif y >= img.shape[0]:
+        y = img.shape[0] - 1
+    if img[y,x]:
+        return (x, y)
+
+    nonzero = cv2.findNonZero(img)
+    distances = np.sqrt((nonzero[:,:,0] - x) ** 2 + (nonzero[:,:,1] - y) ** 2)
+    nearest_index = np.argmin(distances)
+    x, y = nonzero[nearest_index, :, :][0]
+    return (x, y)
+
+@cache
+def get_closest_non_hud_pixel(pos : tuple[int, int], pos_type: str = "abs") -> tuple[int, int]:
+    """
+    Finds the closest non-hud pixel to the target pixel.
+    :param pos: The target pixel
+    :return: The closest non-hud pixel
+    """
+    match pos_type:
+        case "abs":
+            pos = convert_abs_to_screen(pos)
+        case "monitor":
+            pos = convert_monitor_to_screen(pos)
+        case "screen":
+            pass
+        case _:
+            Logger.error(f"Unknown pos type: {pos_type}")
+            return pos
+    screen_pos = _find_nearest_nonzero(get_hud_mask(), pos)
+    match pos_type:
+        case "abs":
+            new_pos = convert_screen_to_abs(screen_pos)
+        case "monitor":
+            new_pos = convert_screen_to_monitor(screen_pos)
+        case "screen":
+            new_pos = screen_pos
+    return new_pos
+
+
 # Testing: Move to whatever ui to test and run
 if __name__ == "__main__":
     import keyboard

test/closest_non_hud_test.py

@@ -0,0 +1,20 @@
+import pytest
+import screen
+from ui_manager import get_closest_non_hud_pixel
+
+
+@pytest.mark.parametrize("pos, pos_type, should_adapt", [
+    ((40, 40), "screen", True), # over merc portrait
+    ((1241, 19), "screen", True), # over clock
+    ((291, 663), "screen", True), # over health globe
+    ((650, 300), "screen", False), # mid screen
+    ((783, 618), "screen", True), # over active skill bar
+    ((1062, 594), "screen", True), # over gargoyle
+    ((0, 0), "abs", False), # mid screen
+    ((221, 327), "abs", True), # near right skill
+])
+def test_get_closest_non_hud_pixel(pos, pos_type, should_adapt):
+    screen.set_window_position(0, 0)
+    new_pos = get_closest_non_hud_pixel(pos, pos_type)
+    is_adapted = not (pos == new_pos)
+    assert(is_adapted == should_adapt)