diff --git a/assets/hud_mask.png b/assets/hud_mask.png index 08e28a38a..36d9c912d 100644 Binary files a/assets/hud_mask.png and b/assets/hud_mask.png differ diff --git a/src/char/bone_necro.py b/src/char/bone_necro.py index ff3ef122f..b45f97b0e 100644 --- a/src/char/bone_necro.py +++ b/src/char/bone_necro.py @@ -14,6 +14,7 @@ import time import os from ui_manager import ScreenObjects +from ui_manager import get_closest_non_hud_pixel class Bone_Necro(IChar): def __init__(self, skill_hotkeys: dict, pather: Pather): @@ -106,8 +107,8 @@ def _cast_circle(self, cast_dir: tuple[float,float],cast_start_angle: float=0.0, angle = self._lerp(cast_start_angle,cast_end_angle,float(i)/cast_div) target = unit_vector(rotate_vec(cast_dir, angle)) Logger.debug(f"Circle cast - current angle: {angle}ยบ") - circle_pos_screen = self._pather._adjust_abs_range_to_screen(target*radius) - circle_pos_monitor = convert_abs_to_monitor(circle_pos_screen) + circle_pos_abs = get_closest_non_hud_pixel(pos = target*radius, pos_type="abs") + circle_pos_monitor = convert_abs_to_monitor(circle_pos_abs) start = time.time() mouse.move(*circle_pos_monitor,delay_factor=[0.95*delay, 1.05*delay]) duration = time.time() - start diff --git a/src/char/i_char.py b/src/char/i_char.py index 49314baa8..4374b43ef 100644 --- a/src/char/i_char.py +++ b/src/char/i_char.py @@ -15,7 +15,7 @@ from config import Config from screen import grab, convert_monitor_to_screen, convert_screen_to_abs, convert_abs_to_monitor, convert_screen_to_monitor import template_finder -from ui_manager import detect_screen_object, ScreenObjects +from ui_manager import detect_screen_object, ScreenObjects, get_closest_non_hud_pixel class IChar: _CrossGameCapabilities: None | CharacterCapabilities = None @@ -309,8 +309,8 @@ def _pre_buff_cta(self): def vec_to_monitor(self, target): - circle_pos_screen = self._pather._adjust_abs_range_to_screen(target) - return convert_abs_to_monitor(circle_pos_screen) + circle_pos_abs = get_closest_non_hud_pixel(pos = target, pos_type="abs") + return convert_abs_to_monitor(circle_pos_abs) def _lerp(self,a: float,b: float, f:float): return a + f * (b - a) diff --git a/src/char/necro.py b/src/char/necro.py index 6a632ffc2..02535d7fa 100644 --- a/src/char/necro.py +++ b/src/char/necro.py @@ -13,7 +13,7 @@ import numpy as np import time import os -from ui_manager import wait_until_visible, ScreenObjects +from ui_manager import wait_until_visible, ScreenObjects, get_closest_non_hud_pixel class Necro(IChar): def __init__(self, skill_hotkeys: dict, pather: Pather): @@ -352,11 +352,10 @@ def _cast_circle(self, cast_dir: tuple[float,float],cast_start_angle: float=0.0, target = unit_vector(rotate_vec(cast_dir, angle)) #Logger.info("current angle ~> "+str(angle)) for j in range(cast_v_div): - circle_pos_screen = self._pather._adjust_abs_range_to_screen((target*120.0*float(j+1.0))*offset) - circle_pos_monitor = convert_abs_to_monitor(circle_pos_screen) + circle_pos_abs = get_closest_non_hud_pixel(pos = (target*120.0*float(j+1.0))*offset, pos_type="abs") + circle_pos_monitor = convert_abs_to_monitor(circle_pos_abs) mouse.move(*circle_pos_monitor,delay_factor=[0.3*delay, .6*delay]) - #Logger.info("circle move") mouse.release(button="right") keyboard.send(Config().char["stand_still"], do_press=False) diff --git a/src/char/poison_necro.py b/src/char/poison_necro.py index f33df185b..72ae64cac 100644 --- a/src/char/poison_necro.py +++ b/src/char/poison_necro.py @@ -13,6 +13,7 @@ import numpy as np import time import os +from ui_manager import get_closest_non_hud_pixel class Poison_Necro(IChar): def __init__(self, skill_hotkeys: dict, pather: Pather): @@ -372,8 +373,8 @@ def _cast_circle(self, cast_dir: tuple[float,float],cast_start_angle: float=0.0, target = unit_vector(rotate_vec(cast_dir, angle)) #Logger.info("current angle ~> "+str(angle)) for j in range(cast_v_div): - circle_pos_screen = self._pather._adjust_abs_range_to_screen((target*120.0*float(j+1.0))*offset) - circle_pos_monitor = screen.convert_abs_to_monitor(circle_pos_screen) + circle_pos_abs = get_closest_non_hud_pixel(pos = ((target*120.0*float(j+1.0))*offset), pos_type="abs") + circle_pos_monitor = screen.convert_abs_to_monitor(circle_pos_abs) mouse.move(*circle_pos_monitor,delay_factor=[0.3*delay, .6*delay]) diff --git a/src/d2r_image/processing_data.py b/src/d2r_image/processing_data.py index 057f41a03..66ff68baf 100644 --- a/src/d2r_image/processing_data.py +++ b/src/d2r_image/processing_data.py @@ -2,9 +2,6 @@ from d2r_image.data_models import ItemQuality import cv2 -HUD_MASK = cv2.imread(f"assets/hud_mask.png", cv2.IMREAD_GRAYSCALE) -HUD_MASK = cv2.threshold(HUD_MASK, 1, 255, cv2.THRESH_BINARY)[1] - ITEM_COLORS = ['white', 'gray', 'blue', 'green', 'yellow', 'gold', 'orange'] GAUS_FILTER = (21, 1) EXPECTED_HEIGHT_RANGE = [round(num) for num in [x / 1.5 for x in [14, 40]]] diff --git a/src/d2r_image/processing_helpers.py b/src/d2r_image/processing_helpers.py index 84a389355..cc8c499df 100644 --- a/src/d2r_image/processing_helpers.py +++ b/src/d2r_image/processing_helpers.py @@ -12,10 +12,11 @@ from d2r_image.processing_data import Runeword import d2r_image.d2data_lookup as d2data_lookup from d2r_image.d2data_lookup import fuzzy_base_item_match -from d2r_image.processing_data import EXPECTED_HEIGHT_RANGE, EXPECTED_WIDTH_RANGE, GAUS_FILTER, ITEM_COLORS, QUALITY_COLOR_MAP, Runeword, HUD_MASK, BOX_EXPECTED_HEIGHT_RANGE, BOX_EXPECTED_WIDTH_RANGE +from d2r_image.processing_data import EXPECTED_HEIGHT_RANGE, EXPECTED_WIDTH_RANGE, GAUS_FILTER, ITEM_COLORS, QUALITY_COLOR_MAP, Runeword, BOX_EXPECTED_HEIGHT_RANGE, BOX_EXPECTED_WIDTH_RANGE from d2r_image.strings_store import base_items from utils.misc import color_filter, erode_to_black, slugify from d2r_image.ocr import image_to_text +from ui_manager import get_hud_mask from screen import convert_screen_to_monitor from utils.misc import color_filter, cut_roi, roi_center @@ -163,8 +164,8 @@ def _contains_color(img: np.ndarray, color: str) -> bool: def clean_img(inp_img: np.ndarray, black_thresh: int = 14) -> np.ndarray: img = inp_img[:, :, :] - if img.shape[0] == HUD_MASK.shape[0] and img.shape[1] == HUD_MASK.shape[1]: - img = cv2.bitwise_and(img, img, mask=HUD_MASK) + if img.shape[0] == get_hud_mask().shape[0] and img.shape[1] == get_hud_mask().shape[1]: + img = cv2.bitwise_and(img, img, mask=get_hud_mask()) # In order to not filter out highlighted items, change their color to black highlight_mask = color_filter(img, Config().colors["item_highlight"])[0] img[highlight_mask > 0] = (0, 0, 0) diff --git a/src/pather.py b/src/pather.py index b320fac9c..87b9ba6e8 100644 --- a/src/pather.py +++ b/src/pather.py @@ -13,7 +13,7 @@ from screen import convert_screen_to_monitor, convert_abs_to_screen, convert_abs_to_monitor, convert_screen_to_abs, grab, stop_detecting_window import template_finder from char import IChar -from ui_manager import detect_screen_object, ScreenObjects, is_visible, select_screen_object_match +from ui_manager import detect_screen_object, ScreenObjects, is_visible, select_screen_object_match, get_closest_non_hud_pixel class Location: # A5 Town @@ -535,44 +535,6 @@ def traverse_nodes_fixed(self, key: str | list[tuple[float, float]], char: IChar # cv2.imwrite(f"./log/screenshots/info/nil_path_{key}_" + time.strftime("%Y%m%d_%H%M%S") + ".png", grab()) return True - def _adjust_abs_range_to_screen(self, abs_pos: tuple[float, float]) -> tuple[float, float]: - """ - Adjust an absolute coordinate so it will not go out of screen or click on any ui which will not move the char - :param abs_pos: Absolute position of the desired position to move to - :return: Absolute position of a valid position that can be clicked on - """ - f = 1.0 - # Check for x-range - if abs_pos[0] > self._range_x[1]: - f = min(f, abs(self._range_x[1] / float(abs_pos[0]))) - elif abs_pos[0] < self._range_x[0]: - f = min(f, abs(self._range_x[0] / float(abs_pos[0]))) - # Check y-range - if abs_pos[1] > self._range_y[1]: - f = min(f, abs(self._range_y[1] / float(abs_pos[1]))) - if abs_pos[1] < self._range_y[0]: - f = min(f, abs(self._range_y[0] / float(abs_pos[1]))) - # Scale the position by the factor f - if f < 1.0: - abs_pos = (int(abs_pos[0] * f), int(abs_pos[1] * f)) - # Check if adjusted position is "inside globe" - screen_pos = convert_abs_to_screen(abs_pos) - if is_in_roi(Config().ui_roi["mana_globe"], screen_pos) or is_in_roi(Config().ui_roi["health_globe"], screen_pos): - # convert any of health or mana roi top coordinate to abs (x-coordinate is just a dummy 0 value) - new_range_y_bottom = convert_screen_to_abs((0, Config().ui_roi["mana_globe"][1]))[1] - f = abs(new_range_y_bottom / float(abs_pos[1])) - abs_pos = (int(abs_pos[0] * f), int(abs_pos[1] * f)) - # Check if clicking on merc img - screen_pos = convert_abs_to_screen(abs_pos) - if is_in_roi(Config().ui_roi["merc_icon"], screen_pos): - width = Config().ui_roi["merc_icon"][2] - height = Config().ui_roi["merc_icon"][3] - w_abs, h_abs = convert_screen_to_abs((width, height)) - fw = abs(w_abs / float(abs_pos[0])) - fh = abs(h_abs / float(abs_pos[1])) - f = max(fw, fh) - abs_pos = (int(abs_pos[0] * f), int(abs_pos[1] * f)) - return abs_pos def find_abs_node_pos(self, node_idx: int, img: np.ndarray, threshold: float = 0.68) -> tuple[float, float]: node = self._nodes[node_idx] @@ -590,7 +552,7 @@ def find_abs_node_pos(self, node_idx: int, img: np.ndarray, threshold: float = 0 # Calc the abs node position with the relative coordinates (relative to ref) node_pos_rel = self._get_node(node_idx, template_match.name) node_pos_abs = self._convert_rel_to_abs(node_pos_rel, ref_pos_abs) - node_pos_abs = self._adjust_abs_range_to_screen(node_pos_abs) + node_pos_abs = get_closest_non_hud_pixel(pos = node_pos_abs, pos_type="abs") return node_pos_abs return None @@ -668,7 +630,7 @@ def traverse_nodes( else: angle = random.random() * math.pi * 2 pos_abs = (math.cos(angle) * 150, math.sin(angle) * 150) - pos_abs = self._adjust_abs_range_to_screen(pos_abs) + pos_abs = get_closest_non_hud_pixel(pos = node_pos_abs, pos_type="abs") Logger.debug(f"Pather: taking a random guess towards " + str(pos_abs)) x_m, y_m = convert_abs_to_monitor(pos_abs) char.move((x_m, y_m), force_move=True) @@ -733,7 +695,7 @@ def display_all_nodes(pather: Pather, filter: str = None): # Calc the abs node position with the relative coordinates (relative to ref) node_pos_rel = pather._get_node(node_idx, template_type) node_pos_abs = pather._convert_rel_to_abs(node_pos_rel, ref_pos_abs) - node_pos_abs = pather._adjust_abs_range_to_screen(node_pos_abs) + node_pos_abs = get_closest_non_hud_pixel(pos = node_pos_abs, pos_type="abs") x, y = convert_abs_to_screen(node_pos_abs) cv2.circle(display_img, (x, y), 5, (255, 0, 0), 3) cv2.putText(display_img, str(node_idx), (x, y), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA) diff --git a/src/target_detect.py b/src/target_detect.py index 0639a3800..914bd7143 100644 --- a/src/target_detect.py +++ b/src/target_detect.py @@ -7,6 +7,7 @@ from utils.misc import color_filter, is_in_roi import json from dataclasses import dataclass +from ui_manager import get_hud_mask FILTER_RANGES=[ {"erode": 1, "blur": 3, "lh": 38, "ls": 169, "lv": 50, "uh": 70, "us": 255, "uv": 255}, # poison @@ -120,8 +121,7 @@ def _process_image(img, mask_char:bool=False, mask_hud:bool=True, erode:int=None """ img = img if mask_hud: - hud_mask = cv2.imread(f"./assets/hud_mask.png", cv2.IMREAD_GRAYSCALE) - img = cv2.bitwise_and(img, img, mask=hud_mask) + img = cv2.bitwise_and(img, img, mask=get_hud_mask()) if mask_char: img = cv2.rectangle(img, (600,250), (700,400), (0,0,0), -1) # black out character by drawing a black box above him (e.g. ignore set glow) if erode: kernel = np.ones((erode, erode), 'uint8') @@ -180,8 +180,6 @@ class LiveViewer: def __init__(self): with open("./src/utils/live-view-last-settings.json") as f: self.settings = json.loads(f.read()) - self.hud_mask = cv2.imread(f"./assets/hud_mask.png", cv2.IMREAD_GRAYSCALE) - self.hud_mask = cv2.threshold(self.hud_mask, 1, 255, cv2.THRESH_BINARY)[1] self.use_existing_image = False self.existing_image_path = "test/assets/mobs.png" self.setup() @@ -209,7 +207,7 @@ def value_update(self, ignore: bool = False): self.image = grab() else: self.image = cv2.imread(self.existing_image_path) - self.image = cv2.bitwise_and(self.image, self.image, mask=self.hud_mask) + self.image = cv2.bitwise_and(self.image, self.image, mask=get_hud_mask()) # black out character self.image = cv2.rectangle(self.image, (550,250), (700,400), (0,0,0), -1) try: diff --git a/src/ui_manager.py b/src/ui_manager.py index 09bff9406..efceebdf3 100644 --- a/src/ui_manager.py +++ b/src/ui_manager.py @@ -3,12 +3,14 @@ import numpy as np import time import cv2 +from functools import cache + from typing import TypeVar, Callable from utils.custom_mouse import mouse from utils.misc import wait, cut_roi, image_is_equal from logger import Logger from config import Config -from screen import grab, convert_abs_to_monitor +from screen import convert_abs_to_screen, convert_monitor_to_screen, convert_screen_to_abs, convert_screen_to_monitor, grab, convert_abs_to_monitor import template_finder from template_finder import TemplateMatch from dataclasses import dataclass @@ -331,6 +333,65 @@ def center_mouse(delay_factor: list = None): else: mouse.move(*center_m, randomize=20, delay_factor = [0.1, 0.2]) +@cache +def get_hud_mask() -> np.ndarray: + mask = cv2.imread(f"assets/hud_mask.png", cv2.IMREAD_GRAYSCALE) + return cv2.threshold(mask, 1, 255, cv2.THRESH_BINARY)[1] + +def _find_nearest_nonzero(img: np.ndarray, pos: tuple[int, int]) -> tuple[int, int]: + """ + Finds the nearest nonzero pixel to the target pixel. + :param img: The image to search in. + :param pos: The target pixel + :return: The nearest nonzero pixel + """ + + x, y = pos + if x < 0: + x = 0 + elif x >= img.shape[1]: + x = img.shape[1] - 1 + if y < 0: + y = 0 + elif y >= img.shape[0]: + y = img.shape[0] - 1 + if img[y,x]: + return (x, y) + + nonzero = cv2.findNonZero(img) + distances = np.sqrt((nonzero[:,:,0] - x) ** 2 + (nonzero[:,:,1] - y) ** 2) + nearest_index = np.argmin(distances) + x, y = nonzero[nearest_index, :, :][0] + return (x, y) + +@cache +def get_closest_non_hud_pixel(pos : tuple[int, int], pos_type: str = "abs") -> tuple[int, int]: + """ + Finds the closest non-hud pixel to the target pixel. + :param pos: The target pixel + :return: The closest non-hud pixel + """ + match pos_type: + case "abs": + pos = convert_abs_to_screen(pos) + case "monitor": + pos = convert_monitor_to_screen(pos) + case "screen": + pass + case _: + Logger.error(f"Unknown pos type: {pos_type}") + return pos + screen_pos = _find_nearest_nonzero(get_hud_mask(), pos) + match pos_type: + case "abs": + new_pos = convert_screen_to_abs(screen_pos) + case "monitor": + new_pos = convert_screen_to_monitor(screen_pos) + case "screen": + new_pos = screen_pos + return new_pos + + # Testing: Move to whatever ui to test and run if __name__ == "__main__": import keyboard diff --git a/test/closest_non_hud_test.py b/test/closest_non_hud_test.py new file mode 100644 index 000000000..f05e5962c --- /dev/null +++ b/test/closest_non_hud_test.py @@ -0,0 +1,20 @@ +import pytest +import screen +from ui_manager import get_closest_non_hud_pixel + + +@pytest.mark.parametrize("pos, pos_type, should_adapt", [ + ((40, 40), "screen", True), # over merc portrait + ((1241, 19), "screen", True), # over clock + ((291, 663), "screen", True), # over health globe + ((650, 300), "screen", False), # mid screen + ((783, 618), "screen", True), # over active skill bar + ((1062, 594), "screen", True), # over gargoyle + ((0, 0), "abs", False), # mid screen + ((221, 327), "abs", True), # near right skill +]) +def test_get_closest_non_hud_pixel(pos, pos_type, should_adapt): + screen.set_window_position(0, 0) + new_pos = get_closest_non_hud_pixel(pos, pos_type) + is_adapted = not (pos == new_pos) + assert(is_adapted == should_adapt) \ No newline at end of file diff --git a/test/pather_test.py b/test/pather_test.py deleted file mode 100644 index 4cd789db6..000000000 --- a/test/pather_test.py +++ /dev/null @@ -1,29 +0,0 @@ -import pytest -from logger import Logger -from pather import Pather -import screen - - -class TestPather: - def setup_method(self): - Logger.init() - Logger.remove_file_logger() - - self.pather = Pather() - - @pytest.mark.parametrize("test_input, expected", [ - ((90, 90), True), - ((25, 70), True), - ((150, 120), False), - ((500, 400), False), - ((400, 1300), True), - ]) - def test_adjust_abs_range_to_screen(self, test_input, expected): - screen.set_window_position(0, 0) - should_be_adapted = expected - pos_abs = screen.convert_screen_to_abs(test_input) - new_pos_abs = self.pather._adjust_abs_range_to_screen(pos_abs) - is_adapted = new_pos_abs != pos_abs - assert(should_be_adapted == is_adapted) - new_pos_abs_2 = self.pather._adjust_abs_range_to_screen(new_pos_abs) - assert(new_pos_abs == new_pos_abs_2)