diff --git a/.gitignore b/.gitignore index 29092c0e..42093446 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ !config/bnip/.gitkeep *.bak +*.lock *.log *.pyc *.pyo diff --git a/README.md b/README.md index 9788d96b..3f0c69a1 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Documentation is not yet finished. For now, it should be self-explanatory. Just Current functionality: - Import builds from maxroll/d4builds/mobalytics -- Create profiles based off of searches for diablo.trade +- Create profiles based off of searches for diablo.trade (requires chrome) - Complete management of your params.ini through the config tab Each tab gives further instructions on how to use it and what kind of input it expects. diff --git a/requirements.txt b/requirements.txt index a28fd4f0..41535aa4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,6 +30,7 @@ pyyaml rapidfuzz ruff selenium +seleniumbase tk typing_extensions webdriver-manager diff --git a/src/__init__.py b/src/__init__.py index 2febd0b6..2c1cb92b 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -2,4 +2,4 @@ TP = concurrent.futures.ThreadPoolExecutor() -__version__ = "5.7.9" +__version__ = "5.7.10" diff --git a/src/gui/importer/common.py b/src/gui/importer/common.py index 9acc20e0..dfe3e1b3 100644 --- a/src/gui/importer/common.py +++ b/src/gui/importer/common.py @@ -14,6 +14,7 @@ from selenium.webdriver.remote.webdriver import WebDriver from selenium.webdriver.remote.webelement import WebElement from selenium.webdriver.support.wait import WebDriverWait +from seleniumbase import Driver from src import __version__ from src.config.loader import IniConfigLoader @@ -139,12 +140,12 @@ def match_to_enum(enum_class, target_string: str, check_keys: bool = False): return None -def retry_importer(func=None, inject_webdriver: bool = False): +def retry_importer(func=None, inject_webdriver: bool = False, uc=False): def decorator_retry_importer(wrap_function): @functools.wraps(wrap_function) def wrapper(*args, **kwargs): if inject_webdriver and "driver" not in kwargs and not args: - kwargs["driver"] = setup_webdriver() + kwargs["driver"] = setup_webdriver(uc=uc) for _ in range(5): try: res = wrap_function(*args, **kwargs) @@ -180,7 +181,9 @@ def save_as_profile(file_name: str, profile: ProfileModel, url: str): LOGGER.info(f"Created profile {save_path}") -def setup_webdriver() -> ChromiumDriver: +def setup_webdriver(uc: bool = False) -> ChromiumDriver: + if uc: + return Driver(uc=uc, headless2=True) match IniConfigLoader().general.browser: case BrowserType.edge: options = webdriver.EdgeOptions() diff --git a/src/gui/importer/d4builds.py b/src/gui/importer/d4builds.py index d880dc9b..7e4eef00 100644 --- a/src/gui/importer/d4builds.py +++ b/src/gui/importer/d4builds.py @@ -44,7 +44,7 @@ class D4BuildsException(Exception): @retry_importer(inject_webdriver=True) -def import_d4builds(driver: ChromiumDriver = None, url: str = None): +def import_d4builds(url: str, driver: ChromiumDriver = None): url = url.strip().replace("\n", "") if BASE_URL not in url: LOGGER.error("Invalid url, please use a d4builds url") diff --git a/src/gui/importer/diablo_trade.py b/src/gui/importer/diablo_trade.py index cc691f76..23c5aa36 100644 --- a/src/gui/importer/diablo_trade.py +++ b/src/gui/importer/diablo_trade.py @@ -5,6 +5,8 @@ from typing import Any from urllib.parse import parse_qs, urlencode, urlparse, urlunparse +import lxml.html +import seleniumbase from pydantic import ValidationError import src.logger @@ -13,7 +15,6 @@ from src.dataloader import Dataloader from src.gui.importer.common import ( format_number_as_short_string, - get_with_retry, match_to_enum, retry_importer, save_as_profile, @@ -50,8 +51,8 @@ class DiabloTradeException(Exception): pass -@retry_importer -def import_diablo_trade(url: str, max_listings: int): +@retry_importer(inject_webdriver=True, uc=True) +def import_diablo_trade(url: str, max_listings: int, driver: seleniumbase.Driver = None): url = url.strip().replace("\n", "") if BASE_URL not in url: LOGGER.error("Invalid url, please use a diablo.trade filter url") @@ -62,11 +63,12 @@ def import_diablo_trade(url: str, max_listings: int): while True: api_url = _construct_api_url(listing_url=url, cursor=cursor) try: - r = get_with_retry(url=api_url) - except ConnectionError: - LOGGER.error("Can't fetch listings, saving current data") + driver.default_get(url=api_url) + source = lxml.html.fromstring(driver.get_page_source()) + data = json.loads(source.text_content().strip()) + except Exception: + LOGGER.exception("Can't fetch listings, saving current data") break - data = r.json() if not (listings := data["data"]): LOGGER.debug("Reached end") break diff --git a/tests/gui/importer/test_diablo_trade.py b/tests/gui/importer/test_diablo_trade.py index 62dd44d5..cc24e808 100644 --- a/tests/gui/importer/test_diablo_trade.py +++ b/tests/gui/importer/test_diablo_trade.py @@ -10,7 +10,7 @@ @pytest.mark.parametrize("url", URLS) -@pytest.mark.requests +@pytest.mark.selenium def test_import_diablo_trade(url: str, mock_ini_loader: MockerFixture, mocker: MockerFixture): Dataloader() # need to load data first or the mock will make it impossible mocker.patch("builtins.open", new=mocker.mock_open())