Fix diablo.trade (#375)

.gitignore

@@ -1,5 +1,6 @@
 !config/bnip/.gitkeep
 *.bak
+*.lock
 *.log
 *.pyc
 *.pyo

README.md

@@ -111,7 +111,7 @@ Documentation is not yet finished. For now, it should be self-explanatory. Just
 Current functionality:
 
 - Import builds from maxroll/d4builds/mobalytics
-- Create profiles based off of searches for diablo.trade
+- Create profiles based off of searches for diablo.trade (requires chrome)
 - Complete management of your params.ini through the config tab
 
 Each tab gives further instructions on how to use it and what kind of input it expects.

requirements.txt

@@ -30,6 +30,7 @@ pyyaml
 rapidfuzz
 ruff
 selenium
+seleniumbase
 tk
 typing_extensions
 webdriver-manager

src/__init__.py

@@ -2,4 +2,4 @@
 
 TP = concurrent.futures.ThreadPoolExecutor()
 
-__version__ = "5.7.9"
+__version__ = "5.7.10"

src/gui/importer/common.py

@@ -14,6 +14,7 @@
 from selenium.webdriver.remote.webdriver import WebDriver
 from selenium.webdriver.remote.webelement import WebElement
 from selenium.webdriver.support.wait import WebDriverWait
+from seleniumbase import Driver
 
 from src import __version__
 from src.config.loader import IniConfigLoader
@@ -139,12 +140,12 @@ def match_to_enum(enum_class, target_string: str, check_keys: bool = False):
     return None
 
 
-def retry_importer(func=None, inject_webdriver: bool = False):
+def retry_importer(func=None, inject_webdriver: bool = False, uc=False):
     def decorator_retry_importer(wrap_function):
         @functools.wraps(wrap_function)
         def wrapper(*args, **kwargs):
             if inject_webdriver and "driver" not in kwargs and not args:
-                kwargs["driver"] = setup_webdriver()
+                kwargs["driver"] = setup_webdriver(uc=uc)
             for _ in range(5):
                 try:
                     res = wrap_function(*args, **kwargs)
@@ -180,7 +181,9 @@ def save_as_profile(file_name: str, profile: ProfileModel, url: str):
     LOGGER.info(f"Created profile {save_path}")
 
 
-def setup_webdriver() -> ChromiumDriver:
+def setup_webdriver(uc: bool = False) -> ChromiumDriver:
+    if uc:
+        return Driver(uc=uc, headless2=True)
     match IniConfigLoader().general.browser:
         case BrowserType.edge:
             options = webdriver.EdgeOptions()

src/gui/importer/d4builds.py

@@ -44,7 +44,7 @@ class D4BuildsException(Exception):
 
 
 @retry_importer(inject_webdriver=True)
-def import_d4builds(driver: ChromiumDriver = None, url: str = None):
+def import_d4builds(url: str, driver: ChromiumDriver = None):
     url = url.strip().replace("\n", "")
     if BASE_URL not in url:
         LOGGER.error("Invalid url, please use a d4builds url")

src/gui/importer/diablo_trade.py

@@ -5,6 +5,8 @@
 from typing import Any
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
+import lxml.html
+import seleniumbase
 from pydantic import ValidationError
 
 import src.logger
@@ -13,7 +15,6 @@
 from src.dataloader import Dataloader
 from src.gui.importer.common import (
     format_number_as_short_string,
-    get_with_retry,
     match_to_enum,
     retry_importer,
     save_as_profile,
@@ -50,8 +51,8 @@ class DiabloTradeException(Exception):
     pass
 
 
-@retry_importer
-def import_diablo_trade(url: str, max_listings: int):
+@retry_importer(inject_webdriver=True, uc=True)
+def import_diablo_trade(url: str, max_listings: int, driver: seleniumbase.Driver = None):
     url = url.strip().replace("\n", "")
     if BASE_URL not in url:
         LOGGER.error("Invalid url, please use a diablo.trade filter url")
@@ -62,11 +63,12 @@ def import_diablo_trade(url: str, max_listings: int):
     while True:
         api_url = _construct_api_url(listing_url=url, cursor=cursor)
         try:
-            r = get_with_retry(url=api_url)
-        except ConnectionError:
-            LOGGER.error("Can't fetch listings, saving current data")
+            driver.default_get(url=api_url)
+            source = lxml.html.fromstring(driver.get_page_source())
+            data = json.loads(source.text_content().strip())
+        except Exception:
+            LOGGER.exception("Can't fetch listings, saving current data")
             break
-        data = r.json()
         if not (listings := data["data"]):
             LOGGER.debug("Reached end")
             break

tests/gui/importer/test_diablo_trade.py

@@ -10,7 +10,7 @@
 
 
 @pytest.mark.parametrize("url", URLS)
-@pytest.mark.requests
+@pytest.mark.selenium
 def test_import_diablo_trade(url: str, mock_ini_loader: MockerFixture, mocker: MockerFixture):
     Dataloader()  # need to load data first or the mock will make it impossible
     mocker.patch("builtins.open", new=mocker.mock_open())