general: set min versionto 3.9, since 3.8 reached EOL

also update tox file and other ci files

.github/workflows/main.yml

@@ -24,20 +24,23 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
         exclude: [
             # windows runners are pretty scarce, so let's only run lowest and highest python version
             {platform: windows-latest, python-version: '3.10'},
             {platform: windows-latest, python-version: '3.11'},
+            {platform: windows-latest, python-version: '3.12'},
 
             # same, macos is a bit too slow and ubuntu covers python quirks well
             {platform: macos-latest  , python-version: '3.10'},
             {platform: macos-latest  , python-version: '3.11'},
+            {platform: macos-latest  , python-version: '3.12'},
         ]
 
     runs-on: ${{ matrix.platform }}
 
-    #continue-on-error: ${{ matrix.platform == 'windows-latest' }}
+    # useful for 'optional' pipelines
+    # continue-on-error: ${{ matrix.platform == 'windows-latest' }}
 
     steps:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
@@ -61,11 +64,13 @@ jobs:
     - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
       uses: actions/upload-artifact@v4
       with:
+        include-hidden-files: true
         name: .coverage.mypy-core_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy-core/
     - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
       uses: actions/upload-artifact@v4
       with:
+        include-hidden-files: true
         name: .coverage.mypy-misc_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy-misc/
 

ruff.toml

@@ -1,4 +1,4 @@
-target-version = "py38"  # NOTE: inferred from pyproject.toml if present
+target-version = "py39"  # NOTE: inferred from pyproject.toml if present
 
 lint.extend-select = [
     "F",    # flakes rules -- default, but extend just in case
@@ -9,7 +9,7 @@ lint.extend-select = [
     "C4",   # flake8-comprehensions -- unnecessary list/map/dict calls
     "COM",  # trailing commas
     "EXE",  # various checks wrt executable files
-    # "I",    # sort imports
+    "I",    # sort imports
     "ICN",  # various import conventions
     "FBT",  # detect use of boolean arguments
     "FURB", # various rules

setup.py

@@ -32,7 +32,7 @@ def main() -> None:
         author_email='[email protected]',
         description='Enhancement of your browsing history',
 
-        python_requires='>=3.8',
+        python_requires='>=3.9',
         install_requires=[
             'appdirs', # for portable user directories detection
             'tzlocal',

src/promnesia/__main__.py

@@ -8,10 +8,11 @@
 import shlex
 import shutil
 import sys
+from collections.abc import Iterable, Iterator, Sequence
 from pathlib import Path
 from subprocess import Popen, check_call, run
 from tempfile import TemporaryDirectory, gettempdir
-from typing import Callable, Iterable, Iterator, Sequence
+from typing import Callable
 
 from . import config, server
 from .common import (

src/promnesia/cannon.py

@@ -14,10 +14,11 @@
 import re
 import typing
 import urllib.parse
+from collections.abc import Collection, Iterable, Sequence
 
 # TODO eh?? they fixed mobile.twitter.com?
 from itertools import chain
-from typing import Any, Collection, Iterable, NamedTuple, Sequence, Tuple, Union
+from typing import Any, NamedTuple, Union
 from urllib.parse import SplitResult, parse_qsl, urlencode, urlsplit, urlunsplit
 
 # this has some benchmark, but quite a few librarires seem unmaintained, sadly
@@ -217,7 +218,7 @@ def get_spec(dom: str) -> Spec:
 
 # TODO this should be a map
 Frag = Any
-Parts = Sequence[Tuple[str, str]]
+Parts = Sequence[tuple[str, str]]
 
 
 def _yc(domain: str, path: str, qq: Parts, frag: Frag) -> tuple[Any, Any, Parts, Frag]:
@@ -284,7 +285,7 @@ def transform_split(split: SplitResult):
     REST = r'(?P<rest>.*)'
 
     Left = Union[str, Sequence[str]]
-    Right = Tuple[str, str, str]
+    Right = tuple[str, str, str]
     # the idea is that we can unify certain URLs here and map them to the 'canonical' one
     # this is a dict only for grouping but should be a list really.. todo
     rules: dict[Left, Right] = {

src/promnesia/common.py

@@ -7,6 +7,7 @@
 import shutil
 import tempfile
 import warnings
+from collections.abc import Iterable, Sequence
 from contextlib import contextmanager
 from copy import copy
 from datetime import date, datetime
@@ -16,7 +17,7 @@
 from subprocess import PIPE, Popen, run
 from timeit import default_timer as timer
 from types import ModuleType
-from typing import Callable, Iterable, NamedTuple, Optional, Sequence, TypeVar, Union
+from typing import Callable, NamedTuple, Optional, TypeVar, Union
 
 import pytz
 from more_itertools import intersperse

src/promnesia/compare.py

@@ -4,8 +4,9 @@
 import argparse
 import logging
 import sys
+from collections.abc import Iterator, Sequence
 from pathlib import Path
-from typing import Iterator, Sequence, TypeVar
+from typing import TypeVar
 
 from .common import DbVisit, PathWithMtime, Url
 from .database.load import row_to_db_visit

src/promnesia/compat.py

@@ -7,9 +7,6 @@
     from typing import Literal, Protocol  # noqa: F401
     ##
 
-
-# can deprecate after python3.9
-def removeprefix(text: str, prefix: str) -> str:
-    if text.startswith(prefix):
-        return text[len(prefix):]
-    return text
+    # todo deprecate properly
+    def removeprefix(text: str, prefix: str) -> str:
+        return text.removeprefix(prefix)

src/promnesia/config.py

@@ -4,9 +4,10 @@
 import importlib.util
 import os
 import warnings
+from collections.abc import Iterable
 from pathlib import Path
 from types import ModuleType
-from typing import Callable, Iterable, NamedTuple, Union
+from typing import Callable, NamedTuple, Union
 
 from .common import DbVisit, PathIsh, Res, Source, default_cache_dir, default_output_dir
 

src/promnesia/database/common.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
+from collections.abc import Sequence
 from datetime import datetime
-from typing import Sequence
 
 from sqlalchemy import (
     Column,

src/promnesia/database/dump.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import sqlite3
+from collections.abc import Iterable
 from pathlib import Path
-from typing import Dict, Iterable, Optional
+from typing import Optional
 
 from more_itertools import chunked
 from sqlalchemy import (
@@ -50,7 +51,7 @@ def begin_immediate_transaction(conn):
     conn.exec_driver_sql('BEGIN IMMEDIATE')
 
 
-Stats = Dict[Optional[SourceName], int]
+Stats = dict[Optional[SourceName], int]
 
 
 # returns critical warnings

src/promnesia/database/load.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import Tuple
 
 from sqlalchemy import (
     Engine,
@@ -14,7 +13,7 @@
 
 from .common import DbVisit, get_columns, row_to_db_visit
 
-DbStuff = Tuple[Engine, Table]
+DbStuff = tuple[Engine, Table]
 
 
 def get_db_stuff(db_path: Path) -> DbStuff:

src/promnesia/extract.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import re
+from collections.abc import Iterable, Sequence
 from functools import lru_cache
-from typing import Iterable, Sequence
 
 from .cannon import CanonifyException
 from .common import (

src/promnesia/kjson.py

@@ -5,17 +5,17 @@
 """
 from __future__ import annotations
 
-from typing import Any, Dict, List, Tuple, Union, cast
+from typing import Any, Union, cast
 
-JDict = Dict[str, Any] # TODO not sure if we can do recursive..
-JList = List[Any]
+JDict = dict[str, Any] # TODO not sure if we can do recursive..
+JList = list[Any]
 JPrim = Union[str, int, float] # , type(None)]
 
 Json = Union[JDict, JList, JPrim]
 
-JPathPart = Tuple[Json, Union[str, int]]
+JPathPart = tuple[Json, Union[str, int]]
 
-JPath = Tuple[JPathPart, ...]
+JPath = tuple[JPathPart, ...]
 
 
 class JsonProcessor:

src/promnesia/server.py

@@ -9,7 +9,7 @@
 from datetime import timedelta
 from functools import lru_cache
 from pathlib import Path
-from typing import Any, Dict, List, NamedTuple, Optional, Protocol
+from typing import Any, NamedTuple, Optional, Protocol
 
 import fastapi
 import pytz
@@ -39,7 +39,7 @@
 )
 from .database.load import DbStuff, get_db_stuff, row_to_db_visit
 
-Json = Dict[str, Any]
+Json = dict[str, Any]
 
 app = fastapi.FastAPI()
 
@@ -348,10 +348,10 @@ def as_version(version: str) -> tuple[int, int, int]:
 
 @dataclass
 class VisitedRequest:
-    urls: List[str]  # noqa: UP006  # pydantic doesn't like list[str] on 3.8 -- remove later
+    urls: list[str]
     client_version: str = ''
 
-VisitedResponse = List[Optional[Json]]
+VisitedResponse = list[Optional[Json]]
 
 @app.get ('/visited', response_model=VisitedResponse)
 @app.post('/visited', response_model=VisitedResponse)

src/promnesia/sources/auto.py

@@ -11,12 +11,13 @@
 import itertools
 import json
 import os
+from collections.abc import Iterable, Iterator, Sequence
 from concurrent.futures import ProcessPoolExecutor as Pool
 from contextlib import nullcontext
 from fnmatch import fnmatch
 from functools import wraps
 from pathlib import Path
-from typing import Any, Callable, Iterable, Iterator, NamedTuple, Optional, Sequence
+from typing import Any, Callable, NamedTuple, Optional
 
 from promnesia.common import (
     Loc,

src/promnesia/sources/browser.py

@@ -6,7 +6,8 @@
 
 import re
 import warnings
-from typing import TYPE_CHECKING, Any, Iterator
+from collections.abc import Iterator
+from typing import TYPE_CHECKING, Any
 
 from promnesia.common import Loc, PathIsh, Results, Second, Visit, is_sqlite_db, logger
 

src/promnesia/sources/browser_legacy.py

@@ -122,15 +122,16 @@ def _index_db(db: Path, emitted: set):
 ColType = str
 
 
-from typing import NamedTuple, Sequence, Tuple, Union
+from collections.abc import Sequence
+from typing import NamedTuple, Union
 
 
 class Schema(NamedTuple):
     cols: Sequence[tuple[Col, ColType]]
     key: Sequence[str]
 
 
-SchemaCheck = Tuple[str, Union[str, Sequence[str]]] # todo Union: meh
+SchemaCheck = tuple[str, Union[str, Sequence[str]]] # todo Union: meh
 
 from dataclasses import dataclass
 

src/promnesia/sources/filetypes.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
+from collections.abc import Iterable, Sequence
 from functools import lru_cache
 from pathlib import Path
-from typing import Callable, Iterable, NamedTuple, Sequence, Union
+from typing import Callable, NamedTuple, Union
 
 from ..common import Results, Url
 

src/promnesia/sources/guess.py

@@ -1,6 +1,7 @@
 # TODO eh. confusing how guess and auto are different...
 # maybe merge them later?
-from typing import Any, Iterable
+from collections.abc import Iterable
+from typing import Any
 
 from ..common import Extraction, PathIsh
 

src/promnesia/sources/html.py

@@ -4,8 +4,8 @@
 
 from __future__ import annotations
 
+from collections.abc import Iterator
 from pathlib import Path
-from typing import Iterator, Tuple
 
 from bs4 import BeautifulSoup
 
@@ -15,7 +15,7 @@
 # TODO on some exceptions, request a fallback to text?
 
 
-Url = Tuple[str, str]
+Url = tuple[str, str]
 
 
 def extract_urls_from_html(s: str) -> Iterator[Url]:

src/promnesia/sources/markdown.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+from collections.abc import Iterator
 from pathlib import Path
-from typing import Iterator, NamedTuple
+from typing import NamedTuple
 
 import mistletoe  # type: ignore
 import mistletoe.block_token as BT  # type: ignore

src/promnesia/sources/org.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
 import re
+from collections.abc import Iterable, Iterator
 from datetime import datetime
 from pathlib import Path
-from typing import Iterable, Iterator, NamedTuple, Optional, cast
+from typing import NamedTuple, Optional, cast
 
 import orgparse
 from orgparse.date import OrgDate, gene_timestamp_regex