update poetry version, switch to ruff format

.github/workflows/build-docs.yml

@@ -9,8 +9,8 @@ on:
   workflow_dispatch:
 
 env:
-  PYTHON_VERSION: "3.10"
-  POETRY_VERSION: "1.5.1"
+  PYTHON_VERSION: "3.11"
+  POETRY_VERSION: "1.6.1"
 
 jobs:
   documentation:

.github/workflows/run-tests.yml

@@ -11,7 +11,7 @@ on:
   workflow_dispatch:
 
 env:
-  POETRY_VERSION: "1.5.1"
+  POETRY_VERSION: "1.6.1"
 
 jobs:
   deploy:
@@ -114,7 +114,7 @@ jobs:
       - name: Style checking
         run: |
           poetry run ruff src tests
-          poetry run black --check src tests
+          poetry run ruff format --check src tests
 
       - name: Type checking
         run: |

.pre-commit-config.yaml

@@ -21,14 +21,12 @@ repos:
     hooks:
       - id: sync_with_poetry
         args: [] # optional args
-  - repo: https://github.com/psf/black
-    rev: 23.10.1
-    hooks:
-      - id: black
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.0.280"
+    rev: v0.1.4
     hooks:
       - id: ruff
+      - id: ruff-format
+        args: [--check]
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.6.1
     hooks:
@@ -37,6 +35,7 @@ repos:
         additional_dependencies:
           - types-pkg-resources
           - types-requests
+          - types-paramiko
           - types-flask
           - types-waitress
           - impunity

pyproject.toml

@@ -91,7 +91,6 @@ rtlsdr =  ["pyrtlsdr"]
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = ">=2.13.0"
-black = ">=21.6"
 mypy = ">=0.981"
 ruff = ">=0.0.247"
 Sphinx = ">=5.1"
@@ -105,6 +104,7 @@ codecov = ">=2.1.11"
 ipykernel = "^6.25.0"
 types-pkg-resources = ">=0.1.3"
 types-requests = ">=2.25.0"
+types-paramiko = ">=3.3.0.0"
 types-flask = ">=1.1.6"
 types-waitress = ">=2.0.8"
 requests = { extras = ["socks"], version = ">=2.27" }
@@ -114,21 +114,6 @@ requests = { extras = ["socks"], version = ">=2.27" }
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
 build-backend = "poetry_dynamic_versioning.backend"
 
-[tool.black]
-line-length = 80
-target_version = ['py38', 'py39', 'py310', 'py311']
-include = '\.pyi?$'
-exclude = '''
-/(
-    __pycache__
-  | \.git
-  | \.mypy_cache
-  | \.ipynb_checkpoints
-  | build
-  | dist
-)/
-'''
-
 [tool.ruff]
 select = [
   "E", "W",  # pycodestyle

src/traffic/algorithms/navigation.py

@@ -531,9 +531,11 @@ def most_probable_navpoints(traj: "Flight") -> Iterator[pd.DataFrame]:
             for block in groupby_intervals(table):
                 d_max = block.eval("duration.max()")
                 t_threshold = d_max - pd.Timedelta("30s")  # noqa: F841
-                yield block.sort_values("shift_mean").query(
-                    "duration >= @t_threshold"
-                ).head(1)
+                yield (
+                    block.sort_values("shift_mean")
+                    .query("duration >= @t_threshold")
+                    .head(1)
+                )
 
         return pd.concat(list(most_probable_navpoints(self))).merge(
             navaids_.data, left_on="navaid", right_on="name"

src/traffic/console/decode.py

@@ -94,11 +94,11 @@ def render(self) -> Table:
                     ac_elt.icao24,
                     format(tail, "%typecode %registration") if tail else "",
                     ac_elt.callsign,
-                    str(ac_elt.lat),
-                    str(ac_elt.lon),
+                    f"{ac_elt.lat:.5}",
+                    f"{ac_elt.lon:.5}",
                     str(ac_elt.alt),
                     str(ac_elt.spd),
-                    str(ac_elt.trk),
+                    f"{ac_elt.trk:.0}",
                     str(len(cumul)),
                     format(cumul[-1]["timestamp"], "%H:%M:%S"),
                 )
@@ -145,8 +145,7 @@ def get_icao24(icao24: str) -> dict[str, list[Entry]]:
     if aircraft_or_none:
         return {
             icao24: list(
-                entry
-                | dict(timestamp=entry["timestamp"].timestamp())  # type: ignore
+                entry | dict(timestamp=entry["timestamp"].timestamp())  # type: ignore
                 for entry in aircraft_or_none.cumul
             )
         }
@@ -198,6 +197,7 @@ def get_icao24(icao24: str) -> dict[str, list[Entry]]:
     default=False,
     help="Display aircraft table in text user interface mode",
 )
+@click.option("-t", "--reference-time", default="now")
 @click.option("-v", "--verbose", count=True, help="Verbosity level")
 def main(
     source: str,
@@ -211,6 +211,7 @@ def main(
     log_file: str | None = None,
     tui: bool = True,
     verbose: int = 0,
+    reference_time: str = "now",
 ) -> None:
     """Decode ADS-B data from a source of raw binary data.
 
@@ -243,6 +244,26 @@ def main(
 
     dump_file = Path(filename).with_suffix(".csv").as_posix()
 
+    if source == "-":
+        assert initial_reference is not None
+        decoder = Decoder.from_dump1090_output(
+            filename="/dev/stdin",
+            reference=initial_reference,
+            uncertainty=decode_uncertainty,
+            reference_time=reference_time,
+        )
+        traffic = decoder.traffic
+        if traffic is not None:
+            console = Console()
+            console.print(traffic)
+            icao24 = traffic.basic_stats.reset_index().iloc[0].icao24
+            flight = traffic[icao24]
+            console.print(flight)
+            output = Path(filename).with_suffix(".parquet")
+            traffic.to_parquet(output)
+            logging.warning(f"File written: {output}")
+        return None
+
     if Path(source).expanduser().exists():
         assert initial_reference is not None
         decoder = Decoder.from_file(source, initial_reference)

src/traffic/core/flight.py

@@ -1637,7 +1637,8 @@ def handle_last_position(self) -> Flight:
                 data.assign(
                     _mark=lambda df: df.last_position
                     != df.shift(1).last_position
-                ).assign(
+                )
+                .assign(
                     latitude=lambda df: df.latitude * df._mark / df._mark,
                     longitude=lambda df: df.longitude * df._mark / df._mark,
                     altitude=lambda df: df.altitude * df._mark / df._mark,

src/traffic/data/adsb/decode.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import codecs
 import heapq
 import logging
 import os
@@ -45,7 +46,7 @@
 MSG_SIZES = {0x31: 11, 0x32: 16, 0x33: 23, 0x34: 23}
 
 
-def next_beast_msg(chunk_it: Iterator[bytes]) -> Iterator[bytes]:
+def next_beast_msg(chunk_it: Iterable[bytes]) -> Iterator[bytes]:
     """Iterate in Beast binary feed.
 
     <esc> "1" : 6 byte MLAT timestamp, 1 byte signal level,
@@ -137,7 +138,7 @@ def decode_time_dump1090(
 
 decode_time: dict[str, Callable[[str, Optional[datetime]], datetime]] = {
     "radarcape": decode_time_radarcape,
-    "dump1090": decode_time_dump1090,
+    "dump1090": decode_time_default,
     "default": decode_time_default,
 }
 
@@ -914,8 +915,10 @@ def __init__(self, template: str, sep: str = ",") -> None:
         self.sep = sep
         self.cols = list(x.strip() for x in template.split(sep))
         time_gen = (i for i, elt in enumerate(self.cols) if elt == "time")
-        self.time_index = next(time_gen, None)
-        if self.time_index is None:
+        time_index = next(time_gen, None)
+        if time_index is not None:
+            self.time_index = time_index
+        else:
             msg = "Format invalid: must contain 'time'"
             raise ValueError(msg)
 
@@ -937,7 +940,8 @@ def __init__(self, template: str, sep: str = ",") -> None:
     def get_timestamp(self, line: str) -> datetime:
         elts = line.split(self.sep)
         return datetime.fromtimestamp(
-            float(elts[self.time_index].strip()), timezone.utc  # type: ignore
+            float(elts[self.time_index].strip()),
+            timezone.utc,
         )
 
     def get_msg(self, line: str) -> str:
@@ -1337,6 +1341,83 @@ def stop(self) -> None:
     def __del__(self) -> None:
         self.stop()
 
+    @classmethod
+    def from_dump1090_output(
+        cls: Type[D],
+        filename: str | Path,
+        reference: Union[str, Airport, tuple[float, float]],
+        uncertainty: bool = False,
+        file_pattern: None | str = "~/ADSB_EHS_RAW_%Y%m%d_dump1090.csv",
+        crc_check: bool = False,
+        reference_time: str = "now",
+    ) -> D:  # coverage: ignore
+        """Decode raw messages dumped from `dump1090
+        <https://github.com/MalcolmRobb/dump1090/>`_ with option mlat
+
+        :param filename: the path to the file containing the data
+        :param reference: the reference location, as specified above
+        :param file_pattern: the filename where to dump received hexadecimal
+            messages
+
+            Timestamp format specifiers are accepted.
+
+            | Default value: ``"~/ADSB_EHS_RAW_%Y%m%d_dump1090.csv"``
+            | (The ``~`` character gets expanded as your home directory)
+
+        :param uncertainty: if True, decode also `uncertainty information
+            <https://mode-s.org/decode/content/ads-b/7-uncertainty.html>`_
+
+        :param crc_check: if True, perform CRC check on messages and discard
+            invalid messages. DF 4, 5, 20 and 21 messages don't have CRC so the
+            parameter should be set to False if you only have those messages.
+
+        .. warning::
+
+            dump1090 must be run the ``--mlat`` option.
+
+        """
+        # TODO edit
+        now = pd.Timestamp(reference_time, tz="utc")
+
+        filename = Path(filename)
+        decoder = cls(reference)
+        b_content = filename.read_bytes()
+        try:
+            all_lines = b_content.decode()
+        except UnicodeDecodeError:
+            all_lines = codecs.encode(b_content, "hex").decode()
+
+        if all_lines.startswith("@"):  # dump1090 with --mlat option
+            decoder.process_msgs(
+                list(
+                    (
+                        now + timedelta(seconds=int(line[1:13], 16) / 12e6),
+                        line[13:-1],
+                    )
+                    for line in all_lines.split("\n")
+                    if line.startswith("@")
+                ),
+                uncertainty=uncertainty,
+                crc_check=crc_check,
+            )
+
+        elif all_lines.startswith("1a"):  # xxd -p on radarcape dump
+            content = bytes.fromhex(all_lines.replace("\n", ""))
+            for bin_msg in next_beast_msg([content]):
+                msg = "".join(["{:02x}".format(t) for t in bin_msg])
+                print(msg)
+                if len(bin_msg) < 23:
+                    continue
+                t = decode_time_radarcape(msg, now)
+                decoder.process(
+                    t,
+                    msg[18:],
+                    uncertainty=uncertainty,
+                    crc_check=crc_check,
+                )
+
+        return decoder
+
     @classmethod
     def from_dump1090(
         cls: Type[D],

src/traffic/data/adsb/opensky.py

@@ -472,8 +472,8 @@ def extended(
     ) -> None | RawData:
         kwargs = {
             **kwargs,
+            # this one is with impala
             **(
-                # this one is with impala
                 dict(table_name="rollcall_replies_data4")
                 if isinstance(self.db_client, impala.Impala)
                 # this one is with Trino

traffic.code-workspace

@@ -9,7 +9,7 @@
 
     "[python]": {
       "files.insertFinalNewline": true,
-      "editor.defaultFormatter": "ms-python.black-formatter",
+      "editor.defaultFormatter": "charliermarsh.ruff",
       "editor.codeActionsOnSave": {
         "source.fixAll": true,
         "source.organizeImports": true