Merge pull request #115 from ooni/add-support-for-parsing-echcheck

Add support for parsing echcheck related metadata in pipeline

oonidata/src/oonidata/models/nettests/__init__.py

@@ -13,6 +13,7 @@
 from .whatsapp import Whatsapp
 from .http_invalid_request_line import HTTPInvalidRequestLine
 from .http_header_field_manipulation import HTTPHeaderFieldManipulation
+from .echcheck import ECHCheck
 
 SUPPORTED_CLASSES = [
     HTTPHeaderFieldManipulation,
@@ -28,6 +29,7 @@
     FacebookMessenger,
     Whatsapp,
     BaseMeasurement,
+    ECHCheck,
 ]
 SupportedDataformats = Union[
     HTTPHeaderFieldManipulation,

oonidata/src/oonidata/models/nettests/echcheck.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from oonidata.models.nettests.web_connectivity import WebConnectivityControl
+
+from ...compat import add_slots
+from ..dataformats import (
+    BaseTestKeys,
+    DNSQuery,
+    NetworkEvent,
+    TCPConnect,
+    TLSHandshake,
+)
+from oonidata.models.nettests.base_measurement import BaseMeasurement
+
+
+@add_slots
+@dataclass
+class ECHCheckTestKeys(BaseTestKeys):
+    tls_handshakes: Optional[List[TLSHandshake]] = None
+    control: Optional[TLSHandshake] = None
+    target: Optional[TLSHandshake] = None
+    network_events: Optional[List[NetworkEvent]] = None
+    queries: Optional[List[DNSQuery]] = None
+    tcp_connect: Optional[List[TCPConnect]] = None
+
+
+@add_slots
+@dataclass
+class ECHCheck(BaseMeasurement):
+    __test_name__ = "echcheck"
+
+    test_keys: ECHCheckTestKeys

oonipipeline/src/oonipipeline/transforms/nettests/echcheck.py

@@ -0,0 +1,38 @@
+from typing import List, Tuple
+
+from oonidata.models.nettests import ECHCheck
+from oonidata.models.observations import WebObservation
+
+from ..measurement_transformer import MeasurementTransformer
+
+
+class ECHCheckTransformer(MeasurementTransformer):
+    def make_observations(self, msmt: ECHCheck) -> Tuple[List[WebObservation]]:
+        dns_observations = []
+        tcp_observations = []
+        tls_observations = []
+
+        if msmt.test_keys.queries:
+           dns_observations = self.make_dns_observations(msmt.test_keys.queries)
+        if msmt.test_keys.tcp_connect:
+           tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect)
+
+        if msmt.test_keys.tls_handshakes:
+           tls_observations = self.make_tls_observations(
+              tls_handshakes=msmt.test_keys.tls_handshakes,
+              network_events=msmt.test_keys.network_events,
+           )
+        elif msmt.test_keys.control and msmt.test_keys.target:
+            target = msmt.test_keys.target
+            target.echconfig = "GREASE"
+            tls_observations = self.make_tls_observations(
+                  tls_handshakes=[msmt.test_keys.control, target],
+                  network_events=msmt.test_keys.network_events,
+            )
+
+        return (self.consume_web_observations(
+            dns_observations=dns_observations,
+            tcp_observations=tcp_observations,
+            tls_observations=tls_observations,
+            http_observations=[]
+        ), )

oonipipeline/src/oonipipeline/transforms/observations.py

@@ -37,6 +37,7 @@
 from .nettests.http_invalid_request_line import (
     HTTPInvalidRequestLineTransformer,
 )
+from .nettests.echcheck import ECHCheckTransformer
 
 from ..netinfo import NetinfoDB
 
@@ -53,6 +54,7 @@
     "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer,
     "http_invalid_request_line": HTTPInvalidRequestLineTransformer,
     "web_connectivity": WebConnectivityTransformer,
+    "echcheck": ECHCheckTransformer,
 }
 
 TypeWebConnectivityObservations = Tuple[

oonipipeline/tests/_fixtures.py

@@ -39,6 +39,8 @@
     "20240302000305.316064_EG_webconnectivity_397bca9091b07444",  # nxdomain blocked, unknown_failure and from the future
     "20240309112858.009725_SE_webconnectivity_dce757ef4ec9b6c8",  # blockpage for Iran in Sweden
     "20241101171509.547086_CN_webconnectivity_f0ec3f0e369cec9b",  # web_connectivity 0.5 which was failing
+    "20250120145930.582606_US_echcheck_899a304b7beef05c", # echcheck tls_handshakes
+    "20240714111032.898994_GB_echcheck_f10079cac5cdf770" # echcheck control and target
 ]
 
 SAMPLE_POSTCANS = ["2024030100_AM_webconnectivity.n1.0.tar.gz"]

oonipipeline/tests/test_transforms.py

@@ -2,6 +2,7 @@
 
 from oonidata.dataclient import load_measurement
 from oonidata.models.nettests.dnscheck import DNSCheck
+from oonidata.models.nettests.echcheck import ECHCheck
 from oonidata.models.nettests.telegram import Telegram
 from oonidata.models.nettests.signal import Signal
 from oonidata.models.nettests.facebook_messenger import FacebookMessenger
@@ -408,3 +409,35 @@ def test_facebook_messenger_obs(netinfodb, measurements):
         hostname_set.add(wo.hostname)
     assert hostname_set == spec_hostname_set
     assert len(web_obs) == 14
+
+
+def test_echcheck_obs_tls_handshakes(netinfodb, measurements):
+    msmt = load_measurement(
+        msmt_path=measurements["20250120145930.582606_US_echcheck_899a304b7beef05c"]
+    )
+    assert isinstance(msmt, ECHCheck)
+    assert msmt.test_version == '0.2.0'
+
+    obs_tup = measurement_to_observations(
+        msmt=msmt, netinfodb=netinfodb, bucket_date="2022-10-13"
+    )
+    assert len(obs_tup) == 1
+    web_obs = obs_tup[0]
+    assert len(web_obs) == 3
+    assert any(wo.tls_echconfig == "GREASE" for wo in web_obs)
+
+
+def test_echcheck_obs_control_and_target(netinfodb, measurements):
+    msmt = load_measurement(
+        msmt_path=measurements["20240714111032.898994_GB_echcheck_f10079cac5cdf770"]
+    )
+    assert isinstance(msmt, ECHCheck)
+    assert msmt.test_version == '0.1.2'
+
+    obs_tup = measurement_to_observations(
+        msmt=msmt, netinfodb=netinfodb, bucket_date="2022-10-13"
+    )
+    assert len(obs_tup) == 1
+    web_obs = obs_tup[0]
+    assert len(web_obs) == 2
+    assert any(wo.tls_echconfig == "GREASE" for wo in web_obs)