Merge pull request #50 from DecFox/issue-32-stunreachability

feat: introduce observation generation for stunreachability, urlgetter, browser_web

oonidata/models/nettests/__init__.py

@@ -6,7 +6,10 @@
 from .facebook_messenger import FacebookMessenger
 from .telegram import Telegram
 from .tor import Tor
+from .urlgetter import UrlGetter
 from .web_connectivity import WebConnectivity
+from .stun_reachability import StunReachability
+from .browser_web import BrowserWeb
 from .whatsapp import Whatsapp
 from .http_invalid_request_line import HTTPInvalidRequestLine
 from .http_header_field_manipulation import HTTPHeaderFieldManipulation
@@ -15,8 +18,11 @@
     HTTPHeaderFieldManipulation,
     HTTPInvalidRequestLine,
     WebConnectivity,
+    StunReachability,
+    BrowserWeb,
     Telegram,
     Tor,
+    UrlGetter,
     DNSCheck,
     Signal,
     FacebookMessenger,
@@ -27,8 +33,11 @@
     HTTPHeaderFieldManipulation,
     HTTPInvalidRequestLine,
     WebConnectivity,
+    StunReachability,
+    BrowserWeb,
     Telegram,
     Tor,
+    UrlGetter,
     DNSCheck,
     Signal,
     FacebookMessenger,

oonidata/models/nettests/browser_web.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from typing import List, Optional
+from oonidata.compat import add_slots
+from oonidata.models.dataformats import (
+    BaseTestKeys,
+)
+from oonidata.models.nettests.base_measurement import BaseMeasurement
+
+
+@add_slots
+@dataclass
+class BrowserWebTestKeys(BaseTestKeys):
+    result: str = None
+    load_time_ms: Optional[float] = None
+    browser: Optional[str] = None
+
+
+@add_slots
+@dataclass
+class BrowserWeb(BaseMeasurement):
+    __test_name__ = "browser_web"
+
+    test_keys: BrowserWebTestKeys

oonidata/models/nettests/stun_reachability.py

@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+from typing import List, Optional
+from oonidata.compat import add_slots
+from oonidata.models.dataformats import (
+    BaseTestKeys,
+    DNSQuery,
+    NetworkEvent,
+    Failure,
+)
+from oonidata.models.nettests.base_measurement import BaseMeasurement
+
+
+@add_slots
+@dataclass
+class StunReachabilityTestKeys(BaseTestKeys):
+    failure: Failure = None 
+    endpoint: Optional[str] = None
+
+    network_events: Optional[List[NetworkEvent]] = None
+    queries: Optional[List[DNSQuery]] = None
+
+
+@add_slots
+@dataclass
+class StunReachability(BaseMeasurement):
+    __test_name__ = "stunreachability"
+
+    test_keys: StunReachabilityTestKeys

oonidata/models/nettests/urlgetter.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+from typing import List, Optional
+from oonidata.compat import add_slots
+from oonidata.models.dataformats import (
+    BaseTestKeys,
+    DNSQuery,
+    NetworkEvent,
+    TCPConnect,
+    TLSHandshake,
+    HTTPTransaction,
+    Failure,
+)
+from oonidata.models.nettests.base_measurement import BaseMeasurement
+
+
+@add_slots
+@dataclass
+class UrlGetterTestKeys(BaseTestKeys):
+    failure: Failure = None
+    failed_operation: Optional[str] = None
+    agent: Optional[str] = None
+
+    socksproxy: Optional[str] = None
+    network_events: Optional[List[NetworkEvent]] = None
+    queries: Optional[List[DNSQuery]] = None
+    requests: Optional[List[HTTPTransaction]] = None
+    tcp_connect: Optional[List[TCPConnect]] = None
+    tls_handshakes: Optional[List[TLSHandshake]] = None
+
+
+@add_slots
+@dataclass
+class UrlGetter(BaseMeasurement):
+    __test_name__ = "urlgetter"
+
+    test_keys: UrlGetterTestKeys

oonidata/transforms/__init__.py

@@ -8,7 +8,10 @@
 from oonidata.transforms.nettests.facebook_messenger import FacebookMessengerTransformer
 from oonidata.transforms.nettests.whatsapp import WhatsappTransformer
 from oonidata.transforms.nettests.telegram import TelegramTransformer
+from oonidata.transforms.nettests.stun_reachability import StunReachabilityTransformer
 from oonidata.transforms.nettests.tor import TorTransformer
+from oonidata.transforms.nettests.browser_web import BrowserWebTransformer
+from oonidata.transforms.nettests.urlgetter import UrlGetterTransformer
 from oonidata.transforms.nettests.web_connectivity import WebConnectivityTransformer
 from oonidata.transforms.nettests.http_invalid_request_line import (
     HTTPInvalidRequestLineTransformer,
@@ -20,7 +23,10 @@
     "facebook_messenger": FacebookMessengerTransformer,
     "whatsapp": WhatsappTransformer,
     "telegram": TelegramTransformer,
+    "stunreachability": StunReachabilityTransformer,
     "tor": TorTransformer,
+    "browser_web": BrowserWebTransformer,
+    "urlgetter": UrlGetterTransformer,
     "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer,
     "http_invalid_request_line": HTTPInvalidRequestLineTransformer,
     "web_connectivity": WebConnectivityTransformer,

oonidata/transforms/nettests/browser_web.py

@@ -0,0 +1,19 @@
+from typing import List, Tuple
+import dataclasses
+from oonidata.models.nettests import BrowserWeb
+from oonidata.models.observations import WebObservation
+from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer, make_web_observation
+
+
+class BrowserWebTransformer(MeasurementTransformer):
+    def make_observations(self, msmt: BrowserWeb) -> Tuple[List[WebObservation]]:
+        bw_obs = WebObservation(
+            **dataclasses.asdict(self.measurement_meta)
+        )
+
+        bw_obs.http_failure = msmt.test_keys.result
+        bw_obs.http_runtime = msmt.test_keys.load_time_ms
+
+        return (
+            ([bw_obs],)
+        )

oonidata/transforms/nettests/stun_reachability.py

@@ -0,0 +1,16 @@
+from typing import List, Tuple
+
+from oonidata.models.nettests import StunReachability
+from oonidata.models.observations import WebObservation
+from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer
+
+
+class StunReachabilityTransformer(MeasurementTransformer):
+    def make_observations(self, msmt: StunReachability) -> Tuple[List[WebObservation]]:
+        dns_observations = self.make_dns_observations(msmt.test_keys.queries)
+
+        return (
+            self.consume_web_observations(
+                dns_observations=dns_observations
+            ),
+        )

oonidata/transforms/nettests/urlgetter.py

@@ -0,0 +1,25 @@
+from typing import List, Tuple
+
+from oonidata.models.nettests import UrlGetter
+from oonidata.models.observations import WebObservation
+from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer
+
+
+class UrlGetterTransformer(MeasurementTransformer):
+    def make_observations(self, msmt: UrlGetter) -> Tuple[List[WebObservation]]:
+        dns_observations = self.make_dns_observations(msmt.test_keys.queries)
+        tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect)
+        tls_observations = self.make_tls_observations(
+            msmt.test_keys.tls_handshakes, 
+            msmt.test_keys.network_events
+        )
+        http_observations = self.make_http_observations(msmt.test_keys.requests)
+
+        return (
+            self.consume_web_observations(
+                dns_observations=dns_observations,
+                tcp_observations=tcp_observations,
+                tls_observations=tls_observations,
+                http_observations=http_observations,
+            ),
+        )

tests/_sample_measurements.py

@@ -23,6 +23,10 @@
     "20210101181154.037019_CH_webconnectivity_68ce38aa9e3182c2",
     "20210101190046.780850_US_webconnectivity_3296f126f79ca186",
     "20231031032643.267235_GR_dnscheck_abcbfc460b9424b6",
+    "20221224235924.922622_BR_stunreachability_905c61a34356a9b2",
+    "20211224011542.635260_IR_urlgetter_38d73cdfee442409",  # "dnslookup://example.org"
+    "20221224180301.892770_VE_urlgetter_0a02e27d0c651b8f",  # "https://dolartoday.com/"  
+    "20231222154141.824397_US_browserweb_615428b4802b5297", 
     "20231101164541.763506_NP_httpinvalidrequestline_0cf676868fa36cc4",
     "20231101164544.534107_BR_httpheaderfieldmanipulation_4caa0b0556f0b141",
     "20231101164649.235575_RU_tor_ccf7519bf683c022",

tests/test_transforms.py

@@ -6,6 +6,9 @@
 from oonidata.models.nettests.facebook_messenger import FacebookMessenger
 from oonidata.models.nettests.whatsapp import Whatsapp
 from oonidata.models.nettests.web_connectivity import WebConnectivity
+from oonidata.models.nettests.stun_reachability import StunReachability
+from oonidata.models.nettests.urlgetter import UrlGetter
+from oonidata.models.nettests.browser_web import BrowserWeb
 from oonidata.models.observations import WebObservation
 from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer
 from oonidata.transforms import measurement_to_observations
@@ -178,6 +181,20 @@ def test_telegram_obs(netinfodb, measurements):
     assert len(web_obs) == 33
 
 
+def test_stunreachability_obs(netinfodb, measurements):
+    msmt = load_measurement(
+        msmt_path=measurements["20221224235924.922622_BR_stunreachability_905c61a34356a9b2"]
+    )
+    assert isinstance(msmt, StunReachability)
+    web_obs: List[WebObservation] = measurement_to_observations(
+        msmt=msmt, netinfodb=netinfodb
+    )[0]
+    assert isinstance(web_obs[0], WebObservation)
+    assert len(web_obs) == 1
+    assert web_obs[0].dns_engine == "system"
+    assert web_obs[0].dns_answer == "206.53.159.130"
+
+
 def test_signal_obs(netinfodb, measurements):
     msmt = load_measurement(
         msmt_path=measurements["20221016235944.266268_GB_signal_1265ff650ee17b44"]
@@ -188,7 +205,6 @@ def test_signal_obs(netinfodb, measurements):
     )[0]
     for wo in web_obs:
         if wo.dns_engine:
-            print(wo.dns_engine)
             assert wo.dns_t
         if wo.tcp_success is not None:
             assert wo.tcp_t
@@ -199,6 +215,44 @@ def test_signal_obs(netinfodb, measurements):
     assert len(web_obs) == 19
 
 
+def test_urlgetter_obs(netinfodb, measurements):
+    msmt = load_measurement(
+        msmt_path=measurements["20211224011542.635260_IR_urlgetter_38d73cdfee442409"]
+    )
+    assert isinstance(msmt, UrlGetter)
+    web_obs: List[WebObservation] = measurement_to_observations(
+        msmt=msmt, netinfodb=netinfodb
+    )[0]
+    for wo in web_obs:
+        if wo.dns_engine:
+            assert wo.dns_t
+        if wo.tcp_success is not None:
+            assert wo.tcp_t
+        if wo.http_request_url:
+            assert wo.http_t
+        if wo.tls_cipher_suite:
+            assert wo.tls_t
+    assert len(web_obs) == 6
+
+    msmt = load_measurement(
+        msmt_path=measurements["20221224180301.892770_VE_urlgetter_0a02e27d0c651b8f"]
+    )
+    assert isinstance(msmt, UrlGetter)
+    web_obs: List[WebObservation] = measurement_to_observations(
+        msmt=msmt, netinfodb=netinfodb
+    )[0]
+    for wo in web_obs:
+        if wo.dns_engine:
+            assert wo.dns_t
+        if wo.tcp_success is not None:
+            assert wo.tcp_t
+        if wo.http_request_url:
+            assert wo.http_t
+        if wo.tls_cipher_suite:
+            assert wo.tls_t
+    assert len(web_obs) == 2
+
+
 def test_whatsapp_obs(netinfodb, measurements):
     msmt = load_measurement(
         msmt_path=measurements["20211018232506.972850_IN_whatsapp_44970a56806dbfb3"]
@@ -219,6 +273,19 @@ def test_whatsapp_obs(netinfodb, measurements):
     assert len(web_obs) == 137
 
 
+def test_browserweb_obs(netinfodb, measurements):
+    msmt = load_measurement(
+        msmt_path=measurements["20231222154141.824397_US_browserweb_615428b4802b5297"]
+    )
+    assert isinstance(msmt, BrowserWeb)
+    web_obs: List[WebObservation] = measurement_to_observations(
+        msmt=msmt, netinfodb=netinfodb
+    )[0]
+    assert len(web_obs) == 1
+    assert isinstance(web_obs[0], WebObservation)
+    assert web_obs[0].http_failure == "error"
+
+
 def test_facebook_messenger_obs(netinfodb, measurements):
     msmt = load_measurement(
         msmt_path=measurements[