feat: introduce observation generation for messaging nettests

oonidata/models/nettests/__init__.py

@@ -3,6 +3,7 @@
 from .base_measurement import BaseMeasurement
 from .dnscheck import DNSCheck
 from .signal import Signal
+from .facebook_messenger import FacebookMessenger
 from .telegram import Telegram
 from .tor import Tor
 from .web_connectivity import WebConnectivity
@@ -18,6 +19,7 @@
     Tor,
     DNSCheck,
     Signal,
+    FacebookMessenger,
     Whatsapp,
     BaseMeasurement,
 ]
@@ -29,6 +31,7 @@
     Tor,
     DNSCheck,
     Signal,
+    FacebookMessenger,
     Whatsapp,
     BaseMeasurement,
 ]

oonidata/models/nettests/facebook_messenger.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass
+from typing import List, Optional
+from oonidata.compat import add_slots
+from oonidata.models.base_model import BaseModel
+from oonidata.models.dataformats import (
+    TCPConnect,
+    DNSQuery
+)
+
+from .base_measurement import BaseMeasurement
+
+
+@add_slots
+@dataclass
+class FacebookMessengerTestKeys(BaseModel):
+    facebook_b_api_dns_consistent: Optional[bool] = None
+    facebook_b_api_reachable: Optional[bool] = None
+    facebook_b_graph_dns_consistent: Optional[bool] = None
+    facebook_b_graph_reachable: Optional[bool] = None
+    facebook_dns_blocking: Optional[bool] = None
+    facebook_edge_dns_consistent: Optional[bool] = None
+    facebook_edge_reachable: Optional[bool] = None
+    facebook_external_cdn_dns_consistent: Optional[bool] = None
+    facebook_external_cdn_reachable: Optional[bool] = None
+    facebook_scontent_cdn_dns_consistent: Optional[bool] = None
+    facebook_scontent_cdn_reachable: Optional[bool] = None
+    facebook_star_dns_consistent: Optional[bool] = None
+    facebook_star_reachable: Optional[bool] = None
+    facebook_stun_dns_consistent: Optional[bool] = None
+    facebook_stun_reachable: Optional[bool] = None
+    facebook_tcp_blocking: Optional[bool] = None
+
+    socksproxy: Optional[str] = None,
+    tcp_connect: Optional[List[TCPConnect]] = None
+    queries: Optional[List[DNSQuery]] = None
+
+
+@add_slots
+@dataclass
+class FacebookMessenger(BaseMeasurement):
+    __test_name__ = "facebook_messenger"
+
+    test_keys: FacebookMessengerTestKeys

oonidata/transforms/__init__.py

@@ -5,6 +5,8 @@
     HTTPHeaderFieldManipulationTransformer,
 )
 from oonidata.transforms.nettests.signal import SignalTransformer
+from oonidata.transforms.nettests.facebook_messenger import FacebookMessengerTransformer
+from oonidata.transforms.nettests.whatsapp import WhatsappTransformer
 from oonidata.transforms.nettests.telegram import TelegramTransformer
 from oonidata.transforms.nettests.tor import TorTransformer
 from oonidata.transforms.nettests.web_connectivity import WebConnectivityTransformer
@@ -15,6 +17,8 @@
 NETTEST_TRANSFORMERS = {
     "dnscheck": DNSCheckTransformer,
     "signal": SignalTransformer,
+    "facebook_messenger": FacebookMessengerTransformer,
+    "whatsapp": WhatsappTransformer,
     "telegram": TelegramTransformer,
     "tor": TorTransformer,
     "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer,

oonidata/transforms/nettests/facebook_messenger.py

@@ -0,0 +1,17 @@
+from typing import List, Tuple
+from oonidata.models.nettests import FacebookMessenger
+from oonidata.models.observations import WebObservation
+from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer
+
+
+class FacebookMessengerTransformer(MeasurementTransformer):
+    def make_observations(self, msmt: FacebookMessenger) -> Tuple[List[WebObservation]]:
+        dns_observations = self.make_dns_observations(msmt.test_keys.queries)
+        tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect)
+
+        return (
+            self.consume_web_observations(
+                dns_observations=dns_observations,
+                tcp_observations=tcp_observations,
+            ),
+        )

oonidata/transforms/nettests/whatsapp.py

@@ -0,0 +1,24 @@
+from typing import List, Tuple
+from oonidata.models.nettests import Whatsapp
+from oonidata.models.observations import WebObservation
+from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer
+
+
+class WhatsappTransformer(MeasurementTransformer):
+    def make_observations(self, msmt: Whatsapp) -> Tuple[List[WebObservation]]:
+        dns_observations = self.make_dns_observations(msmt.test_keys.queries)
+        tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect)
+        tls_observations = self.make_tls_observations(
+            msmt.test_keys.tls_handshakes, 
+            msmt.test_keys.network_events,
+        )
+        http_observations = self.make_http_observations(msmt.test_keys.requests)
+
+        return (
+            self.consume_web_observations(
+                dns_observations=dns_observations,
+                tcp_observations=tcp_observations,
+                tls_observations=tls_observations,
+                http_observations=http_observations,
+            ),
+        )

tests/_sample_measurements.py

@@ -18,6 +18,8 @@
     "20221013000000.517636_US_dnscheck_bfd6d991e70afa0e",
     "20221114002335.786418_BR_webconnectivity_6b203219ec4ded0e",
     "20230427235943.206438_US_telegram_ac585306869eca7b",
+    "20211018232506.972850_IN_whatsapp_44970a56806dbfb3",
+    "20220124235953.650143_ES_facebookmessenger_0e048a26b89a9d70",
     "20210101181154.037019_CH_webconnectivity_68ce38aa9e3182c2",
     "20210101190046.780850_US_webconnectivity_3296f126f79ca186",
     "20231031032643.267235_GR_dnscheck_abcbfc460b9424b6",

tests/test_transforms.py

@@ -2,6 +2,9 @@
 from oonidata.analysis.datasources import load_measurement
 from oonidata.models.nettests.dnscheck import DNSCheck
 from oonidata.models.nettests.telegram import Telegram
+from oonidata.models.nettests.signal import Signal
+from oonidata.models.nettests.facebook_messenger import FacebookMessenger
+from oonidata.models.nettests.whatsapp import Whatsapp
 from oonidata.models.nettests.web_connectivity import WebConnectivity
 from oonidata.models.observations import WebObservation
 from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer
@@ -173,3 +176,79 @@ def test_telegram_obs(netinfodb, measurements):
         if wo.tls_cipher_suite:
             assert wo.tls_t
     assert len(web_obs) == 33
+
+
+def test_signal_obs(netinfodb, measurements):
+    msmt = load_measurement(
+        msmt_path=measurements["20221016235944.266268_GB_signal_1265ff650ee17b44"]
+    )
+    assert isinstance(msmt, Signal)
+    web_obs: List[WebObservation] = measurement_to_observations(
+        msmt=msmt, netinfodb=netinfodb
+    )[0]
+    for wo in web_obs:
+        if wo.dns_engine:
+            print(wo.dns_engine)
+            assert wo.dns_t
+        if wo.tcp_success is not None:
+            assert wo.tcp_t
+        if wo.http_request_url:
+            assert wo.http_t
+        if wo.tls_cipher_suite:
+            assert wo.tls_t
+    assert len(web_obs) == 19
+
+
+def test_whatsapp_obs(netinfodb, measurements):
+    msmt = load_measurement(
+        msmt_path=measurements["20211018232506.972850_IN_whatsapp_44970a56806dbfb3"]
+    )
+    assert isinstance(msmt, Whatsapp)
+    web_obs: List[WebObservation] = measurement_to_observations(
+        msmt=msmt, netinfodb=netinfodb
+    )[0]
+    for wo in web_obs:
+        if wo.dns_engine:
+            assert wo.dns_t
+        if wo.tcp_success is not None:
+            assert wo.tcp_t
+        if wo.http_request_url:
+            assert wo.http_t
+        if wo.tls_cipher_suite:
+            assert wo.tls_t
+    assert len(web_obs) == 137
+
+
+def test_facebook_messenger_obs(netinfodb, measurements):
+    msmt = load_measurement(
+        msmt_path=measurements[
+            "20220124235953.650143_ES_facebookmessenger_0e048a26b89a9d70"
+        ]
+    )
+    assert isinstance(msmt, FacebookMessenger)
+    web_obs: List[WebObservation] = measurement_to_observations(
+        msmt=msmt, netinfodb=netinfodb
+    )[0]
+
+    # Based on https://github.com/ooni/spec/blob/master/nettests/ts-019-facebook-messenger.md
+    spec_hostname_set = set(
+        [
+            "stun.fbsbx.com",
+            "b-api.facebook.com",
+            "b-graph.facebook.com",
+            "edge-mqtt.facebook.com",
+            "external.xx.fbcdn.net",
+            "scontent.xx.fbcdn.net",
+            "star.c10r.facebook.com",
+        ]
+    )
+
+    hostname_set = set()
+    for wo in web_obs:
+        if wo.dns_engine:
+            assert wo.dns_t
+        if wo.tcp_success is not None:
+            assert wo.tcp_t
+        hostname_set.add(wo.hostname)
+    assert hostname_set == spec_hostname_set
+    assert len(web_obs) == 14