diff --git a/oonidata/models/nettests/__init__.py b/oonidata/models/nettests/__init__.py index 41750197..edb5082d 100644 --- a/oonidata/models/nettests/__init__.py +++ b/oonidata/models/nettests/__init__.py @@ -3,6 +3,7 @@ from .base_measurement import BaseMeasurement from .dnscheck import DNSCheck from .signal import Signal +from .facebook_messenger import FacebookMessenger from .telegram import Telegram from .tor import Tor from .web_connectivity import WebConnectivity @@ -18,6 +19,7 @@ Tor, DNSCheck, Signal, + FacebookMessenger, Whatsapp, BaseMeasurement, ] @@ -29,6 +31,7 @@ Tor, DNSCheck, Signal, + FacebookMessenger, Whatsapp, BaseMeasurement, ] diff --git a/oonidata/models/nettests/facebook_messenger.py b/oonidata/models/nettests/facebook_messenger.py new file mode 100644 index 00000000..35aefe19 --- /dev/null +++ b/oonidata/models/nettests/facebook_messenger.py @@ -0,0 +1,43 @@ +from dataclasses import dataclass +from typing import List, Optional +from oonidata.compat import add_slots +from oonidata.models.base_model import BaseModel +from oonidata.models.dataformats import ( + TCPConnect, + DNSQuery +) + +from .base_measurement import BaseMeasurement + + +@add_slots +@dataclass +class FacebookMessengerTestKeys(BaseModel): + facebook_b_api_dns_consistent: Optional[bool] = None + facebook_b_api_reachable: Optional[bool] = None + facebook_b_graph_dns_consistent: Optional[bool] = None + facebook_b_graph_reachable: Optional[bool] = None + facebook_dns_blocking: Optional[bool] = None + facebook_edge_dns_consistent: Optional[bool] = None + facebook_edge_reachable: Optional[bool] = None + facebook_external_cdn_dns_consistent: Optional[bool] = None + facebook_external_cdn_reachable: Optional[bool] = None + facebook_scontent_cdn_dns_consistent: Optional[bool] = None + facebook_scontent_cdn_reachable: Optional[bool] = None + facebook_star_dns_consistent: Optional[bool] = None + facebook_star_reachable: Optional[bool] = None + facebook_stun_dns_consistent: Optional[bool] = None + facebook_stun_reachable: Optional[bool] = None + facebook_tcp_blocking: Optional[bool] = None + + socksproxy: Optional[str] = None, + tcp_connect: Optional[List[TCPConnect]] = None + queries: Optional[List[DNSQuery]] = None + + +@add_slots +@dataclass +class FacebookMessenger(BaseMeasurement): + __test_name__ = "facebook_messenger" + + test_keys: FacebookMessengerTestKeys diff --git a/oonidata/transforms/__init__.py b/oonidata/transforms/__init__.py index ec4de232..e7c3f0f8 100644 --- a/oonidata/transforms/__init__.py +++ b/oonidata/transforms/__init__.py @@ -5,6 +5,8 @@ HTTPHeaderFieldManipulationTransformer, ) from oonidata.transforms.nettests.signal import SignalTransformer +from oonidata.transforms.nettests.facebook_messenger import FacebookMessengerTransformer +from oonidata.transforms.nettests.whatsapp import WhatsappTransformer from oonidata.transforms.nettests.telegram import TelegramTransformer from oonidata.transforms.nettests.tor import TorTransformer from oonidata.transforms.nettests.web_connectivity import WebConnectivityTransformer @@ -15,6 +17,8 @@ NETTEST_TRANSFORMERS = { "dnscheck": DNSCheckTransformer, "signal": SignalTransformer, + "facebook_messenger": FacebookMessengerTransformer, + "whatsapp": WhatsappTransformer, "telegram": TelegramTransformer, "tor": TorTransformer, "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer, diff --git a/oonidata/transforms/nettests/facebook_messenger.py b/oonidata/transforms/nettests/facebook_messenger.py new file mode 100644 index 00000000..73d01148 --- /dev/null +++ b/oonidata/transforms/nettests/facebook_messenger.py @@ -0,0 +1,17 @@ +from typing import List, Tuple +from oonidata.models.nettests import FacebookMessenger +from oonidata.models.observations import WebObservation +from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + + +class FacebookMessengerTransformer(MeasurementTransformer): + def make_observations(self, msmt: FacebookMessenger) -> Tuple[List[WebObservation]]: + dns_observations = self.make_dns_observations(msmt.test_keys.queries) + tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect) + + return ( + self.consume_web_observations( + dns_observations=dns_observations, + tcp_observations=tcp_observations, + ), + ) diff --git a/oonidata/transforms/nettests/whatsapp.py b/oonidata/transforms/nettests/whatsapp.py new file mode 100644 index 00000000..bb4773fb --- /dev/null +++ b/oonidata/transforms/nettests/whatsapp.py @@ -0,0 +1,24 @@ +from typing import List, Tuple +from oonidata.models.nettests import Whatsapp +from oonidata.models.observations import WebObservation +from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + + +class WhatsappTransformer(MeasurementTransformer): + def make_observations(self, msmt: Whatsapp) -> Tuple[List[WebObservation]]: + dns_observations = self.make_dns_observations(msmt.test_keys.queries) + tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect) + tls_observations = self.make_tls_observations( + msmt.test_keys.tls_handshakes, + msmt.test_keys.network_events, + ) + http_observations = self.make_http_observations(msmt.test_keys.requests) + + return ( + self.consume_web_observations( + dns_observations=dns_observations, + tcp_observations=tcp_observations, + tls_observations=tls_observations, + http_observations=http_observations, + ), + ) diff --git a/tests/_sample_measurements.py b/tests/_sample_measurements.py index 813ad135..eb2f7996 100644 --- a/tests/_sample_measurements.py +++ b/tests/_sample_measurements.py @@ -18,6 +18,8 @@ "20221013000000.517636_US_dnscheck_bfd6d991e70afa0e", "20221114002335.786418_BR_webconnectivity_6b203219ec4ded0e", "20230427235943.206438_US_telegram_ac585306869eca7b", + "20211018232506.972850_IN_whatsapp_44970a56806dbfb3", + "20220124235953.650143_ES_facebookmessenger_0e048a26b89a9d70", "20210101181154.037019_CH_webconnectivity_68ce38aa9e3182c2", "20210101190046.780850_US_webconnectivity_3296f126f79ca186", "20231031032643.267235_GR_dnscheck_abcbfc460b9424b6", diff --git a/tests/test_transforms.py b/tests/test_transforms.py index ca0ad840..a0b99071 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -2,6 +2,9 @@ from oonidata.analysis.datasources import load_measurement from oonidata.models.nettests.dnscheck import DNSCheck from oonidata.models.nettests.telegram import Telegram +from oonidata.models.nettests.signal import Signal +from oonidata.models.nettests.facebook_messenger import FacebookMessenger +from oonidata.models.nettests.whatsapp import Whatsapp from oonidata.models.nettests.web_connectivity import WebConnectivity from oonidata.models.observations import WebObservation from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer @@ -173,3 +176,79 @@ def test_telegram_obs(netinfodb, measurements): if wo.tls_cipher_suite: assert wo.tls_t assert len(web_obs) == 33 + + +def test_signal_obs(netinfodb, measurements): + msmt = load_measurement( + msmt_path=measurements["20221016235944.266268_GB_signal_1265ff650ee17b44"] + ) + assert isinstance(msmt, Signal) + web_obs: List[WebObservation] = measurement_to_observations( + msmt=msmt, netinfodb=netinfodb + )[0] + for wo in web_obs: + if wo.dns_engine: + print(wo.dns_engine) + assert wo.dns_t + if wo.tcp_success is not None: + assert wo.tcp_t + if wo.http_request_url: + assert wo.http_t + if wo.tls_cipher_suite: + assert wo.tls_t + assert len(web_obs) == 19 + + +def test_whatsapp_obs(netinfodb, measurements): + msmt = load_measurement( + msmt_path=measurements["20211018232506.972850_IN_whatsapp_44970a56806dbfb3"] + ) + assert isinstance(msmt, Whatsapp) + web_obs: List[WebObservation] = measurement_to_observations( + msmt=msmt, netinfodb=netinfodb + )[0] + for wo in web_obs: + if wo.dns_engine: + assert wo.dns_t + if wo.tcp_success is not None: + assert wo.tcp_t + if wo.http_request_url: + assert wo.http_t + if wo.tls_cipher_suite: + assert wo.tls_t + assert len(web_obs) == 137 + + +def test_facebook_messenger_obs(netinfodb, measurements): + msmt = load_measurement( + msmt_path=measurements[ + "20220124235953.650143_ES_facebookmessenger_0e048a26b89a9d70" + ] + ) + assert isinstance(msmt, FacebookMessenger) + web_obs: List[WebObservation] = measurement_to_observations( + msmt=msmt, netinfodb=netinfodb + )[0] + + # Based on https://github.com/ooni/spec/blob/master/nettests/ts-019-facebook-messenger.md + spec_hostname_set = set( + [ + "stun.fbsbx.com", + "b-api.facebook.com", + "b-graph.facebook.com", + "edge-mqtt.facebook.com", + "external.xx.fbcdn.net", + "scontent.xx.fbcdn.net", + "star.c10r.facebook.com", + ] + ) + + hostname_set = set() + for wo in web_obs: + if wo.dns_engine: + assert wo.dns_t + if wo.tcp_success is not None: + assert wo.tcp_t + hostname_set.add(wo.hostname) + assert hostname_set == spec_hostname_set + assert len(web_obs) == 14