Take note of an improvement related to idempotence of activities

ooni · Apr 26, 2024 · 8fe6117 · 8fe6117
1 parent 6d32e00
commit 8fe6117
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 0 deletions.
diff --git a/oonipipeline/src/oonipipeline/temporal/activities/analysis.py b/oonipipeline/src/oonipipeline/temporal/activities/analysis.py
@@ -117,6 +117,9 @@ def make_analysis_in_a_day(params: MakeAnalysisParams) -> dict:
     column_names_wa = [f.name for f in dataclasses.fields(WebAnalysis)]
     column_names_er = [f.name for f in dataclasses.fields(MeasurementExperimentResult)]
 
+    # TODO(art): this previous range search and deletion makes the idempotence
+    # of the activity not 100% accurate.
+    # We should look into fixing it.
     prev_range_list = [
         get_prev_range(
             db=db_lookup,

diff --git a/oonipipeline/src/oonipipeline/temporal/activities/observations.py b/oonipipeline/src/oonipipeline/temporal/activities/observations.py
@@ -141,6 +141,9 @@ def make_observations_for_file_entry_batch(
 def make_observation_in_day(params: MakeObservationsParams) -> dict:
     day = datetime.strptime(params.bucket_date, "%Y-%m-%d").date()
 
+    # TODO(art): this previous range search and deletion makes the idempotence
+    # of the activity not 100% accurate.
+    # We should look into fixing it.
     with ClickhouseConnection(params.clickhouse, row_buffer_size=10_000) as db:
         prev_ranges = []
         for table_name in ["obs_web"]: