Merge pull request #285 from SubstraFoundation/fix-subtuple-check

Fix subtuple check when replaying events

backend/substrapp/ledger_utils.py

@@ -147,7 +147,7 @@ def _wrapper(*args, **kwargs):
                     return fn(*args, **kwargs)
                 except exceptions_to_retry as e:
                     _nbtries -= 1
-                    if not nbtries:
+                    if not _nbtries:
                         raise
                     _delay += _backoff
                     time.sleep(_delay)

backend/substrapp/tasks/tasks.py

@@ -485,7 +485,10 @@ def prepare_tuple(subtuple, tuple_type):
 
     # Early return if subtuple status is not todo
     # Can happen if we re-process all events
-    if subtuple['status'] != 'todo':
+    # But we need to fetch the subtuple again to get the last
+    # version of it
+    _, subtuple_check = find_training_step_tuple_from_key(subtuple['key'])
+    if subtuple_check['status'] != 'todo':
         logger.error(f'Tuple task ({tuple_type}) not in "todo" state ({subtuple["status"]}).\n{subtuple}')
         return
 

backend/substrapp/tests/tests_tasks.py

@@ -381,7 +381,8 @@ def __init__(self):
                 mock.patch('substrapp.tasks.tasks.uncompress_content'), \
                 mock.patch('substrapp.tasks.tasks.json.loads') as mjson_loads, \
                 mock.patch('substrapp.tasks.tasks.AsyncResult') as masyncres, \
-                mock.patch('substrapp.tasks.tasks.get_owner') as get_owner:
+                mock.patch('substrapp.tasks.tasks.get_owner') as get_owner,\
+                mock.patch('substrapp.tasks.tasks.find_training_step_tuple_from_key') as gettuple:
 
             msettings.return_value = FakeSettings()
             mget_hash.return_value = 'owkinhash'
@@ -392,6 +393,7 @@ def __init__(self):
             mprepare_opener.return_value = 'opener'
             mprepare_data_sample.return_value = 'data'
             get_owner.return_value = 'foo'
+            gettuple.return_value = None, subtuple[0]
 
             masyncres.return_value.state = 'PENDING'