Fix parsing nested structures in response (#1199)

CHANGELOG.md

@@ -8,6 +8,10 @@ Releases prior to 7.0 has been removed from this file to declutter search result
 
 ## [Unreleased]
 
+### Fixed
+
+- substrate.subsquid: Fixed parsing nested structures in response.
+
 ### Changed
 
 - project: Set default PostgreSQL password and Hasura secret (both are `changeme`) for new projects.

docs/9.release-notes/_8.0_changelog.md

@@ -46,6 +46,7 @@
 - performance: Fixed estimation indexing speed in levels per second.
 - starknet.events: Fixed filtering events by key.
 - subsquid: Fixed missing entry in `dipdup_head` internal table.
+- substrate.subsquid: Fixed parsing nested structures in response.
 - tezos.big_maps: Fixed logging status message in `skip_history` mode.
 - tezos.big_maps: Respect order of handlers in `skip_history` mode.
 - tezos.operations: Fixed `sr_cement` operation index subscription.

src/dipdup/models/substrate.py

@@ -114,14 +114,15 @@ def payload(self) -> PayloadT:
         # NOTE: We receive decoded args from node datasource and encoded from subsquid datasource
         if self.data.decoded_args is not None:
             payload = self.data.decoded_args
-        elif self.data.args is not None and self.data.header_extra is not None:
+        elif self.data.args and self.data.header_extra:
             payload = self.runtime.decode_event_args(
                 name=self.data.name,
                 args=self.data.args,
                 spec_version=str(self.data.header_extra['specVersion']),
             )
         else:
             raise NotImplementedError
+
         return cast(PayloadT, payload)
 
     @property

src/dipdup/runtimes.py

@@ -263,4 +263,35 @@ def parse(value: Any, type_: str) -> Any:
                 new_key = pascal_to_snake(key)
                 payload[new_key] = payload.pop(key)
 
-        return payload
+        # NOTE: Also, we need to unpack TypeScript structures to the original form
+        payload = extract_subsquid_payload(payload)
+
+        return payload  # noqa: RET504
+
+
+def extract_subsquid_payload(data: Any) -> Any:
+    if isinstance(data, list | tuple):
+        return tuple(extract_subsquid_payload(item) for item in data)
+
+    if isinstance(data, dict):
+
+        if (kind := data.get('__kind')) is None:
+            return {key: extract_subsquid_payload(value) for key, value in data.items()}
+
+        if 'value' in data:
+            value = data['value']
+            if isinstance(value, list | tuple):
+                # Handle list of values
+                value = tuple(extract_subsquid_payload(item) for item in value)
+            # FIXME: We probably shouldn't do this
+            elif isinstance(value, str):
+                value = int(value)
+            return {kind: value}
+
+        # NOTE: Special case
+        if 'key' in data:
+            return {kind: data['key']}
+
+        return kind
+
+    return data

tests/test_datasources/test_substrate.py

@@ -0,0 +1,114 @@
+from dipdup.runtimes import extract_subsquid_payload
+
+path_1 = [
+    [
+        {
+            'parents': 0,
+            'interior': {
+                '__kind': 'X2',
+                'value': [
+                    {
+                        '__kind': 'PalletInstance',
+                        'value': 50,
+                    },
+                    {
+                        '__kind': 'GeneralIndex',
+                        'value': '1337',
+                    },
+                ],
+            },
+        },
+        84640,
+    ],
+    [
+        {
+            'parents': 1,
+            'interior': {
+                '__kind': 'Here',
+            },
+        },
+        122612710,
+    ],
+]
+path_2 = [
+    {
+        'interior': {
+            '__kind': 'X3',
+            'value': [
+                {'__kind': 'Parachain', 'value': 1000},
+                {'__kind': 'GeneralIndex', 'value': 50},
+                {'__kind': 'GeneralIndex', 'value': 42069},
+            ],
+        },
+        'parents': 1,
+    }
+]
+
+path_3 = [
+    {
+        'interior': {
+            '__kind': 'X3',
+            'value': [
+                {'__kind': 'Parachain', 'value': 2004},
+                {'__kind': 'PalletInstance', 'value': 110},
+                {'__kind': 'AccountKey20', 'key': 39384093},
+            ],
+        },
+        'parents': 1,
+    }
+]
+
+processed_path_1 = (
+    (
+        {
+            'parents': 0,
+            'interior': {
+                'X2': (
+                    {'PalletInstance': 50},
+                    {'GeneralIndex': 1337},
+                ),
+            },
+        },
+        84640,
+    ),
+    (
+        {
+            'parents': 1,
+            'interior': 'Here',
+        },
+        122612710,
+    ),
+)
+
+processed_path_2 = (
+    {
+        'parents': 1,
+        'interior': {
+            'X3': (
+                {'Parachain': 1000},
+                {'GeneralIndex': 50},
+                {'GeneralIndex': 42069},
+            ),
+        },
+    },
+)
+
+processed_path_3 = (
+    {
+        'parents': 1,
+        'interior': {
+            'X3': (
+                {'Parachain': 2004},
+                {'PalletInstance': 110},
+                {'AccountKey20': 39384093},
+            ),
+        },
+    },
+)
+
+
+def test_extract_subsquid_payload() -> None:
+
+    assert extract_subsquid_payload(path_1) == processed_path_1
+    assert extract_subsquid_payload(path_2) == processed_path_2
+    assert extract_subsquid_payload(path_3) == processed_path_3