Realtime blocks timestamp

dex_screener/dipdup.yaml

@@ -32,6 +32,9 @@ datasources:
     kind: substrate.node
     url: https://rpc.hydradx.cloud
     ws_url: wss://hydration-rpc.n.dwellir.com
+  explorer:
+    kind: http
+    url: https://explorer.hydradx.cloud/
 
 indexes:
   hydradx_events:

dex_screener/handlers/batch.py

@@ -6,7 +6,6 @@
     from collections.abc import Iterable
 
     from dipdup.context import HandlerContext
-    from dipdup.datasources.substrate_subscan import SubstrateSubscanDatasource
     from dipdup.index import MatchedHandler
 
 
@@ -20,10 +19,8 @@ async def batch(
         if handler.level not in batch_levels and isinstance(handler.index, SubstrateEventsIndex):
             try:
                 timestamp = int(handler.args[0].data.header_extra['timestamp'] // 1000)
-            except (KeyError, AttributeError, ValueError):
-                subscan: SubstrateSubscanDatasource = ctx.get_substrate_datasource('subscan')
-                block_data = await subscan.request('post', 'scan/block', json={'block_num': handler.level})
-                timestamp = int(block_data['data']['block_timestamp'])
+            except (KeyError, AttributeError, ValueError, TypeError):
+                timestamp = None
 
             from dex_screener.models import Block
             await Block.create(

dex_screener/hooks/refresh_blocks.py

@@ -1,7 +1,36 @@
+import datetime
+
 from dipdup.context import HookContext
 
+from dex_screener.models import Block
+
 
 async def refresh_blocks(
     ctx: HookContext,
 ) -> None:
-    await ctx.execute_sql_script('refresh_blocks')
+    await ctx.execute_sql_script('refresh_blocks')
+
+    levels: list[int] = await Block.filter(timestamp__isnull=True).order_by('level').limit(100).values_list('level', flat=True)
+    if len(levels) == 0:
+        return
+    request_payload = {
+        'query': ' '.join([
+            'query { blocks(where: {height_in:',
+            str(levels),
+            '}) { height timestamp } }',
+        ]),
+    }
+
+    explorer = ctx.get_http_datasource('explorer')
+    response = await explorer.request('post', 'graphql', json=request_payload)
+
+    blocks = [
+        Block(
+            level=block_data['height'],
+            timestamp=datetime.datetime.fromisoformat(block_data['timestamp']).timestamp(),
+        ) for block_data in response['data']['blocks']
+    ]
+
+    await Block.bulk_update(
+        objects=blocks, fields=['timestamp']
+    )

dex_screener/models/__init__.py

@@ -32,7 +32,7 @@ class Meta:
         model = 'models.Block'
 
     level = fields.IntField(primary_key=True)
-    timestamp = fields.IntField()
+    timestamp = fields.IntField(null=True)
 
 
 class LatestBlock(Model):

dex_screener/models/dto.py

@@ -43,16 +43,5 @@ def _get_tx_index(cls, event: SubstrateEvent) -> int:
             return event.data.extrinsic_index
         return 0
 
-    async def update_latest_block(self):
-        from dex_screener.models import Block
-        from dex_screener.models import LatestBlock
-
-        block = await Block.get(level=self.block_id)
-
-        await LatestBlock.update_or_create(
-            id=True,
-            defaults={'block_number': block.level, 'block_timestamp': block.timestamp},
-        )
-
     def get_explorer_url(self) -> str:
         return f'https://hydration.subscan.io/block/{self.block_id}?tab=event&event={self.event_index}'

dex_screener/sql/refresh_blocks/00_refresh_latest_block.sql

@@ -4,6 +4,7 @@ WITH elb AS (
         b.timestamp AS block_timestamp
     FROM dex_swap_event AS e
     JOIN dex_block AS b ON b.level = e.block_id
+    WHERE b.timestamp IS NOT NULL
     ORDER BY e.id DESC
     LIMIT 1
 )