From 47f18ed1a25c77fba3c0943a9b654c0cb26aa049 Mon Sep 17 00:00:00 2001
From: Igor Sereda <sereda.igor.s@gmail.com>
Date: Tue, 11 Feb 2025 19:39:21 +0300
Subject: [PATCH] Realtime blocks timestamp

---
 dex_screener/dipdup.yaml                      |  3 ++
 dex_screener/handlers/batch.py                |  7 ++---
 dex_screener/hooks/refresh_blocks.py          | 31 ++++++++++++++++++-
 dex_screener/models/__init__.py               |  2 +-
 dex_screener/models/dto.py                    | 11 -------
 .../00_refresh_latest_block.sql               |  1 +
 6 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/dex_screener/dipdup.yaml b/dex_screener/dipdup.yaml
index 50daea7..a8f0cf5 100644
--- a/dex_screener/dipdup.yaml
+++ b/dex_screener/dipdup.yaml
@@ -32,6 +32,9 @@ datasources:
     kind: substrate.node
     url: https://rpc.hydradx.cloud
     ws_url: wss://hydration-rpc.n.dwellir.com
+  explorer:
+    kind: http
+    url: https://explorer.hydradx.cloud/
 
 indexes:
   hydradx_events:
diff --git a/dex_screener/handlers/batch.py b/dex_screener/handlers/batch.py
index 42bb74e..5c2f326 100644
--- a/dex_screener/handlers/batch.py
+++ b/dex_screener/handlers/batch.py
@@ -6,7 +6,6 @@
     from collections.abc import Iterable
 
     from dipdup.context import HandlerContext
-    from dipdup.datasources.substrate_subscan import SubstrateSubscanDatasource
     from dipdup.index import MatchedHandler
 
 
@@ -20,10 +19,8 @@ async def batch(
         if handler.level not in batch_levels and isinstance(handler.index, SubstrateEventsIndex):
             try:
                 timestamp = int(handler.args[0].data.header_extra['timestamp'] // 1000)
-            except (KeyError, AttributeError, ValueError):
-                subscan: SubstrateSubscanDatasource = ctx.get_substrate_datasource('subscan')
-                block_data = await subscan.request('post', 'scan/block', json={'block_num': handler.level})
-                timestamp = int(block_data['data']['block_timestamp'])
+            except (KeyError, AttributeError, ValueError, TypeError):
+                timestamp = None
 
             from dex_screener.models import Block
             await Block.create(
diff --git a/dex_screener/hooks/refresh_blocks.py b/dex_screener/hooks/refresh_blocks.py
index 17bf046..29c1cd8 100644
--- a/dex_screener/hooks/refresh_blocks.py
+++ b/dex_screener/hooks/refresh_blocks.py
@@ -1,7 +1,36 @@
+import datetime
+
 from dipdup.context import HookContext
 
+from dex_screener.models import Block
+
 
 async def refresh_blocks(
     ctx: HookContext,
 ) -> None:
-    await ctx.execute_sql_script('refresh_blocks')
\ No newline at end of file
+    await ctx.execute_sql_script('refresh_blocks')
+
+    levels: list[int] = await Block.filter(timestamp__isnull=True).order_by('level').limit(100).values_list('level', flat=True)
+    if len(levels) == 0:
+        return
+    request_payload = {
+        'query': ' '.join([
+            'query { blocks(where: {height_in:',
+            str(levels),
+            '}) { height timestamp } }',
+        ]),
+    }
+
+    explorer = ctx.get_http_datasource('explorer')
+    response = await explorer.request('post', 'graphql', json=request_payload)
+
+    blocks = [
+        Block(
+            level=block_data['height'],
+            timestamp=datetime.datetime.fromisoformat(block_data['timestamp']).timestamp(),
+        ) for block_data in response['data']['blocks']
+    ]
+
+    await Block.bulk_update(
+        objects=blocks, fields=['timestamp']
+    )
diff --git a/dex_screener/models/__init__.py b/dex_screener/models/__init__.py
index 1211502..08bee66 100644
--- a/dex_screener/models/__init__.py
+++ b/dex_screener/models/__init__.py
@@ -32,7 +32,7 @@ class Meta:
         model = 'models.Block'
 
     level = fields.IntField(primary_key=True)
-    timestamp = fields.IntField()
+    timestamp = fields.IntField(null=True)
 
 
 class LatestBlock(Model):
diff --git a/dex_screener/models/dto.py b/dex_screener/models/dto.py
index b7ed7ae..1af1215 100644
--- a/dex_screener/models/dto.py
+++ b/dex_screener/models/dto.py
@@ -43,16 +43,5 @@ def _get_tx_index(cls, event: SubstrateEvent) -> int:
             return event.data.extrinsic_index
         return 0
 
-    async def update_latest_block(self):
-        from dex_screener.models import Block
-        from dex_screener.models import LatestBlock
-
-        block = await Block.get(level=self.block_id)
-
-        await LatestBlock.update_or_create(
-            id=True,
-            defaults={'block_number': block.level, 'block_timestamp': block.timestamp},
-        )
-
     def get_explorer_url(self) -> str:
         return f'https://hydration.subscan.io/block/{self.block_id}?tab=event&event={self.event_index}'
diff --git a/dex_screener/sql/refresh_blocks/00_refresh_latest_block.sql b/dex_screener/sql/refresh_blocks/00_refresh_latest_block.sql
index 347cb7f..d227f08 100644
--- a/dex_screener/sql/refresh_blocks/00_refresh_latest_block.sql
+++ b/dex_screener/sql/refresh_blocks/00_refresh_latest_block.sql
@@ -4,6 +4,7 @@ WITH elb AS (
         b.timestamp AS block_timestamp
     FROM dex_swap_event AS e
     JOIN dex_block AS b ON b.level = e.block_id
+    WHERE b.timestamp IS NOT NULL
     ORDER BY e.id DESC
     LIMIT 1
 )