Details.

src/npoapi/bin/npo_media_follow_changes.py

@@ -3,6 +3,7 @@
   Simple client to get the changes feed from the NPO Frontend API
 """
 import json
+import logging
 import time
 from datetime import datetime
 
@@ -19,6 +20,7 @@ def media_follow_changes():
     client.add_argument("-s", "--since", type=str, default=None)
     client.add_argument("--sleep", type=int, default=5)
     client.add_argument("--deletes", type=str, default="ID_ONLY")
+    client.add_argument("--tail", action='store_true')
     client.add_argument('-p', "--properties", type=str, default=None,
                         help="properties filtering")
 
@@ -29,26 +31,45 @@ def media_follow_changes():
         since = datetime.now().isoformat()
         client.logger.info("No since given, using %s" % since)
 
-    while True:
-        client.logger.info("since: %s" % since)
-        response = TextIOWrapper(client.changes_raw(
-               profile=args.profile,
-               since=since,
-               properties=args.properties,
-               deletes=args.deletes,
-               stream=True), encoding="UTF-8")
-
-        data = json_stream.load(response)
-        changes = data['changes']
-        for change in changes.persistent():
-            c = json_stream.to_standard_types(change)
-            since = str(c['publishDate'])
-            stdout.write(json.dumps(c))
-            stdout.write("\n")
-            stdout.flush()
-
-        response.close()
-        time.sleep(args.sleep)
+    sinceAsEpoch = int(datetime.fromisoformat(since).timestamp() * 1000) - 60000
+
+    try:
+        while True:
+            client.logger.info("since: %s (%s)" % (sinceAsEpoch, datetime.fromtimestamp(sinceAsEpoch/1000).isoformat()))
+            response = client.changes_raw(
+                   profile=args.profile,
+                   since=sinceAsEpoch,
+                   properties=args.properties,
+                   deletes=args.deletes,
+                   stream=True)
+
+            if response.status != 200:
+                logging.error("Error %d" % response.status)
+                continue
+            data = json_stream.load(response)
+            changes = data['changes']
+            newsince = None
+            for change in changes.persistent():
+                c = json_stream.to_standard_types(change)
+                newsince = c.get('publishDate')
+                if not newsince:
+                    logging.error("No publishDate in %s" % c)
+                    break
+                tail = c.get('tail', False)
+                if not tail or args.tail:
+                    stdout.write(json.dumps(c))
+                    stdout.write("\n")
+                stdout.flush()
+            if newsince is None:
+                raise Exception("No tail received?")
+            if newsince <= sinceAsEpoch:
+                raise Exception("Since doesn't grow")
+            sinceAsEpoch = newsince
+            changes.read_all()
+            response.close()
+            time.sleep(args.sleep)
+    except KeyboardInterrupt:
+        client.logger.info("interrupted")
 
     client.exit()
 

src/npoapi/media.py

@@ -46,15 +46,17 @@ def search(self, form="{}", sort="asc", offset: int = 0, limit: int = 240, profi
     def changes(self, profile=None, limit=10, since=None, properties=None, deletes="ID_ONLY", tail=None) -> Union[None, ijson.items]:
         return ijson.items(self.changes_raw(stream=True, profile=profile, limit=limit, since=since, properties=properties, deletes=deletes, tail=tail), 'changes.item')
 
-    def changes_raw(self, profile=None, order="ASC", stream=False, limit=10, since:Union[str, int, datetime.datetime]=None, force_oldstyle=False, properties=None, deletes="ID_ONLY", tail=None, reason_filter=None) -> Union[None, http.client.HTTPResponse, str]:
+    def changes_raw(self, profile=None, order="ASC", stream=False, limit=10, since:Union[str, int, datetime.datetime]=None, force_oldstyle=False, properties=None, deletes="ID_ONLY", tail=None, reason_filter="") -> Union[None, http.client.HTTPResponse, str]:
         if isinstance(properties, list):
             properties = ",".join(properties)
         sinceLong = None
         sinceDate = None
         if not since is None:
             if isinstance(since, datetime.datetime):
                 since = str(since).replace(" ", "T")
-            if not force_oldstyle and (not since.isdigit() or int(since) > 946681200000):
+            elif type(since) == int:\
+                sinceDate= str(since)
+            elif not force_oldstyle and (not since.isdigit() or int(since) > 946681200000):
                 sinceDate = since
             else:
                 sinceLong = since