Add a flush, for #63

ukwa · Dec 6, 2019 · 08e6a39 · 08e6a39
1 parent c3bbcde
commit 08e6a39
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/requirements.txt b/requirements.txt
@@ -27,7 +27,7 @@ Jinja2==2.10.1
 jsonschema==3.0.1
 jupyter-client==5.3.1
 jupyter-core==4.5.0
-kafka-python==1.4.6
+kafka-python==1.4.7
 kiwisolver==1.1.0
 lockfile==0.12.2
 luigi==2.8.7

diff --git a/tasks/crawl/launch.py b/tasks/crawl/launch.py
@@ -194,6 +194,10 @@ def run(self):
                 else:
                     logger.error("Don't understand crawl frequency " + schedule['frequency'])
 
+        logger.info("Closing the launcher to ensure everything is pushed to Kafka...")
+        self.launcher.flush()
+        #self.launcher.close()
+
         logger.info("Completed. Launches this hour: %s" % self.i_launches)
         # Record that all went well:
         self.output().touch()