Merge branch 'master' into biastemp

download_smartfact_data.py

@@ -0,0 +1,123 @@
+import requests
+import schedule
+import os
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor
+import logging
+from time import sleep
+
+logging.basicConfig(level=logging.INFO)
+
+
+url = 'http://fact-project.org/smartfact/data/{}.data'
+files = [
+    'agilent24.data',
+    'agilent50.data',
+    'biastemp.data',
+    'boardrates.data',
+    'cam-biascontrol-current.bin',
+    'cam-biascontrol-voltage.bin',
+    'cam-fadcontrol-eventdata.bin',
+    'cam-feedback-overvoltage.bin',
+    'cam-fsccontrol-temperature.bin',
+    'cam-ftmcontrol-boardrates.bin',
+    'cam-ftmcontrol-patchrates.bin',
+    'cam-ftmcontrol-thresholds-board.bin',
+    'cam-ftmcontrol-thresholds-patch.bin',
+    'camtemp.data',
+    'current.data',
+    'current-prediction.data',
+    'dew.data',
+    'error.data',
+    'errorhist.data',
+    'fact.data',
+    'fad.data',
+    'feedback.data',
+    'fsc.data',
+    'ftm.data',
+    'ftu.data',
+    'gps.data',
+    'gusts.data',
+    'hist-biascontrol-current.bin',
+    'hist-control-deviation.bin',
+    'hist-current-prediction.bin',
+    'hist-fsccontrol-temperature.bin',
+    'hist-ftmcontrol-triggerrate.bin',
+    'hist-magicweather-dew.bin',
+    'hist-magicweather-gusts.bin',
+    'hist-magicweather-hum.bin',
+    'hist-magicweather-press.bin',
+    'hist-magicweather-temp.bin',
+    'hist-magicweather-wind.bin',
+    'hist-pfmini-hum.bin',
+    'hist-pfmini-temp.bin',
+    'hist-ratecontrol-threshold.bin',
+    'hist-temperaturecontrol.bin',
+    'hist-tng-dust.bin',
+    'hist-visibility.bin',
+    'hum.data',
+    'moon.data',
+    'observations.data',
+    'patchrates.data',
+    'pfmini.data',
+    'pointing.data',
+    'press.data',
+    'scriptlog.data',
+    'source.data',
+    'source-list.data',
+    'sqm.data',
+    'status.data',
+    'sun.data',
+    'temp.data',
+    'temperature.data',
+    'thresholds-board.data',
+    'thresholds.data',
+    'thresholds-patch.data',
+    'tngdust.data',
+    'tracking.data',
+    'trigger.data',
+    'visibility.data',
+    'voltage.data',
+    'weather.data',
+    'wind.data',
+]
+
+
+def download(url, outputfile):
+    try:
+        ret = requests.get(url)
+        ret.raise_for_status()
+
+        with open(outputfile, 'w') as f:
+            f.write(ret.text)
+    except requests.ConnectionError:
+        logging.error('Could not download {}'.format(url))
+
+
+def download_all():
+    logging.info('Start downloading all')
+    now = datetime.utcnow()
+
+    output_directory = '{}/{:02d}/{:02d}/{:%H%M}'.format(
+        now.year, now.month, now.day, now
+    )
+    os.makedirs(output_directory, exist_ok=True)
+
+    with ThreadPoolExecutor(max_workers=len(files)) as executor:
+        for filename in files:
+            executor.submit(
+                download,
+                url.format(filename),
+                os.path.join(
+                    output_directory, '{}.data'.format(filename, now)
+                )
+            )
+
+
+if __name__ == '__main__':
+    download_all()
+
+    schedule.every(5).minutes.do(download_all)
+    while True:
+        schedule.run_pending()
+        sleep(10)

setup.py

@@ -4,7 +4,7 @@
 setup(
     name='smart_fact_crawler',
     version='0.5.0',
-    description='acquieres data published on the smartfact web page',
+    description='acquire data published on the smartfact web page',
     url='https://github.com/fact-project/smart_fact_crawler.git',
     author='Dominik Neise, Sebastian Mueller, Maximilian Nöthe',
     author_email='[email protected]',

smart_fact_crawler/__init__.py

@@ -316,19 +316,13 @@ def main_page(url=None, timeout=None, fallback=False):
     get = partial(get_entry, fallback=fallback)
 
     system_status = get(table, 1, 1)
-    power_val, power_unit = get(table, 6, 3, default='nan nan').split()
-    trigger_val, trigger_unit, _ = get(table, 5, 1, default='nan nan nan').split()
     return to_namedtuple('MainPage', {
         'timestamp_1': sft2dt(get(table, 0, 0)),
         'timestamp_2': sft2dt(get(table, 0, 1)),
         'system_status': system_status,
         'relative_camera_temperature': Quantity(s2f(get(table, 3, 1)), 'deg_C'),
         'humidity': Quantity(s2f(get(table, 4, 1)), '%'),
         'wind_speed': Quantity(s2f(get(table, 4, 2)), 'km/h'),
-        'trigger_rate': Quantity(s2f(trigger_val), trigger_unit),
-        'median_current': Quantity(s2f(get(table, 6, 1)), 'uA'),
-        'max_current': Quantity(s2f(get(table, 6, 2)), 'uA'),
-        'power': Quantity(s2f(power_val), power_unit),
     })