Merge pull request #41 from fact-project/add_gtc_dust

add parsing of gtc dust on status.data page

download_all_once.py

@@ -0,0 +1,86 @@
+'''
+downloads all files from both /data and /struct folder
+downloads all .data, .bin and .page files.
+
+you have to delete the files you do not want to use during testing yourself
+'''
+
+import requests
+import schedule
+import os
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor
+import logging
+from time import sleep
+from functools import partial
+from bs4 import BeautifulSoup
+
+logging.basicConfig(level=logging.INFO)
+
+
+def get_file_urls(url):
+    all_urls = []
+    r = requests.get(url)
+    data = r.text
+    soup = BeautifulSoup(data, features='lxml')
+
+    for link in soup.find_all('a'):
+        full_url = url + link.get('href')
+        if (
+            full_url.endswith('.data') or
+            full_url.endswith('.bin') or
+            full_url.endswith('.page')
+        ):
+            print(full_url)
+            all_urls.append(full_url)
+
+    return all_urls
+
+
+def download(url, outputfile):
+    logging.info(f'{url} -> {outputfile}')
+    try:
+        ret = requests.get(url)
+        ret.raise_for_status()
+
+        with open(outputfile, 'w') as f:
+            f.write(ret.text)
+    except requests.exceptions.RequestException as e:
+        logging.exception('Could not download {}'.format(url))
+
+
+def make_output_dir(suffix=''):
+    now = datetime.utcnow()
+
+    output_directory = '{}_{:02d}_{:02d}_{:%H%M}{}'.format(
+        now.year, now.month, now.day, now, suffix
+    )
+    os.makedirs(output_directory, exist_ok=True)
+
+    return output_directory
+
+
+def make_out_path(output_directory, filename):
+    return os.path.join(output_directory, filename)
+
+
+def download_all():
+    logging.info('Start downloading all')
+    for folder in ('data', 'struct'):
+
+        url = f'http://fact-project.org/smartfact/{folder}/'
+        output_directory = make_output_dir(suffix=f'/{folder}')
+        out_path = partial(make_out_path, output_directory)
+
+        files = get_file_urls(url)
+        with ThreadPoolExecutor(max_workers=len(files)) as executor:
+            for filename in files:
+                executor.submit(
+                    download,
+                    filename,
+                    out_path(os.path.split(filename)[-1])
+                )
+
+
+if __name__ == '__main__':
+    download_all()

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name='smart_fact_crawler',
-    version='0.6.3',
+    version='0.6.4',
     description='acquire data published on the smartfact web page',
     url='https://github.com/fact-project/smart_fact_crawler.git',
     author='Dominik Neise, Sebastian Mueller, Maximilian Nöthe',

smart_fact_crawler/__init__.py

@@ -162,6 +162,7 @@ def weather(url=None, timeout=None, fallback=False):
         'wind_gusts': Quantity(s2f(get(table, 8, 1)), 'km/h'),
         'wind_direction': get(table, 9, 1),
         'dust_tng': Quantity(s2f(get(table, 10, 1)), 'ug/m3'),
+        'dust_gtc': Quantity(s2f(get(table, 11, 1)), 'ug/m3'),
     })
 
 
@@ -240,10 +241,10 @@ def status(url=None, timeout=None, fallback=False):
     table = smartfact2table(url, timeout=timeout)
     get = partial(get_entry, fallback=fallback)
 
-    value, unit = get(table, 30, 1, default='nan nan').split(' ')[:2]
+    value, unit = get(table, 31, 1, default='nan nan').split(' ')[:2]
     storage_newdaq = Quantity(s2f(value), unit)
 
-    value, unit = get(table, 31, 1, default='nan nan').split(' ')[:2]
+    value, unit = get(table, 32, 1, default='nan nan').split(' ')[:2]
     storage_daq = Quantity(s2f(value), unit)
 
     return to_namedtuple('StatusPage', {
@@ -272,14 +273,15 @@ def status(url=None, timeout=None, fallback=False):
         'ratescan': get(table, 22, 1),
         'magic_weather': get(table, 23, 1),
         'tng_weather': get(table, 24, 1),
-        'magic_lidar': get(table, 25, 1),
-        'temperature': get(table, 26, 1),
-        'rain_sensor': get(table, 27, 1),
-        'chat_server': get(table, 28, 1),
-        'skype_client': get(table, 29, 1),
+        'gtc_dust': get(table, 25, 1),
+        'magic_lidar': get(table, 26, 1),
+        'temperature': get(table, 27, 1),
+        'rain_sensor': get(table, 28, 1),
+        'chat_server': get(table, 29, 1),
+        'skype_client': get(table, 30, 1),
         'free_space_newdaq': storage_newdaq,
         'free_space_daq': storage_daq,
-        'smartfact_runtime': get(table, 32, 1),
+        'smartfact_runtime': get(table, 33, 1),
     })
 
 

smart_fact_crawler/resources/2019_10_03_1531/data/agilent24.data

@@ -0,0 +1,5 @@
+1570116668030
+#ffffff	24.0
+#ffffff	24.0
+#ffffff	15.0
+#ffffff	0.9

smart_fact_crawler/resources/2019_10_03_1531/data/agilent50.data

@@ -0,0 +1,5 @@
+1570116667983
+#ffffff	51.1
+#ffffff	51.1
+#ffffff	24.7
+#ffffff	10.9

smart_fact_crawler/resources/2019_10_03_1531/data/agilent80.data

@@ -0,0 +1,5 @@
+1570116668300
+#ffffff	80.0
+#ffffff	80.0
+#ffffff	2.0
+#ffffff	1.0

smart_fact_crawler/resources/2019_10_03_1531/data/biastemp.data

@@ -0,0 +1,4 @@
+1570116669193
+#f0fff0	50416604
+#f0fff0	23.5
+#f0fff0	1.0

smart_fact_crawler/resources/2019_10_03_1531/data/boardrates.data

@@ -0,0 +1,5 @@
+1570116672564
+#ffffff	0
+#ffffff	0
+#ffffff	0
+#ffffff	0

smart_fact_crawler/resources/2019_10_03_1531/data/camtemp.data

@@ -0,0 +1,4 @@
+1570116669638
+#ffffff	36.5
+#ffffff	35.6
+#ffffff	33.9

smart_fact_crawler/resources/2019_10_03_1531/data/current.data

@@ -0,0 +1,7 @@
+1570082969657
+#ffffff	no
+#ffffff	0
+#ffffff	3.66
+#ffffff	252
+#ffffff	4.88
+#ffffff	---

smart_fact_crawler/resources/2019_10_03_1531/data/dew.data

@@ -0,0 +1,5 @@
+1570116656000
+#ffffff	-0.1
+#ffffff	-1.9
+#ffffff	-0.588
+#ffffff	0.4

smart_fact_crawler/resources/2019_10_03_1531/data/error.data

@@ -0,0 +1 @@
+1570104534924

smart_fact_crawler/resources/2019_10_03_1531/data/errorhist.data

@@ -0,0 +1,2 @@
+1570080970048
+#ffffff	<->05:36:10 Trigger rate below 1Hz while trigger switched on<br/>21:36:38 Trigger rate below 1Hz while trigger switched on<br/><->

smart_fact_crawler/resources/2019_10_03_1531/data/fact.data

@@ -0,0 +1,7 @@
+1570116672848	1570104534911		0	0
+#ffffff	Idle [single-pe]
+#ffffff	 ☼ [04:14↓] ⊗
+#fffff0	13
+#f0fff0	22	16
+#ffffff
+#ffffff

smart_fact_crawler/resources/2019_10_03_1531/data/fad.data

@@ -0,0 +1,5 @@
+1570082807640
+#f0fff0	**********
+#f0fff0	**********
+#f0fff0	**********
+#f0fff0	**********

smart_fact_crawler/resources/2019_10_03_1531/data/feedback.data

@@ -0,0 +1,7 @@
+1570082969657
+#ffffff	-0.29
+#ffffff	2.38e-08
+#ffffff	-1.4
+#ffffff	-1.4
+#ffffff	-1.38
+#ffffff	4.18

smart_fact_crawler/resources/2019_10_03_1531/data/fsc.data

@@ -0,0 +1,5 @@
+1570116669638
+#ffffff	24.8
+#ffffff	11.3
+#ffffff	12.3
+#ffffff	13.2

smart_fact_crawler/resources/2019_10_03_1531/data/ftu.data

@@ -0,0 +1,5 @@
+1570039844452
+#ffffff	**********
+#ffffff	**********
+#ffffff	**********
+#ffffff	**********

smart_fact_crawler/resources/2019_10_03_1531/data/gtcdust.data

@@ -0,0 +1 @@
+1570116541000

smart_fact_crawler/resources/2019_10_03_1531/data/gusts.data

@@ -0,0 +1,5 @@
+1570116656000
+#ffffff	15.6
+#ffffff	9.9
+#ffffff	18.3
+#ffffff	28

smart_fact_crawler/resources/2019_10_03_1531/data/hum.data

@@ -0,0 +1,5 @@
+1570116656000
+#ffffff	22
+#ffffff	18.1
+#ffffff	20.2
+#ffffff	22

smart_fact_crawler/resources/2019_10_03_1531/data/patchrates.data

@@ -0,0 +1,5 @@
+1570116674564
+#ffffff	0
+#ffffff	0
+#ffffff	0
+#ffffff	0

smart_fact_crawler/resources/2019_10_03_1531/data/pfmini.data

@@ -0,0 +1,3 @@
+1570116660758
+#f0fff0	37.3
+#f0fff0	20.9

smart_fact_crawler/resources/2019_10_03_1531/data/pointing.data

@@ -0,0 +1,3 @@
+1570082858995
+#ffffff	0	N
+#ffffff	101

smart_fact_crawler/resources/2019_10_03_1531/data/press.data

@@ -0,0 +1,5 @@
+1570116656000
+#ffffff	795
+#ffffff	795
+#ffffff	795
+#ffffff	796

smart_fact_crawler/resources/2019_10_03_1531/data/rainsensor.data

@@ -0,0 +1 @@
+1570116601977

smart_fact_crawler/resources/2019_10_03_1531/data/source.data

@@ -0,0 +1,7 @@
+1570082800987
+#ffffff	Park
+#ffffff	0
+#ffffff	0
+#ffffff	0
+#ffffff	0
+#ffffff	0

smart_fact_crawler/resources/2019_10_03_1531/data/sqm.data

@@ -0,0 +1,6 @@
+1504997326150
+#ffffff	19.99
+#ffffff	1
+#ffffff	507009
+#ffffff	1.100
+#ffffff	36.4

smart_fact_crawler/resources/2019_10_03_1531/data/status.data

@@ -0,0 +1,34 @@
+1570116674858	1570104534911		0	0
+#f0fff0	V20r23
+#f0fff0	Idle
+#f0fff0	Idle
+#f0fff0	WaitForRun
+#f0fff0	Locked
+#f0fff0	Valid
+#f0fff0	Connected
+#f0fff0	Valid
+#fffff0	Disconnected
+#fffff0	Connecting
+#f0fff0	GlobalThresholdSet
+#f0fff0	Connected
+#f0fff0	Receiving
+#f0fff0	Valid
+#f0fff0	Locked
+#fffff0	Disconnected
+#f0fff0	VoltageOn
+#f0fff0	VoltageOn
+#f0fff0	VoltageOn
+#f0fff0	DriveOff
+#fffff0	Closed
+#f0fff0	Connected
+#f0fff0	Valid
+#fffff0	NoConnection
+#f0fff0	Valid
+#f0fff0	Invalid
+#f0fff0	Valid
+#f0fff0	Valid
+#f0fff0	Ready
+#ffffff	—
+#f0fff0	8 TB
+#f0fff0	3 TB
+#f0fff0	03:22:19

smart_fact_crawler/resources/2019_10_03_1531/data/sun.data

@@ -0,0 +1,9 @@
+1570118774940
+#ffffff	05:44
+#ffffff	06:13
+#ffffff	06:41
+#ffffff	07:04
+#f0f0ff	18:54
+#ffffff	19:18
+#ffffff	19:46
+#ffffff	20:14

smart_fact_crawler/resources/2019_10_03_1531/data/temp.data

@@ -0,0 +1,5 @@
+1570116656000
+#ffffff	22.6
+#ffffff	22.4
+#ffffff	23.5
+#ffffff	24.5

smart_fact_crawler/resources/2019_10_03_1531/data/temperature.data

@@ -0,0 +1,4 @@
+1570116616281
+#fffff0	20.4
+#fffff0	21.6
+#fffff0	21.9

smart_fact_crawler/resources/2019_10_03_1531/data/thresholds-board.data

@@ -0,0 +1,4 @@
+1570082807165
+#ffffff	20
+#ffffff	20
+#ffffff	20

smart_fact_crawler/resources/2019_10_03_1531/data/thresholds-patch.data

@@ -0,0 +1,4 @@
+1570082807165
+#ffffff	800
+#ffffff	800
+#ffffff	800

smart_fact_crawler/resources/2019_10_03_1531/data/thresholds.data

@@ -0,0 +1,3 @@
+1570082807165
+#ffffff	20
+#ffffff	800

smart_fact_crawler/resources/2019_10_03_1531/data/tngdata.data

@@ -0,0 +1,10 @@
+1569924950000
+#ffffff	21
+#ffffff	1.93
+#ffffff	-4.61
+#ffffff	16
+#ffffff	778
+#ffffff	6.48
+#ffffff	189
+#ffffff	7.34
+#ffffff	651

smart_fact_crawler/resources/2019_10_03_1531/data/tngdust.data

@@ -0,0 +1 @@
+1565775220000

smart_fact_crawler/resources/2019_10_03_1531/data/tracking.data

@@ -0,0 +1,8 @@
+1570082799976
+#ffffff	
+#ffffff	7.695
+#ffffff	1.6179
+#ffffff	39.6
+#ffffff	-232
+#ffffff	2.58
+#f0fff0	115

smart_fact_crawler/resources/2019_10_03_1531/data/trigger.data

@@ -0,0 +1,2 @@
+1570116674564
+#ffffff	77

smart_fact_crawler/resources/2019_10_03_1531/data/voltage.data

@@ -0,0 +1,5 @@
+1570082969657
+#ffffff	-0.016
+#ffffff	0.001
+#ffffff	0.00843
+#ffffff	0.06

smart_fact_crawler/resources/2019_10_03_1531/data/weather.data

@@ -0,0 +1,12 @@
+1570116656000
+#fff8f0	day time [04:43↓]
+#ffffff	visible 29% [07:16↓]
+#ffffff	22.6
+#ffffff	-0.1
+#ffffff	22
+#ffffff	795
+#ffffff	13.2
+#ffffff	15.6
+#ffffff	N
+#ffffff	7.34	09:33
+#ffffff	19.2	15:29

smart_fact_crawler/resources/2019_10_03_1531/data/wind.data

@@ -0,0 +1,5 @@
+1570116656000
+#ffffff	13.2
+#ffffff	4.3
+#ffffff	12.4
+#ffffff	22.6