Skip to content

Commit

Permalink
Merge pull request #24 from fact-project/add_missing
Browse files Browse the repository at this point in the history
Add missing stuff
maxnoe authored Mar 27, 2017
2 parents 8f7524b + d52f336 commit 6e584ad
Showing 5 changed files with 112 additions and 54 deletions.
4 changes: 2 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@

setup(
name='smart_fact_crawler',
version='0.4.0',
version='0.4.1',
description='acquieres data published on the smartfact web page',
url='https://github.com/fact-project/smart_fact_crawler.git',
author='Dominik Neise, Sebastian Mueller, Maximilian Nöthe',
@@ -21,7 +21,7 @@
install_requires=[
'requests',
],
tests_require=['pytest>=3.0'],
tests_require=['pytest>=3.0', 'freezegun'],
setup_requires=['pytest-runner'],
zip_safe=True,
)
57 changes: 53 additions & 4 deletions smart_fact_crawler/__init__.py
Original file line number Diff line number Diff line change
@@ -6,7 +6,6 @@
from .tools import str2float as s2f
from .tools import smartfact_time2datetime as sft2dt
from .tools import smartfact2table
from .tools import extract_run_id_from_system_status
from .tools import get_entry

import re
@@ -20,6 +19,17 @@
Quantity = namedtuple('Quantity', ['value', 'unit'])


run_re = re.compile(
'([0-9]{2}:[0-9]{2}:[0-9]{2}) ' # match the time part
'<#[a-z]+>' # html color
'([a-zA-Z\-]+) ' # run type
'\[([a-zA-Z0-9 ]+)\] ' # source name
'\(Run (\d+)\)' # run number
'</#>'
)
Run = namedtuple('Run', ['start', 'type', 'source', 'id'])


def to_namedtuple(name, dictionary):
return namedtuple(name, dictionary.keys())(**dictionary)

@@ -305,14 +315,19 @@ def main_page(url=None, timeout=None, fallback=False):
get = partial(get_entry, fallback=fallback)

system_status = get(table, 1, 1)
power_val, power_unit = get(table, 6, 3, default='nan nan').split()
trigger_val, trigger_unit, _ = get(table, 5, 1, default='nan nan nan').split()
return to_namedtuple('MainPage', {
'timestamp_1': sft2dt(get(table, 0, 0)),
'timestamp_2': sft2dt(get(table, 0, 1)),
'system_status': system_status,
'run_id': extract_run_id_from_system_status(system_status),
'relative_camera_temperature': Quantity(s2f(get(table, 3, 1)), 'deg_C'),
'humidity': Quantity(s2f(get(table, 4, 1)), '%'),
'wind_speed': Quantity(s2f(get(table, 4, 2)), 'km/h'),
'trigger_rate': Quantity(s2f(trigger_val), trigger_unit),
'median_current': Quantity(s2f(get(table, 6, 1)), 'uA'),
'max_current': Quantity(s2f(get(table, 6, 2)), 'uA'),
'power': Quantity(s2f(power_val), power_unit),
})


@@ -335,14 +350,48 @@ def errorhist(url=None, timeout=None, fallback=False):

table = smartfact2table(url, timeout=timeout)
get = partial(get_entry, fallback=fallback)
timestamp = get(table, 0, 0)

history = [
h
for h in get(table, 1, 1, default='').split("<->")[1].split("<br/>")
if h
]
return to_namedtuple('ErrorHistPage', {
'timestamp': sft2dt(timestamp) if timestamp else None,
'timestamp': sft2dt(get(table, 0, 0)),
'history': history,
})


def build_run(tup):
start, run_type, source, run_id = tup
run_id = int(run_id)

now = datetime.utcnow()
start = datetime.strptime(start, '%H:%M:%S').replace(
year=now.year, month=now.month, day=now.day
)

if start > now:
start -= timedelta(hours=24)

return Run(start, run_type, source, run_id)


def observations(url=None, timeout=None, fallback=False):
if url is None:
url = os.path.join(smartfacturl, 'observations.data')

table = smartfact2table(url, timeout=timeout)
get = partial(get_entry, fallback=fallback)

run_list = get(table, 1, 1)
if run_list is not None:
runs = list(map(build_run, run_re.findall(table[1][1])))
runs.sort(key=lambda r: r.id)
else:
runs = None

return to_namedtuple('ErrorHistPage', {
'timestamp': sft2dt(get(table, 0, 0)),
'runs': runs,
})
57 changes: 57 additions & 0 deletions smart_fact_crawler/tests/test_observations.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
from pkg_resources import resource_filename
from freezegun import freeze_time
from datetime import datetime

test_file = resource_filename(
'smart_fact_crawler', 'resources/20160703_233149/observations.data'
)

def test_re():
from smart_fact_crawler import run_re

with open(test_file) as f:
m = run_re.findall(f.read())

assert len(m) == 67

timestamp, run_type, source, run_id = m[0]

assert timestamp == '23:31:27'
assert run_type == 'data'
assert source == 'Mrk 501'
assert run_id == '67'


@freeze_time('2016-07-03 23:32')
def test_build_run():
from smart_fact_crawler import run_re, build_run

with open(test_file) as f:
m = run_re.findall(f.read())

run = build_run(m[0])

assert run.id == 67
assert run.start == datetime(2016, 7, 3, 23, 31, 27)


@freeze_time('2016-07-04 00:32')
def test_build_run_after_midnight():
from smart_fact_crawler import run_re, build_run

with open(test_file) as f:
m = run_re.findall(f.read())

run = build_run(m[0])

assert run.id == 67
assert run.start == datetime(2016, 7, 3, 23, 31, 27)


@freeze_time('2016-07-03 23:32')
def test_observations():
from smart_fact_crawler import observations

obs = observations(url='file:' + test_file)

assert obs.runs[-1].id == len(obs.runs)
26 changes: 0 additions & 26 deletions smart_fact_crawler/tests/test_unit.py

This file was deleted.

22 changes: 0 additions & 22 deletions smart_fact_crawler/tools.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@
import requests
import urllib
from datetime import datetime
import re


def str2float(text):
@@ -48,27 +47,6 @@ def smartfact2table(url, timeout=None):
return parse_table(text)


def extract_run_id_from_system_status(system_status):
'''
system_status: string like 'data(47) [1169/279s]' or 'Idle [pedestal]'
returns: integer `run_id`
or None if no number between braces `()` found in system_status.
throws TypeError in case `system_status` is not string like
enough to be used inside re.match()
'''
if system_status is None:
return None

run_id_in_system_status_pattern = r'.*\((\d+)\).*'
match_run_id = re.match(run_id_in_system_status_pattern, system_status)

if match_run_id is None:
return None

return int(match_run_id.groups()[0])


def get_entry(table, row, col, fallback=False, default=None):
'''
Get an element from a two dimensinoal list like the return value

0 comments on commit 6e584ad

Please sign in to comment.