From 3473be8a4face6d6357a1529a66cb4b563091ae1 Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Tue, 4 Jun 2019 18:46:48 +0200 Subject: [PATCH] #221 support OGC:WFS3 and first version WFSv3 Drilldown Probe --- GeoHealthCheck/config_main.py | 4 + GeoHealthCheck/enums.py | 3 + GeoHealthCheck/healthcheck.py | 3 + GeoHealthCheck/models.py | 2 +- GeoHealthCheck/plugins/probe/wfs3.py | 197 +++++++++++++++++++++++++++ GeoHealthCheck/probe.py | 1 + GeoHealthCheck/result.py | 2 + docs/index.rst | 5 +- docs/plugins.rst | 4 + docs/userguide.rst | 1 + requirements.txt | 2 +- 11 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 GeoHealthCheck/plugins/probe/wfs3.py diff --git a/GeoHealthCheck/config_main.py b/GeoHealthCheck/config_main.py index 141e971d..bc63a1d8 100644 --- a/GeoHealthCheck/config_main.py +++ b/GeoHealthCheck/config_main.py @@ -105,6 +105,7 @@ 'GeoHealthCheck.plugins.probe.http', 'GeoHealthCheck.plugins.probe.sta', 'GeoHealthCheck.plugins.probe.wmsdrilldown', + 'GeoHealthCheck.plugins.probe.wfs3', # Checkers 'GeoHealthCheck.plugins.check.checks', @@ -145,6 +146,9 @@ 'OGC:STA': { 'probe_class': 'GeoHealthCheck.plugins.probe.sta.StaCaps' }, + 'OGC:WFS3': { + 'probe_class': 'GeoHealthCheck.plugins.probe.wfs3.WFS3Drilldown' + }, 'urn:geoss:waf': { 'probe_class': 'GeoHealthCheck.plugins.probe.http.HttpGet' }, diff --git a/GeoHealthCheck/enums.py b/GeoHealthCheck/enums.py index 49e375da..ab333a1a 100644 --- a/GeoHealthCheck/enums.py +++ b/GeoHealthCheck/enums.py @@ -71,6 +71,9 @@ 'label': 'SensorThings API (STA)', 'versions': ['1.0'] }, + 'OGC:WFS3': { + 'label': 'OGC API Features (WFS3)' + }, 'urn:geoss:waf': { 'label': 'Web Accessible Folder (WAF)' }, diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index 285f8158..0985eca6 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -154,6 +154,7 @@ def sniff_test_resource(config, resource_type, url): 'OGC:WPS': [WebProcessingService], 'OGC:CSW': [CatalogueServiceWeb], 'OGC:SOS': [SensorObservationService], + 'OGC:WFS3': [urlopen], 'OGC:STA': [urlopen], 'WWW:LINK': [urlopen], 'FTP': [urlopen], @@ -233,6 +234,8 @@ def sniff_test_resource(config, resource_type, url): elif resource_type.startswith(('OGC:', 'OSGeo')): if resource_type == 'OGC:STA': title = 'OGC STA' + elif resource_type == 'OGC:WFS3': + title = 'OGC WFS3' else: title = ows.identification.title if title is None: diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index a9c75eeb..1ce0efd3 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -385,7 +385,7 @@ def __repr__(self): @property def get_capabilities_url(self): if self.resource_type.startswith('OGC:') \ - and self.resource_type != 'OGC:STA': + and self.resource_type not in ['OGC:STA', 'OGC:WFS3']: url = '%s%s' % (bind_url(self.url), RESOURCE_TYPES[self.resource_type]['capabilities']) else: diff --git a/GeoHealthCheck/plugins/probe/wfs3.py b/GeoHealthCheck/plugins/probe/wfs3.py new file mode 100644 index 00000000..bc4fc37f --- /dev/null +++ b/GeoHealthCheck/plugins/probe/wfs3.py @@ -0,0 +1,197 @@ +from owslib.wfs import WebFeatureService + +from GeoHealthCheck.probe import Probe +from GeoHealthCheck.result import Result + + +class WFS3Drilldown(Probe): + """ + Probe for WFS3 endpoint "drilldown": starting + with top endpoint: get Collections and do + GetItems on them etc. Using OWSLib.WebFeatureService. + + TODO: needs finalization. + """ + + NAME = 'WFS3 Drilldown' + DESCRIPTION = 'Traverses a OGC WFS3 (REST) API endpoint by drilling down' + RESOURCE_TYPE = 'OGC:WFS3' + + REQUEST_METHOD = 'GET' + + # PARAM_DEFS = { + # 'drilldown_level': { + # 'type': 'string', + # 'description': 'How heavy the drilldown should be.', + # 'default': 'minor', + # 'required': True, + # 'range': ['minor', 'moderate', 'full'] + # } + # } + """Param defs""" + + def __init__(self): + Probe.__init__(self) + + def perform_request(self): + """ + Perform the drilldown. + See https://github.com/geopython/OWSLib/blob/ + master/tests/doctests/wfs3_GeoServerCapabilities.txt + """ + wfs3 = None + collections = None + + # 1. Test top endpoints existence + result = Result(True, 'Test Top Endpoints') + result.start() + try: + wfs3 = WebFeatureService(self._resource.url, version='3.0') + wfs3.conformance() + collections = wfs3.collections() + except Exception as err: + result.set(False, str(err)) + + result.stop() + self.result.add_result(result) + + # 2. Test layers + # TODO: use parameters to work on less/more drilling + # "full" could be all layers. + result = Result(True, 'Test Collections') + result.start() + coll_name = '' + try: + for collection in collections: + coll_name = collection['name'] + coll_name = coll_name.encode('utf-8') + + try: + items = wfs3.collection_items(coll_name, limit=2) + except Exception as e: + msg = 'GetItems %s: OWSLib err: %s ' % (str(e), coll_name) + result = self.add_result(result, + False, msg, 'Test GetItems') + continue + + features = items.get('features', None) + if not features: + msg = 'GetItems %s: No features attr' % coll_name + result = self.add_result(result, + False, msg, 'Test GetItems') + continue + + if len(items['features']) > 0: + + fid = items['features'][0]['id'] + try: + item = wfs3.collection_item(coll_name, fid) + except Exception as e: + msg = 'GetItem %s: OWSLib err: %s' \ + % (str(e), coll_name) + result = self.add_result(result, + False, msg, 'Test GetItem') + continue + + for attr in ['id', 'links', 'properties', 'type']: + val = item.get(attr, None) + if not val: + msg = '%s:%s no attr=%s' \ + % (coll_name, str(fid), attr) + result = self.add_result( + result, False, msg, 'Test GetItem') + continue + + except Exception as err: + result.set(False, 'Collection err: %s : e=%s' + % (coll_name, str(err))) + + result.stop() + + # Add to overall Probe result + self.result.add_result(result) + + def add_result(self, result, val, msg, new_result_name): + result.set(val, msg) + result.stop() + self.result.add_result(result) + result = Result(True, new_result_name) + result.start() + return result + +# class WFS3Caps(Probe): +# """Probe for OGC WFS3 API main endpoint url""" +# +# NAME = 'OGC WFS3 API Capabilities' +# DESCRIPTION = 'Perform OGC WFS3 API Capabilities +# Operation and check validity' +# RESOURCE_TYPE = 'OGC:WFS3' +# +# REQUEST_METHOD = 'GET' +# +# # e.g. https://demo.pygeoapi.io/master/collections?f=json +# REQUEST_TEMPLATE = '/{endpoint}?f=json' +# +# def __init__(self): +# Probe.__init__(self) +# +# PARAM_DEFS = { +# 'endpoint': { +# 'type': 'string', +# 'description': 'The OGC WFS3 API service endpoint type', +# 'default': '/collections', +# 'required': True, +# 'range': ['collections', 'conformance', 'api'] +# } +# } +# """Param defs""" +# +# CHECKS_AVAIL = { +# 'GeoHealthCheck.plugins.check.checks.HttpStatusNoError': { +# 'default': True +# }, +# 'GeoHealthCheck.plugins.check.checks.JsonParse': { +# 'default': True +# } +# } +# """Check for OGC WFS3 API OGC WFS3 API service endpoint +# availability""" +# +# +# class WFS3Collection(Probe): +# """Probe for OGC WFS3 API main endpoint url""" +# +# NAME = 'OGC WFS3 API Capabilities' +# DESCRIPTION = 'Perform OGC WFS3 API Capabilities Operation and +# check validity' +# RESOURCE_TYPE = 'OGC:WFS3' +# +# REQUEST_METHOD = 'GET' +# +# def __init__(self): +# Probe.__init__(self) +# +# CHECKS_AVAIL = { +# 'GeoHealthCheck.plugins.check.checks.HttpStatusNoError': { +# 'default': True +# }, +# 'GeoHealthCheck.plugins.check.checks.JsonParse': { +# 'default': True +# }, +# 'GeoHealthCheck.plugins.check.checks.ContainsStrings': { +# 'default': True, +# 'set_params': { +# 'strings': { +# 'name': 'Must contain links to at least WFS3 Collections, +# Conformance and OpenAPI endpoint', +# 'value': ['links', 'href', '/collections', +# '/conformance', '/api'] +# } +# } +# }, +# } +# """ +# Checks avail for all specific Caps checks. +# Optionally override Check.PARAM_DEFS using set_params +# e.g. with specific `value` or even `name`. +# """ diff --git a/GeoHealthCheck/probe.py b/GeoHealthCheck/probe.py index 800fe658..1bca4204 100644 --- a/GeoHealthCheck/probe.py +++ b/GeoHealthCheck/probe.py @@ -318,6 +318,7 @@ def run_checks(self): # by Check instances. for check_var in self._check_vars: check = None + check_class = '' try: check_class = check_var.check_class check = Factory.create_obj(check_class) diff --git a/GeoHealthCheck/result.py b/GeoHealthCheck/result.py index ef361b34..4476f124 100644 --- a/GeoHealthCheck/result.py +++ b/GeoHealthCheck/result.py @@ -45,6 +45,8 @@ def stop(self): self.response_time_str = '%s.%s' % (delta.seconds, delta.microseconds) def __str__(self): + if self.message: + self.message = self.message.encode('utf-8') return "success=%s msg=%s response_time=%s" % \ (self.success, self.message, self.response_time_str) diff --git a/docs/index.rst b/docs/index.rst index 10b967af..561b6727 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,8 +7,9 @@ Overview GeoHealthCheck (GHC) is a Python application to support monitoring OGC services uptime, availability and Quality of Service (QoS). -It can be used to monitor overall health of OGC services like WMS, WFS, WCS, WMTS, SOS, CSW -and more, but also standard web(-API) URLs. +GHC can be used to monitor overall health of OGC services (OWS) like WMS, WFS, WCS, WMTS, SOS, CSW +and more, plus some recent OGC APIs like SensorThings API and WFS v3 (OGC Features API). +But also standard web REST APIs and ordinary URLs can be monitored. Features -------- diff --git a/docs/plugins.rst b/docs/plugins.rst index 30cc94a3..3e12394c 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -299,6 +299,7 @@ See an example for both below from `config_main.py` for **GHC_PLUGINS** and **GH 'GeoHealthCheck.plugins.probe.http', 'GeoHealthCheck.plugins.probe.sta', 'GeoHealthCheck.plugins.probe.wmsdrilldown', + 'GeoHealthCheck.plugins.probe.wfs3', # Checks 'GeoHealthCheck.plugins.check.checks', @@ -333,6 +334,9 @@ See an example for both below from `config_main.py` for **GHC_PLUGINS** and **GH 'OGC:STA': { 'probe_class': 'GeoHealthCheck.plugins.probe.sta.StaCaps' }, + 'OGC:WFS3': { + 'probe_class': 'GeoHealthCheck.plugins.probe.wfs3.WFS3Drilldown' + }, 'urn:geoss:waf': { 'probe_class': 'GeoHealthCheck.plugins.probe.http.HttpGet' }, diff --git a/docs/userguide.rst b/docs/userguide.rst index 1887ce39..428f2dd1 100644 --- a/docs/userguide.rst +++ b/docs/userguide.rst @@ -102,6 +102,7 @@ The following Resource Types are available: - Web Processing Service (WPS) - Sensor Observation Service (SOS) - `SensorThings API `_ (STA) +- OGC Features API (WFS3) - Web Accessible Folder (WAF) - Web Address (URL) - File Transfer Protocol (FTP) diff --git a/requirements.txt b/requirements.txt index 82c80d1c..042ea504 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ SQLAlchemy==1.3.4 Flask-SQLAlchemy==2.1 itsdangerous==1.1.0 pyproj==1.9.6 -OWSLib==0.17.0 +OWSLib==0.17.1 Sphinx==1.8.4 sphinx-rtd-theme==0.4.3 requests>=2.20.0