From 0895e0a84f29a0878c8aa7f1047dc390b4bf3228 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Jan 2024 10:25:02 +0100 Subject: [PATCH 01/44] Removed get_outputfiles, filter4oldfiles, get_xml_outputfiles, filter4oldfiles, create_xml_timestat_from_scene, get_time_control_ascii_filename, get_time_control_ascii_filename_candidates, get_product_statistics_files --- nwcsafpps_runner/utils.py | 145 -------------------------------------- 1 file changed, 145 deletions(-) diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index 315e12d..dbceb8e 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -503,60 +503,6 @@ def get_pps_inputfile(platform_name, ppsfiles): return None -def get_outputfiles(path, platform_name, orb, st_time='', **kwargs): - """Finds outputfiles depending on certain input criteria. - - From the directory path and satellite id and orbit number, - scan the directory and find all pps output files matching that scene and - return the full filenames. Since the orbit number is unstable there might be - more than one scene with the same orbit number and platform name. In order - to avoid picking up an older scene we check the file modifcation time, and - if the file is too old we discard it! For a more specific search patern the - start time can be used, just add st_time=start-time - """ - - filelist = [] - h5_output = kwargs.get('h5_output') - if h5_output: - h5_output = (os.path.join(path, 'S_NWC') + '*' + - str(METOP_NAME_LETTER.get(platform_name, platform_name)) + - '_' + '%.5d' % int(orb) + '_%s*.h5' % st_time) - LOG.info( - "Match string to do a file globbing on hdf5 output files: " + str(h5_output)) - filelist = filelist + glob(h5_output) - - nc_output = kwargs.get('nc_output') - if nc_output: - nc_output = (os.path.join(path, 'S_NWC') + '*' + - str(METOP_NAME_LETTER.get(platform_name, platform_name)) + - '_' + '%.5d' % int(orb) + '_%s*.nc' % st_time) - LOG.info( - "Match string to do a file globbing on netcdf output files: " + str(nc_output)) - filelist = filelist + glob(nc_output) - - xml_output = kwargs.get('xml_output') - if xml_output: - filelist = filelist + get_xml_outputfiles(path, platform_name, orb, st_time) - - return filter4oldfiles(filelist) - - -def filter4oldfiles(filelist, minutes_thr=90.): - """Check if the PPS file is older than threshold and only consider fresh ones.""" - - now = datetime.utcnow() - time_threshold = timedelta(minutes=minutes_thr) - filtered_flist = [] - for fname in filelist: - mtime = datetime.utcfromtimestamp(os.stat(fname)[stat.ST_MTIME]) - if (now - mtime) < time_threshold: - filtered_flist.append(fname) - else: - LOG.info("Found old PPS result: %s", fname) - - return filtered_flist - - def get_xml_outputfiles(path, platform_name, orb, st_time=''): """Finds xml outputfiles depending on certain input criteria. @@ -606,16 +552,6 @@ def create_xml_timestat_from_lvl1c(scene, pps_control_path): return [] -def create_xml_timestat_from_scene(scene, pps_control_path): - """From lvl1c file create XML file and return a file list, v2018.""" - try: - txt_time_control = get_time_control_ascii_filename(scene, pps_control_path) - except FindTimeControlFileError: - LOG.exception('No XML Time statistics file created!') - return [] - return create_xml_timestat_from_ascii(txt_time_control, pps_control_path) - - def find_product_statistics_from_lvl1c(scene, pps_control_path): """From lvl1c file find product XML files and return a file list.""" try: @@ -657,87 +593,6 @@ def create_xml_timestat_from_ascii(infile, pps_control_path): return [infile.replace('.txt', '.xml')] -def get_time_control_ascii_filename(scene, pps_control_path): - """From the scene object and a file path get the time-control-ascii-filename (with path).""" - infiles = get_time_control_ascii_filename_candidates(scene, pps_control_path) - LOG.info("Time control ascii file candidates: " + str(infiles)) - if len(infiles) == 0: - raise FindTimeControlFileError("No time control ascii file candidate found!") - elif len(infiles) > 1: - msg = "More than one time control ascii file candidate found - unresolved ambiguity!" - raise FindTimeControlFileError(msg) - - return infiles[0] - - -def get_time_control_ascii_filename_candidates(scene, pps_control_path): - """From directory path, sensor and platform name get possible time-control filenames.""" - sensors = SENSOR_LIST.get(scene['platform_name'], scene['platform_name']) - platform_id = SATELLITE_NAME.get(scene['platform_name'], scene['platform_name']) - LOG.info("pps platform_id = %s", str(platform_id)) - - #: Create the start time (format dateTtime) to be used in file findings - st_times = [] - if sensors == 'seviri': - st_times.append(scene['starttime'].strftime("%Y%m%dT%H%M%S.%f")) - elif sensors in ['viirs', ]: - st_times.append(scene['starttime'].strftime("%Y%m%dT%H%M%S")) - elif 'avhrr/3' in sensors or sensors in ['modis', ]: - # At least for the AVHRR data the PPS filenames differ by a few - # seconds from the start time in the message - it seems sufficient - # to truncate the seconds, but we check for minutes +- 1 - atime = scene['starttime'] - timedelta(seconds=60) - while atime < scene['starttime'] + timedelta(seconds=120): - st_times.append(atime.strftime("%Y%m%dT%H%M")) - atime = atime + timedelta(seconds=60) - - # For VIIRS we often see a orbit number difference of 1: - norbit_candidates = [scene['orbit_number']] - for idx in [1, -1]: - norbit_candidates.append(int(scene['orbit_number']) + idx) - - # PPSv2018 MODIS files have the orbit number set to "00000"! - # Level1c4pps files have the orbit number configurable with default "00000"! - norbit_candidates.append(0) - - infiles = [] - for norbit in norbit_candidates: - for st_time in st_times: - txt_time_file = (os.path.join(pps_control_path, 'S_NWC_timectrl_') + - str(METOP_NAME_LETTER.get(platform_id, platform_id)) + - '_' + '%.5d' % norbit + '_' + - st_time + - '*.txt') - LOG.info("glob string = " + str(txt_time_file)) - infiles = infiles + glob(txt_time_file) - - return infiles - - -def get_product_statistics_files(pps_control_path, scene, product_statistics_filename, - max_abs_deviation_minutes): - """From directory path, sensor and platform name get possible product statistics filenames.""" - - platform_id = SATELLITE_NAME.get(scene['platform_name'], scene['platform_name']) - platform_id = METOP_NAME_LETTER.get(platform_id, platform_id) - product_stat_flist = [] - scene_start_time = scene['starttime'] - possible_filetimes = [scene_start_time] - for nmin in range(1, max_abs_deviation_minutes + 1): - possible_filetimes.append(scene_start_time - timedelta(seconds=60 * nmin)) - possible_filetimes.append(scene_start_time + timedelta(seconds=60 * nmin)) - - for product_name in ['CMA', 'CT', 'CTTH', 'CPP', 'CMAPROB']: - for start_time in possible_filetimes: - glbify = globify(product_statistics_filename, {'product': product_name, - 'satellite': platform_id, - 'orbit': '%.5d' % scene['orbit_number'], - 'starttime': start_time}) - product_stat_flist = product_stat_flist + glob(os.path.join(pps_control_path, glbify)) - - return product_stat_flist - - def publish_pps_files(input_msg, publish_q, scene, result_files, **kwargs): """ Publish messages for the files provided. From a0593c28e8dae842e46a62afe623f72fae22e722 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Jan 2024 10:25:41 +0100 Subject: [PATCH 02/44] remove old runner --- bin/pps_runner.py | 321 ---------------------------------------------- 1 file changed, 321 deletions(-) delete mode 100644 bin/pps_runner.py diff --git a/bin/pps_runner.py b/bin/pps_runner.py deleted file mode 100644 index d774b28..0000000 --- a/bin/pps_runner.py +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright (c) 2014 - 2021 Adam.Dybbroe - -# Author(s): - -# Adam.Dybbroe - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""Posttroll runner for PPS -""" -import os -import sys -from glob import glob -from subprocess import Popen, PIPE -import threading -from six.moves.queue import Queue -from datetime import datetime, timedelta - -from nwcsafpps_runner.config import get_config -from nwcsafpps_runner.config import MODE -from nwcsafpps_runner.config import CONFIG_FILE -from nwcsafpps_runner.config import CONFIG_PATH -from nwcsafpps_runner.utils import ready2run, publish_pps_files -from nwcsafpps_runner.utils import (terminate_process, - create_pps_call_command_sequence, - PpsRunError, logreader, get_outputfiles, - message_uid) -from nwcsafpps_runner.utils import (SENSOR_LIST, - SATELLITE_NAME, - METOP_NAME_LETTER) -from nwcsafpps_runner.publish_and_listen import FileListener, FilePublisher - -from nwcsafpps_runner.prepare_nwp import update_nwp - -import logging -LOG = logging.getLogger(__name__) - - -PPS_SCRIPT = os.environ['PPS_SCRIPT'] -LOG.debug("PPS_SCRIPT = %s", str(PPS_SCRIPT)) - -NWP_FLENS = [3, 6, 9, 12, 15, 18, 21, 24] - - -#: Default time format -_DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S' - -#: Default log format -_DEFAULT_LOG_FORMAT = '[%(levelname)s: %(asctime)s : %(name)s] %(message)s' - -LOG.debug("PYTHONPATH: %s", str(sys.path)) - - -class ThreadPool(object): - - def __init__(self, max_nthreads=None): - - self.jobs = set() - self.sema = threading.Semaphore(max_nthreads) - self.lock = threading.Lock() - - def new_thread(self, job_id, group=None, target=None, name=None, args=(), kwargs={}): - - def new_target(*args, **kwargs): - with self.sema: - result = target(*args, **kwargs) - - self.jobs.remove(job_id) - return result - - with self.lock: - if job_id in self.jobs: - LOG.info("Job with id %s already running!", str(job_id)) - return - - self.jobs.add(job_id) - - thread = threading.Thread(group, new_target, name, args, kwargs) - thread.start() - - -def pps_worker(scene, publish_q, input_msg, options): - """Start PPS on a scene - - scene = {'platform_name': platform_name, - 'orbit_number': orbit_number, - 'satday': satday, 'sathour': sathour, - 'starttime': starttime, 'endtime': endtime} - """ - - try: - LOG.debug("Starting pps runner for scene %s", str(scene)) - job_start_time = datetime.utcnow() - - pps_call_args = create_pps_call_command_sequence(PPS_SCRIPT, scene, options) - LOG.info("Command: %s", str(pps_call_args)) - - my_env = os.environ.copy() - # for envkey in my_env: - # LOG.debug("ENV: " + str(envkey) + " " + str(my_env[envkey])) - - pps_output_dir = my_env.get('SM_PRODUCT_DIR', options.get(['pps_outdir'], './')) - LOG.debug("PPS_OUTPUT_DIR = %s", str(pps_output_dir)) - LOG.debug("...from config file = %s", str(options['pps_outdir'])) - if not os.path.isfile(PPS_SCRIPT): - raise IOError("PPS script" + PPS_SCRIPT + " is not there!") - if not os.access(PPS_SCRIPT, os.X_OK): - raise IOError( - "PPS script" + PPS_SCRIPT + " cannot be executed!") - - try: - pps_proc = Popen(pps_call_args, shell=False, stderr=PIPE, stdout=PIPE) - except PpsRunError: - LOG.exception("Failed in PPS...") - - min_thr = options.get('maximum_pps_processing_time_in_minutes', 20) - t__ = threading.Timer(min_thr * 60.0, terminate_process, args=(pps_proc, scene, )) - t__.start() - - out_reader = threading.Thread( - target=logreader, args=(pps_proc.stdout, LOG.info)) - err_reader = threading.Thread( - target=logreader, args=(pps_proc.stderr, LOG.info)) - out_reader.start() - err_reader.start() - out_reader.join() - err_reader.join() - - LOG.info("Ready with PPS level-2 processing on scene: %s", str(scene)) - - # Now try perform som time statistics editing with ppsTimeControl.py from - # pps: - do_time_control = True - try: - from pps_time_control import PPSTimeControl - except ImportError: - LOG.warning("Failed to import the PPSTimeControl from pps") - do_time_control = False - - pps_control_path = my_env.get('STATISTICS_DIR', options.get('pps_statistics_dir', './')) - - if do_time_control: - LOG.info("Read time control ascii file and generate XML") - platform_id = SATELLITE_NAME.get( - scene['platform_name'], scene['platform_name']) - LOG.info("pps platform_id = %s", str(platform_id)) - txt_time_file = (os.path.join(pps_control_path, 'S_NWC_timectrl_') + - str(METOP_NAME_LETTER.get(platform_id, platform_id)) + - '_' + '%.5d' % scene['orbit_number'] + '*.txt') - LOG.info("glob string = %s", str(txt_time_file)) - infiles = glob(txt_time_file) - LOG.info("Time control ascii file candidates: %s", str(infiles)) - if len(infiles) == 1: - infile = infiles[0] - LOG.info("Time control ascii file: %s", str(infile)) - ppstime_con = PPSTimeControl(infile) - ppstime_con.sum_up_processing_times() - ppstime_con.write_xml() - - # Now check what netCDF/hdf5 output was produced and publish - # them: - pps_path = pps_output_dir - result_files = get_outputfiles(pps_path, - SATELLITE_NAME[scene['platform_name']], - scene['orbit_number'], - h5_output=True, - nc_output=True) - LOG.info("PPS Output files: %s", str(result_files)) - xml_files = get_outputfiles(pps_control_path, - SATELLITE_NAME[scene['platform_name']], - scene['orbit_number'], - xml_output=True) - LOG.info("PPS summary statistics files: %s", str(xml_files)) - - # Now publish: - publish_pps_files(input_msg, publish_q, scene, - result_files + xml_files, - environment=MODE, - servername=options['servername'], - station=options['station']) - - dt_ = datetime.utcnow() - job_start_time - LOG.info("PPS on scene %s finished. It took: %s", str(scene), str(dt_)) - - t__.cancel() - - except Exception: - LOG.exception('Failed in pps_worker...') - raise - - -def run_nwp_and_pps(scene, flens, publish_q, input_msg, options): - """Run first the nwp-preparation and then pps. No parallel running here!""" - - prepare_nwp4pps(flens) - pps_worker(scene, publish_q, input_msg, options) - - -def prepare_nwp4pps(flens): - """Prepare NWP data for pps""" - - starttime = datetime.utcnow() - timedelta(days=1) - try: - update_nwp(starttime, flens) - LOG.info("Ready with nwp preparation") - LOG.debug("Leaving prepare_nwp4pps...") - except Exception: - LOG.exception("Something went wrong in update_nwp...") - raise - - -def pps(options): - """The PPS runner. Triggers processing of PPS main script once AAPP or CSPP - is ready with a level-1 file""" - - LOG.info("*** Start the PPS level-2 runner:") - - LOG.info("First check if NWP data should be downloaded and prepared") - now = datetime.utcnow() - update_nwp(now - timedelta(days=1), NWP_FLENS) - LOG.info("Ready with nwp preparation...") - - listener_q = Queue.Queue() - publisher_q = Queue.Queue() - - pub_thread = FilePublisher(publisher_q, options['publish_topic'], runner_name='pps_runner') - pub_thread.start() - listen_thread = FileListener(listener_q, options['subscribe_topics']) - listen_thread.start() - - files4pps = {} - thread_pool = ThreadPool(options['number_of_threads']) - while True: - - try: - msg = listener_q.get() - except Queue.Empty: - continue - - LOG.debug( - "Number of threads currently alive: " + str(threading.active_count())) - - orbit_number = int(msg.data['orbit_number']) - platform_name = msg.data['platform_name'] - starttime = msg.data['start_time'] - endtime = msg.data['end_time'] - - satday = starttime.strftime('%Y%m%d') - sathour = starttime.strftime('%H%M') - sensors = SENSOR_LIST.get(platform_name, None) - scene = {'platform_name': platform_name, - 'orbit_number': orbit_number, - 'satday': satday, 'sathour': sathour, - 'starttime': starttime, 'endtime': endtime, - 'sensor': sensors} - - status = ready2run(msg, files4pps) - if status: - - LOG.info('Start a thread preparing the nwp data and run pps...') - thread_pool.new_thread(message_uid(msg), - target=run_nwp_and_pps, args=(scene, NWP_FLENS, - publisher_q, - msg, options)) - - LOG.debug( - "Number of threads currently alive: " + str(threading.active_count())) - - pub_thread.stop() - listen_thread.stop() - - -if __name__ == "__main__": - - from logging import handlers - LOG.debug("Path to pps_runner config file = " + CONFIG_PATH) - LOG.debug("Pps_runner config file = " + CONFIG_FILE) - OPTIONS = get_config(CONFIG_FILE) - - _PPS_LOG_FILE = OPTIONS.get('pps_log_file', - os.environ.get('PPSRUNNER_LOG_FILE', False)) - if _PPS_LOG_FILE: - ndays = int(OPTIONS["log_rotation_days"]) - ncount = int(OPTIONS["log_rotation_backup"]) - handler = handlers.TimedRotatingFileHandler(_PPS_LOG_FILE, - when='midnight', - interval=ndays, - backupCount=ncount, - encoding=None, - delay=False, - utc=True) - - handler.doRollover() - else: - handler = logging.StreamHandler(sys.stderr) - - handler.setLevel(logging.DEBUG) - formatter = logging.Formatter(fmt=_DEFAULT_LOG_FORMAT, - datefmt=_DEFAULT_TIME_FORMAT) - handler.setFormatter(formatter) - logging.getLogger('').addHandler(handler) - logging.getLogger('').setLevel(logging.DEBUG) - logging.getLogger('posttroll').setLevel(logging.INFO) - - LOG = logging.getLogger('pps_runner') - - pps(OPTIONS) From 3f5e8f8494fb99c7e319f9f68f8e99b9e6712876 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Jan 2024 11:02:06 +0100 Subject: [PATCH 03/44] Refactoring --- nwcsafpps_runner/utils.py | 119 ++++---------------------------------- 1 file changed, 12 insertions(+), 107 deletions(-) diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index dbceb8e..8564b71 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -225,43 +225,17 @@ def get_sceneid(platform_name, orbit_number, starttime): return sceneid -def ready2run(msg, files4pps, use_l1c, **kwargs): +def ready2run(msg, files4pps, **kwargs): """Check whether pps is ready to run or not.""" LOG.debug("Ready to run...") LOG.info("Got message: " + str(msg)) - sdr_granule_processing = kwargs.get('sdr_granule_processing') - stream_tag_name = kwargs.get('stream_tag_name', 'variant') - stream_name = kwargs.get('stream_name', 'EARS') destination = msg.data.get('destination') uris = [] - if (msg.type == 'dataset' and - msg.data['platform_name'] in SUPPORTED_MODIS_SATELLITES): - LOG.info('Dataset: ' + str(msg.data['dataset'])) - LOG.info('Got a dataset on an EOS satellite') - LOG.info('\t ...thus we can assume we have everything we need for PPS') - for obj in msg.data['dataset']: - uris.append(obj['uri']) - - elif (sdr_granule_processing and msg.type == 'dataset' and - msg.data['platform_name'] in SUPPORTED_VIIRS_SATELLITES): - LOG.info('Dataset: ' + str(msg.data['dataset'])) - LOG.info('Got a dataset on a JPSS/SNPP satellite') - if destination is None: - for obj in msg.data['dataset']: - uris.append(obj['uri']) - else: - for obj in msg.data['dataset']: - uris.append(os.path.join(destination, obj['uid'])) - elif msg.type == 'collection' and not sdr_granule_processing: - if 'dataset' in msg.data['collection'][0]: - for dataset in msg.data['collection']: - uris.extend([mda['uri'] for mda in dataset['dataset']]) - - elif msg.type == 'file': + if msg.type == 'file': if destination is None: uris = [(msg.data['uri'])] else: @@ -323,18 +297,7 @@ def ready2run(msg, files4pps, use_l1c, **kwargs): LOG.info( 'Sensor ' + str(msg.data['sensor']) + ' not required...') return False - required_mw_sensors = REQUIRED_MW_SENSORS.get( - msg.data['platform_name']) - if (msg.data['sensor'] in required_mw_sensors and - msg.data['data_processing_level'] != '1C'): - if msg.data['data_processing_level'] == '1c': - LOG.warning("Level should be in upper case!") - else: - LOG.info('Level not the required type for PPS for this sensor: ' + - str(msg.data['sensor']) + ' ' + - str(msg.data['data_processing_level'])) - return False - + # The orbit number is mandatory! orbit_number = int(msg.data['orbit_number']) LOG.debug("Orbit number: " + str(orbit_number)) @@ -347,38 +310,16 @@ def ready2run(msg, files4pps, use_l1c, **kwargs): return False starttime = msg.data.get('start_time') - sceneid = get_sceneid(platform_name, orbit_number, starttime) - - if sceneid not in files4pps: - files4pps[sceneid] = [] LOG.debug("level1_files = %s", level1_files) for item in level1_files: files4pps[sceneid].append(item) LOG.debug("files4pps: %s", str(files4pps[sceneid])) - if use_l1c: - if len(files4pps[sceneid]) < 1: - LOG.info("No level1c files!") - return False - elif (stream_tag_name in msg.data and msg.data[stream_tag_name] in [stream_name, ] and - platform_name in SUPPORTED_EARS_AVHRR_SATELLITES): - LOG.info("EARS Metop data. Only require the HRPT/AVHRR level-1b file to be ready!") - elif (platform_name in SUPPORTED_AVHRR_SATELLITES): - if len(files4pps[sceneid]) < len(REQUIRED_MW_SENSORS[platform_name]) + 1: - LOG.info("Not enough NOAA/Metop sensor data available yet...") - return False - elif platform_name in SUPPORTED_MODIS_SATELLITES: - if len(files4pps[sceneid]) < 2: - LOG.info("Not enough MODIS level 1 files available yet...") - return False + if len(files4pps[sceneid]) < 1: + LOG.info("No level1c files!") + return False - if len(files4pps[sceneid]) > 10: - LOG.info( - "Number of level 1 files ready = " + str(len(files4pps[sceneid]))) - LOG.info("Scene = " + str(sceneid)) - else: - LOG.info("Level 1 files ready: " + str(files4pps[sceneid])) if msg.data['platform_name'] in SUPPORTED_PPS_SATELLITES: LOG.info( @@ -400,25 +341,6 @@ def terminate_process(popen_obj, scene): "Process finished before time out - workerScene: " + str(scene)) -def prepare_pps_arguments(platform_name, level1_filepath, **kwargs): - """Prepare the platform specific arguments to be passed to the PPS scripts/modules.""" - - orbit_number = kwargs.get('orbit_number') - pps_args = {} - - if platform_name in SUPPORTED_MODIS_SATELLITES: - pps_args['modisorbit'] = orbit_number - pps_args['modisfile'] = level1_filepath - - elif platform_name in SUPPORTED_VIIRS_SATELLITES: - pps_args['csppfile'] = level1_filepath - - elif platform_name in SUPPORTED_AVHRR_SATELLITES: - pps_args['hrptfile'] = level1_filepath - - return pps_args - - def create_pps_call_command_sequence(pps_script_name, scene, options): """Create the PPS call commnd sequence. @@ -452,32 +374,18 @@ def create_pps_call_command_sequence(pps_script_name, scene, options): return shlex.split(str(cmdstr)) -def create_pps_call_command(python_exec, pps_script_name, scene, use_l1c=False): +def create_pps_call_command(python_exec, pps_script_name, scene): """Create the pps call command. - Supports PPSv2018 and PPSv2021. + Supports PPSv2021. """ - if use_l1c: - cmdstr = ("%s" % python_exec + " %s " % pps_script_name + - "-af %s" % scene['file4pps']) - LOG.debug("PPS call command: %s", str(cmdstr)) - return cmdstr - - if scene['platform_name'] in SUPPORTED_MODIS_SATELLITES: - cmdstr = ("%s " % python_exec + " %s " % pps_script_name + - " --modisfile %s" % scene['file4pps']) - elif scene['platform_name'] in SUPPORTED_VIIRS_SATELLITES: - cmdstr = ("%s " % python_exec + " %s " % pps_script_name + - " --csppfile %s" % scene['file4pps']) - elif scene['platform_name'] in SUPPORTED_AVHRR_SATELLITES: - cmdstr = ("%s " % python_exec + " %s " % pps_script_name + - " --hrptfile %s" % scene['file4pps']) - else: - raise - + cmdstr = ("%s" % python_exec + " %s " % pps_script_name + + "-af %s" % scene['file4pps']) + LOG.debug("PPS call command: %s", str(cmdstr)) return cmdstr + def get_pps_inputfile(platform_name, ppsfiles): """From the set of files picked up in the PostTroll messages decide the input file used in the PPS call @@ -637,9 +545,6 @@ def publish_pps_files(input_msg, publish_q, scene, result_files, **kwargs): if result_file.endswith("nc"): to_send['format'] = 'CF' to_send['type'] = 'netCDF4' - if result_file.endswith("h5"): - to_send['format'] = 'PPS' - to_send['type'] = 'HDF5' to_send['data_processing_level'] = '2' to_send['start_time'], to_send['end_time'] = starttime, endtime From b0c5bade8e031f26ac2bcf6aa9585805c0175ea8 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Jan 2024 11:02:22 +0100 Subject: [PATCH 04/44] Refactoring --- bin/pps2018_runner.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/bin/pps2018_runner.py b/bin/pps2018_runner.py index 3039d14..ac3221f 100644 --- a/bin/pps2018_runner.py +++ b/bin/pps2018_runner.py @@ -40,11 +40,10 @@ from nwcsafpps_runner.utils import (METOP_NAME_LETTER, SATELLITE_NAME, SENSOR_LIST, NwpPrepareError, PpsRunError, create_pps_call_command, - get_outputfiles, get_pps_inputfile, create_xml_timestat_from_lvl1c, find_product_statistics_from_lvl1c, get_sceneid, logreader, message_uid, - prepare_pps_arguments, publish_pps_files, + publish_pps_files, ready2run, terminate_process) from nwcsafpps_runner.utils import create_xml_timestat_from_scene from nwcsafpps_runner.utils import get_product_statistics_files @@ -111,11 +110,6 @@ def pps_worker(scene, publish_q, input_msg, options): LOG.debug("Platform name: %s", scene['platform_name']) LOG.debug("Orbit number: %s", str(scene['orbit_number'])) - kwargs = prepare_pps_arguments(scene['platform_name'], - scene['file4pps'], - orbit_number=scene['orbit_number']) - LOG.debug("pps-arguments: %s", str(kwargs)) - min_thr = options['maximum_pps_processing_time_in_minutes'] LOG.debug("Maximum allowed PPS processing time in minutes: %d", min_thr) @@ -127,7 +121,7 @@ def pps_worker(scene, publish_q, input_msg, options): pps_run_all_flags = [] use_l1c = (options.get('pps_version') == 'v2021') - cmd_str = create_pps_call_command(py_exec, pps_script, scene, use_l1c=use_l1c) + cmd_str = create_pps_call_command(py_exec, pps_script, scene) for flag in pps_run_all_flags: cmd_str = cmd_str + ' %s' % flag @@ -161,7 +155,7 @@ def pps_worker(scene, publish_q, input_msg, options): if options['run_cmask_prob']: pps_script = options.get('run_cmaprob_script') - cmdl = create_pps_call_command(py_exec, pps_script, scene, use_l1c=use_l1c) + cmdl = create_pps_call_command(py_exec, pps_script, scene) LOG.debug("Run command: " + str(cmdl)) try: @@ -182,17 +176,8 @@ def pps_worker(scene, publish_q, input_msg, options): err_reader2.join() pps_control_path = my_env.get('STATISTICS_DIR', options.get('pps_statistics_dir', './')) - if use_l1c: - # v2021 - xml_files = create_xml_timestat_from_lvl1c(scene, pps_control_path) - xml_files += find_product_statistics_from_lvl1c(scene, pps_control_path) - else: - # v2018 - xml_files = create_xml_timestat_from_scene(scene, pps_control_path) - xml_files = xml_files + get_product_statistics_files(pps_control_path, - scene, - options['product_statistics_filename'], - options['pps_filetime_search_minutes']) + xml_files = create_xml_timestat_from_lvl1c(scene, pps_control_path) + xml_files += find_product_statistics_from_lvl1c(scene, pps_control_path) LOG.info("PPS summary statistics files: %s", str(xml_files)) From e5a06d578943a1993a88ceb20f8621c888ea9261 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Jan 2024 11:02:34 +0100 Subject: [PATCH 05/44] Update tests --- nwcsafpps_runner/tests/test_utils.py | 271 +-------------------------- 1 file changed, 1 insertion(+), 270 deletions(-) diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index 04d76cc..535eedf 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -26,13 +26,8 @@ from datetime import datetime from nwcsafpps_runner.utils import (create_xml_timestat_from_lvl1c, find_product_statistics_from_lvl1c, - publish_pps_files, - create_xml_timestat_from_scene) -from nwcsafpps_runner.utils import get_outputfiles -from nwcsafpps_runner.utils import get_time_control_ascii_filename_candidates -from nwcsafpps_runner.utils import get_time_control_ascii_filename + publish_pps_files) from nwcsafpps_runner.utils import FindTimeControlFileError -from nwcsafpps_runner.utils import get_product_statistics_files @pytest.fixture @@ -121,270 +116,6 @@ def test_xml_for_products_no_file4pps(self, fake_file_dir): res = find_product_statistics_from_lvl1c(self.empty_scene, mydir_out) assert res == [] - def test_create_xml_timestat_from_scene(self, fake_file_dir): - """Test xml files for scene (used for versions <= v2018).""" - mydir, mydir_out = fake_file_dir - res = create_xml_timestat_from_scene(self.scene, mydir_out) - expected = [os.path.join(mydir_out, "S_NWC_timectrl_npp_12345_19810305T0715000Z_19810305T0730000Z.xml")] - assert len(res) == len(set(expected)) - assert set(res) == set(expected) - - def test_create_xml_timestat_from_scene_missing_files(self, fake_file_dir): - """Test missing xml files for scene (used for versions <= v2018).""" - mydir, mydir_out = fake_file_dir - res = create_xml_timestat_from_scene(self.scene, mydir) - expected = [] - assert res == expected - -def test_outputfiles(tmp_path): - """Test get_outputfiles. - - get_outputfiles uses os.stat to test if a file is older than 90 min, - and if so disregard it. This behaviour can't be tested at the moment. - TODO: either one file (correct orbit number and start-time) needs to - be created more than 90 mins ago or the os.stat should be modified - so the so.stat thinks the file was created more than 90 mins ago. - The file should than not be found. - """ - #: Create temp_path - mydir = tmp_path / "export" - mydir.mkdir() - - #: Create test files - def create_files(mydir, typ): - #: These files should always be found - f1 = mydir / "S_NWC_CMAPROB_noaa15_12345_19810305T0715000Z_19810305T0730000Z.{}".format(typ) - f1.write_text("correct orbit and time") - #: These files should be found if start time is not given - f2 = mydir / "S_NWC_CMAPROB_noaa15_12345_19810305T0745000Z_19810305T0800000Z.{}".format(typ) - f2.write_text("correct orbit and time within 90 min") - #: These files should not be found although the start time is correct - f3 = mydir / "S_NWC_CMAPROB_noaa15_54321_19810305T0715000Z_19810305T0730000Z.{}".format(typ) - f3.write_text("wrong orbit and correct time") - - #: Test xml files without start time - typ = "xml" - create_files(mydir, typ) - expected = [os.path.join(mydir, "S_NWC_CMAPROB_noaa15_12345_19810305T0715000Z_19810305T0730000Z.{}".format(typ)), - os.path.join(mydir, "S_NWC_CMAPROB_noaa15_12345_19810305T0745000Z_19810305T0800000Z.{}".format(typ))] - res = get_outputfiles(mydir, "noaa15", 12345, xml_output=True) - assert len(res) == len(set(res)) - assert set(res) == set(expected) - #: Test xml files with start time - expected = [os.path.join(mydir, "S_NWC_CMAPROB_noaa15_12345_19810305T0715000Z_19810305T0730000Z.{}".format(typ))] - res = get_outputfiles(mydir, "noaa15", 12345, st_time="19810305T0715", xml_output=True) - assert len(res) == len(set(res)) - assert set(res) == set(expected) - - #: Test h5 files without start time - typ = "h5" - create_files(mydir, typ) - expected = [os.path.join(mydir, "S_NWC_CMAPROB_noaa15_12345_19810305T0715000Z_19810305T0730000Z.{}".format(typ)), - os.path.join(mydir, "S_NWC_CMAPROB_noaa15_12345_19810305T0745000Z_19810305T0800000Z.{}".format(typ))] - res = get_outputfiles(mydir, "noaa15", 12345, h5_output=True) - assert len(res) == len(set(res)) - assert set(res) == set(expected) - #: Test h5 files with start time - expected = [os.path.join(mydir, "S_NWC_CMAPROB_noaa15_12345_19810305T0715000Z_19810305T0730000Z.{}".format(typ))] - res = get_outputfiles(mydir, "noaa15", 12345, st_time="19810305T0715", h5_output=True) - assert len(res) == len(set(res)) - assert set(res) == set(expected) - - #: Test nc files without start time - typ = "nc" - create_files(mydir, typ) - expected = [os.path.join(mydir, "S_NWC_CMAPROB_noaa15_12345_19810305T0715000Z_19810305T0730000Z.{}".format(typ)), - os.path.join(mydir, "S_NWC_CMAPROB_noaa15_12345_19810305T0745000Z_19810305T0800000Z.{}".format(typ))] - res = get_outputfiles(mydir, "noaa15", 12345, nc_output=True) - assert len(res) == len(set(res)) - assert set(res) == set(expected) - #: Test nc files with start time - expected = [os.path.join(mydir, "S_NWC_CMAPROB_noaa15_12345_19810305T0715000Z_19810305T0730000Z.{}".format(typ))] - res = get_outputfiles(mydir, "noaa15", 12345, st_time="19810305T0715", nc_output=True) - assert len(res) == len(set(res)) - assert set(res) == set(expected) - - -class TestTimeControlFiles(unittest.TestCase): - """Testing the handling and generation of time control XML files.""" - - def setUp(self): - self.testscene = {'platform_name': 'Metop-B', 'orbit_number': 46878, - 'satday': '20210930', 'sathour': '0946', - 'starttime': datetime(2021, 9, 30, 9, 46, 24), - 'endtime': datetime(2021, 9, 30, 10, 1, 43), - 'sensor': ['avhrr/3', 'mhs', 'amsu-a'], - 'file4pps': '/data/metop01_20210930_0946_46878/hrpt_metop01_20210930_0946_46878.l1b'} - self.filename1 = 'S_NWC_timectrl_metopb_46878_20210930T0946289Z_20210930T1001458Z.txt' - self.filename2 = 'S_NWC_timectrl_metopb_46878_20210930T0949289Z_20210930T1001459Z.txt' - self.filename3 = 'S_NWC_timectrl_metopb_46878_20210930T0945289Z_20210930T1001459Z.txt' - self.filename4_zero = 'S_NWC_timectrl_metopb_00000_20210930T0945289Z_20210930T1001459Z.txt' - self.filename_timeoff = 'S_NWC_timectrl_metopb_46878_20210930T0950000Z_20210930T1001459Z.txt' - - def test_get_time_control_ascii_filename_ok(self): - - with tempfile.TemporaryDirectory() as tmpdirname: - - self.file1 = os.path.join(tmpdirname, self.filename1) - with open(self.file1, 'w') as _: - pass - self.file2 = os.path.join(tmpdirname, self.filename2) - with open(self.file2, 'w') as _: - pass - - ascii_file = get_time_control_ascii_filename(self.testscene, tmpdirname) - - self.assertEqual(os.path.basename(ascii_file), - 'S_NWC_timectrl_metopb_46878_20210930T0946289Z_20210930T1001458Z.txt') - - def test_get_time_control_ascii_filename_more_than_one_file(self): - - with tempfile.TemporaryDirectory() as tmpdirname: - - self.file1 = os.path.join(tmpdirname, self.filename1) - with open(self.file1, 'w') as _: - pass - self.file2 = os.path.join(tmpdirname, self.filename3) - with open(self.file2, 'w') as _: - pass - - with pytest.raises(FindTimeControlFileError) as exec_info: - ascii_file = get_time_control_ascii_filename(self.testscene, tmpdirname) - - expected = 'More than one time control ascii file candidate found - unresolved ambiguity!' - assert str(exec_info.value) == expected - - def test_get_time_control_ascii_filename_no_files(self): - - with tempfile.TemporaryDirectory() as tmpdirname: - - self.file1 = os.path.join(tmpdirname, self.filename2) - with open(self.file1, 'w') as _: - pass - - with pytest.raises(FindTimeControlFileError) as exec_info: - ascii_file = get_time_control_ascii_filename(self.testscene, tmpdirname) - - expected = 'No time control ascii file candidate found!' - assert str(exec_info.value) == expected - - def test_get_time_control_ascii_filename_candidates_orbit_ok_time_off(self): - - with tempfile.TemporaryDirectory() as tmpdirname: - - self.file1 = os.path.join(tmpdirname, self.filename_timeoff) - with open(self.file1, 'w') as _: - pass - - ascii_files = get_time_control_ascii_filename_candidates(self.testscene, tmpdirname) - - self.assertEqual(len(ascii_files), 0) - - def test_get_time_control_ascii_filename_candidates_orbit_off_by1(self): - - with tempfile.TemporaryDirectory() as tmpdirname: - - self.file1 = os.path.join(tmpdirname, self.filename1) - with open(self.file1, 'w') as _: - pass - - self.testscene['orbit_number'] = self.testscene['orbit_number'] + 1 - ascii_files = get_time_control_ascii_filename_candidates(self.testscene, tmpdirname) - - self.assertEqual(len(ascii_files), 1) - self.assertTrue(os.path.basename(ascii_files[0]) == - 'S_NWC_timectrl_metopb_46878_20210930T0946289Z_20210930T1001458Z.txt') - - def test_get_time_control_ascii_filename_candidates_zero_orbit(self): - - with tempfile.TemporaryDirectory() as tmpdirname: - - self.file1 = os.path.join(tmpdirname, self.filename4_zero) - with open(self.file1, 'w') as _: - pass - self.file2 = os.path.join(tmpdirname, self.filename3) - with open(self.file2, 'w') as _: - pass - ascii_files = get_time_control_ascii_filename_candidates(self.testscene, tmpdirname) - - self.assertEqual(len(ascii_files), 2) - self.assertTrue(os.path.basename(sorted(ascii_files)[0]) == - 'S_NWC_timectrl_metopb_00000_20210930T0945289Z_20210930T1001459Z.txt') - - -class TestProductStatisticsFiles(unittest.TestCase): - """Testing locating the product statistics XML files.""" - - def setUp(self): - self.testscene = {'platform_name': 'Metop-B', 'orbit_number': 46878, 'satday': - '20210930', 'sathour': '0946', - 'starttime': datetime(2021, 9, 30, 9, 46, 24), - 'endtime': datetime(2021, 9, 30, 10, 1, 43)} - - self.filename1 = 'S_NWC_CTTH_metopb_46878_20210930T0946289Z_20210930T1001458Z_statistics.xml' - self.filename2 = 'S_NWC_CTTH_metopb_46878_20210930T0949289Z_20210930T1001459Z_statistics.xml' - self.filename3 = 'S_NWC_CTTH_metopb_46878_20210930T0947019Z_20210930T1001458Z_statistics.xml' - - self.pattern = ('S_NWC_{product:s}_{satellite:s}_{orbit:s}_{starttime:%Y%m%dT%H%M}{seconds1:s}' + - '_{endtime:%Y%m%dT%H%M}{seconds2}_statistics.xml') - - def test_get_product_statistics_files_fewseconds_off(self): - """Test get the list of product statistics files.""" - # get_product_statistics_files - with tempfile.TemporaryDirectory() as tmpdirname: - - file1 = os.path.join(tmpdirname, self.filename1) - with open(file1, 'w') as _: - pass - file2 = os.path.join(tmpdirname, self.filename2) - with open(file2, 'w') as _: - pass - - xmlfiles = get_product_statistics_files(tmpdirname, self.testscene, self.pattern, 1) - self.assertTrue(len(xmlfiles) == 1) - - filename = os.path.basename(xmlfiles[0]) - self.assertEqual(filename, 'S_NWC_CTTH_metopb_46878_20210930T0946289Z_20210930T1001458Z_statistics.xml') - - with tempfile.TemporaryDirectory() as tmpdirname: - - file1 = os.path.join(tmpdirname, self.filename1) - with open(file1, 'w') as _: - pass - file2 = os.path.join(tmpdirname, self.filename2) - with open(file2, 'w') as _: - pass - - xmlfiles = get_product_statistics_files(tmpdirname, self.testscene, self.pattern, 0) - self.assertTrue(len(xmlfiles) == 1) - - filename = os.path.basename(xmlfiles[0]) - self.assertEqual(filename, 'S_NWC_CTTH_metopb_46878_20210930T0946289Z_20210930T1001458Z_statistics.xml') - - def test_get_product_statistics_files_oneminute_off(self): - """Test get the list of product statistics files.""" - # get_product_statistics_files - with tempfile.TemporaryDirectory() as tmpdirname: - - file3 = os.path.join(tmpdirname, self.filename3) - with open(file3, 'w') as _: - pass - - xmlfiles = get_product_statistics_files(tmpdirname, self.testscene, self.pattern, 0) - self.assertTrue(len(xmlfiles) == 0) - - with tempfile.TemporaryDirectory() as tmpdirname: - - file3 = os.path.join(tmpdirname, self.filename3) - with open(file3, 'w') as _: - pass - - xmlfiles = get_product_statistics_files(tmpdirname, self.testscene, self.pattern, 1) - self.assertTrue(len(xmlfiles) == 1) - - filename = os.path.basename(xmlfiles[0]) - self.assertEqual(filename, 'S_NWC_CTTH_metopb_46878_20210930T0947019Z_20210930T1001458Z_statistics.xml') - class TestPublishPPSFiles(unittest.TestCase): """Test publish pps files.""" From 14ecd8f4972af7c607ddec93405fdf680592aac2 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Jan 2024 11:02:44 +0100 Subject: [PATCH 06/44] Remove old pps runner --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5b3411c..d18869a 100644 --- a/setup.py +++ b/setup.py @@ -58,8 +58,7 @@ long_description=long_description, license='GPLv3', packages=find_packages(), - scripts=['bin/pps_runner.py', - 'bin/pps2018_runner.py', + scripts=['bin/pps2018_runner.py', 'bin/level1c_runner.py', ], data_files=[], install_requires=['posttroll', 'trollsift', 'pygrib', ], From a950ad8b050c39da3fa8e56a3101d5db3c9b8a73 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Jan 2024 11:31:06 +0100 Subject: [PATCH 07/44] Remove old functions get_sceneid create_pps_call_command_sequence get_pps_inputfile get_xml_outputfiles --- nwcsafpps_runner/utils.py | 109 -------------------------------------- 1 file changed, 109 deletions(-) diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index 8564b71..ceee776 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -211,20 +211,6 @@ def message_uid(msg): return SceneId(platform_name, orbit_number, starttime) -def get_sceneid(platform_name, orbit_number, starttime): - - if starttime: - sceneid = (str(platform_name) + '_' + - str(orbit_number) + '_' + - str(starttime.strftime('%Y%m%d%H%M%S'))) - else: - sceneid = (str(platform_name) + '_' + - str(orbit_number)) - - LOG.debug("Scene identifier = " + str(sceneid)) - return sceneid - - def ready2run(msg, files4pps, **kwargs): """Check whether pps is ready to run or not.""" @@ -341,39 +327,6 @@ def terminate_process(popen_obj, scene): "Process finished before time out - workerScene: " + str(scene)) -def create_pps_call_command_sequence(pps_script_name, scene, options): - """Create the PPS call commnd sequence. - - Applies to NWCSAF/PPS v2014. - """ - LVL1_NPP_PATH = os.environ.get('LVL1_NPP_PATH', - options.get('LVL1_NPP_PATH', None)) - LVL1_EOS_PATH = os.environ.get('LVL1_EOS_PATH', - options.get('LVL1_EOS_PATH', None)) - - if scene['platform_name'] in SUPPORTED_MODIS_SATELLITES: - cmdstr = "%s %s %s %s %s" % (pps_script_name, - SATELLITE_NAME[ - scene['platform_name']], - scene['orbit_number'], scene[ - 'satday'], - scene['sathour']) - else: - cmdstr = "%s %s %s 0 0" % (pps_script_name, - SATELLITE_NAME[ - scene['platform_name']], - scene['orbit_number']) - - cmdstr = cmdstr + ' ' + str(options['aapp_level1files_max_minutes_old']) - - if scene['platform_name'] in SUPPORTED_VIIRS_SATELLITES and LVL1_NPP_PATH: - cmdstr = cmdstr + ' ' + str(LVL1_NPP_PATH) - elif scene['platform_name'] in SUPPORTED_MODIS_SATELLITES and LVL1_EOS_PATH: - cmdstr = cmdstr + ' ' + str(LVL1_EOS_PATH) - - return shlex.split(str(cmdstr)) - - def create_pps_call_command(python_exec, pps_script_name, scene): """Create the pps call command. @@ -385,68 +338,6 @@ def create_pps_call_command(python_exec, pps_script_name, scene): return cmdstr - -def get_pps_inputfile(platform_name, ppsfiles): - """From the set of files picked up in the PostTroll messages decide the input - file used in the PPS call - """ - - if platform_name in SUPPORTED_MODIS_SATELLITES: - for ppsfile in ppsfiles: - if os.path.basename(ppsfile).find('021km') > 0: - return ppsfile - elif platform_name in SUPPORTED_AVHRR_SATELLITES: - for ppsfile in ppsfiles: - if os.path.basename(ppsfile).find('hrpt_') >= 0: - return ppsfile - elif platform_name in SUPPORTED_VIIRS_SATELLITES: - for ppsfile in ppsfiles: - if os.path.basename(ppsfile).find('SVM01') >= 0: - return ppsfile - elif platform_name in SUPPORTED_SEVIRI_SATELLITES: - for ppsfile in ppsfiles: - if os.path.basename(ppsfile).find('NWC') >= 0: - return ppsfile - - return None - - -def get_xml_outputfiles(path, platform_name, orb, st_time=''): - """Finds xml outputfiles depending on certain input criteria. - - From the directory path and satellite id and orbit number, - scan the directory and find all pps xml output files matching that scene and - return the full filenames. - - The search allow for small deviations in orbit numbers between the actual - filename and the message. - """ - - xml_output = (os.path.join(path, 'S_NWC') + '*' + - str(METOP_NAME_LETTER.get(platform_name, platform_name)) + - '_' + '%.5d' % int(orb) + '_%s*.xml' % st_time) - LOG.info( - "Match string to do a file globbing on xml output files: " + str(xml_output)) - filelist = glob(xml_output) - - if len(filelist) == 0: - # Perhaps there is an orbit number mismatch? - nxmlfiles = 0 - for idx in [1, -1, 2, -2, 3, -3, 4, -4, 5, -5]: - tmp_orbit = int(orb) + idx - LOG.debug('Try with an orbitnumber of %d instead', tmp_orbit) - xml_output = (os.path.join(path, 'S_NWC') + '*' + - str(METOP_NAME_LETTER.get(platform_name, platform_name)) + - '_' + '%.5d' % int(tmp_orbit) + '_%s*.xml' % st_time) - - filelist = glob(xml_output) - nxmlfiles = len(filelist) - if nxmlfiles > 0: - break - - return filelist - - def create_xml_timestat_from_lvl1c(scene, pps_control_path): """From lvl1c file create XML file and return a file list.""" try: From a89cd527873be89d09ced50b8b9952e7421aaf9d Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Jan 2024 11:31:48 +0100 Subject: [PATCH 08/44] Adapt to use always l1c --- bin/pps2018_runner.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/bin/pps2018_runner.py b/bin/pps2018_runner.py index ac3221f..aedc157 100644 --- a/bin/pps2018_runner.py +++ b/bin/pps2018_runner.py @@ -120,7 +120,7 @@ def pps_worker(scene, publish_q, input_msg, options): if not pps_run_all_flags: pps_run_all_flags = [] - use_l1c = (options.get('pps_version') == 'v2021') + cmd_str = create_pps_call_command(py_exec, pps_script, scene) for flag in pps_run_all_flags: cmd_str = cmd_str + ' %s' % flag @@ -178,7 +178,6 @@ def pps_worker(scene, publish_q, input_msg, options): pps_control_path = my_env.get('STATISTICS_DIR', options.get('pps_statistics_dir', './')) xml_files = create_xml_timestat_from_lvl1c(scene, pps_control_path) xml_files += find_product_statistics_from_lvl1c(scene, pps_control_path) - LOG.info("PPS summary statistics files: %s", str(xml_files)) # The PPS post-hooks takes care of publishing the PPS cloud products @@ -264,11 +263,7 @@ def pps(options): """ LOG.info("*** Start the PPS level-2 runner:") - use_l1c = (options.get('pps_version') == 'v2021') - if use_l1c: - LOG.info("Use level-1c file as input (v2021 and greater)") - else: - LOG.info("Use original level-1 file as input (v2018)") + LOG.info("Use level-1c file as input") LOG.info("First check if NWP data should be downloaded and prepared") nwp_handeling_module = options.get("nwp_handeling_module", None) @@ -318,17 +313,13 @@ def pps(options): } status = ready2run(msg, files4pps, - use_l1c, stream_tag_name=options.get('stream_tag_name', 'variant'), stream_name=options.get('stream_name', 'EARS'), sdr_granule_processing=options.get('sdr_processing') == 'granules') if status: sceneid = get_sceneid(platform_name, orbit_number, starttime) LOG.debug(files4pps[sceneid]) - if use_l1c: - scene['file4pps'] = files4pps[sceneid][0] - else: - scene['file4pps'] = get_pps_inputfile(platform_name, files4pps[sceneid]) + scene['file4pps'] = files4pps[sceneid][0] LOG.debug("Files for PPS: %s", str(scene['file4pps'])) LOG.info('Start a thread preparing the nwp data and run pps...') From 744f818e59468becd833d8a5dc832e4691191f3d Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 17 Jan 2024 16:44:27 +0100 Subject: [PATCH 09/44] Use level1c file as identifier --- bin/pps2018_runner.py | 31 ++----- nwcsafpps_runner/utils.py | 184 +++++++++----------------------------- 2 files changed, 52 insertions(+), 163 deletions(-) diff --git a/bin/pps2018_runner.py b/bin/pps2018_runner.py index aedc157..355bea3 100644 --- a/bin/pps2018_runner.py +++ b/bin/pps2018_runner.py @@ -37,12 +37,11 @@ from nwcsafpps_runner.config import CONFIG_FILE, CONFIG_PATH, get_config from nwcsafpps_runner.prepare_nwp import update_nwp from nwcsafpps_runner.publish_and_listen import FileListener, FilePublisher -from nwcsafpps_runner.utils import (METOP_NAME_LETTER, SATELLITE_NAME, - SENSOR_LIST, NwpPrepareError, PpsRunError, +from nwcsafpps_runner.utils import (SENSOR_LIST, NwpPrepareError, PpsRunError, create_pps_call_command, create_xml_timestat_from_lvl1c, find_product_statistics_from_lvl1c, - get_sceneid, logreader, message_uid, + get_sceneid, logreader, publish_pps_files, ready2run, terminate_process) from nwcsafpps_runner.utils import create_xml_timestat_from_scene @@ -62,7 +61,6 @@ LOG.debug("PYTHONPATH: " + str(sys.path)) -SATNAME = {'Aqua': 'EOS-Aqua'} class ThreadPool(object): @@ -106,7 +104,7 @@ def pps_worker(scene, publish_q, input_msg, options): LOG.info("Starting pps runner for scene %s", str(scene)) job_start_time = datetime.utcnow() - LOG.debug("Level-1 file: %s", scene['file4pps']) + LOG.debug("Level-1c file: %s", scene['file4pps']) LOG.debug("Platform name: %s", scene['platform_name']) LOG.debug("Orbit number: %s", str(scene['orbit_number'])) @@ -312,15 +310,11 @@ def pps(options): 'sensor': sensors } - status = ready2run(msg, files4pps, - stream_tag_name=options.get('stream_tag_name', 'variant'), - stream_name=options.get('stream_name', 'EARS'), - sdr_granule_processing=options.get('sdr_processing') == 'granules') + scene['file4pps'] = get_lvl1c_file_from_msg(msg) + status = ready2run(msg, scene) + if status: - sceneid = get_sceneid(platform_name, orbit_number, starttime) - LOG.debug(files4pps[sceneid]) - scene['file4pps'] = files4pps[sceneid][0] - + LOG.debug("Files for PPS: %s", str(scene['file4pps'])) LOG.info('Start a thread preparing the nwp data and run pps...') @@ -328,7 +322,7 @@ def pps(options): run_nwp_and_pps(scene, NWP_FLENS, publisher_q, msg, options, nwp_handeling_module) else: - thread_pool.new_thread(message_uid(msg), + thread_pool.new_thread(scene['file4pps'], target=run_nwp_and_pps, args=(scene, NWP_FLENS, publisher_q, msg, options, @@ -336,15 +330,6 @@ def pps(options): LOG.debug("Number of threads currently alive: %s", str(threading.active_count())) - # Clean the files4pps dict: - LOG.debug("files4pps: " + str(files4pps)) - try: - files4pps.pop(sceneid) - except KeyError: - LOG.warning("Failed trying to remove key " + str(sceneid) + - " from dictionary files4pps") - - LOG.debug("After cleaning: files4pps = " + str(files4pps)) # FIXME! Should I clean up the thread_pool (open threads?) here at the end!? diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index ceee776..d48f838 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -77,46 +77,36 @@ class FindTimeControlFileError(Exception): DATA1KM_PREFIX = {'EOS-Aqua': 'MYD021km', 'EOS-Terra': 'MOD021km'} PPS_SENSORS = ['amsu-a', 'amsu-b', 'mhs', 'avhrr/3', 'viirs', 'modis', 'seviri', 'metimage'] -REQUIRED_MW_SENSORS = {} -REQUIRED_MW_SENSORS['NOAA-15'] = ['amsu-a', 'amsu-b'] -REQUIRED_MW_SENSORS['NOAA-18'] = [] -REQUIRED_MW_SENSORS['NOAA-19'] = ['amsu-a', 'mhs'] -REQUIRED_MW_SENSORS['Metop-A'] = ['amsu-a', 'mhs'] -REQUIRED_MW_SENSORS['Metop-B'] = ['amsu-a', 'mhs'] -REQUIRED_MW_SENSORS['Metop-C'] = ['amsu-a', 'mhs'] NOAA_METOP_PPS_SENSORNAMES = ['avhrr/3', 'amsu-a', 'amsu-b', 'mhs'] METOP_NAME_LETTER = {'metop01': 'metopb', 'metop02': 'metopa', 'metop03': 'metopc'} METOP_NAME = {'metop01': 'Metop-B', 'metop02': 'Metop-A', 'metop03': 'Metop-C'} METOP_NAME_INV = {'metopb': 'metop01', 'metopa': 'metop02', 'metopc': 'metop03'} -SATELLITE_NAME = {'NOAA-19': 'noaa19', 'NOAA-18': 'noaa18', - 'NOAA-15': 'noaa15', - 'Metop-A': 'metop02', 'Metop-B': 'metop01', - 'Metop-C': 'metop03', - 'Metop-SG-A1': 'metopsga1', - 'Metop-SG-A2': 'metopsga2', - 'Metop-SG-A3': 'metopsga3', - 'Suomi-NPP': 'npp', - 'NOAA-20': 'noaa20', 'NOAA-21': 'noaa21', 'NOAA-23': 'noaa23', - 'EOS-Aqua': 'eos2', 'EOS-Terra': 'eos1', - 'Meteosat-09': 'meteosat09', 'Meteosat-10': 'meteosat10', - 'Meteosat-11': 'meteosat11'} -SENSOR_LIST = {} -for sat in SATELLITE_NAME: - if sat in ['NOAA-15']: - SENSOR_LIST[sat] = ['avhrr/3', 'amsu-b', 'amsu-a'] - elif sat in ['EOS-Aqua', 'EOS-Terra']: - SENSOR_LIST[sat] = 'modis' - elif sat in ['Suomi-NPP', 'NOAA-20', 'NOAA-21']: - SENSOR_LIST[sat] = 'viirs' - elif 'Meteosat' in sat: - SENSOR_LIST[sat] = 'seviri' - elif 'Metop-SG' in sat: - SENSOR_LIST[sat] = 'metimage' - else: - SENSOR_LIST[sat] = ['avhrr/3', 'mhs', 'amsu-a'] +# SATELLITE_NAME = {} +# for sat in SUPPORTED_PPS_SATELLITES: +# SATELLITE_NAME[sat] = sat.lower().replace('-', '') +# historic exceptions +# SATELLITE_NAME['Suomi-NPP'] = 'npp' +# SATELLITE_NAME['EOS-Aqua'] = 'eos2' +# SATELLITE_NAME['EOS-Terra'] = 'eos1' +# SATELLITE_NAME['Metop-A']= 'metop02' +# SATELLITE_NAME['Metop-B']= 'metop01' +# SATELLITE_NAME['Metop-C']= 'metop03' +SENSOR_LIST = {} +for sat in SUPPORTED_PPS_SATELLITES: + if sat in SUPPORTED_AVHRR_SATELLITES: + SENSOR_LIST[sat] = ['avhrr/3'] + elif sat in SUPPORTED_MODIS_SATELLITES: + SENSOR_LIST[sat] = ['modis'] + elif sat in SUPPORTED_VIIRS_SATELLITES: + SENSOR_LIST[sat] = ['viirs'] + elif sat in SUPPORTED_SEVIRI_SATELLITES: + SENSOR_LIST[sat] = ['seviri'] + elif sat in SUPPORTED_METIMAGE_SATELLITES: + SENSOR_LIST[sat] = ['metimage'] + METOP_SENSOR = {'amsu-a': 'amsua', 'avhrr/3': 'avhrr', 'amsu-b': 'amsub', 'hirs/4': 'hirs'} @@ -175,48 +165,8 @@ class PpsRunError(Exception): pass -class SceneId(object): - - def __init__(self, platform_name, orbit_number, starttime, threshold=5): - self.platform_name = platform_name - self.orbit_number = orbit_number - self.starttime = starttime - self.threshold = threshold - - def __str__(self): - - return (str(self.platform_name) + '_' + - str(self.orbit_number) + '_' + - str(self.starttime.strftime('%Y%m%d%H%M'))) - - def __hash__(self): - return hash(str(self.platform_name) + '_' + - str(self.orbit_number) + '_' + - str(self.starttime.strftime('%Y%m%d%H%M'))) - - def __eq__(self, other): - - return (self.platform_name == other.platform_name and - self.orbit_number == other.orbit_number and - abs(self.starttime - other.starttime) < timedelta(minutes=self.threshold)) - - -def message_uid(msg): - """Create a unique id/key-name for the scene.""" - - orbit_number = int(msg.data['orbit_number']) - platform_name = msg.data['platform_name'] - starttime = msg.data['start_time'] - - return SceneId(platform_name, orbit_number, starttime) - - -def ready2run(msg, files4pps, **kwargs): - """Check whether pps is ready to run or not.""" - - LOG.debug("Ready to run...") - LOG.info("Got message: " + str(msg)) - +def get_lvl1c_file_from_msg(msg): + """Get level1c file from msg.""" destination = msg.data.get('destination') uris = [] @@ -229,14 +179,28 @@ def ready2run(msg, files4pps, **kwargs): else: LOG.debug( "Ignoring this type of message data: type = " + str(msg.type)) - return False + return [] try: - level1_files = check_uri(uris) + level1c_files = check_uri(uris) except IOError: LOG.info('One or more files not present on this host!') - return False + return [] + + LOG.debug("files4pps: %s", str(level1c_files)) + if len(level1c_files) < 1: + LOG.info("No level1c files!") + return [] + + return level1c_files[0] + + +def ready2run(msg, scene, **kwargs): + """Check whether pps is ready to run or not.""" + LOG.info("Got message: " + str(msg)) + if len(scene['file4pps']) !=1: + return False try: url_ip = socket.gethostbyname(msg.host) if url_ip not in get_local_ips(): @@ -246,73 +210,13 @@ def ready2run(msg, files4pps, **kwargs): LOG.error("Failed checking host! Hostname = %s", socket.gethostname()) LOG.exception(err) - LOG.info("Sat and Sensor: " + str(msg.data['platform_name']) - + " " + str(msg.data['sensor'])) - if msg.data['sensor'] not in PPS_SENSORS: - LOG.info("Data from sensor " + str(msg.data['sensor']) + - " not needed by PPS " + - "Continue...") - return False - - if msg.data['platform_name'] in SUPPORTED_SEVIRI_SATELLITES: - if msg.data['sensor'] not in ['seviri', ]: - LOG.info( - 'Sensor ' + str(msg.data['sensor']) + - ' not required for MODIS PPS processing...') - return False - elif msg.data['platform_name'] in SUPPORTED_MODIS_SATELLITES: - if msg.data['sensor'] not in ['modis', ]: - LOG.info( - 'Sensor ' + str(msg.data['sensor']) + - ' not required for MODIS PPS processing...') - return False - elif msg.data['platform_name'] in SUPPORTED_VIIRS_SATELLITES: - if msg.data['sensor'] not in ['viirs', ]: - LOG.info( - 'Sensor ' + str(msg.data['sensor']) + - ' not required for S-NPP/VIIRS PPS processing...') - return False - elif msg.data['platform_name'] in SUPPORTED_METIMAGE_SATELLITES: - if msg.data['sensor'] not in ['metimage', ]: - LOG.info( - 'Sensor ' + str(msg.data['sensor']) + - ' not required for METIMAGE PPS processing...') - return False - else: - if msg.data['sensor'] not in NOAA_METOP_PPS_SENSORNAMES: - LOG.info( - 'Sensor ' + str(msg.data['sensor']) + ' not required...') - return False - - # The orbit number is mandatory! - orbit_number = int(msg.data['orbit_number']) - LOG.debug("Orbit number: " + str(orbit_number)) - - # sensor = (msg.data['sensor']) - platform_name = msg.data['platform_name'] - - if platform_name not in SATELLITE_NAME: - LOG.warning("Satellite not supported: " + str(platform_name)) - return False - - starttime = msg.data.get('start_time') - - LOG.debug("level1_files = %s", level1_files) - for item in level1_files: - files4pps[sceneid].append(item) - - LOG.debug("files4pps: %s", str(files4pps[sceneid])) - if len(files4pps[sceneid]) < 1: - LOG.info("No level1c files!") - return False - - if msg.data['platform_name'] in SUPPORTED_PPS_SATELLITES: LOG.info( "This is a PPS supported scene. Start the PPS lvl2 processing!") LOG.info("Process the scene (sat, orbit) = " + str(platform_name) + ' ' + str(orbit_number)) - + + LOG.debug("Ready to run...") return True From 020c5d07069ef49f6e76d5d8336a7cced6e23f8e Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 17 Jan 2024 16:44:37 +0100 Subject: [PATCH 10/44] Added test --- nwcsafpps_runner/tests/test_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index 535eedf..b348c21 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -25,10 +25,13 @@ from unittest.mock import patch, MagicMock from datetime import datetime from nwcsafpps_runner.utils import (create_xml_timestat_from_lvl1c, + get_lvl1c_file_from_msg, find_product_statistics_from_lvl1c, publish_pps_files) from nwcsafpps_runner.utils import FindTimeControlFileError +TEST_MSG = """pytroll://segment/EPSSGA/1B/ file safusr.u@lxserv1043.smhi.se 2023-02-17T08:18:15.748831 v1.01 application/json {"start_time": "2023-02-17T08:03:25", "end_time": "2023-02-17T08:15:25", "orbit_number": 99999, "platform_name": "Metop-SG-A1", "sensor": "metimage", "format": "X", "type": "NETCDF", "data_processing_level": "1b", "variant": "DR", "orig_orbit_number": 23218, "uri": "/san1/polar_in/direct_readout/metimage/W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc", "uid": "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc"}""" + @pytest.fixture def fake_file_dir(tmp_path): @@ -137,6 +140,17 @@ def test_publish_pps_files(self): self.assertTrue(file2 in msg_out[1].args[0]) self.assertTrue(file1 in msg_out[0].args[0]) +class TestGetLvl1cFromMsg(unittest.TestCase): + """Test publish pps files.""" + + def test_get_lvl1c_file_from_msg(self): + """Test get_lvl1c_file_from_message.""" + from posttroll.message import Message + input_msg = Message.decode(rawstr=TEST_MSG) + file1 = get_lvl1c_file_from_msg(input_msg) + file_exp = "/san1/polar_in/direct_readout/metimage/W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc" + self.assertEqual(file1, file_exp) + if __name__ == "__main__": pass From 024e4d77e614432770b913eb8571886fe477ef52 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 17 Jan 2024 16:57:49 +0100 Subject: [PATCH 11/44] flake8 --- bin/pps2018_runner.py | 13 ++++--------- nwcsafpps_runner/tests/test_utils.py | 10 +++++++--- nwcsafpps_runner/utils.py | 15 ++++++--------- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/bin/pps2018_runner.py b/bin/pps2018_runner.py index 355bea3..671d2c2 100644 --- a/bin/pps2018_runner.py +++ b/bin/pps2018_runner.py @@ -29,7 +29,6 @@ import sys import threading from datetime import datetime, timedelta -from glob import glob from subprocess import PIPE, Popen from six.moves.queue import Empty, Queue @@ -39,13 +38,12 @@ from nwcsafpps_runner.publish_and_listen import FileListener, FilePublisher from nwcsafpps_runner.utils import (SENSOR_LIST, NwpPrepareError, PpsRunError, create_pps_call_command, + get_lvl1c_file_from_msg, create_xml_timestat_from_lvl1c, find_product_statistics_from_lvl1c, - get_sceneid, logreader, + logreader, publish_pps_files, ready2run, terminate_process) -from nwcsafpps_runner.utils import create_xml_timestat_from_scene -from nwcsafpps_runner.utils import get_product_statistics_files LOG = logging.getLogger(__name__) @@ -118,7 +116,6 @@ def pps_worker(scene, publish_q, input_msg, options): if not pps_run_all_flags: pps_run_all_flags = [] - cmd_str = create_pps_call_command(py_exec, pps_script, scene) for flag in pps_run_all_flags: cmd_str = cmd_str + ' %s' % flag @@ -267,7 +264,6 @@ def pps(options): nwp_handeling_module = options.get("nwp_handeling_module", None) prepare_nwp4pps(NWP_FLENS, nwp_handeling_module) - files4pps = {} LOG.info("Number of threads: %d", options['number_of_threads']) thread_pool = ThreadPool(options['number_of_threads']) @@ -312,9 +308,9 @@ def pps(options): scene['file4pps'] = get_lvl1c_file_from_msg(msg) status = ready2run(msg, scene) - + if status: - + LOG.debug("Files for PPS: %s", str(scene['file4pps'])) LOG.info('Start a thread preparing the nwp data and run pps...') @@ -330,7 +326,6 @@ def pps(options): LOG.debug("Number of threads currently alive: %s", str(threading.active_count())) - # FIXME! Should I clean up the thread_pool (open threads?) here at the end!? pub_thread.stop() diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index b348c21..83bf23a 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -30,7 +30,7 @@ publish_pps_files) from nwcsafpps_runner.utils import FindTimeControlFileError -TEST_MSG = """pytroll://segment/EPSSGA/1B/ file safusr.u@lxserv1043.smhi.se 2023-02-17T08:18:15.748831 v1.01 application/json {"start_time": "2023-02-17T08:03:25", "end_time": "2023-02-17T08:15:25", "orbit_number": 99999, "platform_name": "Metop-SG-A1", "sensor": "metimage", "format": "X", "type": "NETCDF", "data_processing_level": "1b", "variant": "DR", "orig_orbit_number": 23218, "uri": "/san1/polar_in/direct_readout/metimage/W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc", "uid": "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc"}""" +TEST_MSG = """pytroll://segment/EPSSGA/1B/ file safusr.u@lxserv1043.smhi.se 2023-02-17T08:18:15.748831 v1.01 application/json {"start_time": "2023-02-17T08:03:25", "end_time": "2023-02-17T08:15:25", "orbit_number": 99999, "platform_name": "Metop-SG-A1", "sensor": "metimage", "format": "X", "type": "NETCDF", "data_processing_level": "1b", "variant": "DR", "orig_orbit_number": 23218, "uri": "/san1/polar_in/direct_readout/metimage/W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc", "uid": "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc"}""" # noqa: E501 @pytest.fixture @@ -140,6 +140,7 @@ def test_publish_pps_files(self): self.assertTrue(file2 in msg_out[1].args[0]) self.assertTrue(file1 in msg_out[0].args[0]) + class TestGetLvl1cFromMsg(unittest.TestCase): """Test publish pps files.""" @@ -147,8 +148,11 @@ def test_get_lvl1c_file_from_msg(self): """Test get_lvl1c_file_from_message.""" from posttroll.message import Message input_msg = Message.decode(rawstr=TEST_MSG) - file1 = get_lvl1c_file_from_msg(input_msg) - file_exp = "/san1/polar_in/direct_readout/metimage/W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc" + file1 = get_lvl1c_file_from_msg(input_msg) + file_exp = ( + "/san1/polar_in/direct_readout/metimage/" + + "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_" + + "20210314224906_G_D_20070912101704_20070912101804_T_B____.nc") self.assertEqual(file1, file_exp) diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index d48f838..2f2da58 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -25,16 +25,14 @@ import threading from trollsift.parser import parse # @UnresolvedImport -from trollsift.parser import globify # from trollsift import Parser from posttroll.message import Message # @UnresolvedImport from subprocess import Popen, PIPE import os -import stat import shlex from glob import glob import socket -from datetime import datetime, timedelta + #: Python 2/3 differences from six.moves.urllib.parse import urlparse @@ -106,7 +104,6 @@ class FindTimeControlFileError(Exception): SENSOR_LIST[sat] = ['seviri'] elif sat in SUPPORTED_METIMAGE_SATELLITES: SENSOR_LIST[sat] = ['metimage'] - METOP_SENSOR = {'amsu-a': 'amsua', 'avhrr/3': 'avhrr', 'amsu-b': 'amsub', 'hirs/4': 'hirs'} @@ -194,12 +191,12 @@ def get_lvl1c_file_from_msg(msg): return level1c_files[0] - + def ready2run(msg, scene, **kwargs): """Check whether pps is ready to run or not.""" LOG.info("Got message: " + str(msg)) - if len(scene['file4pps']) !=1: + if len(scene['file4pps']) != 1: return False try: url_ip = socket.gethostbyname(msg.host) @@ -213,9 +210,9 @@ def ready2run(msg, scene, **kwargs): if msg.data['platform_name'] in SUPPORTED_PPS_SATELLITES: LOG.info( "This is a PPS supported scene. Start the PPS lvl2 processing!") - LOG.info("Process the scene (sat, orbit) = " + - str(platform_name) + ' ' + str(orbit_number)) - + LOG.info("Process the file = %s" + + os.basename(scene['file4pps'])) + LOG.debug("Ready to run...") return True From 509319f126ac930df878d1189c4ef041896ea7e1 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 26 Jan 2024 14:43:07 +0100 Subject: [PATCH 12/44] Fixing two bugs --- nwcsafpps_runner/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index 2f2da58..ad9315d 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -22,7 +22,6 @@ """Utility functions for NWCSAF/pps runner(s). """ - import threading from trollsift.parser import parse # @UnresolvedImport # from trollsift import Parser @@ -196,8 +195,6 @@ def ready2run(msg, scene, **kwargs): """Check whether pps is ready to run or not.""" LOG.info("Got message: " + str(msg)) - if len(scene['file4pps']) != 1: - return False try: url_ip = socket.gethostbyname(msg.host) if url_ip not in get_local_ips(): @@ -211,7 +208,7 @@ def ready2run(msg, scene, **kwargs): LOG.info( "This is a PPS supported scene. Start the PPS lvl2 processing!") LOG.info("Process the file = %s" + - os.basename(scene['file4pps'])) + os.path.basename(scene['file4pps'])) LOG.debug("Ready to run...") return True From a78d2f81e12b4bb689401837e3d8478547449e78 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 26 Jan 2024 17:57:53 +0100 Subject: [PATCH 13/44] Moved code for checking host to function Added some tests --- nwcsafpps_runner/tests/test_utils.py | 38 ++++++++++++++++++++++++++++ nwcsafpps_runner/utils.py | 26 +++++++++++-------- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index 83bf23a..36ebbe8 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -27,11 +27,16 @@ from nwcsafpps_runner.utils import (create_xml_timestat_from_lvl1c, get_lvl1c_file_from_msg, find_product_statistics_from_lvl1c, + ready2run, publish_pps_files) from nwcsafpps_runner.utils import FindTimeControlFileError TEST_MSG = """pytroll://segment/EPSSGA/1B/ file safusr.u@lxserv1043.smhi.se 2023-02-17T08:18:15.748831 v1.01 application/json {"start_time": "2023-02-17T08:03:25", "end_time": "2023-02-17T08:15:25", "orbit_number": 99999, "platform_name": "Metop-SG-A1", "sensor": "metimage", "format": "X", "type": "NETCDF", "data_processing_level": "1b", "variant": "DR", "orig_orbit_number": 23218, "uri": "/san1/polar_in/direct_readout/metimage/W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc", "uid": "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc"}""" # noqa: E501 +TEST_MSG_UID = """pytroll://segment/EPSSGA/1B/ file safusr.u@lxserv1043.smhi.se 2023-02-17T08:18:15.748831 v1.01 application/json {"start_time": "2023-02-17T08:03:25", "end_time": "2023-02-17T08:15:25", "orbit_number": 99999, "platform_name": "Metop-SG-A1", "sensor": "metimage", "format": "X", "type": "NETCDF", "data_processing_level": "1b", "variant": "DR", "orig_orbit_number": 23218, "uid": "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc", "uid": "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc", "destination": "/san1/polar_in/direct_readout/metimage"}""" # noqa: E501 + +TEST_MSG_BAD = """pytroll://segment/EPSSGA/1B/ dataset safusr.u@lxserv1043.smhi.se 2023-02-17T08:18:15.748831 v1.01 application/json {"start_time": "2023-02-17T08:03:25", "end_time": "2023-02-17T08:15:25", "orbit_number": 99999, "platform_name": "Metop-SG-A1", "sensor": "metimage", "format": "X", "type": "NETCDF", "data_processing_level": "1b", "variant": "DR", "orig_orbit_number": 23218, "uid": "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc", "uid": "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc", "destination": "/san1/polar_in/direct_readout/metimage"}""" # noqa: E501 + @pytest.fixture def fake_file_dir(tmp_path): @@ -120,6 +125,21 @@ def test_xml_for_products_no_file4pps(self, fake_file_dir): assert res == [] +class TestReady2Run(unittest.TestCase): + """Test ready2run function.""" + + def test_ready2run(self): + from posttroll.message import Message + input_msg = Message.decode(rawstr=TEST_MSG) + mymodule = MagicMock() + import sys + sys.modules["check_host_ok"] = mymodule + ok2run = ready2run(input_msg, {"file4pps": "dummy"}) + self.assertTrue(ok2run) + ok2run = ready2run(input_msg, {"file4pps": None}) + self.assertFalse(ok2run) + + class TestPublishPPSFiles(unittest.TestCase): """Test publish pps files.""" @@ -155,6 +175,24 @@ def test_get_lvl1c_file_from_msg(self): "20210314224906_G_D_20070912101704_20070912101804_T_B____.nc") self.assertEqual(file1, file_exp) + def test_get_lvl1c_file_from_msg_uid(self): + """Test get_lvl1c_file_from_message.""" + from posttroll.message import Message + input_msg = Message.decode(rawstr=TEST_MSG_UID) + file1 = get_lvl1c_file_from_msg(input_msg) + file_exp = ( + "/san1/polar_in/direct_readout/metimage/" + + "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_" + + "20210314224906_G_D_20070912101704_20070912101804_T_B____.nc") + self.assertEqual(file1, file_exp) + + def test_get_lvl1c_file_from_msg_bad(self): + """Test get_lvl1c_file_from_message.""" + from posttroll.message import Message + input_msg = Message.decode(rawstr=TEST_MSG_BAD) + file1 = get_lvl1c_file_from_msg(input_msg) + self.assertEqual(file1, None) + if __name__ == "__main__": pass diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index ad9315d..36885e7 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -175,26 +175,20 @@ def get_lvl1c_file_from_msg(msg): else: LOG.debug( "Ignoring this type of message data: type = " + str(msg.type)) - return [] + return None try: level1c_files = check_uri(uris) except IOError: LOG.info('One or more files not present on this host!') - return [] + return None LOG.debug("files4pps: %s", str(level1c_files)) - if len(level1c_files) < 1: - LOG.info("No level1c files!") - return [] - return level1c_files[0] -def ready2run(msg, scene, **kwargs): - """Check whether pps is ready to run or not.""" - - LOG.info("Got message: " + str(msg)) +def check_host_ok(msg): + """Check that host is ok.""" try: url_ip = socket.gethostbyname(msg.host) if url_ip not in get_local_ips(): @@ -203,6 +197,18 @@ def ready2run(msg, scene, **kwargs): except (AttributeError, socket.gaierror) as err: LOG.error("Failed checking host! Hostname = %s", socket.gethostname()) LOG.exception(err) + return True + + +def ready2run(msg, scene, **kwargs): + """Check whether pps is ready to run or not.""" + + LOG.info("Got message: " + str(msg)) + if not check_host_ok(msg): + return False + + if scene['file4pps'] is None: + return False if msg.data['platform_name'] in SUPPORTED_PPS_SATELLITES: LOG.info( From 62c144088b6c384c74f099c3cea4c60fbec02399 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 27 Mar 2024 11:07:23 +0100 Subject: [PATCH 14/44] Rename runner --- bin/{pps2018_runner.py => pps_runner.py} | 0 examples/pps2018_config.ini_template | 105 ----------------------- examples/pps_config.cfg_template | 98 --------------------- setup.py | 2 +- 4 files changed, 1 insertion(+), 204 deletions(-) rename bin/{pps2018_runner.py => pps_runner.py} (100%) delete mode 100644 examples/pps2018_config.ini_template delete mode 100644 examples/pps_config.cfg_template diff --git a/bin/pps2018_runner.py b/bin/pps_runner.py similarity index 100% rename from bin/pps2018_runner.py rename to bin/pps_runner.py diff --git a/examples/pps2018_config.ini_template b/examples/pps2018_config.ini_template deleted file mode 100644 index 18b03b2..0000000 --- a/examples/pps2018_config.ini_template +++ /dev/null @@ -1,105 +0,0 @@ -[DEFAULT] -#: NWP -nhsp_prefix = LL02_NHSP_ -nhsf_prefix = LL02_NHSF_ -ecmwf_prefix = LL02_NHSF -#: Example filename: LL02_NHSF_202003240000+003H00M -nhsf_file_name_sift = {ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M - -nwp_static_surface = /san1/pps/import/NWP_data/lsm_z.grib1 -nwp_output_prefix = LL02_NHSPSF_ -nwp_outdir = /san1/pps/import/NWP_data/source -pps_nwp_requirements = /san1/pps/import/NWP_data/pps_nwp_list_of_required_fields.txt - - -#: Publish/subscribe -publish_topic = PPS -subscribe_topics = AAPP-HRPT,AAPP-PPS,EOS/1B,segment/SDR/1B,1c/nc/0deg -#: Has to do with messegatype -sdr_processing = granules - -[metno-test] -nhsf_prefix = LL02_NHSF_ -nhsf_file_name_sift = {nhsf_prefix:3s}{analyse_time:8s}{forecast_time:8s}{end:1s} - -#: Python and PPS related -maximum_pps_processing_time_in_minutes = 20 -#: Only used in pps2018_runner -python = /bin/python -run_all_script = /local_disk/opt/acpg/v2018/scr/ppsRunAll.py -run_cmaprob_script = /local_disk/opt/acpg/v2018/scr/ppsCmaskProb.py -run_cmask_prob = yes -run_pps_cpp = yes - - -#: Used for PPS log file -log_rotation_days = 1 -log_rotation_backup = 10 - - -#: Uncategorised -number_of_threads = 10 -station = norrkoping - - -#: These are only used for the LOG output and to publish resultfiles and has nothing to do with PPS -#: Incase the PPS environmental variables are sat they will overide these outdir -#: PPS environmental variable SM_PRODUCT_DIR -pps_outdir = /data/24/saf/polar_out/direct_readout -#: PPS environmental variable STATISTICS_DIR -pps_statistics_dir = /data/24/saf/polar_out/monitoring/direct_readout - - -#: Do not seams to be used. -level1_dir = /local_disk/data/pps/import/PPS_data/source -#: In pps_runner.py there is a environmental variable PPS_SCRIPT -pps_script = /usr/bin/pps_run.sh - -#: In case pps runner is started in an environment with nameserver running without multicast, -#: there is need to specify the adress of your nameserver(s). The nameserver is needed in the publisher -#: to tell which nameserver it must publish its (ie the pps-runner) adress to. -#: nameservers = your.local.nameserver-adress1,your.local.nameserver-adress2 - -#: Different path that sets differently depending on environment variable SMHI_MODE. -#: Incase SMHI_MODE is not specified offline is used. -[offline] -#: NWP Paths Needed -nhsp_path = /data/proj/sam/sam-utv/data/data/S/LL02_NHSP -#: NWP Paths Optional, set to None as deafault -nhsf_path = /data/proj/sam/sam-utv/data/data/S/LL02_NHSF -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 9000 - -[prod] -nhsp_path = /data/24/baps/weatherdata/ECNH/ -nhsf_path = /data/24/sam/data/data/S/REDA/ECMWF/ -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 90 - -[test] -nhsp_path = /data/prodtest/baps/weatherdata/ECNH/ -nhsf_path = /data/prodtest/sam/data/data/S/REDA/ECMWF/ -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 90 - -[utv] -nhsp_path = /data/prodtest/baps/weatherdata/ECNH/ -nhsf_path = /data/prodtest/sam/data/data/S/REDA/ECMWF/ -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 9000 - -[bi] -nhsp_path = /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source -nhsf_path = /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 9000 - -[metno-test] -nhsp_path = /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source -nhsf_path = /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 9000 - -nhsf_prefix = LL02_NHSF_ -nhsf_file_name_sift = {nhsf_prefix:3s}{analyse_time:8s}{forecast_time:8s}{end:1s} - diff --git a/examples/pps_config.cfg_template b/examples/pps_config.cfg_template deleted file mode 100644 index 278dcb0..0000000 --- a/examples/pps_config.cfg_template +++ /dev/null @@ -1,98 +0,0 @@ -[DEFAULT] -#: NWP -nhsp_prefix = LL02_NHSP_ -nhsf_prefix = LL02_NHSF_ -ecmwf_prefix = LL02_NHSF -#: Example filename: LL02_NHSF_202003240000+003H00M -nhsf_file_name_sift = {ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M - -nwp_static_surface = /san1/pps/import/NWP_data/lsm_z.grib1 -nwp_output_prefix = LL02_NHSPSF_ -nwp_outdir = /san1/pps/import/NWP_data/source -pps_nwp_requirements = /san1/pps/import/NWP_data/pps_nwp_list_of_required_fields.txt - - -#: Publish/subscribe -publish_topic = PPS -subscribe_topics = AAPP-HRPT,AAPP-PPS,EOS/1B,segment/SDR/1B,1c/nc/0deg -#: Has to do with messegatype -sdr_processing = aggregated - - -#: Python and PPS related -maximum_pps_processing_time_in_minutes = 20 -#: Only used in pps2018_runner -python = /bin/python -run_all_script = /local_disk/opt/acpg/v2018/scr/ppsRunAll.py -run_cmaprob_script = /local_disk/opt/acpg/v2018/scr/ppsCmaskProb.py -run_cmask_prob = yes -run_pps_cpp = yes - - -#: Used for PPS log file -log_rotation_days = 1 -log_rotation_backup = 10 - - -#: Uncategorised -number_of_threads = 5 -station = norrkoping - - -#: These are only used for the LOG output and to publish resultfiles and has nothing to do with PPS -#: Incase the PPS environmental variables are sat they will overide these outdir -#: PPS environmental variable SM_PRODUCT_DIR -pps_outdir = /data/24/saf/polar_out/direct_readout -#: PPS environmental variable STATISTICS_DIR -pps_statistics_dir = /data/24/saf/polar_out/monitoring/direct_readout - - -#: Do not seams to be used. -level1_dir = /local_disk/data/pps/import/PPS_data/source -#: In pps_runner.py there is a environmental variable PPS_SCRIPT -pps_script = /usr/bin/pps_run.sh - - -#: Different path that sets differently depending on environment variable SMHI_MODE. -#: Incase SMHI_MODE is not specified offline is used. -[offline] -#: NWP Paths Needed -nhsp_path = /data/proj/sam/sam-utv/data/data/S/LL02_NHSP -#: NWP Paths Optional, set to None as deafault -nhsf_path = /data/proj/sam/sam-utv/data/data/S/LL02_NHSF -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 9000 - -[prod] -nhsp_path = /data/24/baps/weatherdata/ECNH/ -nhsf_path = /data/24/sam/data/data/S/REDA/ECMWF/ -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 90 - -[test] -nhsp_path = /data/prodtest/baps/weatherdata/ECNH/ -nhsf_path = /data/prodtest/sam/data/data/S/REDA/ECMWF/ -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 90 - -[utv] -nhsp_path = /data/prodtest/baps/weatherdata/ECNH/ -nhsf_path = /data/prodtest/sam/data/data/S/REDA/ECMWF/ -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 9000 - -[bi] -nhsp_path = /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source -nhsf_path = /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 9000 - -[metno-test] -nhsp_path = /data/prodtest/baps/weatherdata/ECNH/ -nhsf_path = /data/prodtest/sam/data/data/S/REDA/ECMWF/ -#: Only used in pps_runner -aapp_level1files_max_minutes_old = 9000 - -nhsf_prefix = LL02_NHSF_ -nhsf_file_name_sift = {nhsf_prefix:3s}{analyse_time:8s}{forecast_time:8s}{end:1s} - diff --git a/setup.py b/setup.py index d18869a..e8a41ff 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ long_description=long_description, license='GPLv3', packages=find_packages(), - scripts=['bin/pps2018_runner.py', + scripts=['bin/pps_runner.py', 'bin/level1c_runner.py', ], data_files=[], install_requires=['posttroll', 'trollsift', 'pygrib', ], From 40f1a39d391fb227a03805bc713b2d245c94d41c Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 27 Mar 2024 12:34:10 +0100 Subject: [PATCH 15/44] Remove non used config options --- examples/pps2018_config.yaml_template | 129 -------------------------- examples/pps2021_config.yaml_template | 10 +- examples/pps_config.yaml | 78 ---------------- 3 files changed, 3 insertions(+), 214 deletions(-) delete mode 100644 examples/pps2018_config.yaml_template delete mode 100644 examples/pps_config.yaml diff --git a/examples/pps2018_config.yaml_template b/examples/pps2018_config.yaml_template deleted file mode 100644 index de13d13..0000000 --- a/examples/pps2018_config.yaml_template +++ /dev/null @@ -1,129 +0,0 @@ -#: NWP -nhsp_prefix: LL02_NHSP_ -nhsf_prefix: LL02_NHSF_ -ecmwf_prefix: LL02_NHSF -#: Example filename: LL02_NHSF_202003240000+003H00M -nhsf_file_name_sift: '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' - -nwp_static_surface: /san1/pps/import/NWP_data/lsm_z.grib1 -nwp_output_prefix: LL02_NHSPSF_ -nwp_outdir: /san1/pps/import/NWP_data/source -pps_nwp_requirements: /san1/pps/import/NWP_data/pps_nwp_list_of_required_fields.txt - - -#: Publish/subscribe -publish_topic: PPS -subscribe_topics: [AAPP-HRPT,AAPP-PPS,EOS/1B,segment/SDR/1B,1c/nc/0deg] -#: Has to do with messegatype -sdr_processing: granules - - -#: Python and PPS related -pps_version: v2018 -maximum_pps_processing_time_in_minutes: 20 -#: Only used in pps2018_runner -python: python - -# In PPS v2021 the ppsRunAll script runs the cmaprob on default before CTTH If -# you want to skip that and run it later the provide a flag 'no_cmaprob' below, -# and possibly run it separately afterwards. Also instead of setting the -# run_pps_cpp to yes you can provide the flag directly below. For PPSv2018 the -# flag is called np_cpp, but in v2021 it called no_cmic. -run_all_script: - name: /path/to/pps/script/ppsRunAll.py - flags: - - 'no_cmic' - - 'no_cmaskprob' - - 'no_cpp' - -run_cmaprob_script: /path/to/pps/script/ppsCmaskProb.py -run_cmask_prob: yes - - -#: Used for PPS log file -log_rotation_days: 1 -log_rotation_backup: 10 - - -#: Uncategorised -number_of_threads: 10 -# Allows to process several VIIRS granules in parallel. For SEVIRI (and -# AVHRR/MODIS) this can be 1 (no parallel processing as one scan (5 min or 15 -# min) is processed before the next one arrives. - -station: norrkoping - - -#: These are only used for the LOG output and to publish resultfiles and has nothing to do with PPS -#: Incase the PPS environmental variables are sat they will overide these outdir -#: PPS environmental variable SM_PRODUCT_DIR -pps_outdir: /data/24/saf/polar_out/direct_readout -#: PPS environmental variable STATISTICS_DIR -pps_statistics_dir: /data/24/saf/polar_out/monitoring/direct_readout - -# Products statistics file patterns -# S_NWC_CTTH_metopc_15096_20211004T1551189Z_20211004T1601487Z_statistics.xml -product_statistics_filename: 'S_NWC_{product:s}_{satellite:s}_{orbit:s}_{starttime:%Y%m%dT%H%M}{seconds1:s}_{endtime:%Y%m%dT%H%M}{seconds2}_statistics.xml' - -# For AVHRR in particular the start time in the message may not match exactly -# with the time in the PPS output file. And a search may be necessary to find -# the matching PPS file to the scene: -pps_filetime_search_minutes: 1 - -#: Do not seams to be used. -level1_dir: /local_disk/data/pps/import/PPS_data/source -#: In pps_runner.py there is a environmental variable PPS_SCRIPT -pps_script: /usr/bin/pps_run.sh - -#: In case pps runner is started in an environment with nameserver running without multicast, -#: there is need to specify the adress of your nameserver(s). The nameserver is needed in the publisher -#: to tell which nameserver it must publish its (ie the pps-runner) adress to. -#: nameservers: -#: - your.local.nameserver-adress1 -#: - your.local.nameserver-adress2 - -#: Different path that sets differently depending on environment variable SMHI_MODE. -#: Incase SMHI_MODE is not specified offline is used. -offline: -#: NWP Paths Needed - nhsp_path: /data/proj/sam/sam-utv/data/data/S/LL02_NHSP -#: NWP Paths Optional, set to None as deafault - nhsf_path: /data/proj/sam/sam-utv/data/data/S/LL02_NHSF - #: Only used in pps_runner - aapp_level1files_max_minutes_old: 9000 - -prod: - nhsp_path: /data/24/baps/weatherdata/ECNH/ - nhsf_path: /data/24/sam/data/data/S/REDA/ECMWF/ - #: Only used in pps_runner - aapp_level1files_max_minutes_old: 90 - -test: - nhsp_path: /data/prodtest/baps/weatherdata/ECNH/ - nhsf_path: /data/prodtest/sam/data/data/S/REDA/ECMWF/ - #: Only used in pps_runner - aapp_level1files_max_minutes_old: 90 - -utv: - nhsp_path: /data/prodtest/baps/weatherdata/ECNH/ - nhsf_path: /data/prodtest/sam/data/data/S/REDA/ECMWF/ - #: Only used in pps_runner - aapp_level1files_max_minutes_old: 9000 - -bi: - nhsp_path: /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source - nhsf_path: /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source - #: Only used in pps_runner - aapp_level1files_max_minutes_old: 9000 - -metno-test: - nhsp_path: /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source - nhsf_path: /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source - #: Only used in pps_runner - aapp_level1files_max_minutes_old: 9000 - - nhsf_prefix: LL02_NHSF_ - nhsf_file_name_sift: '{nhsf_prefix:3s}{analyse_time:8s}{forecast_time:8s}{end:1s}' - -nhsp_path: /path/to/nwp/gribfiles/with/hybrid/and/pressure/level/data -nhsf_path: /path/to/nwp/gribfiles/with/surface/level/data diff --git a/examples/pps2021_config.yaml_template b/examples/pps2021_config.yaml_template index d1b577b..f87d931 100644 --- a/examples/pps2021_config.yaml_template +++ b/examples/pps2021_config.yaml_template @@ -11,12 +11,12 @@ nwp_outdir: /san1/pps/import/NWP_data/source pps_nwp_requirements: /san1/pps/import/NWP_data/pps_nwp_list_of_required_fields.txt +nhsp_path: /satnhsp +nhsf_path: /satnhsf + #: Publish/subscribe publish_topic: PPS subscribe_topics: [/segment/SDR/1C] #[AAPP-HRPT,AAPP-PPS,EOS/1B,segment/SDR/1B,1c/nc/0deg] -#: Has to do with messegatype -sdr_processing: granules - #: Python and PPS related pps_version: v2021 @@ -49,10 +49,6 @@ station: norrkoping #: Incase the PPS environmental variables are sat they will overide these outdir #: PPS environmental variable SM_PRODUCT_DIR pps_outdir: /data/24/saf/polar_out/direct_readout -#: PPS environmental variable STATISTICS_DIR -pps_statistics_dir: /data/24/saf/polar_out/monitoring/direct_readout -nhsp_path: /satnhsp -nhsf_path: /satnhsf diff --git a/examples/pps_config.yaml b/examples/pps_config.yaml deleted file mode 100644 index 2370325..0000000 --- a/examples/pps_config.yaml +++ /dev/null @@ -1,78 +0,0 @@ -#: NWP -nhsp_prefix: LL02_NHSP_ -nhsf_prefix: LL02_NHSF_ -ecmwf_prefix: LL02_NHSF -#: Example filename: LL02_NHSF_202003240000+003H00M -nhsf_file_name_sift: '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' - -nwp_static_surface: /san1/pps/import/NWP_data/lsm_z.grib1 -nwp_output_prefix: LL02_NHSPSF_ -nwp_outdir: /san1/pps/import/NWP_data/source -pps_nwp_requirements: /san1/pps/import/NWP_data/pps_nwp_list_of_required_fields.txt - - -#: Publish/subscribe -publish_topic: PPS -subscribe_topics: [1c/nc/0deg] #[AAPP-HRPT,AAPP-PPS,EOS/1B,segment/SDR/1B,1c/nc/0deg] -#: Has to do with messegatype -sdr_processing: granules - - -#: Python and PPS related -python: python -run_all_script: /local_disk/opt/acpg/v2018_cmsaf/scr/ppsRunAll.py -run_cmaprob_script: /local_disk/opt/acpg/v2018_cmsaf/scr/ppsCmaskProb.py -#run_all_script: /home/pps/opt_bi/ACPG_v2018_patchCMSAF_Sept2019/scr/ppsRunAll.py -#run_cmaprob_script: /home/pps/opt_bi/ACPG_v2018_patchCMSAF_Sept2019/scr/ppsCmaskProb.py -maximum_pps_processing_time_in_minutes: 20 -run_cmask_prob: yes -run_pps_cpp: no - - -#: Used for PPS log file -log_rotation_days: 1 -log_rotation_backup: 10 - - -#: Uncategorised -number_of_threads: 1 -station: norrkoping - - -#: These are only used for the LOG output and to publish resultfiles and has nothing to do with PPS -#: Incase the PPS environmental variables are sat they will overide these outdir -#: PPS environmental variable SM_PRODUCT_DIR -pps_outdir: /data/24/saf/polar_out/direct_readout -#: PPS environmental variable STATISTICS_DIR -pps_statistics_dir: /data/24/saf/polar_out/monitoring/direct_readout - - -#: Do not seams to be used. -level1_dir: /local_disk/data/pps/import/PPS_data/source -#: In pps_runner.py there is a environmental variable PPS_SCRIPT -pps_script: /usr/bin/pps_run.sh - - -#: Different path that sets differently depending on environment variable SMHI_MODE. -#: Incase SMHI_MODE is not specified offline is used. -offline: -#: NWP Paths Needed - nhsp_path: /data/proj/sam/sam-utv/data/data/S/LL02_NHSP -#: NWP Paths Optional, set to None as deafault - nhsf_path: /data/proj/sam/sam-utv/data/data/S/LL02_NHSF - -bi: - nhsp_path: /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source - nhsf_path: /nobackup/smhid15/sm_erjoh/Satsa/import/NWP_data/source - -prod: - nhsp_path: /data/24_nb/ecmwftos/data/T6DtoLL02_NHSP/ - nhsf_path: /data/24_nb/ecmwftos/data/T9DtoLL02_NHSF/ - -test: - nhsp_path: /data/prodtest/ecmwftos/data/T6DtoLL02_NHSP/ - nhsf_path: /data/prodtest/ecmwftos/data/T9DtoLL02_NHSF/ - -utv: - nhsp_path: /data/prodtest/ecmwftos/data/T6DtoLL02_NHSP/ - nhsf_path: /data/prodtest/ecmwftos/data/T9DtoLL02_NHSF/ From f989714ca8fc7a243c38e374deb90490abef0842 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 2 Apr 2024 12:24:25 +0200 Subject: [PATCH 16/44] Extract nwp preparation to seperate script --- bin/pps_runner.py | 68 +++++--------------------- bin/run_nwp_preparation.py | 86 +++++++++++++++++++++++++++++++++ nwcsafpps_runner/config.py | 7 +-- nwcsafpps_runner/prepare_nwp.py | 69 +++++++++++++------------- setup.py | 3 +- 5 files changed, 139 insertions(+), 94 deletions(-) create mode 100644 bin/run_nwp_preparation.py diff --git a/bin/pps_runner.py b/bin/pps_runner.py index 671d2c2..e47f6ed 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -34,9 +34,8 @@ from six.moves.queue import Empty, Queue from nwcsafpps_runner.config import CONFIG_FILE, CONFIG_PATH, get_config -from nwcsafpps_runner.prepare_nwp import update_nwp from nwcsafpps_runner.publish_and_listen import FileListener, FilePublisher -from nwcsafpps_runner.utils import (SENSOR_LIST, NwpPrepareError, PpsRunError, +from nwcsafpps_runner.utils import (SENSOR_LIST, PpsRunError, create_pps_call_command, get_lvl1c_file_from_msg, create_xml_timestat_from_lvl1c, @@ -48,7 +47,6 @@ LOG = logging.getLogger(__name__) -NWP_FLENS = [3, 6, 9, 12, 15, 18, 21, 24] #: Default time format @@ -207,70 +205,27 @@ def check_threads(threads): threads.remove(thread) -def run_nwp_and_pps(scene, flens, publish_q, input_msg, options, nwp_handeling_module): - """Run first the nwp-preparation and then pps. No parallel running here.""" - - prepare_nwp4pps(flens, nwp_handeling_module) +def run_pps(scene, flens, publish_q, input_msg, options): + """Run pps. No parallel running here.""" pps_worker(scene, publish_q, input_msg, options) -def prepare_nwp4pps(flens, nwp_handeling_module): - """Prepare NWP data for pps.""" - - starttime = datetime.utcnow() - timedelta(days=1) - if nwp_handeling_module: - LOG.debug("Use custom nwp_handeling_function provided in config file...") - LOG.debug("nwp_module_name = %s", str(nwp_handeling_module)) - try: - name = "update_nwp" - name = name.replace("/", "") - module = __import__(nwp_handeling_module, globals(), locals(), [name]) - LOG.info("function : {} loaded from module: {}".format([name], nwp_handeling_module)) - except (ImportError, ModuleNotFoundError): - LOG.exception("Failed to import custom compositer for %s", str(name)) - raise - try: - params = {} - params['starttime'] = starttime - params['nlengths'] = flens - params['options'] = OPTIONS - getattr(module, name)(params) - except AttributeError: - LOG.debug("Could not get attribute %s from %s", str(name), str(module)) - else: - LOG.debug("No custom nwp_handeling_function provided in config file...") - LOG.debug("Use build in.") - try: - update_nwp(starttime, flens) - except (NwpPrepareError, IOError): - LOG.exception("Something went wrong in update_nwp...") - raise - - LOG.info("Ready with nwp preparation") - LOG.debug("Leaving prepare_nwp4pps...") - - def pps(options): """The PPS runner. - Triggers processing of PPS main script once AAPP or CSPP - is ready with a level-1 file + Triggers processing of PPS main script for a level1c files. """ LOG.info("*** Start the PPS level-2 runner:") LOG.info("Use level-1c file as input") - LOG.info("First check if NWP data should be downloaded and prepared") - nwp_handeling_module = options.get("nwp_handeling_module", None) - prepare_nwp4pps(NWP_FLENS, nwp_handeling_module) - LOG.info("Number of threads: %d", options['number_of_threads']) thread_pool = ThreadPool(options['number_of_threads']) listener_q = Queue() publisher_q = Queue() - pub_thread = FilePublisher(publisher_q, options['publish_topic'], runner_name='pps2018_runner', + pub_thread = FilePublisher(publisher_q, options['publish_topic'], runner_name='pps_runner', nameservers=options.get('nameservers', None)) pub_thread.start() listen_thread = FileListener(listener_q, options['subscribe_topics']) @@ -312,17 +267,16 @@ def pps(options): if status: LOG.debug("Files for PPS: %s", str(scene['file4pps'])) - LOG.info('Start a thread preparing the nwp data and run pps...') + LOG.info('Start a thread runing pps...') if options['number_of_threads'] == 1: - run_nwp_and_pps(scene, NWP_FLENS, publisher_q, - msg, options, nwp_handeling_module) + run_pps(scene, NWP_FLENS, publisher_q, + msg, options, nwp_handeling_module) else: thread_pool.new_thread(scene['file4pps'], - target=run_nwp_and_pps, args=(scene, NWP_FLENS, - publisher_q, - msg, options, - nwp_handeling_module)) + target=run_pps, args=(scene, NWP_FLENS, + publisher_q, + msg, options)) LOG.debug("Number of threads currently alive: %s", str(threading.active_count())) diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py new file mode 100644 index 0000000..d8ddae8 --- /dev/null +++ b/bin/run_nwp_preparation.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019 - 2021 Pytroll + +# Author(s): + +# Erik Johansson +# Adam Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import argparse +import logging +import sys +import signal + +from datetime import datetime, timedelta +from nwcsafpps_runner.logger import setup_logging +from nwcsafpps_runner.prepare_nwp import update_nwp +from nwcsafpps_runner.utils import (NwpPrepareError) + +NWP_FLENS = [6, 9, 12, 15, 18, 21, 24] + +LOG = logging.getLogger('nwp-preparation') + +def prepare_nwp4pps(config_file_name, flens): + """Prepare NWP data for pps.""" + + starttime = datetime.datetime.now(datetime.UTC) - timedelta(days=1) + LOG.info("Preparing nwp for PPS") + try: + update_nwp(starttime, flens, config_file_name) + except (NwpPrepareError, IOError): + LOG.exception("Something went wrong in update_nwp...") + raise + LOG.info("Ready with nwp preparation") + +def get_arguments(): + """ + Get command line arguments. + Return the config filepath + """ + + parser = argparse.ArgumentParser() + + parser.add_argument("-l", "--log-config", + help="Log config file to use instead of the standard logging.") + parser.add_argument('-c', '--config_file', + type=str, + dest='config_file', + default='pps_nwp_config.yaml', + help="The file containing " + + "configuration parameters e.g. pps_nwp_config.yaml, \n" + + "default = ./pps_nwp_config.yaml", + required=True) + parser.add_argument("-v", "--verbose", dest="verbosity", action="count", default=0, + help="Verbosity (between 1 and 2 occurrences with more leading to more " + "verbose logging). WARN=0, INFO=1, " + "DEBUG=2. This is overridden by the log config file if specified.") + + args = parser.parse_args() + setup_logging(args) + + if 'template' in args.config_file: + raise IOError("Template file given as master config, aborting!") + + return args.config_file + + +if __name__ == '__main__': + + CONFIG_FILENAME = get_arguments() + prepare_nwp4pps(CONFIG_FILENAME, NWP_FLENS) diff --git a/nwcsafpps_runner/config.py b/nwcsafpps_runner/config.py index 3b80b2c..bd51c71 100644 --- a/nwcsafpps_runner/config.py +++ b/nwcsafpps_runner/config.py @@ -39,7 +39,7 @@ def load_config_from_file(filepath): return config -def get_config_from_yamlfile(configfile, service): +def get_config_from_yamlfile(configfile, service=''): """Get the configuration from file.""" config = load_config_from_file(configfile) @@ -50,8 +50,9 @@ def get_config_from_yamlfile(configfile, service): options[item] = config[item] elif item not in [service]: continue - for key in config[service]: - options[key] = config[service][key] + if service in config: + for key in config[service]: + options[key] = config[service][key] return options diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index 61865c9..c07c6db 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -32,36 +32,28 @@ import pygrib # @UnresolvedImport from six.moves.configparser import NoOptionError -from nwcsafpps_runner.config import get_config -from nwcsafpps_runner.config import CONFIG_FILE -from nwcsafpps_runner.config import CONFIG_PATH # @UnresolvedImport +from nwcsafpps_runner.config import get_config_from_yamlfile from nwcsafpps_runner.utils import run_command from nwcsafpps_runner.utils import NwpPrepareError import logging LOG = logging.getLogger(__name__) -LOG.debug("Path to prepare_nwp config file = %s", str(CONFIG_PATH)) -LOG.debug("Prepare_nwp config file = %s", str(CONFIG_FILE)) -OPTIONS = get_config(CONFIG_FILE) +def prepare_config(config_file_name): + """Get config for NWP processing.""" + LOG.debug("Prepare_nwp config file = %s", str(config_file_name)) -try: - nhsp_path = OPTIONS['nhsp_path'] -except KeyError: - LOG.exception('Parameter not set in config file: ' + 'nhsp_path') -try: - nhsp_prefix = OPTIONS['nhsp_prefix'] -except KeyError: - LOG.exception('Parameter not set in config file: ' + 'nhsp_prefix') + OPTIONS = get_config_from_yamlfile(config_file_name, "dummy_service") -nhsf_file_name_sift = OPTIONS.get('nhsf_file_name_sift') - -nhsf_path = OPTIONS.get('nhsf_path', None) -nhsf_prefix = OPTIONS.get('nhsf_prefix', None) -nwp_outdir = OPTIONS.get('nwp_outdir', None) -nwp_lsmz_filename = OPTIONS.get('nwp_static_surface', None) -nwp_output_prefix = OPTIONS.get('nwp_output_prefix', None) -nwp_req_filename = OPTIONS.get('pps_nwp_requirements', None) + try: + nhsp_path = OPTIONS['nhsp_path'] + except KeyError: + LOG.exception('Parameter not set in config file: ' + 'nhsp_path') + try: + nhsp_prefix = OPTIONS['nhsp_prefix'] + except KeyError: + LOG.exception('Parameter not set in config file: ' + 'nhsp_prefix') + return OPTIONS def logreader(stream, log_func): @@ -89,19 +81,28 @@ def make_temp_filename(*args, **kwargs): return tmp_filename -def update_nwp(starttime, nlengths): +def update_nwp(starttime, nlengths, config_file_name): """Prepare NWP grib files for PPS. Consider only analysis times newer than *starttime*. And consider only the forecast lead times in hours given by the list *nlengths* of integers """ - - LOG.info("Path to prepare_nwp config file = %s", str(CONFIG_PATH)) - LOG.info("Prepare_nwp config file = %s", str(CONFIG_FILE)) - LOG.info("Path to nhsf files: %s", str(nhsf_path)) - LOG.info("Path to nhsp files: %s", str(nhsp_path)) + OPTIONS = prepare_config(config_file_name) + nhsf_file_name_sift = OPTIONS.get('nhsf_file_name_sift') + nhsf_path = OPTIONS.get('nhsf_path', None) + nhsf_prefix = OPTIONS.get('nhsf_prefix', None) + nhsp_path = OPTIONS.get('nhsp_path', None) + nhsp_prefix = OPTIONS.get('nhsp_prefix', None) + nwp_outdir = OPTIONS.get('nwp_outdir', None) + nwp_lsmz_filename = OPTIONS.get('nwp_static_surface', None) + nwp_output_prefix = OPTIONS.get('nwp_output_prefix', None) + nwp_req_filename = OPTIONS.get('pps_nwp_requirements', None) + + LOG.info("Path to prepare_nwp config file = %s", config_file_name) + LOG.info("Path to nhsf files: %s", nhsf_path) + LOG.info("Path to nhsp files: %s", nhsp_path) LOG.info("nwp_output_prefix %s", OPTIONS["nwp_output_prefix"]) - + filelist = glob(os.path.join(nhsf_path, nhsf_prefix + "*")) if len(filelist) == 0: LOG.info("No input files! dir = %s", str(nhsf_path)) @@ -209,7 +210,9 @@ def update_nwp(starttime, nlengths): "mask data to grib file") remove_file(tmp_filename) - nwp_file_ok = check_and_reduce_nwp_content(tmp_result_filename, tmp_result_filename_reduced) + nwp_file_ok = check_and_reduce_nwp_content(tmp_result_filename, + tmp_result_filename_reduced, + nwp_req_filename) if nwp_file_ok is None: LOG.info('NWP file content could not be checked, use anyway.') @@ -250,7 +253,7 @@ def get_mandatory_and_all_fields(lines): return mandatory_fields, all_fields -def get_nwp_requirement(): +def get_nwp_requirement(nwp_req_filename): """Read the new requirement file. Return list with mandatory and wanted fields. """ @@ -280,12 +283,12 @@ def check_nwp_requirement(grb_entries, mandatory_fields, result_file): return True -def check_and_reduce_nwp_content(gribfile, result_file): +def check_and_reduce_nwp_content(gribfile, result_file, nwp_req_filename): """Check the content of the NWP file. Create a reduced file. """ LOG.info("Get nwp requirements.") - mandatory_fields, all_fields = get_nwp_requirement() + mandatory_fields, all_fields = get_nwp_requirement(nwp_req_filename) if mandatory_fields is None: return None diff --git a/setup.py b/setup.py index e8a41ff..0271c40 100644 --- a/setup.py +++ b/setup.py @@ -59,9 +59,10 @@ license='GPLv3', packages=find_packages(), scripts=['bin/pps_runner.py', + 'bin/run_nwp_preparation.py', 'bin/level1c_runner.py', ], data_files=[], - install_requires=['posttroll', 'trollsift', 'pygrib', ], + install_requires=['posttroll', 'trollsift', 'pygrib', 'six'], python_requires='>=3.6', zip_safe=False, setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'], From 642e49be8ca51fc858fc5ea1638ce5c684c5fb5d Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 2 Apr 2024 12:24:45 +0200 Subject: [PATCH 17/44] Config template files --- examples/pps2021_config.yaml_template | 16 ---------------- examples/pps_nwp_config.yaml_template | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 16 deletions(-) create mode 100644 examples/pps_nwp_config.yaml_template diff --git a/examples/pps2021_config.yaml_template b/examples/pps2021_config.yaml_template index f87d931..8bc3d13 100644 --- a/examples/pps2021_config.yaml_template +++ b/examples/pps2021_config.yaml_template @@ -1,19 +1,3 @@ -#: NWP -nhsp_prefix: LL02_NHSP_ -nhsf_prefix: LL02_NHSF_ -ecmwf_prefix: LL02_NHSF -#: Example filename: LL02_NHSF_202003240000+003H00M -nhsf_file_name_sift: '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' - -nwp_static_surface: /san1/pps/import/NWP_data/lsm_z.grib1 -nwp_output_prefix: LL02_NHSPSF_ -nwp_outdir: /san1/pps/import/NWP_data/source -pps_nwp_requirements: /san1/pps/import/NWP_data/pps_nwp_list_of_required_fields.txt - - -nhsp_path: /satnhsp -nhsf_path: /satnhsf - #: Publish/subscribe publish_topic: PPS subscribe_topics: [/segment/SDR/1C] #[AAPP-HRPT,AAPP-PPS,EOS/1B,segment/SDR/1B,1c/nc/0deg] diff --git a/examples/pps_nwp_config.yaml_template b/examples/pps_nwp_config.yaml_template new file mode 100644 index 0000000..7ea222d --- /dev/null +++ b/examples/pps_nwp_config.yaml_template @@ -0,0 +1,19 @@ +#: NWP +nhsp_prefix: LL02_NHSP_ +nhsf_prefix: LL02_NHSF_ +ecmwf_prefix: LL02_NHSF +#: Example filename: LL02_NHSF_202003240000+003H00M +nhsf_file_name_sift: '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' + +nwp_static_surface: /san1/pps/import/NWP_data/lsm_z.grib1 +nwp_output_prefix: LL02_NHSPSF_ +nwp_outdir: /san1/pps/import/NWP_data/source +pps_nwp_requirements: /san1/pps/import/NWP_data/pps_nwp_list_of_required_fields.txt + +nhsp_path: /satnhsp +nhsf_path: /satnhsf + + + + + From ab3facfcfd936c50a7d68bd23d09ef4f32056b2f Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 2 Apr 2024 13:08:50 +0200 Subject: [PATCH 18/44] flake8 and tests --- bin/level1c_runner.py | 3 +-- bin/pps_runner.py | 11 +++----- bin/run_nwp_preparation.py | 6 ++--- nwcsafpps_runner/prepare_nwp.py | 24 ++++++++++-------- nwcsafpps_runner/tests/test_nwp_prepare.py | 29 +++++++--------------- nwcsafpps_runner/tests/test_utils.py | 8 +++--- 6 files changed, 34 insertions(+), 47 deletions(-) diff --git a/bin/level1c_runner.py b/bin/level1c_runner.py index 16e7e4f..dbe4555 100644 --- a/bin/level1c_runner.py +++ b/bin/level1c_runner.py @@ -24,7 +24,6 @@ import argparse import logging -import sys import signal from posttroll.subscriber import Subscribe @@ -32,7 +31,7 @@ from nwcsafpps_runner.logger import setup_logging from nwcsafpps_runner.message_utils import publish_l1c, prepare_l1c_message from nwcsafpps_runner.l1c_processing import L1cProcessor -from nwcsafpps_runner.l1c_processing import ServiceNameNotSupported +# from nwcsafpps_runner.l1c_processing import ServiceNameNotSupported from nwcsafpps_runner.l1c_processing import MessageTypeNotSupported LOOP = True diff --git a/bin/pps_runner.py b/bin/pps_runner.py index e47f6ed..2b83143 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -28,7 +28,7 @@ import os import sys import threading -from datetime import datetime, timedelta +from datetime import datetime from subprocess import PIPE, Popen from six.moves.queue import Empty, Queue @@ -47,8 +47,6 @@ LOG = logging.getLogger(__name__) - - #: Default time format _DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S' @@ -205,7 +203,7 @@ def check_threads(threads): threads.remove(thread) -def run_pps(scene, flens, publish_q, input_msg, options): +def run_pps(scene, publish_q, input_msg, options): """Run pps. No parallel running here.""" pps_worker(scene, publish_q, input_msg, options) @@ -270,11 +268,10 @@ def pps(options): LOG.info('Start a thread runing pps...') if options['number_of_threads'] == 1: - run_pps(scene, NWP_FLENS, publisher_q, - msg, options, nwp_handeling_module) + run_pps(scene, publisher_q, msg, options) else: thread_pool.new_thread(scene['file4pps'], - target=run_pps, args=(scene, NWP_FLENS, + target=run_pps, args=(scene, publisher_q, msg, options)) diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index d8ddae8..88b8dc8 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -24,8 +24,6 @@ import argparse import logging -import sys -import signal from datetime import datetime, timedelta from nwcsafpps_runner.logger import setup_logging @@ -36,6 +34,7 @@ LOG = logging.getLogger('nwp-preparation') + def prepare_nwp4pps(config_file_name, flens): """Prepare NWP data for pps.""" @@ -46,7 +45,8 @@ def prepare_nwp4pps(config_file_name, flens): except (NwpPrepareError, IOError): LOG.exception("Something went wrong in update_nwp...") raise - LOG.info("Ready with nwp preparation") + LOG.info("Ready with nwp preparation for pps") + def get_arguments(): """ diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index c07c6db..534276e 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -39,20 +39,17 @@ import logging LOG = logging.getLogger(__name__) + def prepare_config(config_file_name): """Get config for NWP processing.""" LOG.debug("Prepare_nwp config file = %s", str(config_file_name)) OPTIONS = get_config_from_yamlfile(config_file_name, "dummy_service") - try: - nhsp_path = OPTIONS['nhsp_path'] - except KeyError: - LOG.exception('Parameter not set in config file: ' + 'nhsp_path') - try: - nhsp_prefix = OPTIONS['nhsp_prefix'] - except KeyError: - LOG.exception('Parameter not set in config file: ' + 'nhsp_prefix') + for parameter in ['nhsp_path', 'nhsp_prefix', + 'nhsf_path', 'nhsf_prefix']: + if parameter not in OPTIONS: + LOG.exception('Parameter "{:s}" not set in config file: '.format(parameter)) return OPTIONS @@ -82,12 +79,17 @@ def make_temp_filename(*args, **kwargs): def update_nwp(starttime, nlengths, config_file_name): + """Get config options and then prepare nwp.""" + LOG.info("Path to prepare_nwp config file = %s", config_file_name) + OPTIONS = prepare_config(config_file_name) + update_nwp_inner(starttime, nlengths, OPTIONS) + +def update_nwp_inner(starttime, nlengths, OPTIONS): """Prepare NWP grib files for PPS. Consider only analysis times newer than *starttime*. And consider only the forecast lead times in hours given by the list *nlengths* of integers """ - OPTIONS = prepare_config(config_file_name) nhsf_file_name_sift = OPTIONS.get('nhsf_file_name_sift') nhsf_path = OPTIONS.get('nhsf_path', None) nhsf_prefix = OPTIONS.get('nhsf_prefix', None) @@ -98,11 +100,11 @@ def update_nwp(starttime, nlengths, config_file_name): nwp_output_prefix = OPTIONS.get('nwp_output_prefix', None) nwp_req_filename = OPTIONS.get('pps_nwp_requirements', None) - LOG.info("Path to prepare_nwp config file = %s", config_file_name) + LOG.info("Path to nhsf files: %s", nhsf_path) LOG.info("Path to nhsp files: %s", nhsp_path) LOG.info("nwp_output_prefix %s", OPTIONS["nwp_output_prefix"]) - + filelist = glob(os.path.join(nhsf_path, nhsf_prefix + "*")) if len(filelist) == 0: LOG.info("No input files! dir = %s", str(nhsf_path)) diff --git a/nwcsafpps_runner/tests/test_nwp_prepare.py b/nwcsafpps_runner/tests/test_nwp_prepare.py index b16d4b4..e877c5b 100644 --- a/nwcsafpps_runner/tests/test_nwp_prepare.py +++ b/nwcsafpps_runner/tests/test_nwp_prepare.py @@ -77,46 +77,35 @@ def setUp(self): fhand = open(self.OPTIONS["nwp_static_surface"], 'a') fhand.close() - @patch('nwcsafpps_runner.config.get_config') - def test_update_nwp(self, mock_get_config): + def test_update_nwp_inner(self): """Create file.""" - mock_get_config.return_value = self.OPTIONS import nwcsafpps_runner.prepare_nwp as nwc_prep - reload(nwc_prep) date = datetime(year=2022, month=5, day=10, hour=0) - nwc_prep.update_nwp(date - timedelta(days=2), [9]) + nwc_prep.update_nwp_inner(date - timedelta(days=2), [9], self.OPTIONS) # Run again when file is already created - nwc_prep.update_nwp(date - timedelta(days=2), [9]) + nwc_prep.update_nwp_inner(date - timedelta(days=2), [9], self.OPTIONS) self.assertTrue(os.path.exists(self.outfile)) - @patch('nwcsafpps_runner.config.get_config') - def test_update_nwp_no_config_file(self, mock_get_config): - """Create file no config file.""" - mock_get_config.return_value = self.OPTIONS + def test_update_nwp_inner_no_requirement_file(self): + """Create file no requirement file.""" os.remove(self.requirement_name) import nwcsafpps_runner.prepare_nwp as nwc_prep - reload(nwc_prep) date = datetime(year=2022, month=5, day=10, hour=0) - nwc_prep.update_nwp(date - timedelta(days=2), [9]) + nwc_prep.update_nwp_inner(date - timedelta(days=2), [9], self.OPTIONS) self.assertTrue(os.path.exists(self.outfile)) - @patch('nwcsafpps_runner.config.get_config') - def test_update_nwp_missing_fields(self, mock_get_config): + def test_update_nwp_inner_missing_fields(self): """Test that no file without mandatory data is created.""" - mock_get_config.return_value = self.OPTIONS_M import nwcsafpps_runner.prepare_nwp as nwc_prep - reload(nwc_prep) date = datetime(year=2022, month=5, day=10, hour=0) - nwc_prep.update_nwp(date - timedelta(days=2), [9]) + nwc_prep.update_nwp_inner(date - timedelta(days=2), [9], self.OPTIONS) self.assertFalse(os.path.exists("temp_test_dir/PPS_ECMWF_MANDATORY_202205100000+009H00M")) os.remove(self.OPTIONS["nwp_static_surface"]) os.remove(self.requirement_name_m) - @patch('nwcsafpps_runner.config.get_config') - def test_remove_filename(self, mock_get_config): + def test_remove_filename(self): """Test the function for removing files.""" from nwcsafpps_runner.prepare_nwp import remove_file - mock_get_config.return_value = self.OPTIONS remove_file(self.OPTIONS["nwp_static_surface"]) self.assertFalse(os.path.exists(self.OPTIONS["nwp_static_surface"])) # Should be able to run on already removed file without raising exception diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index 36ebbe8..1ea2d05 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -127,13 +127,13 @@ def test_xml_for_products_no_file4pps(self, fake_file_dir): class TestReady2Run(unittest.TestCase): """Test ready2run function.""" + - def test_ready2run(self): + @patch('nwcsafpps_runner.utils.check_host_ok') + def test_ready2run(self, mock_check_host_ok): from posttroll.message import Message input_msg = Message.decode(rawstr=TEST_MSG) - mymodule = MagicMock() - import sys - sys.modules["check_host_ok"] = mymodule + mock_check_host_ok.return_value = True ok2run = ready2run(input_msg, {"file4pps": "dummy"}) self.assertTrue(ok2run) ok2run = ready2run(input_msg, {"file4pps": None}) From 458a652d35b49687ceb1c67ccc5e0821d38b97e3 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 2 Apr 2024 13:36:24 +0200 Subject: [PATCH 19/44] remove unused file --- examples/pps_run.sh.template | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100755 examples/pps_run.sh.template diff --git a/examples/pps_run.sh.template b/examples/pps_run.sh.template deleted file mode 100755 index 72e8782..0000000 --- a/examples/pps_run.sh.template +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# $1 = Satellite -# $2 = Orbit number -# $3 = yyyymmdd -# $4 = hhmm -# $5 = lvl1_dir - -. /local_disk/opt/acpg/cfg/.profile_pps - -if [ "x$5" != "x" ]; then - SM_AAPP_DATA_DIR=$5 - export SM_AAPP_DATA_DIR - echo "Level 1 dir = $SM_AAPP_DATA_DIR" -fi - -if [ "x$3" == "x0" ] && [ "x$4" == "x0" ]; then - python /local_disk/opt/acpg/scr/ppsRunAllParallel.py -p $1 $2 -else - python /local_disk/opt/acpg/scr/ppsRunAllParallel.py -p $1 $2 --satday $3 --sathour $4 -fi From 8e85405fd289397d089deb97a1918686676e8b0e Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 2 Apr 2024 13:36:34 +0200 Subject: [PATCH 20/44] Add level1c4pps dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0271c40..abfa7f9 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ 'bin/run_nwp_preparation.py', 'bin/level1c_runner.py', ], data_files=[], - install_requires=['posttroll', 'trollsift', 'pygrib', 'six'], + install_requires=['posttroll', 'trollsift', 'pygrib', 'six', 'level1c4pps'], python_requires='>=3.6', zip_safe=False, setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'], From 4834af214e6f1f91bc29d8a50a46d27a8610749d Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 2 Apr 2024 13:37:45 +0200 Subject: [PATCH 21/44] Update to newer versions of testing (pytest?) --- nwcsafpps_runner/tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index 1ea2d05..8d740ab 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -72,7 +72,7 @@ def create_files(mydir, file_tag, typ): class TestCreateXmlFromLvl1c: """Test finding xml files form level1c file.""" - def setup(self): + def setup_method(self): """Define the level1c filename.""" self.scene = {'file4pps': "S_NWC_viirs_npp_12345_19810305T0715000Z_19810305T0730000Z.nc", 'starttime': datetime.strptime('19810305T0715000', "%Y%m%dT%H%M%S%f"), From 59c584e170a217a7c96ba2d4b25794be031e5e6b Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 3 Apr 2024 20:14:18 +0200 Subject: [PATCH 22/44] refactoring configuration --- .github/workflows/ci.yaml | 2 +- bin/pps_runner.py | 8 +- nwcsafpps_runner/config.py | 73 +++++++------------ nwcsafpps_runner/l1c_processing.py | 5 +- nwcsafpps_runner/prepare_nwp.py | 6 +- nwcsafpps_runner/tests/test_config.py | 11 ++- nwcsafpps_runner/tests/test_level1c_runner.py | 14 ++-- 7 files changed, 49 insertions(+), 70 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3e99fb6..ab54b3e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,7 +10,7 @@ jobs: fail-fast: true matrix: os: ["ubuntu-latest", "macos-latest"] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.9", "3.11", "3.12"] experimental: [false] include: - python-version: "3.9" diff --git a/bin/pps_runner.py b/bin/pps_runner.py index 2b83143..4f02599 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -33,7 +33,7 @@ from six.moves.queue import Empty, Queue -from nwcsafpps_runner.config import CONFIG_FILE, CONFIG_PATH, get_config +from nwcsafpps_runner.config import get_config from nwcsafpps_runner.publish_and_listen import FileListener, FilePublisher from nwcsafpps_runner.utils import (SENSOR_LIST, PpsRunError, create_pps_call_command, @@ -286,7 +286,11 @@ def pps(options): if __name__ == "__main__": from logging import handlers - OPTIONS = get_config(os.path.join(CONFIG_PATH, CONFIG_FILE)) + + CONFIG_PATH = os.environ.get('PPSRUNNER_CONFIG_DIR', './') + CONFIG_FILE = os.environ.get('PPSRUNNER_CONFIG_FILE', 'pps2018_config.yaml') + + OPTIONS = get_config(os.path.join(CONFIG_PATH, CONFIG_FILE), add_defaults=True) _PPS_LOG_FILE = OPTIONS.get('pps_log_file', os.environ.get('PPSRUNNER_LOG_FILE', False)) diff --git a/nwcsafpps_runner/config.py b/nwcsafpps_runner/config.py index bd51c71..f9d69c5 100644 --- a/nwcsafpps_runner/config.py +++ b/nwcsafpps_runner/config.py @@ -27,62 +27,38 @@ import socket import yaml -CONFIG_PATH = os.environ.get('PPSRUNNER_CONFIG_DIR', './') -CONFIG_FILE = os.environ.get('PPSRUNNER_CONFIG_FILE', 'pps2018_config.yaml') - def load_config_from_file(filepath): """Load the yaml config from file, given the file-path""" with open(filepath, 'r') as fp_: config = yaml.load(fp_, Loader=yaml.FullLoader) - return config -def get_config_from_yamlfile(configfile, service=''): - """Get the configuration from file.""" - - config = load_config_from_file(configfile) - - options = {} - for item in config: - if not isinstance(config[item], dict): - options[item] = config[item] - elif item not in [service]: - continue - if service in config: - for key in config[service]: - options[key] = config[service][key] - +def move_service_dict_attributes_to_top_level(options, service): + if service in options and isinstance(options[service], dict): + service_config = options.pop(service) + for key in service_config: + options[key] = service_config[key] + + +def get_config(configfile, add_defaults=False, service=None): + """Get configuration from the config file.""" + filetype = os.path.splitext(configfile)[1] + if filetype != '.yaml': + raise ValueError("Configfile {:s} should be of type .yaml, not {:s}".format(configfile, + filetype)) + options = load_config_from_file(configfile) + modify_config_vars(options) + if add_defaults: + add_some_default_vars(options) + if service is not None: + move_service_dict_attributes_to_top_level(options, service) return options -def get_config(conf, service=''): - """Get configuration from file using the env for the file-path.""" - configfile = os.path.join(CONFIG_PATH, conf) - filetype = os.path.splitext(conf)[1] - if filetype == '.yaml': - options = get_config_yaml(configfile, service) - else: - print("%s is not a valid extension for the config file" % filetype) - print("Pleas use .yaml") - options = -1 - return options - - -def get_config_yaml(configfile, service=''): - """Get the configuration from file.""" - config = load_config_from_file(configfile) - - options = {} - for item in config: - if not isinstance(config[item], dict) or item not in service: - options[item] = config[item] - elif item in [service]: - for key in config[service]: - if not isinstance(config[service][key], dict): - options[key] = config[service][key] - +def modify_config_vars(options): + """Modify some config options.""" if isinstance(options.get('subscribe_topics'), str): subscribe_topics = options.get('subscribe_topics').split(',') for item in subscribe_topics: @@ -90,11 +66,12 @@ def get_config_yaml(configfile, service=''): subscribe_topics.remove(item) options['subscribe_topics'] = subscribe_topics + +def add_some_default_vars(options): + """Add some default vars.""" + # service = '', probably no items are '' so this is the same as: options['number_of_threads'] = int(options.get('number_of_threads', 5)) options['maximum_pps_processing_time_in_minutes'] = int(options.get('maximum_pps_processing_time_in_minutes', 20)) options['servername'] = options.get('servername', socket.gethostname()) options['station'] = options.get('station', 'unknown') options['run_cmask_prob'] = options.get('run_cmask_prob', True) - options['run_pps_cpp'] = options.get('run_pps_cpp', True) - - return options diff --git a/nwcsafpps_runner/l1c_processing.py b/nwcsafpps_runner/l1c_processing.py index 05104fc..8696188 100644 --- a/nwcsafpps_runner/l1c_processing.py +++ b/nwcsafpps_runner/l1c_processing.py @@ -33,7 +33,7 @@ from level1c4pps.avhrr2pps_lib import process_one_scene as process_avhrr from level1c4pps.metimage2pps_lib import process_one_scene as process_metimage -from nwcsafpps_runner.config import get_config_from_yamlfile as get_config +from nwcsafpps_runner.config import get_config LOG = logging.getLogger(__name__) @@ -77,8 +77,7 @@ class L1cProcessor(object): """Container for the NWCSAF/PPS Level-c processing.""" def __init__(self, config_filename, service_name): - - options = get_config(config_filename, service_name) + options = get_config(config_filename, service=service_name) self.initialize(service_name) self._l1c_processor_call_kwargs = options.get('l1cprocess_call_arguments', {'engine': 'h5netcdf'}) diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index 534276e..8a975df 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -32,7 +32,7 @@ import pygrib # @UnresolvedImport from six.moves.configparser import NoOptionError -from nwcsafpps_runner.config import get_config_from_yamlfile +from nwcsafpps_runner.config import load_config_from_file from nwcsafpps_runner.utils import run_command from nwcsafpps_runner.utils import NwpPrepareError @@ -44,7 +44,7 @@ def prepare_config(config_file_name): """Get config for NWP processing.""" LOG.debug("Prepare_nwp config file = %s", str(config_file_name)) - OPTIONS = get_config_from_yamlfile(config_file_name, "dummy_service") + OPTIONS = load_config_from_file(config_file_name) for parameter in ['nhsp_path', 'nhsp_prefix', 'nhsf_path', 'nhsf_prefix']: @@ -84,6 +84,7 @@ def update_nwp(starttime, nlengths, config_file_name): OPTIONS = prepare_config(config_file_name) update_nwp_inner(starttime, nlengths, OPTIONS) + def update_nwp_inner(starttime, nlengths, OPTIONS): """Prepare NWP grib files for PPS. Consider only analysis times newer than *starttime*. And consider only the forecast lead times in hours given by @@ -100,7 +101,6 @@ def update_nwp_inner(starttime, nlengths, OPTIONS): nwp_output_prefix = OPTIONS.get('nwp_output_prefix', None) nwp_req_filename = OPTIONS.get('pps_nwp_requirements', None) - LOG.info("Path to nhsf files: %s", nhsf_path) LOG.info("Path to nhsp files: %s", nhsp_path) LOG.info("nwp_output_prefix %s", OPTIONS["nwp_output_prefix"]) diff --git a/nwcsafpps_runner/tests/test_config.py b/nwcsafpps_runner/tests/test_config.py index d6184ba..c481e87 100644 --- a/nwcsafpps_runner/tests/test_config.py +++ b/nwcsafpps_runner/tests/test_config.py @@ -27,8 +27,7 @@ import unittest import yaml -from nwcsafpps_runner.config import get_config_from_yamlfile -from nwcsafpps_runner.config import get_config_yaml +from nwcsafpps_runner.config import get_config TEST_YAML_LVL1C_RUNNER_CONTENT_OK = """ @@ -106,9 +105,9 @@ def setUp(self): def test_read_lvl1c_runner_config(self, config): """Test loading and initialising the yaml config""" config.return_value = self.config_lvl1c_complete - myconfig_filename = '/tmp/my/config/file' + myconfig_filename = '/tmp/my/config/file.yaml' - result = get_config_from_yamlfile(myconfig_filename, 'seviri-l1c') + result = get_config(myconfig_filename, service='seviri-l1c') expected = {'message_types': ['/1b/hrit/0deg'], 'publish_topic': ['/1c/nc/0deg'], @@ -126,9 +125,9 @@ def test_read_pps_runner_config(self, gethostname, config): """Test loading and initialising the yaml config""" gethostname.return_value = "my.local.host" config.return_value = self.config_pps_complete - myconfig_filename = '/tmp/my/config/file' + myconfig_filename = '/tmp/my/config/file.yaml' - result = get_config_yaml(myconfig_filename) + result = get_config(myconfig_filename, add_defaults=True) expected = {'nhsp_prefix': 'LL02_NHSP_', 'nhsf_prefix': 'LL02_NHSF_', diff --git a/nwcsafpps_runner/tests/test_level1c_runner.py b/nwcsafpps_runner/tests/test_level1c_runner.py index 98ff763..29c9fa2 100644 --- a/nwcsafpps_runner/tests/test_level1c_runner.py +++ b/nwcsafpps_runner/tests/test_level1c_runner.py @@ -263,7 +263,7 @@ def test_create_l1c_processor_instance(self, cpu_count, config): config.return_value = self.config_complete with tempfile.NamedTemporaryFile() as myconfig_file: - l1c_proc = L1cProcessor(myconfig_file.name, 'seviri-l1c') + l1c_proc = L1cProcessor(myconfig_file.name + ".yaml", 'seviri-l1c') self.assertEqual(l1c_proc.platform_name, 'unknown') self.assertEqual(l1c_proc.sensor, 'unknown') @@ -284,7 +284,7 @@ def test_create_l1c_processor_instance_minimal_config(self, cpu_count, config): config.return_value = self.config_minimum with tempfile.NamedTemporaryFile() as myconfig_file: - l1c_proc = L1cProcessor(myconfig_file.name, 'seviri-l1c') + l1c_proc = L1cProcessor(myconfig_file.name + ".yaml", 'seviri-l1c') self.assertEqual(l1c_proc.platform_name, 'unknown') self.assertEqual(l1c_proc.sensor, 'unknown') @@ -304,7 +304,7 @@ def test_get_level1_files_from_dataset_viirs(self, cpu_count, config): config.return_value = self.config_viirs_ok with tempfile.NamedTemporaryFile() as myconfig_file: - l1c_proc = L1cProcessor(myconfig_file.name, 'viirs-l1c') + l1c_proc = L1cProcessor(myconfig_file.name + ".yaml", 'viirs-l1c') level1_dataset = TEST_VIIRS_MSG_DATA.get('dataset') @@ -325,7 +325,7 @@ def test_orbit_number_from_msg_viirs(self, cpu_count, config): input_msg = Message.decode(rawstr=TEST_INPUT_MESSAGE_VIIRS_MSG) with tempfile.NamedTemporaryFile() as myconfig_file: - l1c_proc = L1cProcessor(myconfig_file.name, 'viirs-l1c') + l1c_proc = L1cProcessor(myconfig_file.name + ".yaml", 'viirs-l1c') l1c_proc.run(input_msg) expected_orbit_number = TEST_VIIRS_MSG_DATA.get('orbit_number') @@ -339,7 +339,7 @@ def test_process_timeout(self, config): config.return_value = self.config_viirs_orbit_number_from_msg_ok input_msg = Message.decode(rawstr=TEST_INPUT_MESSAGE_VIIRS_MSG) with tempfile.NamedTemporaryFile() as myconfig_file: - l1c_proc = L1cProcessor(myconfig_file.name, 'viirs-l1c') + l1c_proc = L1cProcessor(myconfig_file.name + ".yaml", 'viirs-l1c') l1c_proc.run(input_msg) self.assertTrue(time.time() - start_time < 5) @@ -353,7 +353,7 @@ def test_orbit_number_missing_in_msg_viirs(self, cpu_count, config): with self.assertLogs('nwcsafpps_runner.l1c_processing', level='INFO') as cm: with tempfile.NamedTemporaryFile() as myconfig_file: - l1c_proc = L1cProcessor(myconfig_file.name, 'viirs-l1c') + l1c_proc = L1cProcessor(myconfig_file.name + ".yaml", 'viirs-l1c') l1c_proc.run(input_msg) expected_orbit_number = 99999 self.assertEqual(l1c_proc.orbit_number, expected_orbit_number) @@ -369,7 +369,7 @@ def test_create_l1c_processor_instance_nameservers(self, cpu_count, config): config.return_value = self.config_complete_nameservers with tempfile.NamedTemporaryFile() as myconfig_file: - l1c_proc = L1cProcessor(myconfig_file.name, 'seviri-l1c') + l1c_proc = L1cProcessor(myconfig_file.name + ".yaml", 'seviri-l1c') self.assertEqual(l1c_proc.platform_name, 'unknown') self.assertEqual(l1c_proc.sensor, 'unknown') From 85ab48a3ed0f12ef579c7589a47ad66d5143c030 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 3 Apr 2024 20:14:31 +0200 Subject: [PATCH 23/44] flake8 --- nwcsafpps_runner/tests/test_nwp_prepare.py | 2 -- nwcsafpps_runner/tests/test_utils.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/nwcsafpps_runner/tests/test_nwp_prepare.py b/nwcsafpps_runner/tests/test_nwp_prepare.py index e877c5b..453e216 100644 --- a/nwcsafpps_runner/tests/test_nwp_prepare.py +++ b/nwcsafpps_runner/tests/test_nwp_prepare.py @@ -23,12 +23,10 @@ """Unit testing the nwp_prepare runner code.""" -from unittest.mock import patch import unittest from datetime import datetime import os import logging -from importlib import reload from datetime import timedelta LOG = logging.getLogger(__name__) logging.basicConfig( diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index 8d740ab..bd518a7 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -19,7 +19,6 @@ """Test utility functions.""" import os -import tempfile import unittest import pytest from unittest.mock import patch, MagicMock @@ -29,7 +28,6 @@ find_product_statistics_from_lvl1c, ready2run, publish_pps_files) -from nwcsafpps_runner.utils import FindTimeControlFileError TEST_MSG = """pytroll://segment/EPSSGA/1B/ file safusr.u@lxserv1043.smhi.se 2023-02-17T08:18:15.748831 v1.01 application/json {"start_time": "2023-02-17T08:03:25", "end_time": "2023-02-17T08:15:25", "orbit_number": 99999, "platform_name": "Metop-SG-A1", "sensor": "metimage", "format": "X", "type": "NETCDF", "data_processing_level": "1b", "variant": "DR", "orig_orbit_number": 23218, "uri": "/san1/polar_in/direct_readout/metimage/W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc", "uid": "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc"}""" # noqa: E501 @@ -127,7 +125,6 @@ def test_xml_for_products_no_file4pps(self, fake_file_dir): class TestReady2Run(unittest.TestCase): """Test ready2run function.""" - @patch('nwcsafpps_runner.utils.check_host_ok') def test_ready2run(self, mock_check_host_ok): From 130bf2ed82e342e14dc34dc2079900f7cfb45c89 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Thu, 4 Apr 2024 13:14:32 +0200 Subject: [PATCH 24/44] Add possibility to run update_nwp every hour --- bin/run_nwp_preparation.py | 54 ++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index 88b8dc8..72bfaec 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -24,36 +24,48 @@ import argparse import logging +import time from datetime import datetime, timedelta from nwcsafpps_runner.logger import setup_logging from nwcsafpps_runner.prepare_nwp import update_nwp -from nwcsafpps_runner.utils import (NwpPrepareError) +from nwcsafpps_runner.utils import NwpPrepareError NWP_FLENS = [6, 9, 12, 15, 18, 21, 24] LOG = logging.getLogger('nwp-preparation') +# datetime.datetime.utcnow => datetime.datetime.now(datetime.UTC) ~python 3.12 -def prepare_nwp4pps(config_file_name, flens): +def prepare_nwp4pps(options, flens): """Prepare NWP data for pps.""" - starttime = datetime.datetime.now(datetime.UTC) - timedelta(days=1) + config_file_name = options.config_file + every_hour_minute = options.run_every_hour_at_minute + starttime = datetime.utcnow() - timedelta(days=1) LOG.info("Preparing nwp for PPS") - try: - update_nwp(starttime, flens, config_file_name) - except (NwpPrepareError, IOError): - LOG.exception("Something went wrong in update_nwp...") - raise - LOG.info("Ready with nwp preparation for pps") + update_nwp(starttime, flens, config_file_name) + if every_hour_minute > 60: + return + while True: + minute = datetime.utcnow().minute + if minute < every_hour_minute or minute > every_hour_minute + 7: + LOG.info("Not time for nwp preparation for pps yet, waiting 5 minutes") + time.sleep(60 * 5) + else: + starttime = datetime.utcnow() - timedelta(days=1) + LOG.info("Preparing nwp for PPS") + try: + update_nwp(starttime, flens, config_file_name) + except (NwpPrepareError, IOError): + LOG.exception("Something went wrong in update_nwp...") + raise + LOG.info("Ready with nwp preparation for pps, waiting 45 minutes") + time.sleep(45 * 60) def get_arguments(): - """ - Get command line arguments. - Return the config filepath - """ - + """Get command line arguments.""" parser = argparse.ArgumentParser() parser.add_argument("-l", "--log-config", @@ -66,21 +78,25 @@ def get_arguments(): "configuration parameters e.g. pps_nwp_config.yaml, \n" + "default = ./pps_nwp_config.yaml", required=True) + parser.add_argument('--run_every_hour_at_minute', + type=int, + default='99', + help="Rerun preparation every hour approximately at this minute.", + required=False) parser.add_argument("-v", "--verbose", dest="verbosity", action="count", default=0, help="Verbosity (between 1 and 2 occurrences with more leading to more " "verbose logging). WARN=0, INFO=1, " "DEBUG=2. This is overridden by the log config file if specified.") args = parser.parse_args() - setup_logging(args) if 'template' in args.config_file: raise IOError("Template file given as master config, aborting!") - - return args.config_file + return args if __name__ == '__main__': - CONFIG_FILENAME = get_arguments() - prepare_nwp4pps(CONFIG_FILENAME, NWP_FLENS) + options = get_arguments() + setup_logging(options) + prepare_nwp4pps(options, NWP_FLENS) From c9261433c04fbdb81e5bd65a389a54e79d1d71d7 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 5 Apr 2024 18:29:07 +0200 Subject: [PATCH 25/44] More subfunctions in nwp-prepare code --- nwcsafpps_runner/prepare_nwp.py | 318 ++++++++++++++++---------------- 1 file changed, 162 insertions(+), 156 deletions(-) diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index 8a975df..a79735a 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -39,18 +39,71 @@ import logging LOG = logging.getLogger(__name__) +class NWPFileFamily(object): + """Container for a nwp file family.""" + + def __init__(self, cfg, filename): + self.nhsf_file = filename + self.nhsp_file = filename.replace(cfg["nhsf_path"], cfg["nhsp_path"]).replace( + cfg["nhsf_prefix"], cfg["nhsp_prefix"]) + self.file_end = os.path.basename(filename).replace(cfg["nhsf_prefix"], "") + self.tmp_filename = make_temp_filename(suffix="_" + self.file_end, dir=cfg["nwp_outdir"]) + self.tmp_result_filename= self.tmp_filename + "_tmp_result" + self.tmp_result_filename_reduced = self.tmp_filename + "_tmp_result_reduced" + out_name = cfg["nwp_output_prefix"] + self.file_end + self.result_file = os.path.join(cfg["nwp_outdir"], out_name) + self.forecast_step = None + self.analysis_time = None + self.timestamp = None + self.nwp_lsmz_filename = cfg["nwp_static_surface"] + self.nwp_req_filename = cfg["pps_nwp_requirements"] + self.cfg = cfg + self.set_time_info(filename, cfg) + + + def set_time_info(self, filename, cfg): + try: + parser = Parser(cfg["nhsf_file_name_sift"]) + except NoOptionError as noe: + LOG.error("NoOptionError {}".format(noe)) + if not parser.validate(os.path.basename(self.nhsf_file)): + LOG.error("Parser validate on filename: {} failed.".format(self.nhsf_file)) + res = parser.parse("{}".format(os.path.basename(self.nhsf_file))) + if 'analysis_time' in res: + if res['analysis_time'].year == 1900: + res['analysis_time'] = res['analysis_time'].replace(year=datetime.utcnow().year) + self.analysis_time = res['analysis_time'] + self.timestamp = self.analysis_time.strftime("%Y%m%d%H%M") + else: + raise NwpPrepareError("Can not parse analysis_time in file name. Check config and filename timestamp") + if 'forecast_step' in res: + self.forecast_step = res['forecast_step'] + else: + raise NwpPrepareError( + 'Failed parsing forecast_step in file name. Check config and filename timestamp.') + + def prepare_config(config_file_name): """Get config for NWP processing.""" LOG.debug("Prepare_nwp config file = %s", str(config_file_name)) - OPTIONS = load_config_from_file(config_file_name) + cfg = load_config_from_file(config_file_name) for parameter in ['nhsp_path', 'nhsp_prefix', + 'nhsf_file_name_sift', + 'pps_nwp_requirements', + 'nwp_output_prefix', 'nhsf_path', 'nhsf_prefix']: - if parameter not in OPTIONS: + if parameter not in cfg: LOG.exception('Parameter "{:s}" not set in config file: '.format(parameter)) - return OPTIONS + + if not os.path.exists(cfg['nwp_static_surface']): + LOG.error("Config parameter nwp_static_surface: {:s} does not exist." + "Can't prepare NWP data".format(cfg['nwp_static_surface'])) + raise IOError('Failed getting static land-sea mask and topography') + + return cfg def logreader(stream, log_func): @@ -67,8 +120,6 @@ def remove_file(filename): if os.path.exists(filename): LOG.warning("Removing tmp file: %s.", filename) os.remove(filename) - else: - LOG.warning("tmp file %s gone! Cannot remove it...", filename) def make_temp_filename(*args, **kwargs): @@ -81,162 +132,120 @@ def make_temp_filename(*args, **kwargs): def update_nwp(starttime, nlengths, config_file_name): """Get config options and then prepare nwp.""" LOG.info("Path to prepare_nwp config file = %s", config_file_name) - OPTIONS = prepare_config(config_file_name) - update_nwp_inner(starttime, nlengths, OPTIONS) + cfg = prepare_config(config_file_name) + return update_nwp_inner(starttime, nlengths, cfg) -def update_nwp_inner(starttime, nlengths, OPTIONS): - """Prepare NWP grib files for PPS. Consider only analysis times newer than +def should_be_skipped(file_obj, starttime, nlengths): + """Skip some files. Consider only analysis times newer than *starttime*. And consider only the forecast lead times in hours given by - the list *nlengths* of integers - + the list *nlengths* of integers. Never reprocess. + """ - nhsf_file_name_sift = OPTIONS.get('nhsf_file_name_sift') - nhsf_path = OPTIONS.get('nhsf_path', None) - nhsf_prefix = OPTIONS.get('nhsf_prefix', None) - nhsp_path = OPTIONS.get('nhsp_path', None) - nhsp_prefix = OPTIONS.get('nhsp_prefix', None) - nwp_outdir = OPTIONS.get('nwp_outdir', None) - nwp_lsmz_filename = OPTIONS.get('nwp_static_surface', None) - nwp_output_prefix = OPTIONS.get('nwp_output_prefix', None) - nwp_req_filename = OPTIONS.get('pps_nwp_requirements', None) - - LOG.info("Path to nhsf files: %s", nhsf_path) - LOG.info("Path to nhsp files: %s", nhsp_path) - LOG.info("nwp_output_prefix %s", OPTIONS["nwp_output_prefix"]) - - filelist = glob(os.path.join(nhsf_path, nhsf_prefix + "*")) + if file_obj.analysis_time < starttime: + return True + if file_obj.forecast_step not in nlengths: + LOG.debug("Skip step. Forecast step and nlengths: {:s} {:s}".format( + str(file_obj.forecast_step), str(nlengths))) + return True + if not os.path.exists(file_obj.nhsp_file): + LOG.warning("Corresponding nhsp-file not there: {:s}".format(file_obj.nhsp_file)) + return True + if os.path.exists(file_obj.result_file): + LOG.info("File: {:s} already there...".format(file_obj.result_file)) + return True + return False + +def get_files_to_process(cfg): + """Get all nhsf files in nhsf directory.""" + filelist = glob(os.path.join(cfg["nhsf_path"], cfg["nhsf_prefix"] + "*")) if len(filelist) == 0: - LOG.info("No input files! dir = %s", str(nhsf_path)) - return - - LOG.debug('NHSF NWP files found = %s', str(filelist)) - nfiles_error = 0 - for filename in filelist: - if nhsf_file_name_sift is None: - raise NwpPrepareError() - - try: - parser = Parser(nhsf_file_name_sift) - except NoOptionError as noe: - LOG.error("NoOptionError {}".format(noe)) - continue - if not parser.validate(os.path.basename(filename)): - LOG.error("Parser validate on filename: {} failed.".format(filename)) - continue - - res = parser.parse("{}".format(os.path.basename(filename))) - if 'analysis_time' in res: - if res['analysis_time'].year == 1900: - res['analysis_time'] = res['analysis_time'].replace(year=datetime.utcnow().year) - - analysis_time = res['analysis_time'] - timestamp = analysis_time.strftime("%Y%m%d%H%M") - else: - raise NwpPrepareError("Can not parse analysis_time in file name. Check config and filename timestamp") - - if 'forecast_time' in res: - if res['forecast_time'].year == 1900: - res['forecast_time'] = res['forecast_time'].replace(year=datetime.utcnow().year) - forecast_time = res['forecast_time'] - forecast_step = forecast_time - analysis_time - forecast_step = "{:03d}H{:02d}M".format(forecast_step.days * 24 + forecast_step.seconds / 3600, 0) - timeinfo = "{:s}{:s}{:s}".format(analysis_time.strftime( - "%m%d%H%M"), forecast_time.strftime("%m%d%H%M"), res['end']) - else: - # This needs to be done more solid using the sift pattern! FIXME! - timeinfo = filename.rsplit("_", 1)[-1] - # Forecast step in hours: - if 'forecast_step' in res: - forecast_step = res['forecast_step'] - else: - raise NwpPrepareError( - 'Failed parsing forecast_step in file name. Check config and filename timestamp.') - - if analysis_time < starttime: - continue - if forecast_step not in nlengths: - LOG.debug("Skip step. Forecast step and nlengths: %s %s", str(forecast_step), str(nlengths)) - continue - - LOG.debug("Analysis time and start time: %s %s", str(analysis_time), str(starttime)) - LOG.info("timestamp, step: %s %s", str(timestamp), str(forecast_step)) - result_file = os.path.join( - nwp_outdir, nwp_output_prefix + timestamp + "+" + '%.3dH00M' % forecast_step) - if os.path.exists(result_file): - LOG.info("File: " + str(result_file) + " already there...") - continue + LOG.info("No input files! dir = {:s}".format(cfg["nhsf_path"])) + return [] + LOG.debug('NHSF NWP files found = {:s}'.format(str(filelist))) + return filelist + +def create_nwp_file(file_obj): + """Create a new nwp file.""" + + LOG.info("Result and tmp files:\n\t {:s}\n\t {:s}\n\t {:s}\n\t {:s}".format( + file_obj.result_file, + file_obj.tmp_filename, + file_obj.tmp_result_filename, + file_obj.tmp_result_filename_reduced)) + + # probably to make sure files are not written at the moment! + cmd = "grib_copy -w gridType=regular_ll {:s} {:s}".format(file_obj.nhsp_file, + file_obj.tmp_filename) + retv = run_command(cmd) + LOG.debug("Returncode = " + str(retv)) + if retv != 0: + LOG.error( + "Failed doing the grib-copy! Will continue with the next file") + return None - tmp_filename = make_temp_filename(suffix="_" + timestamp + "+" + - '%.3dH00M' % forecast_step, dir=nwp_outdir) + cmd = "cat {:s} {:s} {:s} > {:s}".format(file_obj.tmp_filename, + file_obj.nhsf_file, + file_obj.nwp_lsmz_filename, + file_obj.tmp_result_filename) + LOG.debug("Merge data and add topography and land-sea mask:") + LOG.debug("Command = " + str(cmd)) + _start = time.time() + retv = os.system(cmd) + _end = time.time() + LOG.debug("os.system call took: %f seconds", _end - _start) + LOG.debug("Returncode = " + str(retv)) + if retv != 0: + LOG.warning("Failed generating nwp file {:} ...".format( file_obj.result_file)) + raise IOError("Failed merging grib data") + nwp_file_ok = check_and_reduce_nwp_content(file_obj.tmp_result_filename, + file_obj.tmp_result_filename_reduced, + file_obj.nwp_req_filename) + + if nwp_file_ok is None: + LOG.info('NWP file content could not be checked, use anyway.') + os.rename(file_obj.tmp_result_filename, file_obj.result_file) + LOG.debug("Renamed file {:s} to {:s}".format(file_obj.tmp_result_filename, + file_obj.result_file)) + elif nwp_file_ok: + os.rename(file_obj.tmp_result_filename_reduced, file_obj.result_file) + LOG.debug("Renamed file {:s} to {:s}".format(file_obj.tmp_result_filename_reduced, + file_obj.result_file)) + LOG.info('NWP file with reduced content has been created: {:s}'.format( + file_obj.result_file)) + else: + LOG.warning("Missing important fields. No nwp file ({:s}) created".format( + result_file.result_file)) + return None + return file_obj.result_file + - LOG.info("result and tmp files: " + str(result_file) + " " + str(tmp_filename)) - nhsp_file = os.path.join(nhsp_path, nhsp_prefix + timeinfo) - if not os.path.exists(nhsp_file): - LOG.warning("Corresponding nhsp-file not there: " + str(nhsp_file)) + +def update_nwp_inner(starttime, nlengths, cfg): + """Prepare NWP grib files for PPS. Consider only analysis times newer than + *starttime*. And consider only the forecast lead times in hours given by + the list *nlengths* of integers + """ + + LOG.info("Path to nhsf files: {:s}".format(cfg["nhsf_path"])) + LOG.info("Path to nhsp files: {:s}".format(cfg["nhsp_path"])) + LOG.info("nwp_output_prefix {:s}".format(cfg["nwp_output_prefix"])) + ok_files = [] + for fname in get_files_to_process(cfg): + file_obj = NWPFileFamily(cfg, fname) + if should_be_skipped(file_obj, starttime, nlengths): continue - - cmd = ("grib_copy -w gridType=regular_ll " + nhsp_file + " " + tmp_filename) - retv = run_command(cmd) - LOG.debug("Returncode = " + str(retv)) - if retv != 0: - LOG.error( - "Failed doing the grib-copy! Will continue with the next file") - nfiles_error = nfiles_error + 1 - if nfiles_error > len(filelist) / 2: - LOG.error( - "More than half of the Grib files failed upon grib_copy!") - raise IOError('Failed running grib_copy on many Grib files') - - if not os.path.exists(nwp_lsmz_filename): - LOG.error("No static grib file with land-sea mask and " + - "topography available. Can't prepare NWP data") - raise IOError('Failed getting static land-sea mask and topography') - - tmp_result_filename = tmp_filename + "_tmp_result" - tmp_result_filename_reduced = tmp_result_filename + '_reduced' - cmd = ('cat ' + tmp_filename + " " + - os.path.join(nhsf_path, nhsf_prefix + timeinfo) + - " " + nwp_lsmz_filename + " > " + tmp_result_filename) - LOG.debug("Add topography and land-sea mask to data:") - LOG.debug("Command = " + str(cmd)) - _start = time.time() - retv = os.system(cmd) - _end = time.time() - LOG.debug("os.system call took: %f seconds", _end - _start) - LOG.debug("Returncode = " + str(retv)) - if retv != 0: - LOG.warning("Failed generating nwp file %s ...", result_file) - remove_file(tmp_result_filename) - raise IOError("Failed adding topography and land-sea " + - "mask data to grib file") - remove_file(tmp_filename) - - nwp_file_ok = check_and_reduce_nwp_content(tmp_result_filename, - tmp_result_filename_reduced, - nwp_req_filename) - - if nwp_file_ok is None: - LOG.info('NWP file content could not be checked, use anyway.') - _start = time.time() - os.rename(tmp_result_filename, result_file) - _end = time.time() - LOG.debug("Rename file %s to %s: This took %f seconds", - tmp_result_filename, result_file, _end - _start) - elif nwp_file_ok: - remove_file(tmp_result_filename) - _start = time.time() - os.rename(tmp_result_filename_reduced, result_file) - _end = time.time() - LOG.debug("Rename file %s to %s: This took %f seconds", - tmp_result_filename_reduced, result_file, _end - _start) - LOG.info('NWP file with reduced content has been created: %s', - result_file) - else: - LOG.warning("Missing important fields. No nwp file %s written to disk", - result_file) - remove_file(tmp_result_filename) - return + LOG.debug("Analysis time and start time: {:s} {:s}".format(str(file_obj.analysis_time), + str(starttime))) + LOG.info("timestamp, step: {:s} {:s}".format(file_obj.timestamp, + str(file_obj.forecast_step))) + out_file = create_nwp_file(file_obj) + remove_file(file_obj.tmp_result_filename_reduced) + remove_file(file_obj.tmp_result_filename) + remove_file(file_obj.tmp_filename) + if out_file is not None: + ok_files.append(out_file) + return ok_files, cfg.get("publish_topic", None) def get_mandatory_and_all_fields(lines): @@ -278,8 +287,6 @@ def check_nwp_requirement(grb_entries, mandatory_fields, result_file): for item in mandatory_fields: if item not in grb_entries: LOG.warning("Mandatory field missing in NWP file: %s", str(item)) - if os.path.exists(result_file): - os.remove(result_file) return False LOG.info("NWP file has all required fields for PPS: %s", result_file) return True @@ -308,7 +315,6 @@ def check_and_reduce_nwp_content(gribfile, result_file, nwp_req_filename): msg = grb.tostring() grbout.write(msg) grbout.close() - LOG.info("Check fields in file: %s", result_file) return check_nwp_requirement(grb_entries, mandatory_fields, result_file) From 5354d27a23376b3d2cb7f6695f06602516332669 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 5 Apr 2024 18:30:36 +0200 Subject: [PATCH 26/44] Add publisher --- bin/run_nwp_preparation.py | 32 ++++++++++++++++++++++--------- nwcsafpps_runner/message_utils.py | 12 ++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index 72bfaec..b3f0410 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -30,6 +30,8 @@ from nwcsafpps_runner.logger import setup_logging from nwcsafpps_runner.prepare_nwp import update_nwp from nwcsafpps_runner.utils import NwpPrepareError +from posttroll.publisher import Publish +from nwcsafpps_runner.message_utils import publish_l1c, prepare_nwp_message NWP_FLENS = [6, 9, 12, 15, 18, 21, 24] @@ -37,14 +39,23 @@ # datetime.datetime.utcnow => datetime.datetime.now(datetime.UTC) ~python 3.12 -def prepare_nwp4pps(options, flens): - """Prepare NWP data for pps.""" - +def prepare_and_publish(pub, options, flens): config_file_name = options.config_file - every_hour_minute = options.run_every_hour_at_minute starttime = datetime.utcnow() - timedelta(days=1) + ok_files, publish_topic = update_nwp(starttime, flens, config_file_name) + print(ok_files) + if "publish_topic" is not None: + for filename in ok_files: + publish_msg = prepare_nwp_message(filename, publish_topic) + LOG.debug("Will publish") + LOG.debug("publish_msg") + publish_l1c(pub, publish_msg, publish_topic) + +def _run_subscribe_publisher(pub, options, flens): + """Prepare NWP data for pps.""" + every_hour_minute = options.run_every_hour_at_minute LOG.info("Preparing nwp for PPS") - update_nwp(starttime, flens, config_file_name) + prepare_and_publish(pub, options, flens) if every_hour_minute > 60: return while True: @@ -53,16 +64,19 @@ def prepare_nwp4pps(options, flens): LOG.info("Not time for nwp preparation for pps yet, waiting 5 minutes") time.sleep(60 * 5) else: - starttime = datetime.utcnow() - timedelta(days=1) LOG.info("Preparing nwp for PPS") try: - update_nwp(starttime, flens, config_file_name) + prepare_and_publish(pub, options, flens) except (NwpPrepareError, IOError): LOG.exception("Something went wrong in update_nwp...") raise LOG.info("Ready with nwp preparation for pps, waiting 45 minutes") time.sleep(45 * 60) - + +def prepare_nwp4pps_runner(options, flens): + """Start runner for nwp data preparations.""" + with Publish("pps-nwp-preparation-runner", 0) as pub: + _run_subscribe_publisher(pub, options, flens) def get_arguments(): """Get command line arguments.""" @@ -99,4 +113,4 @@ def get_arguments(): options = get_arguments() setup_logging(options) - prepare_nwp4pps(options, NWP_FLENS) + prepare_nwp4pps_runner(options, NWP_FLENS) diff --git a/nwcsafpps_runner/message_utils.py b/nwcsafpps_runner/message_utils.py index cdf63cb..34a3e02 100644 --- a/nwcsafpps_runner/message_utils.py +++ b/nwcsafpps_runner/message_utils.py @@ -32,6 +32,18 @@ LOG = logging.getLogger(__name__) +def prepare_nwp_message(result_file, publish_topic): + msg = Message(atype='file', subject=publish_topic) + to_send = {} + to_send["uri"] = result_file + filename = os.path.basename(result_file) + to_send["uid"] = filename + to_send['format'] = 'NWP grib' + to_send['type'] = 'grib' + + return Message('/' + publish_topic + '/', + "file", to_send).encode() + def prepare_l1c_message(result_file, mda, **kwargs): """Prepare the output message for the level-1c file creation.""" From 2d2bb56ad49241a06ce1d031665e2df9f320aba9 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Mon, 8 Apr 2024 14:22:15 +0200 Subject: [PATCH 27/44] Added and refactored some tests --- bin/pps_runner.py | 2 +- nwcsafpps_runner/prepare_nwp.py | 2 +- nwcsafpps_runner/tests/test_nwp_prepare.py | 171 ++++++++++++--------- 3 files changed, 99 insertions(+), 76 deletions(-) diff --git a/bin/pps_runner.py b/bin/pps_runner.py index 4f02599..2742c99 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -288,7 +288,7 @@ def pps(options): from logging import handlers CONFIG_PATH = os.environ.get('PPSRUNNER_CONFIG_DIR', './') - CONFIG_FILE = os.environ.get('PPSRUNNER_CONFIG_FILE', 'pps2018_config.yaml') + CONFIG_FILE = os.environ.get('PPSRUNNER_CONFIG_FILE', 'pps_config.yaml') OPTIONS = get_config(os.path.join(CONFIG_PATH, CONFIG_FILE), add_defaults=True) diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index a79735a..6e4ca28 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -215,7 +215,7 @@ def create_nwp_file(file_obj): file_obj.result_file)) else: LOG.warning("Missing important fields. No nwp file ({:s}) created".format( - result_file.result_file)) + file_obj.result_file)) return None return file_obj.result_file diff --git a/nwcsafpps_runner/tests/test_nwp_prepare.py b/nwcsafpps_runner/tests/test_nwp_prepare.py index 453e216..3f44d09 100644 --- a/nwcsafpps_runner/tests/test_nwp_prepare.py +++ b/nwcsafpps_runner/tests/test_nwp_prepare.py @@ -21,13 +21,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Unit testing the nwp_prepare runner code.""" - +"""Test the nwp_prepare runner code.""" +import pytest import unittest from datetime import datetime import os import logging from datetime import timedelta +from nwcsafpps_runner.message_utils import prepare_nwp_message +import nwcsafpps_runner.prepare_nwp as nwc_prep + LOG = logging.getLogger(__name__) logging.basicConfig( format='%(levelname)s |%(asctime)s|: %(message)s', @@ -36,91 +39,111 @@ datefmt='%H:%M:%S') -class NWPprepareRunner(unittest.TestCase): +@pytest.fixture +def fake_file_dir(tmp_path): + """Create directory with test files.""" + my_temp_dir = tmp_path / "temp_test_dir" + my_temp_dir.mkdir() + my_temp_dir = my_temp_dir + + requirement_name = my_temp_dir / 'pps_nwp_req.txt' + req_file = open(requirement_name, 'w') + req_file.write("M 235 Skin temperature 0 surface\n" + + "O 129 Geopotential 350 isobaricInhPa\n") + req_file.close() + requirement_name_m = my_temp_dir / 'pps_nwp_req_mandatory.txt' + req_file = open(requirement_name_m, 'w') + req_file.write("M 235 Skin temperature 0 surface\n" + + "M 129 Geopotential 350 isobaricInhPa\n") + req_file.close() + static_surface = my_temp_dir / 'static_surface' + req_file = open(static_surface, 'a') + req_file.close() + + cfg_file = my_temp_dir / 'pps_config.yaml' + req_file = open(cfg_file, 'w') + req_file.write("pps_nwp_requirements: " + str(requirement_name) + "\n" + "nwp_outdir: " + str(my_temp_dir) + "\n" + "nhsp_path: " + "nwcsafpps_runner/tests/files/" + "\n" + "nhsf_path: " + "nwcsafpps_runner/tests/files/" + "\n" + "nhsp_prefix: " + "LL02_NHSP_" + "\n" + "nhsf_prefix: " + "LL02_NHSF_" + "\n" + "nwp_static_surface: " + str(my_temp_dir) + "/static_surface"+ "\n" + "ecmwf_prefix: " + "LL02_NHSF" + "\n" + "nwp_output_prefix: " + "PPS_ECMWF_" + "\n" + "nhsf_file_name_sift: " + "'" + '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' + "'" + "\n") + + cfg_file = my_temp_dir / 'pps_config_missing_fields.yaml' + req_file = open(cfg_file, 'w') + req_file.write("pps_nwp_requirements: " + str(requirement_name_m) + "\n" + "nwp_outdir: " + str(my_temp_dir) + "\n" + "nhsp_path: " + "nwcsafpps_runner/tests/files/" + "\n" + "nhsf_path: " + "nwcsafpps_runner/tests/files/" + "\n" + "nhsp_prefix: " + "LL02_NHSP_" + "\n" + "nhsf_prefix: " + "LL02_NHSF_" + "\n" + "nwp_static_surface: " + str(my_temp_dir) + "/static_surface"+ "\n" + "ecmwf_prefix: " + "LL02_NHSF" + "\n" + "nwp_output_prefix: " + "PPS_ECMWF_MANDATORY" + "\n" + "nhsf_file_name_sift: " + "'" + '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' + "'" + "\n") + + return str(my_temp_dir) + + +class TestNwpMessage: + """Test the nwp message.""" + + def test_nwp_message(self): + """Test the nwp message.""" + filename = "dummy_dir/PPS_ECMWF_202205100000+009H00M" + publish_msg = prepare_nwp_message(filename, "dummy_topic") + expected_uri = '"uri": "dummy_dir/PPS_ECMWF_202205100000+009H00M"' + assert expected_uri in publish_msg + + +class TestNWPprepareRunner: """Test the nwp prepare runer.""" - def setUp(self): - """Create config options and file.""" - my_tmp_dir = "temp_test_dir" - if os.path.exists(my_tmp_dir): - pass - else: - os.mkdir(my_tmp_dir) - self.requirement_name = my_tmp_dir + '/pps_nwp_req.txt' - req_file = open(self.requirement_name, 'w') - req_file.write("M 235 Skin temperature 0 surface\n" + - "O 129 Geopotential 350 isobaricInhPa\n") - req_file.close() - self.requirement_name_m = my_tmp_dir + '/pps_nwp_req_mandatory.txt' - req_file = open(self.requirement_name_m, 'w') - req_file.write("M 235 Skin temperature 0 surface\n" + - "M 129 Geopotential 350 isobaricInhPa\n") - req_file.close() - self.OPTIONS = { - "pps_nwp_requirements": self.requirement_name, - "nwp_outdir": my_tmp_dir, - "nhsp_path": "nwcsafpps_runner/tests/files/", - "nhsf_path": "nwcsafpps_runner/tests/files/", - "nhsp_prefix": "LL02_NHSP_", - "nhsf_prefix": "LL02_NHSF_", - "nwp_static_surface": my_tmp_dir + '/empty_file', - "ecmwf_prefix": "LL02_NHSF", - "nwp_output_prefix": "PPS_ECMWF_", - "nhsf_file_name_sift": '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' - } - self.OPTIONS_M = dict(self.OPTIONS) - self.OPTIONS_M["pps_nwp_requirements"] = self.requirement_name_m - self.OPTIONS_M["nwp_output_prefix"] = "PPS_ECMWF_MANDATORY_" - self.outfile = my_tmp_dir + "/PPS_ECMWF_202205100000+009H00M" - fhand = open(self.OPTIONS["nwp_static_surface"], 'a') - fhand.close() - - def test_update_nwp_inner(self): - """Create file.""" - import nwcsafpps_runner.prepare_nwp as nwc_prep + def test_update_nwp(self, fake_file_dir): + """Test create file.""" + my_temp_dir = fake_file_dir + outfile = os.path.join(str(my_temp_dir), "PPS_ECMWF_202205100000+009H00M") + cfg_file = my_temp_dir + '/pps_config.yaml' date = datetime(year=2022, month=5, day=10, hour=0) - nwc_prep.update_nwp_inner(date - timedelta(days=2), [9], self.OPTIONS) + nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) # Run again when file is already created - nwc_prep.update_nwp_inner(date - timedelta(days=2), [9], self.OPTIONS) - self.assertTrue(os.path.exists(self.outfile)) + nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) + assert os.path.exists(outfile) - def test_update_nwp_inner_no_requirement_file(self): + def test_update_nwp_no_requirement_file(self, fake_file_dir): """Create file no requirement file.""" - os.remove(self.requirement_name) - import nwcsafpps_runner.prepare_nwp as nwc_prep + my_temp_dir = fake_file_dir + cfg_file = my_temp_dir + '/pps_config.yaml' + requirement_name = str(my_temp_dir) + '/pps_nwp_req.txt' + outfile = os.path.join(str(my_temp_dir), "PPS_ECMWF_202205100000+009H00M") + os.remove(requirement_name) date = datetime(year=2022, month=5, day=10, hour=0) - nwc_prep.update_nwp_inner(date - timedelta(days=2), [9], self.OPTIONS) - self.assertTrue(os.path.exists(self.outfile)) + nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) + assert os.path.exists(outfile) - def test_update_nwp_inner_missing_fields(self): + def test_update_nwp_missing_fields(self, fake_file_dir): """Test that no file without mandatory data is created.""" - import nwcsafpps_runner.prepare_nwp as nwc_prep + my_temp_dir = fake_file_dir + outfile = os.path.join(str(my_temp_dir), "PPS_ECMWF_MANDATORY_202205100000+009H00M") + cfg_file = my_temp_dir + '/pps_config_missing_fields.yaml' + date = datetime(year=2022, month=5, day=10, hour=0) + nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) date = datetime(year=2022, month=5, day=10, hour=0) - nwc_prep.update_nwp_inner(date - timedelta(days=2), [9], self.OPTIONS) - self.assertFalse(os.path.exists("temp_test_dir/PPS_ECMWF_MANDATORY_202205100000+009H00M")) - os.remove(self.OPTIONS["nwp_static_surface"]) - os.remove(self.requirement_name_m) + nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) + assert not (os.path.exists(outfile)) - def test_remove_filename(self): + def test_remove_filename(self, fake_file_dir): """Test the function for removing files.""" from nwcsafpps_runner.prepare_nwp import remove_file - remove_file(self.OPTIONS["nwp_static_surface"]) - self.assertFalse(os.path.exists(self.OPTIONS["nwp_static_surface"])) + my_temp_dir = fake_file_dir + nwp_surface_file = str(my_temp_dir) + '/static_surface' + remove_file(nwp_surface_file) + assert not os.path.exists(nwp_surface_file) # Should be able to run on already removed file without raising exception - remove_file(self.OPTIONS["nwp_static_surface"]) - - def tearDown(self): - """Remove files after testing.""" - for temp_file in [self.OPTIONS["nwp_static_surface"], self.requirement_name_m, - self.requirement_name, self.outfile]: - if os.path.exists(temp_file): - os.remove(temp_file) - + remove_file(nwp_surface_file) -def suite(): - """Create the test suite for test_nwp_prepare.""" - loader = unittest.TestLoader() - mysuite = unittest.TestSuite() - mysuite.addTest(loader.loadTestsFromTestCase(NWPprepareRunner)) - return mysuite From fe4f7598d9d96c198ff83df7467bd4bda66de686 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 9 Apr 2024 10:28:10 +0200 Subject: [PATCH 28/44] flake8 --- bin/run_nwp_preparation.py | 3 +- nwcsafpps_runner/message_utils.py | 5 +-- nwcsafpps_runner/prepare_nwp.py | 27 ++++++----- nwcsafpps_runner/tests/test_nwp_prepare.py | 52 +++++++++++----------- 4 files changed, 42 insertions(+), 45 deletions(-) diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index b3f0410..9dbf0c1 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -41,9 +41,8 @@ def prepare_and_publish(pub, options, flens): config_file_name = options.config_file - starttime = datetime.utcnow() - timedelta(days=1) + starttime = datetime.utcnow() - timedelta(days=8) ok_files, publish_topic = update_nwp(starttime, flens, config_file_name) - print(ok_files) if "publish_topic" is not None: for filename in ok_files: publish_msg = prepare_nwp_message(filename, publish_topic) diff --git a/nwcsafpps_runner/message_utils.py b/nwcsafpps_runner/message_utils.py index 34a3e02..b85a536 100644 --- a/nwcsafpps_runner/message_utils.py +++ b/nwcsafpps_runner/message_utils.py @@ -33,17 +33,16 @@ def prepare_nwp_message(result_file, publish_topic): - msg = Message(atype='file', subject=publish_topic) to_send = {} to_send["uri"] = result_file filename = os.path.basename(result_file) to_send["uid"] = filename to_send['format'] = 'NWP grib' to_send['type'] = 'grib' - return Message('/' + publish_topic + '/', "file", to_send).encode() - + + def prepare_l1c_message(result_file, mda, **kwargs): """Prepare the output message for the level-1c file creation.""" diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index 6e4ca28..2edb127 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -39,6 +39,7 @@ import logging LOG = logging.getLogger(__name__) + class NWPFileFamily(object): """Container for a nwp file family.""" @@ -48,7 +49,7 @@ def __init__(self, cfg, filename): cfg["nhsf_prefix"], cfg["nhsp_prefix"]) self.file_end = os.path.basename(filename).replace(cfg["nhsf_prefix"], "") self.tmp_filename = make_temp_filename(suffix="_" + self.file_end, dir=cfg["nwp_outdir"]) - self.tmp_result_filename= self.tmp_filename + "_tmp_result" + self.tmp_result_filename = self.tmp_filename + "_tmp_result" self.tmp_result_filename_reduced = self.tmp_filename + "_tmp_result_reduced" out_name = cfg["nwp_output_prefix"] + self.file_end self.result_file = os.path.join(cfg["nwp_outdir"], out_name) @@ -59,8 +60,7 @@ def __init__(self, cfg, filename): self.nwp_req_filename = cfg["pps_nwp_requirements"] self.cfg = cfg self.set_time_info(filename, cfg) - - + def set_time_info(self, filename, cfg): try: parser = Parser(cfg["nhsf_file_name_sift"]) @@ -83,7 +83,6 @@ def set_time_info(self, filename, cfg): 'Failed parsing forecast_step in file name. Check config and filename timestamp.') - def prepare_config(config_file_name): """Get config for NWP processing.""" LOG.debug("Prepare_nwp config file = %s", str(config_file_name)) @@ -97,12 +96,11 @@ def prepare_config(config_file_name): 'nhsf_path', 'nhsf_prefix']: if parameter not in cfg: LOG.exception('Parameter "{:s}" not set in config file: '.format(parameter)) - + if not os.path.exists(cfg['nwp_static_surface']): LOG.error("Config parameter nwp_static_surface: {:s} does not exist." "Can't prepare NWP data".format(cfg['nwp_static_surface'])) raise IOError('Failed getting static land-sea mask and topography') - return cfg @@ -140,7 +138,7 @@ def should_be_skipped(file_obj, starttime, nlengths): """Skip some files. Consider only analysis times newer than *starttime*. And consider only the forecast lead times in hours given by the list *nlengths* of integers. Never reprocess. - + """ if file_obj.analysis_time < starttime: return True @@ -156,6 +154,7 @@ def should_be_skipped(file_obj, starttime, nlengths): return True return False + def get_files_to_process(cfg): """Get all nhsf files in nhsf directory.""" filelist = glob(os.path.join(cfg["nhsf_path"], cfg["nhsf_prefix"] + "*")) @@ -165,15 +164,16 @@ def get_files_to_process(cfg): LOG.debug('NHSF NWP files found = {:s}'.format(str(filelist))) return filelist + def create_nwp_file(file_obj): """Create a new nwp file.""" - + LOG.info("Result and tmp files:\n\t {:s}\n\t {:s}\n\t {:s}\n\t {:s}".format( file_obj.result_file, file_obj.tmp_filename, file_obj.tmp_result_filename, file_obj.tmp_result_filename_reduced)) - + # probably to make sure files are not written at the moment! cmd = "grib_copy -w gridType=regular_ll {:s} {:s}".format(file_obj.nhsp_file, file_obj.tmp_filename) @@ -196,7 +196,7 @@ def create_nwp_file(file_obj): LOG.debug("os.system call took: %f seconds", _end - _start) LOG.debug("Returncode = " + str(retv)) if retv != 0: - LOG.warning("Failed generating nwp file {:} ...".format( file_obj.result_file)) + LOG.warning("Failed generating nwp file {:} ...".format(file_obj.result_file)) raise IOError("Failed merging grib data") nwp_file_ok = check_and_reduce_nwp_content(file_obj.tmp_result_filename, file_obj.tmp_result_filename_reduced, @@ -218,15 +218,14 @@ def create_nwp_file(file_obj): file_obj.result_file)) return None return file_obj.result_file - - + def update_nwp_inner(starttime, nlengths, cfg): """Prepare NWP grib files for PPS. Consider only analysis times newer than *starttime*. And consider only the forecast lead times in hours given by the list *nlengths* of integers """ - + LOG.info("Path to nhsf files: {:s}".format(cfg["nhsf_path"])) LOG.info("Path to nhsp files: {:s}".format(cfg["nhsp_path"])) LOG.info("nwp_output_prefix {:s}".format(cfg["nwp_output_prefix"])) @@ -241,7 +240,7 @@ def update_nwp_inner(starttime, nlengths, cfg): str(file_obj.forecast_step))) out_file = create_nwp_file(file_obj) remove_file(file_obj.tmp_result_filename_reduced) - remove_file(file_obj.tmp_result_filename) + remove_file(file_obj.tmp_result_filename) remove_file(file_obj.tmp_filename) if out_file is not None: ok_files.append(out_file) diff --git a/nwcsafpps_runner/tests/test_nwp_prepare.py b/nwcsafpps_runner/tests/test_nwp_prepare.py index 3f44d09..91d3781 100644 --- a/nwcsafpps_runner/tests/test_nwp_prepare.py +++ b/nwcsafpps_runner/tests/test_nwp_prepare.py @@ -30,7 +30,7 @@ from datetime import timedelta from nwcsafpps_runner.message_utils import prepare_nwp_message import nwcsafpps_runner.prepare_nwp as nwc_prep - + LOG = logging.getLogger(__name__) logging.basicConfig( format='%(levelname)s |%(asctime)s|: %(message)s', @@ -62,30 +62,32 @@ def fake_file_dir(tmp_path): cfg_file = my_temp_dir / 'pps_config.yaml' req_file = open(cfg_file, 'w') - req_file.write("pps_nwp_requirements: " + str(requirement_name) + "\n" - "nwp_outdir: " + str(my_temp_dir) + "\n" - "nhsp_path: " + "nwcsafpps_runner/tests/files/" + "\n" - "nhsf_path: " + "nwcsafpps_runner/tests/files/" + "\n" - "nhsp_prefix: " + "LL02_NHSP_" + "\n" - "nhsf_prefix: " + "LL02_NHSF_" + "\n" - "nwp_static_surface: " + str(my_temp_dir) + "/static_surface"+ "\n" - "ecmwf_prefix: " + "LL02_NHSF" + "\n" - "nwp_output_prefix: " + "PPS_ECMWF_" + "\n" - "nhsf_file_name_sift: " + "'" + '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' + "'" + "\n") - + req_file.write( + "pps_nwp_requirements: " + str(requirement_name) + "\n" + "nwp_outdir: " + str(my_temp_dir) + "\n" + "nhsp_path: " + "nwcsafpps_runner/tests/files/" + "\n" + "nhsf_path: " + "nwcsafpps_runner/tests/files/" + "\n" + "nhsp_prefix: " + "LL02_NHSP_" + "\n" + "nhsf_prefix: " + "LL02_NHSF_" + "\n" + "nwp_static_surface: " + str(my_temp_dir) + "/static_surface" + "\n" + "ecmwf_prefix: " + "LL02_NHSF" + "\n" + "nwp_output_prefix: " + "PPS_ECMWF_" + "\n" + "nhsf_file_name_sift: '" + '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' + "'" + "\n") + cfg_file = my_temp_dir / 'pps_config_missing_fields.yaml' req_file = open(cfg_file, 'w') - req_file.write("pps_nwp_requirements: " + str(requirement_name_m) + "\n" - "nwp_outdir: " + str(my_temp_dir) + "\n" - "nhsp_path: " + "nwcsafpps_runner/tests/files/" + "\n" - "nhsf_path: " + "nwcsafpps_runner/tests/files/" + "\n" - "nhsp_prefix: " + "LL02_NHSP_" + "\n" - "nhsf_prefix: " + "LL02_NHSF_" + "\n" - "nwp_static_surface: " + str(my_temp_dir) + "/static_surface"+ "\n" - "ecmwf_prefix: " + "LL02_NHSF" + "\n" - "nwp_output_prefix: " + "PPS_ECMWF_MANDATORY" + "\n" - "nhsf_file_name_sift: " + "'" + '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' + "'" + "\n") - + req_file.write( + "pps_nwp_requirements: " + str(requirement_name_m) + "\n" + "nwp_outdir: " + str(my_temp_dir) + "\n" + "nhsp_path: " + "nwcsafpps_runner/tests/files/" + "\n" + "nhsf_path: " + "nwcsafpps_runner/tests/files/" + "\n" + "nhsp_prefix: " + "LL02_NHSP_" + "\n" + "nhsf_prefix: " + "LL02_NHSF_" + "\n" + "nwp_static_surface: " + str(my_temp_dir) + "/static_surface" + "\n" + "ecmwf_prefix: " + "LL02_NHSF" + "\n" + "nwp_output_prefix: " + "PPS_ECMWF_MANDATORY" + "\n" + "nhsf_file_name_sift: '" + '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' + "'" + "\n") + return str(my_temp_dir) @@ -97,7 +99,7 @@ def test_nwp_message(self): filename = "dummy_dir/PPS_ECMWF_202205100000+009H00M" publish_msg = prepare_nwp_message(filename, "dummy_topic") expected_uri = '"uri": "dummy_dir/PPS_ECMWF_202205100000+009H00M"' - assert expected_uri in publish_msg + assert expected_uri in publish_msg class TestNWPprepareRunner: @@ -145,5 +147,3 @@ def test_remove_filename(self, fake_file_dir): assert not os.path.exists(nwp_surface_file) # Should be able to run on already removed file without raising exception remove_file(nwp_surface_file) - - From 94c040d09ddee1962774dff09ea1c91eb87234cf Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 9 Apr 2024 14:41:18 +0200 Subject: [PATCH 29/44] Added possibility to run with patched subscriber Had to remove the timeout from the subscr.recv, not sure if it is a problem. --- bin/pps_runner.py | 45 ++++++++++++++++++++++---- bin/run_nwp_preparation.py | 10 ++++-- nwcsafpps_runner/publish_and_listen.py | 2 +- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/bin/pps_runner.py b/bin/pps_runner.py index 2742c99..857d893 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -24,6 +24,7 @@ """Posttroll runner for the NWCSAF/PPS version >= v2018. """ +import argparse import logging import os import sys @@ -283,14 +284,34 @@ def pps(options): listen_thread.stop() +def get_arguments(): + """Get command line arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument('--test_with_l1c_file', + type=str, + help="To test for l1c file with patched subscriber", + required=False) + parser.add_argument('-c', '--config_file', + type=str, + dest='config_file', + default='l1c_config.yaml', + help="The file containing " + + "configuration parameters e.g. product_filter_config.yaml, \n" + + "default = ./l1c_config.yaml", + required=True) + + args = parser.parse_args() + return args + + if __name__ == "__main__": from logging import handlers - - CONFIG_PATH = os.environ.get('PPSRUNNER_CONFIG_DIR', './') - CONFIG_FILE = os.environ.get('PPSRUNNER_CONFIG_FILE', 'pps_config.yaml') - OPTIONS = get_config(os.path.join(CONFIG_PATH, CONFIG_FILE), add_defaults=True) + args = get_arguments() + config_file = args.config_file + + OPTIONS = get_config(config_file, add_defaults=True) _PPS_LOG_FILE = OPTIONS.get('pps_log_file', os.environ.get('PPSRUNNER_LOG_FILE', False)) @@ -318,7 +339,17 @@ def pps(options): logging.getLogger('posttroll').setLevel(logging.INFO) LOG = logging.getLogger('pps_runner') - LOG.debug("Path to PPS-runner config file = " + CONFIG_PATH) - LOG.debug("PPS-runner config file = " + CONFIG_FILE) - + LOG.debug("Path to PPS-runner config file = {:s}".format(args.config_file)) + + if args.test_with_l1c_file: + from posttroll.message import Message + from posttroll.testing import patched_subscriber_recv + some_files = [args.test_with_l1c_file] + messages = [Message("some_topic", "file", data={"uri": f, "orbit_number": 00000, "sensor": "avhrr", + 'platform_name': "EOS-Aqua", + "start_time": datetime(2024, 4, 9, 8, 3)}) + for f in some_files] + subscriber_settings = dict(nameserver=False, addresses=["ipc://bla"]) + with patched_subscriber_recv(messages): + pps(OPTIONS) pps(OPTIONS) diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index 9dbf0c1..a044c7b 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -39,17 +39,19 @@ # datetime.datetime.utcnow => datetime.datetime.now(datetime.UTC) ~python 3.12 + def prepare_and_publish(pub, options, flens): config_file_name = options.config_file starttime = datetime.utcnow() - timedelta(days=8) ok_files, publish_topic = update_nwp(starttime, flens, config_file_name) - if "publish_topic" is not None: + if publish_topic is not None: for filename in ok_files: publish_msg = prepare_nwp_message(filename, publish_topic) LOG.debug("Will publish") LOG.debug("publish_msg") publish_l1c(pub, publish_msg, publish_topic) - + + def _run_subscribe_publisher(pub, options, flens): """Prepare NWP data for pps.""" every_hour_minute = options.run_every_hour_at_minute @@ -71,12 +73,14 @@ def _run_subscribe_publisher(pub, options, flens): raise LOG.info("Ready with nwp preparation for pps, waiting 45 minutes") time.sleep(45 * 60) - + + def prepare_nwp4pps_runner(options, flens): """Start runner for nwp data preparations.""" with Publish("pps-nwp-preparation-runner", 0) as pub: _run_subscribe_publisher(pub, options, flens) + def get_arguments(): """Get command line arguments.""" parser = argparse.ArgumentParser() diff --git a/nwcsafpps_runner/publish_and_listen.py b/nwcsafpps_runner/publish_and_listen.py index a403696..59465da 100644 --- a/nwcsafpps_runner/publish_and_listen.py +++ b/nwcsafpps_runner/publish_and_listen.py @@ -51,7 +51,7 @@ def run(self): LOG.debug("Subscribe topics = %s", str(self.subscribe_topics)) with posttroll.subscriber.Subscribe("", self.subscribe_topics, True) as subscr: - for msg in subscr.recv(timeout=90): + for msg in subscr.recv(): if not self.loop: break From c0482430b3ee93490c5808edbc0c172307c35536 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 12 Apr 2024 15:36:30 +0200 Subject: [PATCH 30/44] Remove six --- bin/pps_runner.py | 2 +- nwcsafpps_runner/l1c_processing.py | 2 +- nwcsafpps_runner/prepare_nwp.py | 2 +- nwcsafpps_runner/tests/test_config.py | 60 +++++++++++---------------- setup.py | 2 +- 5 files changed, 28 insertions(+), 40 deletions(-) diff --git a/bin/pps_runner.py b/bin/pps_runner.py index 857d893..d32492e 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -32,7 +32,7 @@ from datetime import datetime from subprocess import PIPE, Popen -from six.moves.queue import Empty, Queue +from queue import Empty, Queue from nwcsafpps_runner.config import get_config from nwcsafpps_runner.publish_and_listen import FileListener, FilePublisher diff --git a/nwcsafpps_runner/l1c_processing.py b/nwcsafpps_runner/l1c_processing.py index 8696188..305503f 100644 --- a/nwcsafpps_runner/l1c_processing.py +++ b/nwcsafpps_runner/l1c_processing.py @@ -24,7 +24,7 @@ """ import logging -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from multiprocessing import cpu_count from multiprocessing import Process, Manager from level1c4pps.seviri2pps_lib import process_one_scan as process_seviri diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index 2edb127..4dbca28 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -30,7 +30,7 @@ import tempfile from trollsift import Parser import pygrib # @UnresolvedImport -from six.moves.configparser import NoOptionError +from configparser import NoOptionError from nwcsafpps_runner.config import load_config_from_file from nwcsafpps_runner.utils import run_command diff --git a/nwcsafpps_runner/tests/test_config.py b/nwcsafpps_runner/tests/test_config.py index c481e87..83b5f63 100644 --- a/nwcsafpps_runner/tests/test_config.py +++ b/nwcsafpps_runner/tests/test_config.py @@ -23,6 +23,7 @@ """Unit testing the config handling. """ +import pytest from unittest.mock import patch import unittest import yaml @@ -45,17 +46,6 @@ """ TEST_YAML_PPS_RUNNER_CONFIG_OK = """ -nhsp_prefix: LL02_NHSP_ -nhsf_prefix: LL02_NHSF_ -ecmwf_prefix: LL02_NHSF - -nhsf_file_name_sift: '{ecmwf_prefix:9s}_{analysis_time:%Y%m%d%H%M}+{forecast_step:d}H00M' - -nwp_static_surface: /san1/pps/import/NWP_data/lsm_z.grib1 -nwp_output_prefix: LL02_NHSPSF_ -nwp_outdir: /san1/pps/import/NWP_data/source -pps_nwp_requirements: /san1/pps/import/NWP_data/pps_nwp_list_of_required_fields.txt - publish_topic: PPS subscribe_topics: [/segment/SDR/1C] #[AAPP-HRPT,AAPP-PPS,EOS/1B,segment/SDR/1B,1c/nc/0deg] sdr_processing: granules @@ -88,24 +78,34 @@ nhsf_path: /path/to/nwp/data/surface_fields/ """ +@pytest.fixture +def fake_files(tmp_path): + """Create directory with test files.""" + my_temp_dir = tmp_path + + file_l1c = tmp_path / 'lvl1c_file.yaml' + file_h = open(file_l1c, 'w') + file_h.write(TEST_YAML_LVL1C_RUNNER_CONTENT_OK) + file_h.close() + + file_pps = tmp_path / 'pps_file.yaml' + file_h = open(file_pps, 'w') + file_h.write(TEST_YAML_PPS_RUNNER_CONFIG_OK) + file_h.close() + return str(file_l1c), str(file_pps) def create_config_from_yaml(yaml_content_str): """Create aapp-runner config dict from a yaml file.""" return yaml.load(yaml_content_str, Loader=yaml.FullLoader) -class TestGetConfig(unittest.TestCase): +class TestGetConfig: """Test getting the yaml config from file""" - def setUp(self): - self.config_lvl1c_complete = create_config_from_yaml(TEST_YAML_LVL1C_RUNNER_CONTENT_OK) - self.config_pps_complete = create_config_from_yaml(TEST_YAML_PPS_RUNNER_CONFIG_OK) - @patch('nwcsafpps_runner.config.load_config_from_file') - def test_read_lvl1c_runner_config(self, config): + def test_read_lvl1c_runner_config(self, fake_files): """Test loading and initialising the yaml config""" - config.return_value = self.config_lvl1c_complete - myconfig_filename = '/tmp/my/config/file.yaml' + myconfig_filename, _ = fake_files result = get_config(myconfig_filename, service='seviri-l1c') @@ -117,28 +117,16 @@ def test_read_lvl1c_runner_config(self, config): 'l1cprocess_call_arguments': {'engine': 'netcdf4', 'rotate': True}} - self.assertDictEqual(result, expected) + assert result == expected - @patch('nwcsafpps_runner.config.load_config_from_file') @patch('nwcsafpps_runner.config.socket.gethostname') - def test_read_pps_runner_config(self, gethostname, config): + def test_read_pps_runner_config(self, gethostname, fake_files): """Test loading and initialising the yaml config""" + _, myconfig_filename = fake_files gethostname.return_value = "my.local.host" - config.return_value = self.config_pps_complete - myconfig_filename = '/tmp/my/config/file.yaml' - result = get_config(myconfig_filename, add_defaults=True) - expected = {'nhsp_prefix': 'LL02_NHSP_', - 'nhsf_prefix': 'LL02_NHSF_', - 'ecmwf_prefix': 'LL02_NHSF', - 'nhsf_file_name_sift': ('{ecmwf_prefix:9s}_' + - '{analysis_time:%Y%m%d%H%M}+' + - '{forecast_step:d}H00M'), - 'nwp_static_surface': '/san1/pps/import/NWP_data/lsm_z.grib1', - 'nwp_output_prefix': 'LL02_NHSPSF_', - 'nwp_outdir': '/san1/pps/import/NWP_data/source', - 'pps_nwp_requirements': '/san1/pps/import/NWP_data/pps_nwp_list_of_required_fields.txt', + expected = { 'publish_topic': 'PPS', 'subscribe_topics': ['/segment/SDR/1C'], 'sdr_processing': 'granules', @@ -162,4 +150,4 @@ def test_read_pps_runner_config(self, gethostname, config): 'nhsf_path': '/path/to/nwp/data/surface_fields/', 'servername': 'my.local.host'} - self.assertDictEqual(result, expected) + assert result == expected diff --git a/setup.py b/setup.py index abfa7f9..6940aca 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ 'bin/run_nwp_preparation.py', 'bin/level1c_runner.py', ], data_files=[], - install_requires=['posttroll', 'trollsift', 'pygrib', 'six', 'level1c4pps'], + install_requires=['posttroll', 'trollsift', 'pygrib', 'level1c4pps'], python_requires='>=3.6', zip_safe=False, setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'], From 45da179459460342a9844775d017af48fd42ca9f Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 12 Apr 2024 15:36:50 +0200 Subject: [PATCH 31/44] Check for NWP files one day back as before --- bin/run_nwp_preparation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index a044c7b..31c32df 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -42,7 +42,7 @@ def prepare_and_publish(pub, options, flens): config_file_name = options.config_file - starttime = datetime.utcnow() - timedelta(days=8) + starttime = datetime.utcnow() - timedelta(days=1) ok_files, publish_topic = update_nwp(starttime, flens, config_file_name) if publish_topic is not None: for filename in ok_files: From f863d7052a6fd47816cf9f99b16492b431349812 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 12 Apr 2024 15:37:17 +0200 Subject: [PATCH 32/44] Do not use pkg_resoruses to find version --- nwcsafpps_runner/__init__.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/nwcsafpps_runner/__init__.py b/nwcsafpps_runner/__init__.py index b9c1d0b..59d2025 100644 --- a/nwcsafpps_runner/__init__.py +++ b/nwcsafpps_runner/__init__.py @@ -20,12 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""nwcsafpps_runner package. -""" - -from pkg_resources import get_distribution, DistributionNotFound -try: - __version__ = get_distribution(__name__).version -except DistributionNotFound: - # package is not installed - pass +"""The nwcsafpps_runner package.""" + +from importlib.metadata import version +__version__ = version(__name__) + From 4070de4bb2d3a647fa85824b98e9847f216e5e76 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 12 Apr 2024 15:37:30 +0200 Subject: [PATCH 33/44] One more usage of six --- nwcsafpps_runner/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index 36885e7..a4d8658 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -32,8 +32,7 @@ from glob import glob import socket -#: Python 2/3 differences -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from posttroll.address_receiver import get_local_ips From 1b1e4d4a011a39bbf0bed973ebe6545699e82a0d Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 12 Apr 2024 16:04:32 +0200 Subject: [PATCH 34/44] Use shell=False with Popen Also allow to test with list of files --- bin/pps_runner.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/pps_runner.py b/bin/pps_runner.py index d32492e..6748fa4 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -127,7 +127,7 @@ def pps_worker(scene, publish_q, input_msg, options): LOG.debug("Run command: " + str(cmd_str)) try: - pps_all_proc = Popen(cmd_str, shell=True, stderr=PIPE, stdout=PIPE) + pps_all_proc = Popen(cmd_str.split(" "), shell=False, stderr=PIPE, stdout=PIPE) except PpsRunError: LOG.exception("Failed in PPS...") @@ -151,7 +151,7 @@ def pps_worker(scene, publish_q, input_msg, options): LOG.debug("Run command: " + str(cmdl)) try: - pps_cmaprob_proc = Popen(cmdl, shell=True, stderr=PIPE, stdout=PIPE) + pps_cmaprob_proc = Popen(cmdl.split(" "), shell=False, stderr=PIPE, stdout=PIPE) except PpsRunError: LOG.exception("Failed when trying to run the PPS Cma-prob") timer_cmaprob = threading.Timer(min_thr * 60.0, terminate_process, @@ -287,10 +287,10 @@ def pps(options): def get_arguments(): """Get command line arguments.""" parser = argparse.ArgumentParser() - parser.add_argument('--test_with_l1c_file', - type=str, - help="To test for l1c file with patched subscriber", - required=False) + parser.add_argument('test_with_l1c_files', + metavar='fileN', type=str, nargs='+', + default=[], + help="To test for l1c file with patched subscriber") parser.add_argument('-c', '--config_file', type=str, dest='config_file', @@ -341,10 +341,10 @@ def get_arguments(): LOG = logging.getLogger('pps_runner') LOG.debug("Path to PPS-runner config file = {:s}".format(args.config_file)) - if args.test_with_l1c_file: + if args.test_with_l1c_files != []: from posttroll.message import Message from posttroll.testing import patched_subscriber_recv - some_files = [args.test_with_l1c_file] + some_files = args.test_with_l1c_files messages = [Message("some_topic", "file", data={"uri": f, "orbit_number": 00000, "sensor": "avhrr", 'platform_name': "EOS-Aqua", "start_time": datetime(2024, 4, 9, 8, 3)}) From 5bc5720a72f7d6c291ccae8ba91a3aa9e1180938 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 12 Apr 2024 16:05:23 +0200 Subject: [PATCH 35/44] Fixed name of env variable --- bin/pps_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pps_runner.py b/bin/pps_runner.py index 6748fa4..bf75324 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -167,7 +167,7 @@ def pps_worker(scene, publish_q, input_msg, options): out_reader2.join() err_reader2.join() - pps_control_path = my_env.get('STATISTICS_DIR', options.get('pps_statistics_dir', './')) + pps_control_path = my_env.get('SM_STATISTICS_DIR', options.get('pps_statistics_dir', './')) xml_files = create_xml_timestat_from_lvl1c(scene, pps_control_path) xml_files += find_product_statistics_from_lvl1c(scene, pps_control_path) LOG.info("PPS summary statistics files: %s", str(xml_files)) From f68f7afab3da36727eacf668724fc5ec63c9d9e6 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 12 Apr 2024 16:09:21 +0200 Subject: [PATCH 36/44] Use timeout for the reciever Needed for threads that are shut down. For now it is not working with the patched reciever --- nwcsafpps_runner/publish_and_listen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nwcsafpps_runner/publish_and_listen.py b/nwcsafpps_runner/publish_and_listen.py index 59465da..a403696 100644 --- a/nwcsafpps_runner/publish_and_listen.py +++ b/nwcsafpps_runner/publish_and_listen.py @@ -51,7 +51,7 @@ def run(self): LOG.debug("Subscribe topics = %s", str(self.subscribe_topics)) with posttroll.subscriber.Subscribe("", self.subscribe_topics, True) as subscr: - for msg in subscr.recv(): + for msg in subscr.recv(timeout=90): if not self.loop: break From 29df5ecffd1855251bdc8e5689d2206e285b647e Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 12 Apr 2024 16:13:07 +0200 Subject: [PATCH 37/44] flake8 --- nwcsafpps_runner/__init__.py | 1 - nwcsafpps_runner/tests/test_config.py | 12 +++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/nwcsafpps_runner/__init__.py b/nwcsafpps_runner/__init__.py index 59d2025..9c42d23 100644 --- a/nwcsafpps_runner/__init__.py +++ b/nwcsafpps_runner/__init__.py @@ -24,4 +24,3 @@ from importlib.metadata import version __version__ = version(__name__) - diff --git a/nwcsafpps_runner/tests/test_config.py b/nwcsafpps_runner/tests/test_config.py index 83b5f63..7d7079f 100644 --- a/nwcsafpps_runner/tests/test_config.py +++ b/nwcsafpps_runner/tests/test_config.py @@ -26,7 +26,6 @@ import pytest from unittest.mock import patch import unittest -import yaml from nwcsafpps_runner.config import get_config @@ -78,31 +77,26 @@ nhsf_path: /path/to/nwp/data/surface_fields/ """ + @pytest.fixture def fake_files(tmp_path): """Create directory with test files.""" - my_temp_dir = tmp_path file_l1c = tmp_path / 'lvl1c_file.yaml' file_h = open(file_l1c, 'w') file_h.write(TEST_YAML_LVL1C_RUNNER_CONTENT_OK) file_h.close() - + file_pps = tmp_path / 'pps_file.yaml' file_h = open(file_pps, 'w') file_h.write(TEST_YAML_PPS_RUNNER_CONFIG_OK) - file_h.close() + file_h.close() return str(file_l1c), str(file_pps) -def create_config_from_yaml(yaml_content_str): - """Create aapp-runner config dict from a yaml file.""" - return yaml.load(yaml_content_str, Loader=yaml.FullLoader) - class TestGetConfig: """Test getting the yaml config from file""" - def test_read_lvl1c_runner_config(self, fake_files): """Test loading and initialising the yaml config""" myconfig_filename, _ = fake_files From 1bde12c98be4bfcf88d7fff6a9e9b1edbabdd17d Mon Sep 17 00:00:00 2001 From: Nina Date: Mon, 15 Apr 2024 09:29:31 +0200 Subject: [PATCH 38/44] Update bin/level1c_runner.py Co-authored-by: Adam Dybbroe --- bin/level1c_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/level1c_runner.py b/bin/level1c_runner.py index dbe4555..fd685fd 100644 --- a/bin/level1c_runner.py +++ b/bin/level1c_runner.py @@ -31,7 +31,6 @@ from nwcsafpps_runner.logger import setup_logging from nwcsafpps_runner.message_utils import publish_l1c, prepare_l1c_message from nwcsafpps_runner.l1c_processing import L1cProcessor -# from nwcsafpps_runner.l1c_processing import ServiceNameNotSupported from nwcsafpps_runner.l1c_processing import MessageTypeNotSupported LOOP = True From f520c0e86f8d2c5dc3a713eae6a72b93ef659b53 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Mon, 15 Apr 2024 11:01:32 +0200 Subject: [PATCH 39/44] Ute timezone aware objects utcnow to be depricated --- bin/pps_runner.py | 6 +++--- bin/run_nwp_preparation.py | 8 ++++---- nwcsafpps_runner/prepare_nwp.py | 8 ++++---- nwcsafpps_runner/tests/test_nwp_prepare.py | 10 ++++------ 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/bin/pps_runner.py b/bin/pps_runner.py index bf75324..cd60a29 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -29,7 +29,7 @@ import os import sys import threading -from datetime import datetime +from datetime import datetime, timezone from subprocess import PIPE, Popen from queue import Empty, Queue @@ -97,7 +97,7 @@ def pps_worker(scene, publish_q, input_msg, options): try: LOG.info("Starting pps runner for scene %s", str(scene)) - job_start_time = datetime.utcnow() + job_start_time = datetime.now(tz=timezone.utc) LOG.debug("Level-1c file: %s", scene['file4pps']) LOG.debug("Platform name: %s", scene['platform_name']) @@ -178,7 +178,7 @@ def pps_worker(scene, publish_q, input_msg, options): servername=options['servername'], station=options['station']) - dt_ = datetime.utcnow() - job_start_time + dt_ = datetime.now(tz=timezone.utc) - job_start_time LOG.info("PPS on scene " + str(scene) + " finished. It took: " + str(dt_)) t__.cancel() diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index 31c32df..52393f4 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -26,7 +26,7 @@ import logging import time -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from nwcsafpps_runner.logger import setup_logging from nwcsafpps_runner.prepare_nwp import update_nwp from nwcsafpps_runner.utils import NwpPrepareError @@ -37,12 +37,12 @@ LOG = logging.getLogger('nwp-preparation') -# datetime.datetime.utcnow => datetime.datetime.now(datetime.UTC) ~python 3.12 +# Eventually timezone.utz => UTC. UTC in python 3.12 not in python 3.9 def prepare_and_publish(pub, options, flens): config_file_name = options.config_file - starttime = datetime.utcnow() - timedelta(days=1) + starttime = datetime.now(tz=timezone.utc) - timedelta(days=1) ok_files, publish_topic = update_nwp(starttime, flens, config_file_name) if publish_topic is not None: for filename in ok_files: @@ -60,7 +60,7 @@ def _run_subscribe_publisher(pub, options, flens): if every_hour_minute > 60: return while True: - minute = datetime.utcnow().minute + minute = datetime.now(tz=timezone.utc).minute if minute < every_hour_minute or minute > every_hour_minute + 7: LOG.info("Not time for nwp preparation for pps yet, waiting 5 minutes") time.sleep(60 * 5) diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index 4dbca28..2854e54 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -25,7 +25,7 @@ from glob import glob import os -from datetime import datetime +from datetime import datetime, timezone import time import tempfile from trollsift import Parser @@ -71,8 +71,8 @@ def set_time_info(self, filename, cfg): res = parser.parse("{}".format(os.path.basename(self.nhsf_file))) if 'analysis_time' in res: if res['analysis_time'].year == 1900: - res['analysis_time'] = res['analysis_time'].replace(year=datetime.utcnow().year) - self.analysis_time = res['analysis_time'] + res['analysis_time'] = res['analysis_time'].replace(year=datetime.now(timezone.utc).year) + self.analysis_time = res['analysis_time'].replace(tzinfo=timezone.utc) self.timestamp = self.analysis_time.strftime("%Y%m%d%H%M") else: raise NwpPrepareError("Can not parse analysis_time in file name. Check config and filename timestamp") @@ -338,5 +338,5 @@ def check_and_reduce_nwp_content(gribfile, result_file, nwp_req_filename): LOG = logging.getLogger('test_update_nwp') from datetime import timedelta - now = datetime.utcnow() + now = datetime.now(tz=timezone.utc) update_nwp(now - timedelta(days=2), [9]) diff --git a/nwcsafpps_runner/tests/test_nwp_prepare.py b/nwcsafpps_runner/tests/test_nwp_prepare.py index 91d3781..e570c90 100644 --- a/nwcsafpps_runner/tests/test_nwp_prepare.py +++ b/nwcsafpps_runner/tests/test_nwp_prepare.py @@ -24,7 +24,7 @@ """Test the nwp_prepare runner code.""" import pytest import unittest -from datetime import datetime +from datetime import datetime, timezone import os import logging from datetime import timedelta @@ -110,7 +110,7 @@ def test_update_nwp(self, fake_file_dir): my_temp_dir = fake_file_dir outfile = os.path.join(str(my_temp_dir), "PPS_ECMWF_202205100000+009H00M") cfg_file = my_temp_dir + '/pps_config.yaml' - date = datetime(year=2022, month=5, day=10, hour=0) + date = datetime(year=2022, month=5, day=10, hour=0, tzinfo=timezone.utc) nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) # Run again when file is already created nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) @@ -123,7 +123,7 @@ def test_update_nwp_no_requirement_file(self, fake_file_dir): requirement_name = str(my_temp_dir) + '/pps_nwp_req.txt' outfile = os.path.join(str(my_temp_dir), "PPS_ECMWF_202205100000+009H00M") os.remove(requirement_name) - date = datetime(year=2022, month=5, day=10, hour=0) + date = datetime(year=2022, month=5, day=10, hour=0, tzinfo=timezone.utc) nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) assert os.path.exists(outfile) @@ -132,9 +132,7 @@ def test_update_nwp_missing_fields(self, fake_file_dir): my_temp_dir = fake_file_dir outfile = os.path.join(str(my_temp_dir), "PPS_ECMWF_MANDATORY_202205100000+009H00M") cfg_file = my_temp_dir + '/pps_config_missing_fields.yaml' - date = datetime(year=2022, month=5, day=10, hour=0) - nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) - date = datetime(year=2022, month=5, day=10, hour=0) + date = datetime(year=2022, month=5, day=10, hour=0, tzinfo=timezone.utc) nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) assert not (os.path.exists(outfile)) From a1bd874b121223e7f6ea9bb82cfedcee8afdd18e Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Mon, 15 Apr 2024 11:11:59 +0200 Subject: [PATCH 40/44] isort --- bin/level1c_runner.py | 9 ++++--- bin/pps_runner.py | 10 +++---- bin/run_nwp_preparation.py | 7 ++--- nwcsafpps_runner/__init__.py | 1 + nwcsafpps_runner/config.py | 1 + nwcsafpps_runner/l1c_processing.py | 10 +++---- nwcsafpps_runner/logger.py | 1 + nwcsafpps_runner/message_utils.py | 3 ++- nwcsafpps_runner/metno_update_nwp.py | 9 ++++--- nwcsafpps_runner/pps_posttroll_hook.py | 11 ++++---- nwcsafpps_runner/prepare_nwp.py | 16 ++++++------ nwcsafpps_runner/publish_and_listen.py | 6 +++-- nwcsafpps_runner/tests/test_config.py | 6 ++--- nwcsafpps_runner/tests/test_level1c_runner.py | 26 +++++++++---------- nwcsafpps_runner/tests/test_nwp_prepare.py | 13 +++++----- nwcsafpps_runner/tests/test_pps_hook.py | 14 +++++----- nwcsafpps_runner/tests/test_utils.py | 14 +++++----- nwcsafpps_runner/utils.py | 17 ++++++------ setup.py | 2 +- 19 files changed, 93 insertions(+), 83 deletions(-) diff --git a/bin/level1c_runner.py b/bin/level1c_runner.py index fd685fd..ee3b5da 100644 --- a/bin/level1c_runner.py +++ b/bin/level1c_runner.py @@ -26,12 +26,13 @@ import logging import signal -from posttroll.subscriber import Subscribe from posttroll.publisher import Publish +from posttroll.subscriber import Subscribe + +from nwcsafpps_runner.l1c_processing import (L1cProcessor, + MessageTypeNotSupported) from nwcsafpps_runner.logger import setup_logging -from nwcsafpps_runner.message_utils import publish_l1c, prepare_l1c_message -from nwcsafpps_runner.l1c_processing import L1cProcessor -from nwcsafpps_runner.l1c_processing import MessageTypeNotSupported +from nwcsafpps_runner.message_utils import prepare_l1c_message, publish_l1c LOOP = True diff --git a/bin/pps_runner.py b/bin/pps_runner.py index cd60a29..4630b81 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -30,20 +30,18 @@ import sys import threading from datetime import datetime, timezone -from subprocess import PIPE, Popen - from queue import Empty, Queue +from subprocess import PIPE, Popen from nwcsafpps_runner.config import get_config from nwcsafpps_runner.publish_and_listen import FileListener, FilePublisher from nwcsafpps_runner.utils import (SENSOR_LIST, PpsRunError, create_pps_call_command, - get_lvl1c_file_from_msg, create_xml_timestat_from_lvl1c, find_product_statistics_from_lvl1c, - logreader, - publish_pps_files, - ready2run, terminate_process) + get_lvl1c_file_from_msg, logreader, + publish_pps_files, ready2run, + terminate_process) LOG = logging.getLogger(__name__) diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index 52393f4..5bba956 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -25,13 +25,14 @@ import argparse import logging import time - from datetime import datetime, timedelta, timezone + +from posttroll.publisher import Publish + from nwcsafpps_runner.logger import setup_logging +from nwcsafpps_runner.message_utils import prepare_nwp_message, publish_l1c from nwcsafpps_runner.prepare_nwp import update_nwp from nwcsafpps_runner.utils import NwpPrepareError -from posttroll.publisher import Publish -from nwcsafpps_runner.message_utils import publish_l1c, prepare_nwp_message NWP_FLENS = [6, 9, 12, 15, 18, 21, 24] diff --git a/nwcsafpps_runner/__init__.py b/nwcsafpps_runner/__init__.py index 9c42d23..5c950fc 100644 --- a/nwcsafpps_runner/__init__.py +++ b/nwcsafpps_runner/__init__.py @@ -23,4 +23,5 @@ """The nwcsafpps_runner package.""" from importlib.metadata import version + __version__ = version(__name__) diff --git a/nwcsafpps_runner/config.py b/nwcsafpps_runner/config.py index f9d69c5..89e63c8 100644 --- a/nwcsafpps_runner/config.py +++ b/nwcsafpps_runner/config.py @@ -25,6 +25,7 @@ import os import socket + import yaml diff --git a/nwcsafpps_runner/l1c_processing.py b/nwcsafpps_runner/l1c_processing.py index 305503f..33a23fc 100644 --- a/nwcsafpps_runner/l1c_processing.py +++ b/nwcsafpps_runner/l1c_processing.py @@ -24,14 +24,14 @@ """ import logging +from multiprocessing import Manager, Process, cpu_count from urllib.parse import urlparse -from multiprocessing import cpu_count -from multiprocessing import Process, Manager -from level1c4pps.seviri2pps_lib import process_one_scan as process_seviri -from level1c4pps.viirs2pps_lib import process_one_scene as process_viirs -from level1c4pps.modis2pps_lib import process_one_scene as process_modis + from level1c4pps.avhrr2pps_lib import process_one_scene as process_avhrr from level1c4pps.metimage2pps_lib import process_one_scene as process_metimage +from level1c4pps.modis2pps_lib import process_one_scene as process_modis +from level1c4pps.seviri2pps_lib import process_one_scan as process_seviri +from level1c4pps.viirs2pps_lib import process_one_scene as process_viirs from nwcsafpps_runner.config import get_config diff --git a/nwcsafpps_runner/logger.py b/nwcsafpps_runner/logger.py index 8410d55..c6459ac 100644 --- a/nwcsafpps_runner/logger.py +++ b/nwcsafpps_runner/logger.py @@ -26,6 +26,7 @@ import logging import logging.config import logging.handlers + import yaml LOG_FORMAT = "[%(asctime)s %(levelname)-8s] %(message)s" diff --git a/nwcsafpps_runner/message_utils.py b/nwcsafpps_runner/message_utils.py index b85a536..2aeee1f 100644 --- a/nwcsafpps_runner/message_utils.py +++ b/nwcsafpps_runner/message_utils.py @@ -23,10 +23,11 @@ """Message utilities. """ -import os import logging +import os import socket from urllib.parse import urlunsplit + from posttroll.message import Message LOG = logging.getLogger(__name__) diff --git a/nwcsafpps_runner/metno_update_nwp.py b/nwcsafpps_runner/metno_update_nwp.py index 2037140..2b89697 100644 --- a/nwcsafpps_runner/metno_update_nwp.py +++ b/nwcsafpps_runner/metno_update_nwp.py @@ -24,12 +24,13 @@ """ import logging -import tempfile -from glob import glob import os +import tempfile from datetime import datetime -import numpy as np +from glob import glob + import eccodes as ecc +import numpy as np LOG = logging.getLogger(__name__) @@ -198,8 +199,8 @@ def update_nwp(params): LOG.info("File: " + str(result_file) + " already there...") continue - import fcntl import errno + import fcntl import time rfl = open(_result_file_lock, 'w+') # do some locking diff --git a/nwcsafpps_runner/pps_posttroll_hook.py b/nwcsafpps_runner/pps_posttroll_hook.py index 9badac4..b757822 100644 --- a/nwcsafpps_runner/pps_posttroll_hook.py +++ b/nwcsafpps_runner/pps_posttroll_hook.py @@ -45,15 +45,16 @@ """ +import logging import os import socket -import logging -from posttroll.publisher import Publish -from posttroll.message import Message -from multiprocessing import Manager import threading -from datetime import timedelta import time +from datetime import timedelta +from multiprocessing import Manager + +from posttroll.message import Message +from posttroll.publisher import Publish LOG = logging.getLogger(__name__) diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index 2854e54..c755e39 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -23,20 +23,20 @@ """Prepare NWP data for PPS """ -from glob import glob +import logging import os -from datetime import datetime, timezone -import time import tempfile -from trollsift import Parser -import pygrib # @UnresolvedImport +import time from configparser import NoOptionError +from datetime import datetime, timezone +from glob import glob + +import pygrib # @UnresolvedImport +from trollsift import Parser from nwcsafpps_runner.config import load_config_from_file -from nwcsafpps_runner.utils import run_command -from nwcsafpps_runner.utils import NwpPrepareError +from nwcsafpps_runner.utils import NwpPrepareError, run_command -import logging LOG = logging.getLogger(__name__) diff --git a/nwcsafpps_runner/publish_and_listen.py b/nwcsafpps_runner/publish_and_listen.py index a403696..60e5c62 100644 --- a/nwcsafpps_runner/publish_and_listen.py +++ b/nwcsafpps_runner/publish_and_listen.py @@ -23,13 +23,15 @@ """Publisher and Listener classes for the PPS runners. """ +import logging +import threading + import posttroll.subscriber from posttroll.publisher import Publish -import threading + from nwcsafpps_runner.utils import (SUPPORTED_PPS_SATELLITES, SUPPORTED_SEVIRI_SATELLITES) -import logging LOG = logging.getLogger(__name__) diff --git a/nwcsafpps_runner/tests/test_config.py b/nwcsafpps_runner/tests/test_config.py index 7d7079f..c0d1247 100644 --- a/nwcsafpps_runner/tests/test_config.py +++ b/nwcsafpps_runner/tests/test_config.py @@ -23,12 +23,12 @@ """Unit testing the config handling. """ -import pytest -from unittest.mock import patch import unittest +from unittest.mock import patch -from nwcsafpps_runner.config import get_config +import pytest +from nwcsafpps_runner.config import get_config TEST_YAML_LVL1C_RUNNER_CONTENT_OK = """ seviri-l1c: diff --git a/nwcsafpps_runner/tests/test_level1c_runner.py b/nwcsafpps_runner/tests/test_level1c_runner.py index 29c9fa2..bd59ced 100644 --- a/nwcsafpps_runner/tests/test_level1c_runner.py +++ b/nwcsafpps_runner/tests/test_level1c_runner.py @@ -23,23 +23,23 @@ """Unit testing the level-1c runner code """ -import pytest -from unittest.mock import patch -import unittest -from posttroll.message import Message -from datetime import datetime -import yaml import tempfile import time +import unittest +from datetime import datetime +from unittest.mock import patch -from nwcsafpps_runner.message_utils import publish_l1c, prepare_l1c_message -from nwcsafpps_runner.l1c_processing import check_message_okay -from nwcsafpps_runner.l1c_processing import check_service_is_supported -from nwcsafpps_runner.l1c_processing import L1cProcessor -from nwcsafpps_runner.l1c_processing import ServiceNameNotSupported -from nwcsafpps_runner.l1c_processing import MessageTypeNotSupported -from nwcsafpps_runner.l1c_processing import MessageContentMissing +import pytest +import yaml +from posttroll.message import Message +from nwcsafpps_runner.l1c_processing import (L1cProcessor, + MessageContentMissing, + MessageTypeNotSupported, + ServiceNameNotSupported, + check_message_okay, + check_service_is_supported) +from nwcsafpps_runner.message_utils import prepare_l1c_message, publish_l1c TEST_YAML_CONTENT_OK = """ seviri-l1c: diff --git a/nwcsafpps_runner/tests/test_nwp_prepare.py b/nwcsafpps_runner/tests/test_nwp_prepare.py index e570c90..e4441fa 100644 --- a/nwcsafpps_runner/tests/test_nwp_prepare.py +++ b/nwcsafpps_runner/tests/test_nwp_prepare.py @@ -22,14 +22,15 @@ # along with this program. If not, see . """Test the nwp_prepare runner code.""" -import pytest -import unittest -from datetime import datetime, timezone -import os import logging -from datetime import timedelta -from nwcsafpps_runner.message_utils import prepare_nwp_message +import os +import unittest +from datetime import datetime, timedelta, timezone + +import pytest + import nwcsafpps_runner.prepare_nwp as nwc_prep +from nwcsafpps_runner.message_utils import prepare_nwp_message LOG = logging.getLogger(__name__) logging.basicConfig( diff --git a/nwcsafpps_runner/tests/test_pps_hook.py b/nwcsafpps_runner/tests/test_pps_hook.py index d1e7697..6daaa17 100644 --- a/nwcsafpps_runner/tests/test_pps_hook.py +++ b/nwcsafpps_runner/tests/test_pps_hook.py @@ -25,18 +25,18 @@ The message is created from metadata partly read from a yaml config file. """ -from datetime import datetime, timedelta import unittest -from unittest.mock import patch, Mock, MagicMock +from datetime import datetime, timedelta +from multiprocessing import Manager +from unittest.mock import MagicMock, Mock, patch + import pytest import yaml -from multiprocessing import Manager import nwcsafpps_runner -from nwcsafpps_runner.pps_posttroll_hook import MANDATORY_FIELDS_FROM_YAML -from nwcsafpps_runner.pps_posttroll_hook import SEC_DURATION_ONE_GRANULE -from nwcsafpps_runner.pps_posttroll_hook import PPSPublisher - +from nwcsafpps_runner.pps_posttroll_hook import (MANDATORY_FIELDS_FROM_YAML, + SEC_DURATION_ONE_GRANULE, + PPSPublisher) START_TIME1 = datetime.fromisoformat("2020-12-17T14:08:25.800000") END_TIME1 = datetime.fromisoformat("2020-12-17T14:09:50") diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index bd518a7..44cc716 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -20,14 +20,15 @@ """Test utility functions.""" import os import unittest -import pytest -from unittest.mock import patch, MagicMock from datetime import datetime +from unittest.mock import MagicMock, patch + +import pytest + from nwcsafpps_runner.utils import (create_xml_timestat_from_lvl1c, - get_lvl1c_file_from_msg, find_product_statistics_from_lvl1c, - ready2run, - publish_pps_files) + get_lvl1c_file_from_msg, publish_pps_files, + ready2run) TEST_MSG = """pytroll://segment/EPSSGA/1B/ file safusr.u@lxserv1043.smhi.se 2023-02-17T08:18:15.748831 v1.01 application/json {"start_time": "2023-02-17T08:03:25", "end_time": "2023-02-17T08:15:25", "orbit_number": 99999, "platform_name": "Metop-SG-A1", "sensor": "metimage", "format": "X", "type": "NETCDF", "data_processing_level": "1b", "variant": "DR", "orig_orbit_number": 23218, "uri": "/san1/polar_in/direct_readout/metimage/W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc", "uid": "W_XX-EUMETSAT-Darmstadt,SAT,SGA1-VII-1B-RAD_C_EUMT_20210314224906_G_D_20070912101704_20070912101804_T_B____.nc"}""" # noqa: E501 @@ -142,8 +143,9 @@ class TestPublishPPSFiles(unittest.TestCase): def test_publish_pps_files(self): """Test publish pps files.""" - from posttroll.message import Message from multiprocessing import Manager + + from posttroll.message import Message file1 = "S_NWC_CTTH_metopb_46878_20210930T0947019Z_20210930T1001458Z_statistics.xml" file2 = "S_NWC_CMA_metopb_46878_20210930T0947019Z_20210930T1001458Z_statistics.xml" scene = {'instrument': 'avhrr', 'platform_name': 'EOS-Terra', 'orbit_number': "46878"} diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index a4d8658..fead421 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -22,21 +22,20 @@ """Utility functions for NWCSAF/pps runner(s). """ -import threading -from trollsift.parser import parse # @UnresolvedImport -# from trollsift import Parser -from posttroll.message import Message # @UnresolvedImport -from subprocess import Popen, PIPE +import logging import os import shlex -from glob import glob import socket - +import threading +from glob import glob +from subprocess import PIPE, Popen from urllib.parse import urlparse from posttroll.address_receiver import get_local_ips +# from trollsift import Parser +from posttroll.message import Message # @UnresolvedImport +from trollsift.parser import parse # @UnresolvedImport -import logging LOG = logging.getLogger(__name__) @@ -265,7 +264,7 @@ def find_product_statistics_from_lvl1c(scene, pps_control_path): def create_pps_file_from_lvl1c(l1c_file_name, pps_control_path, name_tag, file_type): """From lvl1c file create name_tag-file of type file_type.""" - from trollsift import parse, compose + from trollsift import compose, parse f_pattern = 'S_NWC_{name_tag}_{platform_id}_{orbit_number}_{start_time}Z_{end_time}Z{file_type}' l1c_path, l1c_file = os.path.split(l1c_file_name) data = parse(f_pattern, l1c_file) diff --git a/setup.py b/setup.py index 6940aca..d8a95a3 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ """Setup for pps-runner. """ -from setuptools import setup, find_packages +from setuptools import find_packages, setup try: # HACK: https://github.com/pypa/setuptools_scm/issues/190#issuecomment-351181286 From 1f6f8ba3e5392fa5463f3ee1b3a3dafc9280b00b Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Mon, 15 Apr 2024 11:31:36 +0200 Subject: [PATCH 41/44] flake8 --- bin/pps_runner.py | 3 +-- bin/run_nwp_preparation.py | 2 ++ nwcsafpps_runner/prepare_nwp.py | 27 +++++++++---------- nwcsafpps_runner/tests/test_config.py | 10 +++---- nwcsafpps_runner/tests/test_level1c_runner.py | 8 +++--- nwcsafpps_runner/tests/test_utils.py | 1 + 6 files changed, 24 insertions(+), 27 deletions(-) diff --git a/bin/pps_runner.py b/bin/pps_runner.py index 4630b81..81d313a 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -21,8 +21,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Posttroll runner for the NWCSAF/PPS version >= v2018. -""" +"""Posttroll runner for the NWCSAF/PPS version >= v2018.""" import argparse import logging diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index 5bba956..4c60d1b 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -20,6 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +"""Runner for filtering NWP data for PPS.""" import argparse @@ -42,6 +43,7 @@ def prepare_and_publish(pub, options, flens): + """Prepare NWP files and publish.""" config_file_name = options.config_file starttime = datetime.now(tz=timezone.utc) - timedelta(days=1) ok_files, publish_topic = update_nwp(starttime, flens, config_file_name) diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index c755e39..ac9169d 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -20,8 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Prepare NWP data for PPS -""" +"""Prepare NWP data for PPS.""" import logging import os @@ -44,6 +43,7 @@ class NWPFileFamily(object): """Container for a nwp file family.""" def __init__(self, cfg, filename): + """Container for nwp-file object.""" self.nhsf_file = filename self.nhsp_file = filename.replace(cfg["nhsf_path"], cfg["nhsp_path"]).replace( cfg["nhsf_prefix"], cfg["nhsp_prefix"]) @@ -62,6 +62,7 @@ def __init__(self, cfg, filename): self.set_time_info(filename, cfg) def set_time_info(self, filename, cfg): + """Parse time info from a file.""" try: parser = Parser(cfg["nhsf_file_name_sift"]) except NoOptionError as noe: @@ -135,7 +136,9 @@ def update_nwp(starttime, nlengths, config_file_name): def should_be_skipped(file_obj, starttime, nlengths): - """Skip some files. Consider only analysis times newer than + """Skip some files. + + Consider only analysis times newer than *starttime*. And consider only the forecast lead times in hours given by the list *nlengths* of integers. Never reprocess. @@ -167,7 +170,6 @@ def get_files_to_process(cfg): def create_nwp_file(file_obj): """Create a new nwp file.""" - LOG.info("Result and tmp files:\n\t {:s}\n\t {:s}\n\t {:s}\n\t {:s}".format( file_obj.result_file, file_obj.tmp_filename, @@ -221,11 +223,12 @@ def create_nwp_file(file_obj): def update_nwp_inner(starttime, nlengths, cfg): - """Prepare NWP grib files for PPS. Consider only analysis times newer than + """Prepare NWP grib files for PPS. + + Consider only analysis times newer than *starttime*. And consider only the forecast lead times in hours given by the list *nlengths* of integers """ - LOG.info("Path to nhsf files: {:s}".format(cfg["nhsf_path"])) LOG.info("Path to nhsp files: {:s}".format(cfg["nhsp_path"])) LOG.info("nwp_output_prefix {:s}".format(cfg["nwp_output_prefix"])) @@ -264,9 +267,7 @@ def get_mandatory_and_all_fields(lines): def get_nwp_requirement(nwp_req_filename): - """Read the new requirement file. Return list with mandatory and wanted fields. - - """ + """Read the new requirement file. Return list with mandatory and wanted fields.""" try: with open(nwp_req_filename, 'r') as fpt: lines = fpt.readlines() @@ -279,9 +280,7 @@ def get_nwp_requirement(nwp_req_filename): def check_nwp_requirement(grb_entries, mandatory_fields, result_file): - """Check nwp file all mandatory enteries should be present. - - """ + """Check nwp file all mandatory enteries should be present.""" grb_entries.sort() for item in mandatory_fields: if item not in grb_entries: @@ -292,9 +291,7 @@ def check_nwp_requirement(grb_entries, mandatory_fields, result_file): def check_and_reduce_nwp_content(gribfile, result_file, nwp_req_filename): - """Check the content of the NWP file. Create a reduced file. - - """ + """Check the content of the NWP file. Create a reduced file.""" LOG.info("Get nwp requirements.") mandatory_fields, all_fields = get_nwp_requirement(nwp_req_filename) if mandatory_fields is None: diff --git a/nwcsafpps_runner/tests/test_config.py b/nwcsafpps_runner/tests/test_config.py index c0d1247..804f2a0 100644 --- a/nwcsafpps_runner/tests/test_config.py +++ b/nwcsafpps_runner/tests/test_config.py @@ -20,8 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Unit testing the config handling. -""" +"""Testing the config handling.""" import unittest from unittest.mock import patch @@ -81,7 +80,6 @@ @pytest.fixture def fake_files(tmp_path): """Create directory with test files.""" - file_l1c = tmp_path / 'lvl1c_file.yaml' file_h = open(file_l1c, 'w') file_h.write(TEST_YAML_LVL1C_RUNNER_CONTENT_OK) @@ -95,10 +93,10 @@ def fake_files(tmp_path): class TestGetConfig: - """Test getting the yaml config from file""" + """Test getting the yaml config from file.""" def test_read_lvl1c_runner_config(self, fake_files): - """Test loading and initialising the yaml config""" + """Test loading and initialising the yaml config.""" myconfig_filename, _ = fake_files result = get_config(myconfig_filename, service='seviri-l1c') @@ -115,7 +113,7 @@ def test_read_lvl1c_runner_config(self, fake_files): @patch('nwcsafpps_runner.config.socket.gethostname') def test_read_pps_runner_config(self, gethostname, fake_files): - """Test loading and initialising the yaml config""" + """Test loading and initialising the yaml config.""" _, myconfig_filename = fake_files gethostname.return_value = "my.local.host" result = get_config(myconfig_filename, add_defaults=True) diff --git a/nwcsafpps_runner/tests/test_level1c_runner.py b/nwcsafpps_runner/tests/test_level1c_runner.py index bd59ced..5803cef 100644 --- a/nwcsafpps_runner/tests/test_level1c_runner.py +++ b/nwcsafpps_runner/tests/test_level1c_runner.py @@ -20,8 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Unit testing the level-1c runner code -""" +"""Testing the level-1c runner code.""" import tempfile import time @@ -118,6 +117,7 @@ class MyFakePublisher(object): + """Fake publisher for testing.""" def __init__(self): pass @@ -137,11 +137,11 @@ def create_config_from_yaml(yaml_content_str): class TestPublishMessage(unittest.TestCase): + """Test publication of messages.""" @patch('nwcsafpps_runner.message_utils.socket.gethostname') def test_create_publish_message(self, gethostname): """Test the creation of the publish message.""" - gethostname.return_value = "my_local_server" my_fake_level1c_file = '/my/level1c/file/path/level1c.nc' input_msg = Message.decode(rawstr=TEST_INPUT_MSG) @@ -167,7 +167,6 @@ def test_create_publish_message(self, gethostname): @patch('nwcsafpps_runner.message_utils.Message.encode') def test_publish_messages(self, mock_message): """Test the sending the messages.""" - my_fake_publisher = MyFakePublisher() mock_message.return_value = "some pytroll message" @@ -196,6 +195,7 @@ class TestL1cProcessing(unittest.TestCase): """Test the L1c processing module.""" def setUp(self): + """Set up method.""" self.config_complete = create_config_from_yaml(TEST_YAML_CONTENT_OK) self.config_minimum = create_config_from_yaml(TEST_YAML_CONTENT_OK_MINIMAL) self.config_viirs_ok = create_config_from_yaml(TEST_YAML_CONTENT_VIIRS_OK) diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index 44cc716..f3c7ebe 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -129,6 +129,7 @@ class TestReady2Run(unittest.TestCase): @patch('nwcsafpps_runner.utils.check_host_ok') def test_ready2run(self, mock_check_host_ok): + """Test the ready to run function.""" from posttroll.message import Message input_msg = Message.decode(rawstr=TEST_MSG) mock_check_host_ok.return_value = True From c22870c3d05896d91cfd163983719a4223244e4e Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Mon, 15 Apr 2024 12:43:03 +0200 Subject: [PATCH 42/44] flake8 docstrings --- nwcsafpps_runner/config.py | 6 +++--- nwcsafpps_runner/l1c_processing.py | 6 +----- nwcsafpps_runner/logger.py | 3 +-- nwcsafpps_runner/message_utils.py | 8 ++------ nwcsafpps_runner/publish_and_listen.py | 15 +++++++------- nwcsafpps_runner/utils.py | 27 +++++++++----------------- setup.py | 3 +-- 7 files changed, 25 insertions(+), 43 deletions(-) diff --git a/nwcsafpps_runner/config.py b/nwcsafpps_runner/config.py index 89e63c8..346ca4c 100644 --- a/nwcsafpps_runner/config.py +++ b/nwcsafpps_runner/config.py @@ -20,8 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Reading configuration settings for NWCSAF/pps runner(s). -""" +"""Reading configuration settings for NWCSAF/PPS runner(s).""" import os import socket @@ -30,13 +29,14 @@ def load_config_from_file(filepath): - """Load the yaml config from file, given the file-path""" + """Load the yaml config from file, given the file-path.""" with open(filepath, 'r') as fp_: config = yaml.load(fp_, Loader=yaml.FullLoader) return config def move_service_dict_attributes_to_top_level(options, service): + """Mover attributes in dict service to top level.""" if service in options and isinstance(options[service], dict): service_config = options.pop(service) for key in service_config: diff --git a/nwcsafpps_runner/l1c_processing.py b/nwcsafpps_runner/l1c_processing.py index 33a23fc..f53e71b 100644 --- a/nwcsafpps_runner/l1c_processing.py +++ b/nwcsafpps_runner/l1c_processing.py @@ -20,8 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""The level-1c processing tools -""" +"""The level-1c processing tools.""" import logging from multiprocessing import Manager, Process, cpu_count @@ -170,7 +169,6 @@ def _get_message_data(self, message): def get_level1_files_from_dataset(self, level1_dataset): """Get the level-1 files from the dataset.""" - if self.service in ['seviri-l1c']: self.level1_files = get_seviri_level1_files_from_dataset(level1_dataset) else: @@ -179,7 +177,6 @@ def get_level1_files_from_dataset(self, level1_dataset): def check_platform_name_consistent_with_service(self): """Check that the platform name is consistent with the service name.""" - if self.platform_name.lower() not in SUPPORTED_SATELLITES.get(self.service, []): errmsg = ("%s: Platform name not supported for this service: %s", str(self.platform_name), self.service) @@ -188,7 +185,6 @@ def check_platform_name_consistent_with_service(self): def get_seviri_level1_files_from_dataset(level1_dataset): """Get the seviri level-1 filenames from the dataset and return as list.""" - pro_files = False epi_files = False level1_files = [] diff --git a/nwcsafpps_runner/logger.py b/nwcsafpps_runner/logger.py index c6459ac..76a820c 100644 --- a/nwcsafpps_runner/logger.py +++ b/nwcsafpps_runner/logger.py @@ -20,8 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""The log handling. -""" +"""The log handling.""" import logging import logging.config diff --git a/nwcsafpps_runner/message_utils.py b/nwcsafpps_runner/message_utils.py index 2aeee1f..157c0d7 100644 --- a/nwcsafpps_runner/message_utils.py +++ b/nwcsafpps_runner/message_utils.py @@ -20,13 +20,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Message utilities. -""" +"""Message utilities.""" import logging import os -import socket -from urllib.parse import urlunsplit from posttroll.message import Message @@ -34,6 +31,7 @@ def prepare_nwp_message(result_file, publish_topic): + """Prepare message for NWP files.""" to_send = {} to_send["uri"] = result_file filename = os.path.basename(result_file) @@ -46,7 +44,6 @@ def prepare_nwp_message(result_file, publish_topic): def prepare_l1c_message(result_file, mda, **kwargs): """Prepare the output message for the level-1c file creation.""" - if not result_file: return @@ -79,7 +76,6 @@ def prepare_l1c_message(result_file, mda, **kwargs): def publish_l1c(publisher, publish_msg, publish_topic): """Publish the messages that l1c files are ready.""" - LOG.debug('Publish topic = %s', publish_topic) for topic in publish_topic: msg = Message(topic, "file", publish_msg).encode() diff --git a/nwcsafpps_runner/publish_and_listen.py b/nwcsafpps_runner/publish_and_listen.py index 60e5c62..0595ad8 100644 --- a/nwcsafpps_runner/publish_and_listen.py +++ b/nwcsafpps_runner/publish_and_listen.py @@ -20,8 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Publisher and Listener classes for the PPS runners. -""" +"""Publisher and Listener classes for the PPS runner.""" import logging import threading @@ -38,18 +37,19 @@ class FileListener(threading.Thread): def __init__(self, queue, subscribe_topics): + """Init the file listener.""" threading.Thread.__init__(self) self.loop = True self.queue = queue self.subscribe_topics = subscribe_topics def stop(self): - """Stops the file listener.""" + """Stop the file listener.""" self.loop = False self.queue.put(None) def run(self): - + """Run the file listener.""" LOG.debug("Subscribe topics = %s", str(self.subscribe_topics)) with posttroll.subscriber.Subscribe("", self.subscribe_topics, True) as subscr: @@ -64,7 +64,7 @@ def run(self): self.queue.put(msg) def check_message(self, msg): - + """Check the message.""" if not msg: return False @@ -93,6 +93,7 @@ class FilePublisher(threading.Thread): """ def __init__(self, queue, publish_topic, **kwargs): + """Init the file publisher.""" threading.Thread.__init__(self) self.loop = True self.queue = queue @@ -107,12 +108,12 @@ def __init__(self, queue, publish_topic, **kwargs): self.nameservers = [self.nameservers] def stop(self): - """Stops the file publisher.""" + """Stop the file publisher.""" self.loop = False self.queue.put(None) def run(self): - + """Run the file publisher.""" with Publish(self.runner_name, 0, self.publish_topic, nameservers=self.nameservers) as publisher: while self.loop: diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index fead421..87b76b9 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -20,8 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Utility functions for NWCSAF/pps runner(s). -""" +"""Utility functions for NWCSAF/pps runner(s).""" + import logging import os import shlex @@ -47,6 +47,10 @@ class FindTimeControlFileError(Exception): pass +class PpsRunError(Exception): + pass + + PPS_OUT_PATTERN = ("S_NWC_{segment}_{orig_platform_name}_{orbit_number:05d}_" + "{start_time:%Y%m%dT%H%M%S%f}Z_{end_time:%Y%m%dT%H%M%S%f}Z.{extention}") PPS_OUT_PATTERN_MULTIPLE = ("S_NWC_{segment1}_{segment2}_{orig_platform_name}_{orbit_number:05d}_" + @@ -131,9 +135,7 @@ def run_command(cmdstr): def check_uri(uri): - """Check that the provided *uri* is on the local host and return the - file path. - """ + """Check that the provided *uri* is on the local host and return the file path.""" if isinstance(uri, (list, set, tuple)): paths = [check_uri(ressource) for ressource in uri] return paths @@ -155,10 +157,6 @@ def check_uri(uri): return url.path -class PpsRunError(Exception): - pass - - def get_lvl1c_file_from_msg(msg): """Get level1c file from msg.""" destination = msg.data.get('destination') @@ -200,7 +198,6 @@ def check_host_ok(msg): def ready2run(msg, scene, **kwargs): """Check whether pps is ready to run or not.""" - LOG.info("Got message: " + str(msg)) if not check_host_ok(msg): return False @@ -230,10 +227,7 @@ def terminate_process(popen_obj, scene): def create_pps_call_command(python_exec, pps_script_name, scene): - """Create the pps call command. - - Supports PPSv2021. - """ + """Create the pps call command.""" cmdstr = ("%s" % python_exec + " %s " % pps_script_name + "-af %s" % scene['file4pps']) LOG.debug("PPS call command: %s", str(cmdstr)) @@ -295,10 +289,7 @@ def create_xml_timestat_from_ascii(infile, pps_control_path): def publish_pps_files(input_msg, publish_q, scene, result_files, **kwargs): - """ - Publish messages for the files provided. - """ - + """Publish messages for the files provided.""" servername = kwargs.get('servername') station = kwargs.get('station', 'unknown') diff --git a/setup.py b/setup.py index d8a95a3..e7121b4 100644 --- a/setup.py +++ b/setup.py @@ -21,8 +21,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Setup for pps-runner. -""" +"""Setup for pps-runner.""" from setuptools import find_packages, setup try: From 13bb3d2b7228480fd4b03662d021911342fc87e4 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Mon, 15 Apr 2024 12:55:39 +0200 Subject: [PATCH 43/44] fix tests after flake8 --- nwcsafpps_runner/tests/test_level1c_runner.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nwcsafpps_runner/tests/test_level1c_runner.py b/nwcsafpps_runner/tests/test_level1c_runner.py index 5803cef..f6a86f7 100644 --- a/nwcsafpps_runner/tests/test_level1c_runner.py +++ b/nwcsafpps_runner/tests/test_level1c_runner.py @@ -139,10 +139,8 @@ def create_config_from_yaml(yaml_content_str): class TestPublishMessage(unittest.TestCase): """Test publication of messages.""" - @patch('nwcsafpps_runner.message_utils.socket.gethostname') - def test_create_publish_message(self, gethostname): + def test_create_publish_message(self): """Test the creation of the publish message.""" - gethostname.return_value = "my_local_server" my_fake_level1c_file = '/my/level1c/file/path/level1c.nc' input_msg = Message.decode(rawstr=TEST_INPUT_MSG) From c97711765c810af3bf6f88bab6b050c533ee92d6 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Mon, 15 Apr 2024 12:57:06 +0200 Subject: [PATCH 44/44] Remove unused duplicate function There is a logreader function also in the utils module. --- nwcsafpps_runner/prepare_nwp.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index ac9169d..608b20d 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -105,15 +105,6 @@ def prepare_config(config_file_name): return cfg -def logreader(stream, log_func): - while True: - s = stream.readline() - if not s: - break - log_func(s.strip()) - stream.close() - - def remove_file(filename): """Remove a temporary file.""" if os.path.exists(filename):