diff --git a/README.md b/README.md index c3bb1d4..e46604c 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,14 @@ Command-line interface created by setup-py. * Encoding JSON-formated values to BUFR. ## News +`Vers. 0.10.0` +All code supports Python2 *and* Python3, without code-conversion (i.e. by 2to3). + +With Python3 the `setup.py` installs the command-line script with a different +name `trollbufr3`, to reflect its usage of Python3 and to have both versions +callable. + +`Vers. 0.6.0` With version 0.6.0 a feature for encoding a JSON formatted file into binary BUFR is added. @@ -38,5 +46,4 @@ New is `-j` to write any output in a JSON format. ## To-Do There are still things to do: -* Support Python3 * Implement the remaining obscure operators diff --git a/setup.py b/setup.py index 8798af3..fb40286 100644 --- a/setup.py +++ b/setup.py @@ -29,32 +29,53 @@ version = imp.load_source("trollbufr.version", "trollbufr/version.py") -requires = ["bitstring"] +requires = ["bitstring", "six"] -if sys.version_info < (2, 7): - requires.append("argparse") - -setup(name="trollbufr", - version=version.__version__, - description="Reading meteorological data format BUFR in pure Python", - author="Alexander Maul", - author_email="alexander.maul@dwd.de", - classifiers=["Development Status :: 4 - Beta", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: GNU Lesser General Public License v3 " + - "or later (LGPLv3+)", - "Operating System :: OS Independent", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Topic :: Scientific/Engineering" - ], - test_suite="bufr.tests.suite", - entry_points={ - "console_scripts": ["trollbufr = trollbufr.bufr_main:run", - "trollbufr_update = trollbufr.update:run"]}, - packages=["trollbufr", "trollbufr.coder"], - install_requires=requires, - python_requires=">=2.6, <3", - zip_safe=False, - ) +if sys.version_info >= (3, 0): + setup(name="trollbufr", + version=version.version, + description="Reading meteorological data format BUFR in pure Python", + author="Alexander Maul", + author_email="alexander.maul@dwd.de", + classifiers=["Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU Lesser General Public License v3 " + + "or later (LGPLv3+)", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Topic :: Scientific/Engineering" + ], + test_suite="bufr.tests.suite", + entry_points={ + "console_scripts": ["trollbufr3 = trollbufr.bufr_main:run", + "trollbufr3_update = trollbufr.update:run"]}, + packages=["trollbufr", "trollbufr.coder"], + install_requires=requires, + python_requires=">=3.4", + zip_safe=False, + ) +else: + setup(name="trollbufr", + version=version.version, + description="Reading meteorological data format BUFR in pure Python", + author="Alexander Maul", + author_email="alexander.maul@dwd.de", + classifiers=["Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU Lesser General Public License v3 " + + "or later (LGPLv3+)", + "Operating System :: OS Independent", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Topic :: Scientific/Engineering" + ], + test_suite="bufr.tests.suite", + entry_points={ + "console_scripts": ["trollbufr = trollbufr.bufr_main:run", + "trollbufr_update = trollbufr.update:run"]}, + packages=["trollbufr", "trollbufr.coder"], + install_requires=requires, + python_requires=">=2.6, <3", + zip_safe=False, + ) diff --git a/trollbufr/bufr.py b/trollbufr/bufr.py index ee7daa4..356c77a 100644 --- a/trollbufr/bufr.py +++ b/trollbufr/bufr.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016-2018 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -30,15 +32,15 @@ which has the iterator function `next_data()` to iterate over all bin_data elements in this subset. """ -from coder.load_tables import TableCache -import coder.bufr_sect as sect -from coder.subset import SubsetReader, SubsetWriter -from coder.bdata import Blob -from coder.tables import TabBElem -from coder.functions import (descr_is_data, descr_is_loop, descr_is_oper, - descr_is_seq, descr_is_nil, get_descr_list) -from coder.errors import (SUPPORTED_BUFR_EDITION, BufrDecodeError, - BufrDecodeWarning, BufrTableError, BufrEncodeError) +from trollbufr.coder.load_tables import TableCache +from trollbufr.coder import bufr_sect as sect +from trollbufr.coder.subset import SubsetReader, SubsetWriter +from trollbufr.coder.bdata import Blob +from trollbufr.coder.tables import TabBElem +from trollbufr.coder.functions import (descr_is_data, descr_is_loop, descr_is_oper, + descr_is_seq, descr_is_nil, get_descr_list) +from trollbufr.coder.errors import (SUPPORTED_BUFR_EDITION, BufrDecodeError, + BufrDecodeWarning, BufrTableError, BufrEncodeError) import logging logger = logging.getLogger("trollbufr") @@ -321,7 +323,7 @@ def decode_meta(self, bin_data, load_tables=True): if load_tables: try: self._tables = self.load_tables() - except StandardError or Warning as exc: + except Exception or Warning as exc: tables_fail = exc # # Section 2 @@ -525,7 +527,7 @@ def encode(self, json_data, load_tables=True): if load_tables and not self._tables: try: self._tables = self.load_tables() - except StandardError or Warning as exc: + except Exception or Warning as exc: raise exc # # Section 2 diff --git a/trollbufr/bufr_main.py b/trollbufr/bufr_main.py index 0b9fc9c..58b00e9 100644 --- a/trollbufr/bufr_main.py +++ b/trollbufr/bufr_main.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016,2017 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -26,15 +28,16 @@ if present) from file(s) and writes human-readable to stdout. ''' from __future__ import print_function +from __future__ import absolute_import import sys import os from argparse import ArgumentParser, RawDescriptionHelpFormatter -from version import __version__ -from bufr import Bufr -from coder.bufr_types import TabBType -import load_file -import coder.load_tables +from trollbufr.version import version +from trollbufr.bufr import Bufr +from trollbufr.coder.bufr_types import TabBType +from trollbufr import load_file +from trollbufr.coder import load_tables import logging logger = logging.getLogger("trollbufr") @@ -48,7 +51,7 @@ def read_bufr_data(args): per all subsets at once, which improves performance for compressed BUFR. """ try: - fh_out = open(args.out_file, "wb") + fh_out = open(args.out_file, "w") except: fh_out = sys.stdout bufr = Bufr(args.tables_type, args.tables_path) @@ -137,7 +140,7 @@ def read_bufr_data(args): d_name, str(descr_entry.value), d_unit), file=fh_out) - except StandardError as e: + except Exception as e: print("ERROR\t%s" % e, file=fh_out) if logger.isEnabledFor(logging.DEBUG): logger.exception(e) @@ -169,7 +172,7 @@ def read_bufr_to_json(args): json_bufr = bufr.decode(blob, load_tables=True, as_array=args.array) - except StandardError as e: + except Exception as e: logger.error(e, exc_info=1 and logger.isEnabledFor(logging.DEBUG)) json_data_item["error"] = str(e) else: @@ -178,7 +181,7 @@ def read_bufr_to_json(args): finally: json_data.append(json_data_item) import json - out_fh = open(args.out_file, "wb") or sys.stdout + out_fh = open(args.out_file, "w") or sys.stdout with out_fh as fh_out: if args.sparse: json.dump(json_data, fh_out) @@ -190,7 +193,7 @@ def read_bufr_desc(args): """Read BUFR(s), decode meta-data and descriptor list, write to file-handle. """ try: - fh_out = open(args.out_file, "wb") + fh_out = open(args.out_file, "w") except: fh_out = sys.stdout for fn_in in args.in_file: @@ -212,7 +215,7 @@ def read_bufr_desc(args): else: d = bufr.get_descr_full() print("DESC :\n%s" % "\n".join(d), file=fh_out) - except StandardError as e: + except Exception as e: print("ERROR\t%s" % e, file=fh_out) if logger.isEnabledFor(logging.DEBUG): logger.exception(e) @@ -230,7 +233,7 @@ def write_bufr(args): fh_out = sys.stdout multi_bul = False for fn_in in args.in_file: - with open(fn_in, "rb") as fh_in: + with open(fn_in, "r") as fh_in: json_data = json.load(fh_in) for json_data_msg in json_data: if not "bufr" in json_data_msg or json_data_msg["bufr"] is None: @@ -240,9 +243,9 @@ def write_bufr(args): bin_data = bufr.encode(json_data_msg["bufr"], load_tables=True) if json_data_msg["heading"] is not None: - multi_bul and print("\r\r\n\r\r\n", end="", file=fh_out) - print("%s\r\r" % json_data_msg["heading"], file=fh_out) - print(bin_data, end="", file=fh_out) + multi_bul and fh_out.write(b"\r\r\n\r\r\n") + fh_out.write(("%s\r\r\n" % json_data_msg["heading"]).encode()) + fh_out.write(bin_data) multi_bul = True if fh_out is not sys.stdout: fh_out.close() @@ -254,7 +257,7 @@ def run(argv=None): argv = sys.argv else: sys.argv.extend(argv) - program_version = __version__ + program_version = version try: # Setup argument parser parser = ArgumentParser(description=__import__('__main__').__doc__, @@ -306,11 +309,11 @@ def run(argv=None): metavar="path" ) group_tab.add_argument("-T", "--tables_type", - default=coder.load_tables.list_parser()[0], - choices=coder.load_tables.list_parser(), + default=load_tables.list_parser()[0], + choices=load_tables.list_parser(), help="type of table format [%s], default: %s" % ( - "|".join(coder.load_tables.list_parser()), - coder.load_tables.list_parser()[0] + "|".join(load_tables.list_parser()), + load_tables.list_parser()[0] ), metavar="name" ) @@ -375,7 +378,7 @@ def run(argv=None): except KeyboardInterrupt: return 0 - except StandardError as e: + except Exception as e: if logger.isEnabledFor(logging.DEBUG): logger.exception(e) else: diff --git a/trollbufr/coder/bdata.py b/trollbufr/coder/bdata.py index 5274f59..4629772 100644 --- a/trollbufr/coder/bdata.py +++ b/trollbufr/coder/bdata.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -48,7 +50,7 @@ def __init__(self, bin_data=None, rw=False): self.reset() def __str__(self): - return "%dB %d/%d" % (len(self._data) / 8, self._data.pos // 8, self._data.pos % 8) + return "%dB %d/%d" % (len(self._data) // 8, self._data.pos // 8, self._data.pos % 8) def __len__(self): return len(self._data) @@ -93,7 +95,7 @@ def writelist(self, fmt, json_data): def read_align(self, even=False): p = self._data.pos self._data.bytealign() - if even and (self._data.pos / 8) & 1: + if even and (self._data.pos // 8) & 1: self._data.pos += 8 return self._data.pos - p diff --git a/trollbufr/coder/bufr_sect.py b/trollbufr/coder/bufr_sect.py index 22b37a4..e5eb23d 100644 --- a/trollbufr/coder/bufr_sect.py +++ b/trollbufr/coder/bufr_sect.py @@ -24,11 +24,12 @@ No bin_data json_data from section 4 are decoded. Created on Nov 18, 2016 +Ported to Py3 09/2018 @author: amaul ''' -from errors import SUPPORTED_BUFR_EDITION, BufrEncodeError -from functions import str2dtg, dtg2str +from .errors import SUPPORTED_BUFR_EDITION, BufrEncodeError +from .functions import str2dtg, dtg2str """ Section 0 @@ -45,16 +46,21 @@ def decode_sect0(bin_data, offset): """ keys = ["bufr", "size", "edition"] vals = bin_data.readlist("bytes:4, uintbe:24, uint:8") - if vals[0] != "BUFR": - return -1, {} - return bin_data.get_point(), 8, dict(zip(keys[1:], vals[1:])) + if vals[0] != b"BUFR": + return -1, -1, {} + return bin_data.get_point(), 8, dict(list(zip(keys[1:], vals[1:]))) def encode_sect0(bin_data, edition=4): """ :return: section start offset, meta-dict """ - bin_data.writelist("bytes:4={}, uintbe:24={}, uint:8={}", ("BUFR", 0, edition)) + # Originally: + # bin_data.writelist("bytes:4={}, uintbe:24={}, uint:8={}", ("BUFR", 0, edition)) + # The next two lines are a workaround, since in Py3 bitstring seems to + # evaluate "bytes:" incorrectly. + bin_data.write_bytes("BUFR") + bin_data.writelist("uintbe:24={}, uint:8={}", (0, edition)) return 0, {"edition": edition} @@ -121,12 +127,12 @@ def decode_sect1(bin_data, offset, edition=4): ), } vals = bin_data.readlist(key_offs[edition][1]) - rd = dict(zip(key_offs[edition][0], vals)) + rd = dict(list(zip(key_offs[edition][0], vals))) rd["datetime"] = str2dtg(rd["datetime"], ed=edition) l = rd.pop("length") if bin_data.get_point() < offset + l: rd["sect1_local_use"] = bin_data.readlist("hex:8," * (offset + l - bin_data.get_point())) - if edition == 3 and rd["sect1_local_use"] == ["00"]: + if edition == 3 and rd["sect1_local_use"] == [b"00"]: rd.pop("sect1_local_use") bin_data.reset(offset + l) return offset + l, l, rd @@ -134,8 +140,8 @@ def decode_sect1(bin_data, offset, edition=4): def encode_sect1(bin_data, json_data, edition=4): """ - :param json_data: list or tuple with slots - (master, center, subcenter, update, sect2, cat, cat_int, cat_loc, mver, lver, + :param json_data: list or tuple with slots + (master, center, subcenter, update, sect2, cat, cat_int, cat_loc, mver, lver, str2dtg-yy, str2dtg-mo, str2dtg-dy, str2dtg-hh, str2dtg-mi, str2dtg-ss) :return: section start offset, meta-dict """ @@ -170,7 +176,7 @@ def encode_sect1(bin_data, json_data, edition=4): loc_use = json_data[-1] json_data = json_data[:-1] ord_val = json_data - rd = dict(zip(key_offs[edition][0], json_data[:-6])) + rd = dict(list(zip(key_offs[edition][0], json_data[:-6]))) rd["datetime"] = dtg2str(json_data[-6:], edition) section_start = len(bin_data) bin_data.writelist(key_offs[edition][1], ord_val) @@ -202,7 +208,7 @@ def decode_sect2(bin_data, offset): l = bin_data.readlist("uint:24, pad:8")[0] s2data = bin_data.readlist("hex:8," * (l - 4)) bin_data.reset(offset + l) - return offset + l, l, {'data_start': offset + 4, 'data_end': offset + l, "sect2_data": s2data} + return offset + l, l, {"data_start": offset + 4, "data_end": offset + l, "sect2_data": s2data} def encode_sect2(bin_data, json_data): @@ -238,7 +244,7 @@ def decode_sect3(bin_data, offset): """ keys = ("length", "subsets", "obs", "comp") vals = bin_data.readlist("uint:24, pad:8, uint:16, bool, bool, pad:6") - rd = dict(zip(keys, vals)) + rd = dict(list(zip(keys, vals))) l = rd.pop("length") desc = [] while bin_data.get_point() < offset + l - 1: @@ -286,7 +292,7 @@ def decode_sect4(bin_data, offset): """ l = bin_data.read("uint:24") bin_data.reset(offset + l) - return offset + l, l, {'data_start': offset + 4, 'data_end': offset + l} + return offset + l, l, {"data_start": offset + 4, "data_end": offset + l} def encode_sect4(bin_data, edition=4): @@ -316,7 +322,7 @@ def decode_sect5(bin_data, offset): """ :return: offset, length, {} """ - if bin_data.read("bytes:4") == "7777": + if bin_data.read("bytes:4") == b"7777": return offset + 4, 4, {} else: return -1, -1, {} @@ -327,5 +333,5 @@ def encode_sect5(bin_data): :return: section start offset """ section_start = len(bin_data) - bin_data.write_bytes("7777") + bin_data.write_bytes(b"7777") return section_start // 8 diff --git a/trollbufr/coder/bufr_types.py b/trollbufr/coder/bufr_types.py index 7606411..b6f915d 100644 --- a/trollbufr/coder/bufr_types.py +++ b/trollbufr/coder/bufr_types.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016-2018 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -144,7 +146,7 @@ def apply(self, bitmap): if bitmap[len(bitmap) - i] == 0] self._stack_idx = 0 - def next(self): + def __next__(self): """Return next descriptor/alter pair from stack.""" if self._stack_idx >= len(self._backref_stack): raise StopIteration @@ -152,6 +154,10 @@ def next(self): self._stack_idx += 1 return r + def next(self): + """Return next descriptor/alter pair from stack.""" + return self.__next__() + def pause(self, paused=True): """Pause or re-activate recording.""" self._recording = not paused diff --git a/trollbufr/coder/errors.py b/trollbufr/coder/errors.py index 994fe64..ba17465 100644 --- a/trollbufr/coder/errors.py +++ b/trollbufr/coder/errors.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -29,7 +31,7 @@ """List of supported BUFR editions.""" -class BufrDecodeError(StandardError): +class BufrDecodeError(Exception): '''Error class, raised if anything prevents further decoding''' def __init__(self, msg): @@ -57,7 +59,7 @@ def __unicode__(self): return self.msg -class BufrEncodeError(StandardError): +class BufrEncodeError(Exception): '''Error class, raised if anything prevents further encoding''' def __init__(self, msg): @@ -85,7 +87,7 @@ def __unicode__(self): return self.msg -class BufrTableError(StandardError): +class BufrTableError(Exception): '''Error class, raised if anything prevents further decoding''' def __init__(self, msg): diff --git a/trollbufr/coder/functions.py b/trollbufr/coder/functions.py index a24c2a2..e7cdf86 100644 --- a/trollbufr/coder/functions.py +++ b/trollbufr/coder/functions.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016-2018 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -26,15 +28,22 @@ @author: amaul """ - +import sys import datetime import logging import struct -from errors import BufrDecodeError, BufrEncodeError, BufrTableError -from bufr_types import AlterState, TabBType +from .errors import BufrDecodeError, BufrEncodeError, BufrTableError +from .bufr_types import AlterState, TabBType logger = logging.getLogger("trollbufr") +if sys.version_info >= (3, 0): + def ordx(x): + return x +else: + def ordx(x): + return ord(x) + def octets2num(bin_data, offset, count): """Convert character slice of length count from bin_data (high->low) to int. @@ -47,7 +56,7 @@ def octets2num(bin_data, offset, count): v = 0 i = count - 1 for b in bin_data[offset: offset + count]: - v |= ord(b) << 8 * i + v |= ordx(b) << 8 * i i -= 1 return offset + count, v @@ -205,7 +214,7 @@ def rval2str(rval): rval >>= 8 octets.reverse() val = "".join(octets) - return val.decode("latin1").encode("utf8") + return val _IEEE_INF = {32: ("f", 0x7f7fffff), 64: ("d", 0x7fefffffffffffff)} @@ -249,7 +258,8 @@ def rval2num(tab_b_elem, alter, rval): # First, test if all bits are set, which usually means "missing value". # The delayed replication and repetition descr are special nut-cases. logger.debug("rval %d ==_(1<<%d)%d #%06d/%d", rval, loc_width, - all_one(loc_width), tab_b_elem.descr, tab_b_elem.descr / 1000) + all_one(loc_width), tab_b_elem.descr, + tab_b_elem.descr // 1000) val = None elif alter.ieee and (tab_b_elem.typ == TabBType.DOUBLE or tab_b_elem.typ == TabBType.LONG): @@ -265,7 +275,7 @@ def rval2num(tab_b_elem, alter, rval): val = float(rval + loc_refval) / 10 ** loc_scale elif tab_b_elem.typ == TabBType.LONG: # Integer: add reference, divide by scale - val = (rval + loc_refval) / 10 ** loc_scale + val = int((rval + loc_refval) / 10 ** loc_scale) elif tab_b_elem.typ == TabBType.STRING: val = rval2str(rval) else: @@ -345,7 +355,13 @@ def num2cval(tab_b_elem, alter, fix_width, value_list): :return: loc_width, min_value, min_width, recal_val """ rval_list = [] - if not any(True for x in value_list if x is not None) or max(value_list) == min(value_list): + value_list_sansnone = [x for x in value_list if x is not None] + logger.debug("value_list:\n%s\nvalue_list_sansnone:\n%s\n", value_list, value_list_sansnone) + if (not any(True for x in value_list if x is not None) + or (max(value_list_sansnone) == min(value_list_sansnone) + and len(value_list) == len(value_list_sansnone) + ) + ): # All values are "missing", or all are equal if tab_b_elem and alter: min_value, loc_width = num2rval(tab_b_elem, alter, value_list[0]) @@ -374,11 +390,11 @@ def num2cval(tab_b_elem, alter, fix_width, value_list): for v in value_list: rval_list.extend((v, fix_width)) loc_width = rval_list[1] - min_value = min(rval_list[::2]) + min_value = min(x for x in rval_list[::2] if x is not None) min_width = 0 recal_val = [(v - min_value if v != all_one(loc_width) else None) for v in rval_list[::2]] - recal_max_val = max(recal_val) + recal_max_val = max(x for x in recal_val if x is not None) min_width = recal_max_val.bit_length() if recal_max_val == all_one(min_width): min_width += 1 @@ -468,7 +484,7 @@ def mk_value_list(value_list, value_list_idx): # Build a list of this value from all subsets. try: val_l = [x[value_list_idx] for x in value_list] - except StandardError as e: + except Exception as e: logger.error("%d # %s", value_list_idx, value_list) raise e else: diff --git a/trollbufr/coder/load_tables.py b/trollbufr/coder/load_tables.py index b4ddbc1..c5f73fe 100644 --- a/trollbufr/coder/load_tables.py +++ b/trollbufr/coder/load_tables.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -27,8 +29,8 @@ import logging import os from importlib import import_module -from errors import BufrTableError -from tables import Tables +from .errors import BufrTableError +from .tables import Tables logger = logging.getLogger("trollbufr") @@ -94,7 +96,7 @@ def load_all(master, center, subcenter, master_vers, local_vers, base_path, tabf mp, _ = tparse.get_file("A", base_path, master, center, subcenter, master_vers, local_vers) tparse.load_tab_a(tables, mp) logger.info(_text_tab_loaded, mp) - except StandardError as e: + except Exception as e: logger.warning(e) # # Table B (elements) @@ -107,7 +109,7 @@ def load_all(master, center, subcenter, master_vers, local_vers, base_path, tabf if local_vers: tparse.load_tab_b(tables, lp) logger.info(_text_tab_loaded, lp) - except StandardError as e: + except Exception as e: logger.error(e) raise e # @@ -116,7 +118,7 @@ def load_all(master, center, subcenter, master_vers, local_vers, base_path, tabf mp, _ = tparse.get_file("C", base_path, master, center, subcenter, master_vers, local_vers) tparse.load_tab_c(tables, mp) logger.info(_text_tab_loaded, mp) - except StandardError as e: + except Exception as e: logger.warning(e) # # Table D (sequences) @@ -129,7 +131,7 @@ def load_all(master, center, subcenter, master_vers, local_vers, base_path, tabf if local_vers: tparse.load_tab_d(tables, lp) logger.info(_text_tab_loaded, lp) - except StandardError as e: + except Exception as e: logger.error(e) raise e # @@ -143,7 +145,7 @@ def load_all(master, center, subcenter, master_vers, local_vers, base_path, tabf if local_vers: tparse.load_tab_cf(tables, lp) logger.info(_text_tab_loaded, lp) - except StandardError as er: + except Exception as er: logger.warning(er) return tables diff --git a/trollbufr/coder/operator.py b/trollbufr/coder/operator.py index 830bb09..542fb99 100644 --- a/trollbufr/coder/operator.py +++ b/trollbufr/coder/operator.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016-2018 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -26,9 +28,9 @@ @author: amaul """ -import functions as fun -from errors import BufrDecodeError, BufrEncodeError -from bufr_types import DescrDataEntry, TabBType +from . import functions as fun +from .errors import BufrDecodeError, BufrEncodeError +from .bufr_types import DescrDataEntry, TabBType import logging logger = logging.getLogger("trollbufr") @@ -226,7 +228,7 @@ def fun_07(subset, descr): else: subset._alter.scale = an subset._alter.refmul = 10 ^ an - subset._alter.wnum = ((10 * an) + 2) / 3 + subset._alter.wnum = ((10 * an) + 2) // 3 return None @@ -294,7 +296,7 @@ def fun_statistic_read(subset, descr, an): l_rval = DescrDataEntry(descr, "OPR", en[0], None) elif an == 255: """Statistical values marker operator.""" - bar = subset._backref_record.next() + bar = next(subset._backref_record) if bar: val = subset.get_val(subset._blob, subset.subs_num, @@ -315,7 +317,7 @@ def fun_statistic_write(subset, descr, an): # Filter back-references by bitmap elif an == 255: """Statistical values marker operator.""" - bar = subset._backref_record.next() + bar = next(subset._backref_record) subset.add_val(subset._blob, subset._vl, subset._vi, tab_b_elem=bar[0], alter=bar[1]) subset._vi += 1 else: diff --git a/trollbufr/coder/parse_bufrdc.py b/trollbufr/coder/parse_bufrdc.py index 9fcc133..7e79305 100644 --- a/trollbufr/coder/parse_bufrdc.py +++ b/trollbufr/coder/parse_bufrdc.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -28,8 +30,8 @@ import logging import os -from errors import BufrTableError -from tables import TabBElem +from .errors import BufrTableError +from .tables import TabBElem logger = logging.getLogger("trollbufr") @@ -110,7 +112,7 @@ def load_tab_a(tables, fname): """Load table A (data category) from 'fname' into object Tables.""" # if not os.path.exists(fname): # raise BufrTableError(_text_file_not_found % fname) -# with open(fname, "rb") as fh: +# with open(fname, "r") as fh: # for line in fh: # if line[0]=="#" or len(line) < 3: # continue @@ -131,10 +133,10 @@ def load_tab_b(tables, fname): """Load table B (elements) from 'fname' into object Tables.""" if not os.path.exists(fname): raise BufrTableError(_text_file_not_found % fname) - with open(fname, "rb") as fh: + with open(fname, "r") as fh: for line in fh: try: - if line[0]=="#" or len(line) < 3: + if line[0] == "#" or len(line) < 3: continue e = None el_descr = int(line[1:7]) @@ -152,7 +154,7 @@ def load_tab_b(tables, fname): # descr, typ, unit, abbrev, full_name, scale, refval, width e = TabBElem(el_descr, el_typ, el_unit, None, el_full_name, el_scale, el_refval, el_width) tables.tab_b[int(el_descr)] = e - except StandardError as exc: + except Exception as exc: logger.warning("Corrupt table %s (%s)", fname, line[0:8]) logger.warning(exc) return True @@ -162,7 +164,7 @@ def load_tab_c(tables, fname): """Load table C (operators) from 'fname' into object Tables.""" # if not os.path.exists(fname): # raise BufrTableError(_text_file_not_found % fname) -# with open(fname, "rb") as fh: +# with open(fname, "r") as fh: # for line in fh: # if line[0]=="#" or len(line) < 3: # continue @@ -185,11 +187,11 @@ def load_tab_d(tables, fname): """Load table D (sequences) from 'fname' into object Tables.""" if not os.path.exists(fname): raise BufrTableError(_text_file_not_found % fname) - with open(fname, "rb") as fh: + with open(fname, "r") as fh: desc = None e = [] for line in fh: - if line[0]=="#" or len(line) < 3: + if line[0] == "#" or len(line) < 3: continue try: le = (line[1:7], line[7:10], line[10:17]) @@ -215,10 +217,10 @@ def load_tab_cf(tables, fname): """ if not os.path.exists(fname): raise BufrTableError(_text_file_not_found % fname) - with open(fname, "rb") as fh: + with open(fname, "r") as fh: la = ["" * 5] for line in fh: - if line[0]=="#" or len(line) < 3: + if line[0] == "#" or len(line) < 3: continue l = line.rstrip() try: diff --git a/trollbufr/coder/parse_eccodes.py b/trollbufr/coder/parse_eccodes.py index cac526d..4fae10a 100644 --- a/trollbufr/coder/parse_eccodes.py +++ b/trollbufr/coder/parse_eccodes.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -30,8 +32,8 @@ import os import re -from errors import BufrTableError -from tables import TabBElem +from .errors import BufrTableError +from .tables import TabBElem logger = logging.getLogger("trollbufr") @@ -81,7 +83,7 @@ def load_tab_a(tables, fname): """Load table A (data category) from 'fname' into object Tables.""" if not os.path.exists(fname): raise BufrTableError(_text_file_not_found % fname) - with open(fname, "rb") as fh: + with open(fname, "r") as fh: for line in fh: if line[0] == "#" or len(line) < 3: continue @@ -101,7 +103,7 @@ def load_tab_b(tables, fname): """Load table B (elements) from 'fname' into object Tables.""" if not os.path.exists(fname): raise BufrTableError(_text_file_not_found % fname) - with open(fname, "rb") as fh: + with open(fname, "r") as fh: for line in fh: if line[0] == "#" or len(line) < 3: continue @@ -128,7 +130,7 @@ def load_tab_c(tables, fname): """Load table C (operators) from 'fname' into object Tables.""" if not os.path.exists(fname): raise BufrTableError(_text_file_not_found % fname) - with open(fname, "rb") as fh: + with open(fname, "r") as fh: for line in fh: if line[0] == "#" or len(line) < 3: continue @@ -153,7 +155,7 @@ def load_tab_d(tables, fname): raise BufrTableError(_text_file_not_found % fname) # Using regex for eccodes' sequence.tab, for libdwd we split on tabulator. re_fl = re.compile("\"(?P\d+)\"\s*=\s*\[(?P[0-9, ]+)\]") - with open(fname, "rb") as fh: + with open(fname, "r") as fh: desc = None e = [] cline = "" @@ -187,7 +189,7 @@ def load_tab_cf(tables, fname): raise BufrTableError(_text_file_not_found % fname) for fn_etab in glob.glob(os.path.join(fname, "*.table")): desc = os.path.basename(fn_etab).split('.') - with open(fn_etab, "rb") as fh: + with open(fn_etab, "r") as fh: for line in fh: if line[0] == "#" or len(line) < 3: continue diff --git a/trollbufr/coder/parse_libdwd.py b/trollbufr/coder/parse_libdwd.py index 55ee88f..4861f0b 100644 --- a/trollbufr/coder/parse_libdwd.py +++ b/trollbufr/coder/parse_libdwd.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -29,8 +31,8 @@ import os import re -from errors import BufrTableError -from tables import TabBElem +from .errors import BufrTableError +from .tables import TabBElem logger = logging.getLogger("trollbufr") @@ -84,9 +86,9 @@ def load_tab_a(tables, fname): """Load table A (data category) from 'fname' into object Tables.""" if not os.path.exists(fname): raise BufrTableError(_text_file_not_found % fname) - with open(fname, "rb") as fh: + with open(fname, "r") as fh: for line in fh: - if line[0]=="#" or len(line) < 3: + if line[0] == "#" or len(line) < 3: continue d = None e = None @@ -109,9 +111,9 @@ def load_tab_b(tables, fname): # "FXYlibDWDTypeunitscalereferenceValuedataWidth_Bitsdescriptor_name" re_fl = re.compile( r"^(\d+)(?:\t|\s+)(\w)(?:\t|\s+)(.+?)(?:\t|\s+)([0-9-]+)(?:\t|\s+)([0-9-]+)(?:\t|\s+)([0-9-]+)(?:\t|\s+)(.+)$") - with open(fname, "rb") as fh: + with open(fname, "r") as fh: for line in fh: - if line[0]=="#" or len(line) < 3: + if line[0] == "#" or len(line) < 3: continue m = re_fl.match(line) if m is None: @@ -120,7 +122,7 @@ def load_tab_b(tables, fname): e = TabBElem(int(m.group(1)), m.group(2), m.group(3), None, m.group(7), int(m.group(4)), int(m.group(5)), int(m.group(6))) tables.tab_b[int(m.group(1))] = e - except StandardError as err: + except Exception as err: logger.error(err, exc_info=1) return True @@ -129,9 +131,9 @@ def load_tab_c(tables, fname): """Load table C (operators) from 'fname' into object Tables.""" if not os.path.exists(fname): raise BufrTableError(_text_file_not_found % fname) - with open(fname, "rb") as fh: + with open(fname, "r") as fh: for line in fh: - if line[0]=="#" or len(line) < 3: + if line[0] == "#" or len(line) < 3: continue d = None e = None @@ -151,11 +153,11 @@ def load_tab_d(tables, fname): """Load table D (sequences) from 'fname' into object Tables.""" if not os.path.exists(fname): raise BufrTableError(_text_file_not_found % fname) - with open(fname, "rb") as fh: + with open(fname, "r") as fh: desc = None e = [] for line in fh: - if line[0]=="#" or len(line) < 3: + if line[0] == "#" or len(line) < 3: continue try: le = line.split('\t') @@ -180,9 +182,9 @@ def load_tab_cf(tables, fname): """ if not os.path.exists(fname): raise BufrTableError(_text_file_not_found % fname) - with open(fname, "rb") as fh: + with open(fname, "r") as fh: for line in fh: - if line[0]=="#" or len(line) < 3: + if line[0] == "#" or len(line) < 3: continue e = line.rstrip().split('\t') if e[4].startswith("Reserved") or e[4].startswith("Not used"): diff --git a/trollbufr/coder/subset.py b/trollbufr/coder/subset.py index 476176a..cc384a1 100644 --- a/trollbufr/coder/subset.py +++ b/trollbufr/coder/subset.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016-2018 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -28,10 +30,10 @@ @author: amaul """ -import functions as fun -import operator as op -from errors import BufrDecodeError, BufrEncodeError -from bufr_types import DescrDataEntry, AlterState, BackrefRecord +from . import functions as fun +from . import operator as op +from .errors import BufrDecodeError, BufrEncodeError +from .bufr_types import DescrDataEntry, AlterState, BackrefRecord import logging logger = logging.getLogger("trollbufr") @@ -391,7 +393,7 @@ def extract_loop_list(vl, vi): """Compression: define local function to eval loop count and list of loop lists. """ - lst = zip(*[x[vi] for x in vl]) + lst = list(zip(*[x[vi] for x in vl])) cnt = len(lst) return cnt, lst diff --git a/trollbufr/coder/tables.py b/trollbufr/coder/tables.py index 6953441..af18cb7 100644 --- a/trollbufr/coder/tables.py +++ b/trollbufr/coder/tables.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -25,7 +27,7 @@ @author: amaul ''' -from bufr_types import TabBType +from .bufr_types import TabBType import logging logger = logging.getLogger("trollbufr") @@ -87,7 +89,7 @@ def lookup_codeflag(self, descr, val): logger.debug("CODE %06d: %d -> %s", descr, val, sval) elif b.typ == TabBType.FLAG: vl = [] - for k, v in self.tab_cf[descr].items(): + for k, v in list(self.tab_cf[descr].items()): if val & (1 << (b.width - k)): vl.append(v) sval = "|".join(vl) diff --git a/trollbufr/load_file.py b/trollbufr/load_file.py index 8805568..95aca50 100644 --- a/trollbufr/load_file.py +++ b/trollbufr/load_file.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -25,15 +27,15 @@ @author: amaul ''' import re -import coder.functions as f -from coder.bdata import Blob -from coder.errors import BufrDecodeWarning +from trollbufr.coder import functions as f +from trollbufr.coder.bdata import Blob +from trollbufr.coder.errors import BufrDecodeWarning import logging logger = logging.getLogger("trollbufr") """This RE matches any Abbreviated Heading Line""" -_re_ahl = re.compile(r"[^A-Z0-9]*?([A-Z]{4}[0-9]{2} [A-Z]{4} [0-9]{6}(?: [ACR][A-Z]{2})?)[^A-Z0-9]+") +_re_ahl = re.compile(b"[^A-Z0-9]*?([A-Z]{4}[0-9]{2} [A-Z]{4} [0-9]{6}(?: [ACR][A-Z]{2})?)[^A-Z0-9]+") def next_bufr(path=None, bin_data=None): @@ -59,7 +61,7 @@ def next_bufr(path=None, bin_data=None): while offs < len(bin_data): # Search for next BUFR bstart = offs - while bin_data[bstart: bstart + 4] != "BUFR": + while bin_data[bstart: bstart + 4] != b"BUFR": bstart += 1 if bstart >= len(bin_data) - 30: # reached end-of-bin_data @@ -74,7 +76,7 @@ def next_bufr(path=None, bin_data=None): 0 if m is None else m.groups()[0] ) if m is not None: - header = m.groups()[0] + header = (m.groups()[0]).decode() else: header = None # Bufr starts here @@ -86,7 +88,7 @@ def next_bufr(path=None, bin_data=None): bend = bstart + size offs = bend # Check if end is correct - if bin_data[bend - 4: bend] != "7777": + if bin_data[bend - 4: bend] != b"7777": # The bufr is corrupt if section5 is not correct logger.error("End '7777' not found") raise BufrDecodeWarning("Bufr offset/length error!") @@ -99,6 +101,6 @@ def next_bufr(path=None, bin_data=None): if __name__ == "__main__": import sys - print sys.argv + print(sys.argv) for b in next_bufr(path=sys.argv[1]): - print b + print(b) diff --git a/trollbufr/update.py b/trollbufr/update.py index 8c91249..feec16b 100644 --- a/trollbufr/update.py +++ b/trollbufr/update.py @@ -3,6 +3,8 @@ # # Copyright (c) 2016,2018 Alexander Maul # +# Ported to Py3 09/2018 +# # Author(s): # # Alexander Maul @@ -22,10 +24,12 @@ """ TrollBUFR - table update. """ +from __future__ import print_function +from __future__ import absolute_import + import sys import os - -import urllib2 +import urllib.request, urllib.error, urllib.parse import tarfile import zipfile from argparse import ArgumentParser @@ -34,7 +38,7 @@ import logging logger = logging.getLogger("trollbufr") -__version__ = "0.1.0" +__version__ = "0.2.0" E_OK = 0 E_ARG = 1 @@ -112,9 +116,9 @@ def download_all(args, url_list): return E_ARG with open(arc_dest, "w") as dest: logger.info("Download %s", url) - response = urllib2.urlopen(url) + response = urllib.request.urlopen(url) dest.write(response.read()) - except StandardError as e: + except Exception as e: logger.warning("%s : %s", url, e) else: arc_list.append(arc_dest) @@ -184,7 +188,7 @@ def run(argv=None): logger.error("URL or URL-file missing!\n") return E_ERR try: - print args + print(args) logger.debug("Sources: %s", url_list) logger.debug("Destination: %s", args.tables_path) arc_list = download_all(args, url_list) @@ -197,14 +201,14 @@ def run(argv=None): un_zip(args, arc_dest) else: logger.warning("Unkown archive format: %s", arc_dest) - except StandardError as e: + except Exception as e: logger.warning("Extract %s : %s", arc_dest, e) else: os.remove(arc_dest) except KeyboardInterrupt: ### handle keyboard interrupt ### return E_OK - except StandardError as e: + except Exception as e: logger.error(e) return E_ERR return E_OK diff --git a/trollbufr/version.py b/trollbufr/version.py index 9bb6f1e..fb78633 100644 --- a/trollbufr/version.py +++ b/trollbufr/version.py @@ -21,10 +21,11 @@ # along with this program. If not, see . """Version file. +Ported to Py3 09/2018 """ __major__ = "0" -__minor__ = "6" -__patch__ = "1" +__minor__ = "10" +__patch__ = "0" -__version__ = ".".join([__major__, __minor__, __patch__]) +version = ".".join([__major__, __minor__, __patch__])