Merge pull request #606 from steersbob/feature/read-mode

use ReadMode in requests and codec

brewblox_devcon_spark/broadcast.py

@@ -31,7 +31,8 @@ async def run(self):
 
         try:
             if state.is_synchronized():
-                blocks, logged_blocks = await self.api.read_all_broadcast_blocks()
+                blocks = await self.api.read_all_blocks()
+                logged_blocks = await self.api.read_all_logged_blocks()
 
                 # Convert list to key/value format suitable for history
                 history_data = {

brewblox_devcon_spark/codec/__init__.py

@@ -11,10 +11,8 @@
 
 from .. import exceptions, utils
 from ..models import (DecodedPayload, EncodedPayload, IntermediateRequest,
-                      IntermediateResponse)
+                      IntermediateResponse, ReadMode)
 from . import lookup, pb2, time_utils, unit_conversion
-from .opts import (DateFormatOpt, DecodeOpts, FilterOpt, MetadataOpt,
-                   ProtoEnumOpt)
 from .processor import ProtobufProcessor
 
 DEPRECATED_TYPE_INT = 65533
@@ -41,8 +39,8 @@ def join_type(blockType: str, subtype: Optional[str]) -> str:
 
 
 class Codec:
-    def __init__(self, strip_readonly=True):
-        self._processor = ProtobufProcessor(strip_readonly)
+    def __init__(self, filter_values=True):
+        self._processor = ProtobufProcessor(filter_values)
 
     def encode_request(self, request: IntermediateRequest) -> str:
         try:
@@ -106,7 +104,9 @@ def decode_response(self, b64_encoded: str) -> IntermediateResponse:
             LOGGER.debug(msg, exc_info=True)
             raise exceptions.DecodeException(msg)
 
-    def encode_payload(self, payload: DecodedPayload) -> EncodedPayload:
+    def encode_payload(self,
+                       payload: DecodedPayload,
+                       filter_values: bool | None = None) -> EncodedPayload:
         try:
             # No encoding required
             if payload.blockType is None:
@@ -139,7 +139,8 @@ def encode_payload(self, payload: DecodedPayload) -> EncodedPayload:
 
             message = impl.message_cls()
             payload = self._processor.pre_encode(message.DESCRIPTOR,
-                                                 payload.model_copy(deep=True))
+                                                 payload.model_copy(deep=True),
+                                                 filter_values=filter_values)
             json_format.ParseDict(payload.content, message)
             content: str = b64encode(message.SerializeToString()).decode()
 
@@ -164,7 +165,8 @@ def encode_payload(self, payload: DecodedPayload) -> EncodedPayload:
 
     def decode_payload(self,
                        payload: EncodedPayload, /,
-                       opts: DecodeOpts = DecodeOpts(),
+                       mode: ReadMode = ReadMode.DEFAULT,
+                       filter_values: bool | None = None,
                        ) -> DecodedPayload:
         try:
             if payload.blockType == DEPRECATED_TYPE_INT:
@@ -183,14 +185,13 @@ def decode_payload(self,
 
             if impl:
                 # We have an object lookup, and can decode the content
-                int_enum = opts.enums == ProtoEnumOpt.INT
                 message = impl.message_cls()
                 message.ParseFromString(b64decode(payload.content))
                 content: dict = json_format.MessageToDict(
                     message=message,
                     preserving_proto_field_name=True,
                     including_default_value_fields=True,
-                    use_integers_for_enums=int_enum,
+                    use_integers_for_enums=(mode in (ReadMode.STORED, ReadMode.LOGGED)),
                 )
                 decoded = DecodedPayload(
                     blockId=payload.blockId,
@@ -200,7 +201,10 @@ def decode_payload(self,
                     maskMode=payload.maskMode,
                     maskFields=payload.maskFields
                 )
-                return self._processor.post_decode(message.DESCRIPTOR, decoded, opts)
+                return self._processor.post_decode(message.DESCRIPTOR,
+                                                   decoded,
+                                                   mode=mode,
+                                                   filter_values=filter_values)
 
             # No object lookup found. Try the interfaces.
             intf_impl = next((v for v in lookup.CV_INTERFACES.get()
@@ -252,11 +256,6 @@ def setup():
     'setup',
     'CV',
 
-    'DecodeOpts',
-    'ProtoEnumOpt',
-    'FilterOpt',
-    'MetadataOpt',
-    'DateFormatOpt',
     'ProtobufProcessor'
 
     # utils

brewblox_devcon_spark/codec/opts.py

@@ -2,15 +2,9 @@
 Codec options
 """
 
-from dataclasses import dataclass
 from enum import Enum, auto
 
 
-class FilterOpt(Enum):
-    ALL = auto()
-    LOGGED = auto()
-
-
 class MetadataOpt(Enum):
     POSTFIX = auto()
     TYPED = auto()
@@ -25,11 +19,3 @@ class DateFormatOpt(Enum):
     MILLISECONDS = auto()
     SECONDS = auto()
     ISO8601 = auto()
-
-
-@dataclass(frozen=True)
-class DecodeOpts():
-    filter: FilterOpt = FilterOpt.ALL
-    metadata: MetadataOpt = MetadataOpt.TYPED
-    enums: ProtoEnumOpt = ProtoEnumOpt.STR
-    dates: DateFormatOpt = DateFormatOpt.ISO8601

brewblox_devcon_spark/codec/pb2.py

@@ -42,7 +42,6 @@
     import TempSensorExternal_pb2
     import TempSensorMock_pb2
     import TempSensorOneWire_pb2
-    import TouchSettings_pb2
     import Variables_pb2
     import WiFiSettings_pb2
 
@@ -78,7 +77,6 @@
     'TempSensorExternal_pb2',
     'TempSensorMock_pb2',
     'TempSensorOneWire_pb2',
-    'TouchSettings_pb2',
     'Variables_pb2',
     'WiFiSettings_pb2',
 ]

brewblox_devcon_spark/codec/processor.py

@@ -15,10 +15,11 @@
 from google.protobuf import json_format
 from google.protobuf.descriptor import Descriptor, FieldDescriptor
 
-from brewblox_devcon_spark.models import DecodedPayload, MaskField, MaskMode
+from brewblox_devcon_spark.models import (DecodedPayload, MaskField, MaskMode,
+                                          ReadMode)
 
 from . import unit_conversion
-from .opts import DateFormatOpt, DecodeOpts, FilterOpt, MetadataOpt
+from .opts import DateFormatOpt, MetadataOpt
 from .pb2 import brewblox_pb2
 from .time_utils import serialize_datetime
 
@@ -87,9 +88,9 @@ class OptionElement():
 class ProtobufProcessor():
     _BREWBLOX_PROVIDER: FieldDescriptor = brewblox_pb2.field
 
-    def __init__(self, strip_readonly=True):
+    def __init__(self, filter_values=True):
         self._converter = unit_conversion.CV.get()
-        self._strip_readonly = strip_readonly
+        self._filter_values = filter_values
 
         symbols = re.escape('[]<>')
         self._postfix_pattern = re.compile(''.join([
@@ -227,7 +228,8 @@ def _encode_unit(self, value: float | dict, unit_type: str, postfix: str | None)
 
     def pre_encode(self,
                    desc: Descriptor,
-                   payload: DecodedPayload,
+                   payload: DecodedPayload, /,
+                   filter_values: bool | None = None
                    ) -> DecodedPayload:
         """
         Modifies `payload` based on Protobuf options and dict key postfixes.
@@ -290,14 +292,17 @@ def pre_encode(self,
                 }
             }
         """
+        if filter_values is None:
+            filter_values = self._filter_values
+
         for element in self._walk_elements(desc, payload.content):
             options = self._field_options(element.field)
 
             if options.ignored:
                 del element.obj[element.key]
                 continue
 
-            if options.readonly and self._strip_readonly:
+            if filter_values and options.readonly:
                 del element.obj[element.key]
                 continue
 
@@ -360,8 +365,9 @@ def _convert_value(value: Any) -> str | int | float:
 
     def post_decode(self,
                     desc: Descriptor,
-                    payload: DecodedPayload,
-                    opts: DecodeOpts,
+                    payload: DecodedPayload, /,
+                    mode: ReadMode = ReadMode.DEFAULT,
+                    filter_values: bool | None = None,
                     ) -> DecodedPayload:
         """
         Post-processes protobuf data based on protobuf / codec options.
@@ -376,14 +382,10 @@ def post_decode(self,
         * ipv4address:  Converts integer IP address to dot string notation.
         * readonly:     Ignored: decoding means reading from controller.
         * ignored:      Strip value from output.
-        * logged:       Tag for filtering output data.
+        * logged:       Tag for filtering output data when using ReadMode.LOGGED.
+        * stored:       Tag for filtering output data when using ReadMode.STORED.
         * *_invalid_if: Sets value to None if equal to invalid value.
 
-        Supported codec options:
-        * filter:       If opts.filter == LOGGED, all values without options.logged are excluded from output.
-        * metadata:     Format used to serialize object metadata.
-                        Determines whether units/links are postfixed or rendered as typed object.
-
         Example:
             >>> values = {
                 'settings': {
@@ -397,7 +399,7 @@ def post_decode(self,
             >>> post_decode(
                     ExampleMessage_pb2.ExampleMessage(),
                     values,
-                    DecodeOpts(metadata=MetadataOpt.POSTFIX))
+                    mode=ReadMode.LOGGED)
 
             # ExampleMessage.proto:
             #
@@ -422,6 +424,16 @@ def post_decode(self,
                 }
             }
         """
+        metadata_opt = MetadataOpt.TYPED
+        date_fmt_opt = DateFormatOpt.ISO8601
+
+        if mode == ReadMode.LOGGED:
+            metadata_opt = MetadataOpt.POSTFIX
+            date_fmt_opt = DateFormatOpt.SECONDS
+
+        if filter_values is None:
+            filter_values = self._filter_values
+
         for element in self._walk_elements(desc, payload.content):
             options = self._field_options(element.field)
 
@@ -438,9 +450,11 @@ def post_decode(self,
                 del element.obj[element.key]
                 continue
 
-            if opts.filter == FilterOpt.LOGGED and not options.logged:
-                del element.obj[element.key]
-                continue
+            if filter_values:
+                if (mode == ReadMode.STORED and not options.stored) \
+                        or (mode == ReadMode.LOGGED and not options.logged):
+                    del element.obj[element.key]
+                    continue
 
             link_type = self.type_name(options.objtype)
             qty_system_unit = self.unit_name(options.unit)
@@ -458,7 +472,7 @@ def _convert_value(value: float | int | str) -> float | int | str | None:
                     else:
                         value = self._converter.to_user_value(value, qty_system_unit)
 
-                    if opts.metadata == MetadataOpt.TYPED:
+                    if metadata_opt == MetadataOpt.TYPED:
                         value = {
                             '__bloxtype': 'Quantity',
                             'unit': qty_user_unit,
@@ -474,7 +488,7 @@ def _convert_value(value: float | int | str) -> float | int | str | None:
                     if excluded or null_value:
                         value = None
 
-                    if opts.metadata == MetadataOpt.TYPED:
+                    if metadata_opt == MetadataOpt.TYPED:
                         value = {
                             '__bloxtype': 'Link',
                             'type': link_type,
@@ -496,15 +510,15 @@ def _convert_value(value: float | int | str) -> float | int | str | None:
                     return self.int_to_ipv4(value)
 
                 if options.datetime:
-                    return serialize_datetime(value, opts.dates)
+                    return serialize_datetime(value, date_fmt_opt)
 
                 return value
 
             new_key = element.key
             new_value = element.obj[element.key]
 
             # If metadata is postfixed, we may need to update the key
-            if opts.metadata == MetadataOpt.POSTFIX:
+            if metadata_opt == MetadataOpt.POSTFIX:
                 if options.objtype:
                     new_key = f'{element.key}<{link_type}>'
                 if options.unit: