A lot of error map refactoring

common/src/main/java/com/google/daq/mqtt/util/ExceptionMap.java

@@ -25,12 +25,18 @@ public class ExceptionMap extends RuntimeException {
   private static final byte[] SEPARATOR_BYTES = ": ".getBytes();
   private static final String ERROR_FORMAT_INDENT = "  ";
 
-  final Map<String, Exception> exceptions = new TreeMap<>();
+  final Map<ExceptionCategory, Exception> exceptions = new TreeMap<>();
 
   public ExceptionMap(String description) {
     super(description);
   }
 
+  public enum ExceptionCategory {
+    missing, extra, out, validation, loading, writing, site_metadata, initializing, sample,
+    EXCEPTION_REGISTERING, EXCEPTION_ENVELOPE, EXCEPTION_CREDENTIALS, EXCEPTION_SAMPLES,
+    EXCEPTION_FILES, EXCEPTION_BINDING, creating, updating, schema, configuring
+  }
+
   /**
    * Format the given exception with indicated level.
    *
@@ -50,8 +56,8 @@ private static ErrorTree format(Throwable e, final String prefix, final String i
       if (e.getCause() != null) {
         errorTree.child = format(e.getCause(), newPrefix, indent);
       }
-      ((ExceptionMap) e)
-          .forEach((key, sub) -> errorTree.children.put(key, format(sub, newPrefix, indent)));
+      ((ExceptionMap) e).forEach((key, sub) ->
+          errorTree.children.put(key.toString(), format(sub, newPrefix, indent)));
     } else if (e instanceof ValidationException) {
       ((ValidationException) e)
           .getCausingExceptions()
@@ -69,7 +75,7 @@ private static ErrorTree format(Throwable e, final String prefix, final String i
     return errorTree;
   }
 
-  private void forEach(BiConsumer<String, Exception> consumer) {
+  private void forEach(BiConsumer<ExceptionCategory, Exception> consumer) {
     exceptions.forEach(consumer);
   }
 
@@ -96,13 +102,13 @@ public void throwIfNotEmpty() {
    * @param key       entry key
    * @param exception exception to add
    */
-  public void put(String key, Exception exception) {
+  public void put(ExceptionCategory key, Exception exception) {
     if (exceptions.put(key, exception) != null) {
       throw new IllegalArgumentException("Exception key already defined: " + key);
     }
   }
 
-  public Stream<Map.Entry<String, Exception>> stream() {
+  public Stream<Map.Entry<ExceptionCategory, Exception>> stream() {
     return exceptions.entrySet().stream();
   }
 
@@ -118,7 +124,7 @@ public int size() {
   /**
    * Execute the action and capture into the map if it throws an exception.
    */
-  public void capture(String category, Runnable action) {
+  public void capture(ExceptionCategory category, Runnable action) {
     try {
       action.run();
     } catch (Exception e) {

common/src/main/java/com/google/udmi/util/GeneralUtils.java

@@ -298,18 +298,18 @@ private static JsonGenerator getPrettyPrinterGenerator(OutputStream outputStream
     }
   }
 
+  @SuppressWarnings("unchecked")
   public static Map<String, Object> getSubMap(Map<String, Object> input, String field) {
-    //noinspection unchecked
     return (Map<String, Object>) input.get(field);
   }
 
+  @SuppressWarnings("unchecked")
   public static Map<String, Object> getSubMapNull(Map<String, Object> input, String field) {
-    //noinspection unchecked
     return ifNotNullGet(input, map -> (Map<String, Object>) map.get(field));
   }
 
+  @SuppressWarnings("unchecked")
   public static Map<String, Object> getSubMapDefault(Map<String, Object> input, String field) {
-    //noinspection unchecked
     return ifNotNullGet(input,
         map -> (Map<String, Object>) map.computeIfAbsent(field, k -> new HashMap<>()));
   }

common/src/main/java/com/google/udmi/util/SiteModel.java

@@ -36,6 +36,7 @@
 import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
 import com.google.common.collect.ImmutableList;
 import com.google.daq.mqtt.util.ExceptionMap;
+import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory;
 import java.io.File;
 import java.util.Arrays;
 import java.util.Collection;
@@ -343,7 +344,7 @@ public SiteMetadata loadSiteMetadata() {
       siteMetadataObject = loadFileRequired(ObjectNode.class, siteMetadataFile);
       return convertToStrict(SiteMetadata.class, siteMetadataObject);
     } catch (Exception e) {
-      siteMetadataExceptionMap.put(SITE_METADATA_KEY, e);
+      siteMetadataExceptionMap.put(ExceptionCategory.site_metadata, e);
       return convertTo(SiteMetadata.class, siteMetadataObject);
     }
   }

etc/Category.java

@@ -12,7 +12,7 @@
 // This class is manually curated, auto-generated, and then copied into the gencode directory.
 // Look for the proper source and don't be fooled! Ultimately sourced from docs/specs/categories.md
 public class Category {
-    public static final Map<String, Level> LEVEL = new HashMap();
+    public static final Map<String, Level> LEVEL = new HashMap<>();
 
     @@ gencode stuff goes here
 }

validator/build.gradle

@@ -27,6 +27,10 @@ java {
     }
 }
 
+compileJava {
+    options.compilerArgs = ['-Xlint:unchecked']
+}
+
 sourceSets {
     main {
         java {

validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java

@@ -20,6 +20,7 @@
 import static com.google.udmi.util.GeneralUtils.isTrue;
 import static com.google.udmi.util.GeneralUtils.writeString;
 import static com.google.udmi.util.JsonUtil.OBJECT_MAPPER;
+import static com.google.udmi.util.JsonUtil.getNowInstant;
 import static com.google.udmi.util.JsonUtil.loadFile;
 import static com.google.udmi.util.JsonUtil.unquoteJson;
 import static com.google.udmi.util.SiteModel.CLOUD_MODEL_FILE;
@@ -43,8 +44,10 @@
 import com.google.daq.mqtt.util.DeviceExceptionManager;
 import com.google.daq.mqtt.util.ExceptionMap;
 import com.google.daq.mqtt.util.ExceptionMap.ErrorTree;
+import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory;
 import com.google.daq.mqtt.util.ValidationError;
 import com.google.daq.mqtt.util.ValidationException;
+import com.google.daq.mqtt.validator.ErrorMap;
 import com.google.daq.mqtt.validator.Validator;
 import com.google.udmi.util.JsonUtil;
 import com.google.udmi.util.MessageDowngrader;
@@ -62,6 +65,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
@@ -201,6 +205,7 @@ class LocalDevice {
   private Date lastActive;
   private boolean blocked;
   private CloudModel cloudModel;
+  private Instant lastUpdated;
 
   LocalDevice(
       SiteModel siteModel, String deviceId, Map<String, JsonSchema> schemas,
@@ -254,27 +259,31 @@ private void prepareOutDir() {
   }
 
   public void validateExpectedFiles() {
-    ExceptionMap exceptionMap = new ExceptionMap("expected files");
+    if (isExtraKind()) {
+      return;
+    }
+
+    ExceptionMap exceptionMap = new ExceptionMap("expected files mismatch");
 
     String[] files = deviceDir.list();
     checkNotNull(files, "No files found in " + deviceDir.getAbsolutePath());
     Set<String> actualFiles = ImmutableSet.copyOf(files);
     Set<String> expectedFiles = Sets.union(DEVICE_FILES, keyFiles());
     SortedSet<String> missing = new TreeSet<>(Sets.difference(expectedFiles, actualFiles));
     if (!missing.isEmpty()) {
-      exceptionMap.put("missing", new RuntimeException("Missing files: " + missing));
+      exceptionMap.put(ExceptionCategory.missing, new RuntimeException("Missing files: " + missing));
     }
     SortedSet<String> extra = new TreeSet<>(
         Sets.difference(Sets.difference(actualFiles, expectedFiles), OPTIONAL_FILES));
     if (!extra.isEmpty()) {
-      exceptionMap.put("extra", new RuntimeException("Extra files: " + extra));
+      exceptionMap.put(ExceptionCategory.extra, new RuntimeException("Extra files: " + extra));
     }
     String[] outFiles = outDir.list();
     if (outFiles != null) {
       Set<String> outSet = ImmutableSet.copyOf(outFiles);
       SortedSet<String> extraOut = new TreeSet<>(Sets.difference(outSet, OUT_FILES));
       if (!extraOut.isEmpty()) {
-        exceptionMap.put("out", new RuntimeException("Extra out files: " + extraOut));
+        exceptionMap.put(ExceptionCategory.out, new RuntimeException("Extra out files: " + extraOut));
       }
     }
 
@@ -288,7 +297,7 @@ private void validateMetadata() {
       ProcessingReport report = schemas.get(METADATA_SCHEMA_JSON).validate(metadataObject);
       parseMetadataValidateProcessingReport(report);
     } catch (ProcessingException | ValidationException e) {
-      exceptionMap.put(EXCEPTION_VALIDATING, e);
+      exceptionMap.put(ExceptionCategory.validation, e);
     }
   }
 
@@ -317,7 +326,7 @@ private Metadata readMetadata() {
           ids -> ifTrueThen(ids.isEmpty(), () -> deviceMetadata.gateway.proxy_ids = null));
       return deviceMetadata;
     } catch (Exception exception) {
-      exceptionMap.put(EXCEPTION_LOADING, exception);
+      exceptionMap.put(ExceptionCategory.loading, exception);
       return null;
     }
   }
@@ -497,7 +506,7 @@ void initializeSettings() {
       settings.keyBytes = getKeyBytes();
       settings.config = deviceConfigString();
     } catch (Exception e) {
-      captureError(EXCEPTION_INITIALIZING, e);
+      captureError(ExceptionCategory.initializing, e);
     }
   }
 
@@ -516,6 +525,11 @@ private List<String> getCloudModelProxyList() {
   public void updateModel(CloudModel device) {
     setDeviceNumId(checkNotNull(device.num_id, "missing deviceNumId for " + deviceId));
     setLastActive(device.last_event_time);
+    setLastUpdated(getNowInstant());
+  }
+
+  private void setLastUpdated(Instant lastUpdated) {
+    this.lastUpdated = lastUpdated;
   }
 
   public String getLastActive() {
@@ -554,7 +568,11 @@ private String deviceConfigString() {
     return runInContext("While converting device config", () -> {
       Object fromValue = config.deviceConfigJson();
 
-      config.getSchemaViolationsMap().forEach(this::captureError);
+      ifNotNullThen(config.getSchemaViolationsMap(), map -> {
+        ErrorMap schemaValidationErrors = new ErrorMap("schema validation errors");
+        schemaValidationErrors.putAll(map);
+        captureError(ExceptionCategory.schema, schemaValidationErrors.asException());
+      });
 
       if (fromValue instanceof String stringValue) {
         return stringValue;
@@ -699,7 +717,7 @@ void writeNormalized() {
     try {
       writeString(metadataFile, compressJsonString(metadata, MAX_JSON_LENGTH));
     } catch (Exception e) {
-      exceptionMap.put(EXCEPTION_WRITING, e);
+      exceptionMap.put(ExceptionCategory.writing, e);
     }
   }
 
@@ -752,7 +770,7 @@ public DeviceStatus getStatus() {
     return DeviceStatus.ERROR;
   }
 
-  public void captureError(String exceptionType, Exception exception) {
+  public void captureError(ExceptionCategory exceptionType, Exception exception) {
     exceptionMap.put(exceptionType, exception);
     File exceptionLog = new File(outDir, EXCEPTION_LOG_FILE);
     try {
@@ -793,7 +811,7 @@ public void validateSamples() {
       } catch (Exception e) {
         Exception scopedException =
             new RuntimeException("While validating sample file " + sampleName, e);
-        samplesMap.put(sampleName, scopedException);
+        samplesMap.put(ExceptionCategory.sample, scopedException);
       }
     }
     samplesMap.throwIfNotEmpty();

validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java

@@ -55,6 +55,7 @@
 import com.google.daq.mqtt.util.DeviceGatewayBoundException;
 import com.google.daq.mqtt.util.ExceptionMap;
 import com.google.daq.mqtt.util.ExceptionMap.ErrorTree;
+import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory;
 import com.google.daq.mqtt.util.PubSubPusher;
 import com.google.daq.mqtt.util.ValidationError;
 import com.google.udmi.util.CommandLineOption;
@@ -794,7 +795,7 @@ private boolean processLocalDevice(String localName, AtomicInteger processedDevi
       }
     } catch (Exception e) {
       System.err.printf("Error processing %s: %s%n", localDevice.getDeviceId(), e);
-      localDevice.captureError(LocalDevice.EXCEPTION_REGISTERING, e);
+      localDevice.captureError(ExceptionCategory.EXCEPTION_REGISTERING, e);
     }
     return created;
   }
@@ -1058,7 +1059,7 @@ private void bindGatewayDevices(Map<String, LocalDevice> localDevices) {
             cloudIotManager.bindDevices(toBind, gatewayId, true);
           } catch (Exception e) {
             proxiedDevices.forEach(
-                localDevice -> localDevice.captureError(LocalDevice.EXCEPTION_BINDING, e));
+                localDevice -> localDevice.captureError(ExceptionCategory.EXCEPTION_BINDING, e));
           }
         });
       });
@@ -1173,7 +1174,7 @@ private void validateSamples(Map<String, LocalDevice> localDevices) {
       try {
         device.validateSamples();
       } catch (Exception e) {
-        device.captureError(LocalDevice.EXCEPTION_SAMPLES, e);
+        device.captureError(ExceptionCategory.EXCEPTION_SAMPLES, e);
       }
     }
   }
@@ -1183,7 +1184,7 @@ private void validateExpected(Map<String, LocalDevice> localDevices) {
       try {
         device.validateExpectedFiles();
       } catch (Exception e) {
-        device.captureError(LocalDevice.EXCEPTION_FILES, e);
+        device.captureError(ExceptionCategory.EXCEPTION_FILES, e);
       }
     }
   }
@@ -1213,7 +1214,7 @@ private void validateKeys(Map<String, LocalDevice> localDevices) {
                       new RuntimeException(
                           format(
                               "Duplicate credentials found for %s & %s", previous, deviceName));
-                  localDevice.captureError(LocalDevice.EXCEPTION_CREDENTIALS, exception);
+                  localDevice.captureError(ExceptionCategory.EXCEPTION_CREDENTIALS, exception);
                 }
               }
             });
@@ -1230,13 +1231,13 @@ private void initializeLocalDevices() {
     initializeDevices(workingDevices);
     initializeSettings(workingDevices);
     writeNormalized(workingDevices);
-    outputMetadataModels(workingDevices);
+    previewModels(workingDevices);
     validateExpected(workingDevices);
     validateSamples(workingDevices);
     validateKeys(workingDevices);
   }
 
-  private void outputMetadataModels(Map<String, LocalDevice> localDevices) {
+  private void previewModels(Map<String, LocalDevice> localDevices) {
     if (!metadataModelOut) {
       return;
     }
@@ -1267,15 +1268,15 @@ private void initializeDevices(Map<String, LocalDevice> localDevices) {
       } catch (ValidationError error) {
         throw new RuntimeException("While initializing device", error);
       } catch (Exception e) {
-        localDevice.captureError(LocalDevice.EXCEPTION_CREDENTIALS, e);
+        localDevice.captureError(ExceptionCategory.EXCEPTION_CREDENTIALS, e);
       }
 
       if (cloudIotManager != null) {
         try {
           localDevice.validateEnvelope(
               cloudIotManager.getRegistryId(), cloudIotManager.getSiteName());
         } catch (Exception e) {
-          localDevice.captureError(LocalDevice.EXCEPTION_ENVELOPE,
+          localDevice.captureError(ExceptionCategory.EXCEPTION_ENVELOPE,
               new RuntimeException("While validating envelope", e));
         }
       }