Fix new device num_id creation, and warning for non-strict metadata p…

…rocessing, and some other tweaks (#942)

bin/test_regclean

@@ -29,6 +29,7 @@ site_path=sites/udmi_site_model
 iot_config=$site_path/cloud_iot_config.json
 output_file=$site_path/out/registration_summary.json
 csv_file=$site_path/out/registration_summary.csv
+expected_num_id=1234567890
 pubber_config=/tmp/pubber_config.json
 serial_no=clean-$RANDOM
 device_id=AHU-1
@@ -37,6 +38,21 @@ fake_device=$site_path/devices/$fake_id
 fake_metadata=$fake_device/metadata.json
 device_dir=$site_path/devices/$device_id
 
+function check_csv_file {
+    expected=$1
+    more $csv_file | cat
+
+    fake_state=`fgrep $fake_id $csv_file | cut -d, -f3`
+    [[ ${fake_state# } == $expected ]] || fail Device $fake_id not $expected
+    [[ `fgrep BLOCK $csv_file | wc -l` -le 1 ]] || fail More than one blocked device
+
+    device_num_id=`fgrep AHU-22 $csv_file | cut -d, -f2`
+    [[ $(echo $device_num_id) == $expected_num_id ]] || fail Device num_id $device_num_id does not match expected $expected_num_id
+
+    sns_line=`fgrep -n SNS-4 $csv_file | cut -d: -f1`
+    [[ $sns_line == 6 ]] || fail Device SNS-4 not found on expected output csv line, was $sns_line expected 6
+}
+
 echo bin/keygen CERT $device_dir
 bin/keygen CERT $device_dir || true
 
@@ -171,19 +187,23 @@ cat $pubber_config
 echo Check for failure in running pubber...
 echo pubber/bin/run $pubber_config
 result=0
-timeout 20s pubber/bin/run $pubber_config || result=$?
+timeout 30s pubber/bin/run $pubber_config || result=$?
 [[ $result == 255 ]] || fail Improper pubber failure code $result
 
 echo Now recreate the registry from scratch!
 echo bin/registrar $site_args
 bin/registrar $site_args
 
+check_csv_file ERRORS
+
 echo Remove fake device from site model, so it will be blocked
 rm -rf $fake_device
 echo $UDMI_ROOT/bin/registrar $full_args -b
 # Test from other-directory for proper path handling
 (cd /tmp; $UDMI_ROOT/bin/registrar $full_args -b)
 
+check_csv_file BLOCK
+
 echo Check $device_id again, but this time with success...
 echo pubber/bin/run $pubber_config
 result=0
@@ -192,20 +212,6 @@ timeout 20s pubber/bin/run $pubber_config || result=$?
 
 echo Successful pubber check passed on clean registry.
 
-more sites/udmi_site_model/out/registration_summary.csv | cat
-
-fake_state=`fgrep $fake_id $csv_file | cut -d, -f3`
-[[ ${fake_state# } == BLOCK ]] || fail Device $fake_id not blocked
-[[ `fgrep BLOCK $csv_file | wc -l` == 1 ]] || fail More than one blocked device
-
-expected_num_id=1234567890
-device_num_id=`fgrep AHU-22 $csv_file | cut -d, -f2`
-device_num_id=`echo $device_num_id`
-[[ $device_num_id == $expected_num_id ]] || warn Device num_id $device_num_id does not match expected $expected_num_id
-
-sns_line=`fgrep -n SNS-4 sites/udmi_site_model/out/registration_summary.csv | cut -d: -f1`
-[[ $sns_line == 6 ]] || fail Device SNS-4 not found on expected output line, was $sns_line expected 6
-
 if [[ -n $solo_arg ]]; then
     echo Deleting singular entry $solo_arg...
     echo bin/registrar $site_args -d $solo_arg

common/src/main/java/com/google/udmi/util/SiteModel.java

@@ -314,7 +314,7 @@ public SiteMetadata loadSiteMetadata(){
     File siteMetadataFile = new File(new File(sitePath), SITE_METADATA_FILE);
     siteMetadataExceptionMap = new ExceptionMap(SITE_METADATA_KEY);
     try {
-       siteMetadataObject= loadFileRequired(ObjectNode.class, siteMetadataFile);
+       siteMetadataObject = loadFileRequired(ObjectNode.class, siteMetadataFile);
        return convertToStrict(SiteMetadata.class, siteMetadataObject);
     } catch (Exception e) {
       siteMetadataExceptionMap.put(SITE_METADATA_KEY, e);
@@ -569,6 +569,10 @@ public String getDevicePassword(String deviceId) {
     return sha256(getFileBytes(keyFile)).substring(0, 8);
   }
 
+  public String getSiteName() {
+    return exeConfig.site_name;
+  }
+
   public static class MetadataException extends Metadata {
 
     public final File file;

etc/shell_common.sh

@@ -32,9 +32,13 @@ UDMI_ROOT=$(realpath $UDMI_ROOT)
 
 UDMI_JAR=$UDMI_ROOT/validator/build/libs/validator-1.0-SNAPSHOT-all.jar
 
-udmi_version=$(cd $UDMI_ROOT; git describe --dirty) || true
+VERSION_BASE='1.*'
 
-[[ -z $udmi_version ]] && udmi_version=git-$(cd $UDMI_ROOT; git describe --dirty --always) || true
+# Ignore non-version branches (e.g. something like 'develop')
+udmi_version=$(cd $UDMI_ROOT; git describe --dirty --match $VERSION_BASE) || true
+
+# No luck... just generate any viable version.
+[[ -n $udmi_version ]] || udmi_version=git-$(cd $UDMI_ROOT; git describe --dirty --match $VERSION_BASE --always) || true
 
 [[ $udmi_version == git- ]] && udmi_version=unknown
 

udmis/bin/build

@@ -28,10 +28,6 @@ cp -a ../bin/ build/udmi_bin/
 cp -a ../etc/ build/udmi_etc/
 cp -a ../schema build/schema/
 
-echo Copying default mosquitto config files...
-mkdir -p var/
-cp /etc/mosquitto/mosquitto.conf var/ || true
-
 jarfile=build/libs/udmis-1.0-SNAPSHOT-all.jar
 
 newer_files=$([[ -f $jarfile ]] && find src/ $BASE/gencode/java -type f -newer $jarfile || echo missing)

udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java

@@ -336,9 +336,10 @@ private CloudModel createDevice(String registryId, Device device) {
       CreateDeviceRequest request =
           CreateDeviceRequest.Builder.newBuilder().setParent(parent).setDevice(device)
               .build();
-      requireNonNull(deviceManager.createDevice(request),
-          "create device failed for " + parent);
-      cloudModel.num_id = hashedDeviceId(registryId, device.toBuilder().getId());
+      requireNonNull(deviceManager.createDevice(request), "create device failed for " + parent);
+      String numId = device.toBuilder().getNumId();
+      cloudModel.num_id =
+          ofNullable(numId).orElseGet(() -> hashedDeviceId(registryId, device.toBuilder().getId()));
       return cloudModel;
     } catch (ApplicationException applicationException) {
       if (applicationException.getMessage().contains("ALREADY_EXISTS")) {

udmis/src/main/java/com/google/bos/udmi/service/access/DynamicIotAccessProvider.java

@@ -141,6 +141,8 @@ public CloudModel listDevices(String registryId) {
 
   @Override
   public CloudModel modelDevice(String registryId, String deviceId, CloudModel cloudModel) {
+    debug("%s iot device %s/%s, %s %s", cloudModel.operation, registryId, deviceId,
+        cloudModel.blocked, cloudModel.num_id);
     return getProviderFor(registryId).modelDevice(registryId, deviceId, cloudModel);
   }
 

udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java

@@ -25,6 +25,7 @@
 import static com.google.udmi.util.JsonUtil.convertTo;
 import static com.google.udmi.util.JsonUtil.convertToStrict;
 import static com.google.udmi.util.JsonUtil.fromString;
+import static com.google.udmi.util.JsonUtil.fromStringStrict;
 import static com.google.udmi.util.JsonUtil.isoConvert;
 import static com.google.udmi.util.JsonUtil.mapCast;
 import static com.google.udmi.util.JsonUtil.stringify;
@@ -67,6 +68,7 @@ public class ReflectProcessor extends ProcessorBase {
 
   public static final String PAYLOAD_KEY = "payload";
   private static final Date START_TIME = new Date();
+  private static final String LEGACY_METADATA_STRING = "{ }";
 
   public ReflectProcessor(EndpointConfiguration config) {
     super(config);
@@ -137,7 +139,23 @@ private Object extractModel(CloudModel request) {
   }
 
   private SiteMetadataUpdate asSiteMetadataUpdate(String metadataString) {
-    return fromString(SiteMetadataUpdate.class, metadataString);
+    SiteMetadataUpdate siteMetadataUpdate =
+        fromStringDynamic(SiteMetadataUpdate.class, metadataString);
+    // TODO: Remove the LEGACY_METADATA_STRING check once tool deployments have matured.
+    if (!LEGACY_METADATA_STRING.equals(metadataString)) {
+      requireNonNull(siteMetadataUpdate.name, "missing site model name");
+    }
+    return siteMetadataUpdate;
+  }
+
+  private <T> T fromStringDynamic(Class<T> targetClass, String metadataString) {
+    try {
+      return fromStringStrict(targetClass, metadataString);
+    } catch (Exception e) {
+      warn("String conversion failed, trying again with relaxed conversion: ",
+          friendlyStackTrace(e));
+      return fromString(targetClass, metadataString);
+    }
   }
 
   private Envelope extractMessageEnvelope(Object message) {
@@ -241,16 +259,17 @@ private CloudModel reflectModel(Envelope attributes, CloudModel request) {
     }
   }
 
-  private static ModelUpdate asModelUpdate(String modelString) {
+  private ModelUpdate asModelUpdate(String modelString) {
     // If it's not a valid JSON object, then fall back to a string description alternate.
     if (modelString == null || !modelString.startsWith(JsonUtil.JSON_OBJECT_LEADER)) {
       ModelUpdate modelUpdate = new ModelUpdate();
       modelUpdate.system = new SystemModel();
       modelUpdate.system.description = modelString;
       return modelUpdate;
     }
-    // Not strict because registrar could publish a metadata which fails strictly
-    return fromString(ModelUpdate.class, modelString);
+    ModelUpdate modelUpdate = fromStringDynamic(ModelUpdate.class, modelString);
+    requireNonNull(modelUpdate.system, "missing system block in model message");
+    return modelUpdate;
   }
 
   private CloudModel reflectProcess(Envelope attributes, Object payload) {

udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/SimpleMqttPipe.java

@@ -163,8 +163,9 @@ protected void publishRaw(Bundle bundle) {
       String topic = makeMqttTopic(bundle);
       MqttMessage message = makeMqttMessage(bundle);
       mqttClient.publish(topic, message);
-      debug("Client has %d inFlight tokens with %s", mqttClient.getPendingDeliveryTokens().length,
-          topic);
+      int tokens = mqttClient.getPendingDeliveryTokens().length;
+      ifTrueThen(tokens > 2, () ->
+          debug("Client has %d inFlight tokens, from %s", tokens, topic));
     } catch (Exception e) {
       throw new RuntimeException("While publishing to mqtt client " + clientId, e);
     }

validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java

@@ -323,8 +323,8 @@ protected Map<String, LocalDevice> getLocalDevices() {
   private void processSiteMetadata() {
     SiteMetadata siteMetadata = ofNullable(siteModel.loadSiteMetadata()).orElseGet(
         SiteMetadata::new);
-
-    if (siteMetadata != null && updateCloudIoT) {
+    siteMetadata.name = ofNullable(siteMetadata.name).orElse(siteModel.getSiteName());
+    if (updateCloudIoT) {
       cloudIotManager.updateRegistry(siteMetadata);
     }
   }