Parser.pm has a memory leak (well really the MongoDB driver does) whe…

…n getting the list of Indexes for a collection. We work around this memory leak by using the metadata (presuming that the metadata is properly indexed in MongoDB)

lib/GRNOC/TSDS/Parser.pm

@@ -1151,14 +1151,14 @@ sub _query_database {
             return;
         }
 
-        # we need to make sure that any fields in the where clause actually exist and are
-        # properly indexed to help guide mongo away from doing any massive table scans
-        return if (! $self->_verify_where_fields($database, $where_names));
-
         # load the metadata about this collection so we can use it to figure out
         # things like needing to unwind later
         $metadata = $database->get_collection(METADATA)->find_one();
 
+        # we need to make sure that any fields in the where clause actually exist and are
+        # properly indexed to help guide mongo away from doing any massive table scans
+        return if (! $self->_verify_where_fields($metadata, $where_names));
+
         # we need start/end times to do some calculations on the values
         $queried_field_names->{'start'}      = 1;
         $queried_field_names->{'end'}        = 1;
@@ -2096,28 +2096,21 @@ sub _get_unwind_operations {
     return @unwind;
 }
 
-sub _verify_where_fields {
+sub _verify_where_fields{
     my $self         = shift;
-    my $database     = shift;
+    my $metadata     = shift;
     my $where_names  = shift;
 
-    my @indexes = $database->get_collection(MEASUREMENTS)->indexes->list->all;
+    my %indexes;
+    flatten_keys_hash($metadata->{'meta_fields'}, undef, \%indexes);
 
-    foreach my $where_name (keys %$where_names){
-	my $found = 0;
+    $indexes{'identifier'} = 1;
 
-	foreach my $index (@indexes){
-	    if (exists $index->{'key'}->{$where_name}){
-		$found = 1;
-		last;
-	    }
-	}
-	if (! $found){
-	    $self->error("Unable to execute query on unknown or unindexed field \"$where_name\"");
-	    return;
-	}
+    foreach my $where_name (keys %{$where_names}){
+        if(!defined($indexes{$where_name})){
+            return 0;
+        }
     }
-
     return 1;
 }
 
@@ -4033,4 +4026,33 @@ sub update_constraints_file {
 
 }
 
+#Flatten the hash of keys of MetaData into what we need for our "where" clause
+#validation... essentially pop.location.state etc...
+#oh its recursive too!
+sub flatten_keys_hash {
+    my ($hash, $prefix, $results) = @_;
+
+    for my $key (keys %$hash) {
+
+        if($key eq 'fields'){
+
+            flatten_keys_hash($hash->{$key}, $prefix, $results);
+        }else{
+
+            my $new_prefix;
+            if(defined($prefix)){
+                $new_prefix = "$prefix.$key";
+            }else{
+                $new_prefix = $key;
+            }
+
+            if (ref $hash->{$key} eq 'HASH') {
+                flatten_keys_hash( $hash->{$key}, $new_prefix, $results );
+            }
+
+            $results->{$new_prefix} = 1;
+        }
+    }
+}
+
 1;