General Optimizations to new application design

docker-compose.yml

@@ -8,15 +8,15 @@ services:
     ports:
       - ${DOCKER_POSTGRES_PORT-5432}:5432
     volumes:
-      - ./postgres:/var/lib/postgresql/data
+      - postgres:/var/lib/postgresql/data
 
   rabbitmq:
     image: rabbitmq:management
     ports:
       - ${DOCKER_RABBITMQ_PORT-5672}:5672
       - ${DOCKER_RABBITMQ_CONSOLE_PORT-15672}:15672
     volumes:
-      - ./rabbitmq:/var/lib/rabbitmq/mnesia
+      - rabbitmq:/var/lib/rabbitmq/mnesia
     logging:
       driver: none
 
@@ -38,7 +38,7 @@ services:
       - ${DOCKER_MINIO_PORT-9000}:9000
       - ${DOCKER_MINIO_CONSOLE_PORT-9001}:9001
     volumes:
-      - ./minio:/data
+      - minio:/data
     logging:
       driver: none
 

sample_data/ingest_sample_data.py

@@ -1,7 +1,7 @@
+from datetime import datetime
 import json
 import os
 from pathlib import Path
-from datetime import datetime
 
 from django.contrib.gis.geos import Point
 from django.core.files.base import ContentFile
@@ -105,18 +105,31 @@ def ingest_datasets(include_large=False, dataset_indexes=None):
     with open('sample_data/datasets.json') as datasets_json:
         data = json.load(datasets_json)
         for index, dataset in enumerate(data):
+            # Grab fields specific to dataset classification
+            network_options = dataset.get('network_options')
+            region_options = dataset.get('region_options')
+
             if dataset_indexes is None or index in dataset_indexes:
                 print('\t- ', dataset['name'])
                 existing = Dataset.objects.filter(name=dataset['name'])
                 if existing.count():
                     dataset_for_conversion = existing.first()
                 else:
+                    # Determine classification
+                    classification = Dataset.Classification.OTHER
+                    if network_options:
+                        classification = Dataset.Classification.NETWORK
+                    elif region_options:
+                        classification = Dataset.Classification.REGION
+
+                    # Create dataset
                     new_dataset = Dataset.objects.create(
                         name=dataset['name'],
                         description=dataset['description'],
                         category=dataset['category'],
                         dataset_type=dataset.get('type', 'vector').upper(),
                         metadata=dataset.get('metadata', {}),
+                        classification=classification,
                     )
                     print('\t', f'Dataset {new_dataset.name} created.')
                     for index, file_info in enumerate(dataset.get('files', [])):
@@ -132,8 +145,8 @@ def ingest_datasets(include_large=False, dataset_indexes=None):
                     print('\t', f'Converting data for {dataset_for_conversion.name}...')
                     dataset_for_conversion.spawn_conversion_task(
                         style_options=dataset.get('style_options'),
-                        network_options=dataset.get('network_options'),
-                        region_options=dataset.get('region_options'),
+                        network_options=network_options,
+                        region_options=region_options,
                         asynchronous=False,
                     )
                 else:

uvdat/core/migrations/0001_models_redesign.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.1 on 2023-10-19 13:30
+# Generated by Django 4.1 on 2023-11-15 21:41
 
 import django.contrib.gis.db.models.fields
 from django.db import migrations, models
@@ -64,6 +64,14 @@ class Migration(migrations.Migration):
                         choices=[('VECTOR', 'Vector'), ('RASTER', 'Raster')], max_length=6
                     ),
                 ),
+                (
+                    'classification',
+                    models.CharField(
+                        choices=[('Network', 'Network'), ('Region', 'Region'), ('Other', 'Other')],
+                        default='Other',
+                        max_length=16,
+                    ),
+                ),
             ],
         ),
         migrations.CreateModel(
@@ -138,8 +146,7 @@ class Migration(migrations.Migration):
                 ('metadata', models.JSONField(blank=True, null=True)),
                 ('default_style', models.JSONField(blank=True, null=True)),
                 ('index', models.IntegerField(null=True)),
-                ('geojson_data', models.JSONField(blank=True, null=True)),
-                ('large_geojson_data', s3_file_field.fields.S3FileField(null=True)),
+                ('geojson_file', s3_file_field.fields.S3FileField(null=True)),
                 (
                     'file_item',
                     models.ForeignKey(
@@ -394,6 +401,16 @@ class Migration(migrations.Migration):
                 to='core.context',
             ),
         ),
+        migrations.AddIndex(
+            model_name='vectortile',
+            index=models.Index(fields=['z', 'x', 'y'], name='vectortile-coordinates-index'),
+        ),
+        migrations.AddConstraint(
+            model_name='vectortile',
+            constraint=models.UniqueConstraint(
+                fields=('map_layer', 'z', 'x', 'y'), name='unique-map-layer-index'
+            ),
+        ),
         migrations.AddConstraint(
             model_name='sourceregion',
             constraint=models.UniqueConstraint(

uvdat/core/models/dataset.py

@@ -6,6 +6,11 @@ class DatasetType(models.TextChoices):
         VECTOR = 'VECTOR', 'Vector'
         RASTER = 'RASTER', 'Raster'
 
+    class Classification(models.TextChoices):
+        NETWORK = 'Network'
+        REGION = 'Region'
+        OTHER = 'Other'
+
     name = models.CharField(max_length=255, unique=True)
     description = models.TextField(null=True, blank=True)
     category = models.CharField(max_length=25)
@@ -15,6 +20,9 @@ class DatasetType(models.TextChoices):
         max_length=max(len(choice[0]) for choice in DatasetType.choices),
         choices=DatasetType.choices,
     )
+    classification = models.CharField(
+        max_length=16, choices=Classification.choices, default=Classification.OTHER
+    )
 
     def is_in_context(self, context_id):
         from uvdat.core.models import Context
@@ -72,23 +80,52 @@ def get_network_gcc(self, exclude_nodes):
         return get_dataset_network_gcc(self, exclude_nodes)
 
     def get_map_layers(self):
+        """Return a queryset of either RasterMapLayer, or VectorMapLayer."""
         from uvdat.core.models import RasterMapLayer, VectorMapLayer
 
-        ret = []
-        for vector_map_layer in VectorMapLayer.objects.filter(file_item__dataset=self):
-            ret.append(
-                {
-                    'id': vector_map_layer.id,
-                    'index': vector_map_layer.index,
-                    'type': 'vector',
-                }
-            )
-        for raster_map_layer in RasterMapLayer.objects.filter(file_item__dataset=self):
-            ret.append(
-                {
-                    'id': raster_map_layer.id,
-                    'index': raster_map_layer.index,
-                    'type': 'raster',
-                }
+        if self.dataset_type == self.DatasetType.RASTER:
+            return RasterMapLayer.objects.filter(file_item__dataset=self)
+        if self.dataset_type == self.DatasetType.VECTOR:
+            return VectorMapLayer.objects.filter(file_item__dataset=self)
+
+        raise NotImplementedError(f'Dataset Type {self.dataset_type}')
+
+    def get_map_layer_tile_extents(self):
+        """
+        Return the extents of all vector map layers of this dataset.
+
+        Returns `None` if the dataset is not a vector dataset.
+        """
+        if self.dataset_type != self.DatasetType.VECTOR:
+            return None
+
+        from uvdat.core.models import VectorMapLayer, VectorTile
+
+        # Retrieve all layers
+        layer_ids = VectorMapLayer.objects.filter(file_item__dataset=self).values_list(
+            'id', flat=True
+        )
+
+        # Return x/y extents by layer id and z depth
+        vals = (
+            VectorTile.objects.filter(map_layer_id__in=layer_ids)
+            .values('map_layer_id', 'z')
+            .annotate(
+                min_x=models.Min('x'),
+                min_y=models.Min('y'),
+                max_x=models.Max('x'),
+                max_y=models.Max('y'),
             )
-        return ret
+            .order_by('map_layer_id')
+        )
+
+        # Deconstruct query into response format
+        layers = {}
+        for entry in vals:
+            map_layer_id = entry.pop('map_layer_id')
+            if map_layer_id not in layers:
+                layers[map_layer_id] = {}
+
+            layers[map_layer_id][entry.pop('z')] = entry
+
+        return layers

uvdat/core/models/map_layers.py

@@ -42,55 +42,57 @@ def get_image_data(self, resolution: float = 1.0):
 
 
 class VectorMapLayer(AbstractMapLayer):
-    geojson_data = models.JSONField(blank=True, null=True)
-    large_geojson_data = S3FileField(null=True)
-
-    def get_geojson_data(self):
-        if self.geojson_data:
-            if isinstance(self.geojson_data, dict):
-                return self.geojson_data
-            return json.loads(self.geojson_data)
-        else:
-            return json.loads(json.load(self.large_geojson_data.open()))
+    geojson_file = S3FileField(null=True)
 
-    def save_geojson_data(self, content):
-        if isinstance(content, dict):
-            geojson = content
+    def write_geojson_data(self, content: str | dict):
+        if isinstance(content, str):
+            data = content
+        elif isinstance(content, dict):
+            data = json.dumps(content)
         else:
-            geojson = content.to_json()
-
-        geojson_size = len(json.dumps(geojson).encode())
-
-        # JSONField limited to 268435455 bytes
-        if geojson_size < 268000000:
-            self.geojson_data = geojson
-        else:
-            self.large_geojson_data.save(
-                'vectordata.geojson',
-                ContentFile(json.dumps(geojson).encode()),
-            )
-
-    def get_available_tile_coords(self):
-        # TODO: compute this once and save it on the object;
-        # Query can take a while, and this is called on the RasterMapLayerSerializer
-        tile_coords = []
-        for vector_tile in VectorTile.objects.filter(map_layer=self):
-            tile_coords.append(
-                dict(
-                    x=vector_tile.x,
-                    y=vector_tile.y,
-                    z=vector_tile.z,
+            raise Exception(f'Invalid content type supplied: {type(content)}')
+
+        self.geojson_file.save('vectordata.geojson', ContentFile(data.encode()))
+
+    def read_geojson_data(self) -> dict:
+        """Read and load the data from geojson_file into a dict."""
+        return json.load(self.geojson_file.open())
+
+    def get_tile_extents(self):
+        """Return a dict that maps z tile values to the x/y extent at that depth."""
+        return {
+            entry.pop('z'): entry
+            for entry in (
+                VectorTile.objects.filter(map_layer=self)
+                .values('z')
+                .annotate(
+                    min_x=models.Min('x'),
+                    min_y=models.Min('y'),
+                    max_x=models.Max('x'),
+                    max_y=models.Max('y'),
                 )
+                .order_by()
             )
-        return tile_coords
-
-    def get_vector_tile(self, x, y, z):
-        return VectorTile.objects.get(map_layer=self, x=x, y=y, z=z)
+        }
 
 
 class VectorTile(models.Model):
+    EMPTY_TILE_DATA = {
+        'type': 'FeatureCollection',
+        'features': [],
+    }
+
     map_layer = models.ForeignKey(VectorMapLayer, on_delete=models.CASCADE)
     geojson_data = models.JSONField(blank=True, null=True)
     x = models.IntegerField(default=0)
     y = models.IntegerField(default=0)
     z = models.IntegerField(default=0)
+
+    class Meta:
+        constraints = [
+            # Ensure that a full index only ever resolves to one record
+            models.UniqueConstraint(
+                name='unique-map-layer-index', fields=['map_layer', 'z', 'x', 'y']
+            )
+        ]
+        indexes = [models.Index(fields=('z', 'x', 'y'), name='vectortile-coordinates-index')]

uvdat/core/rest/dataset.py

@@ -2,6 +2,7 @@
 
 from django.http import HttpResponse
 from rest_framework.decorators import action
+from rest_framework.response import Response
 from rest_framework.viewsets import ModelViewSet
 
 from uvdat.core.models import Dataset
@@ -19,6 +20,28 @@ def get_queryset(self):
         else:
             return Dataset.objects.all()
 
+    @action(detail=True, methods=['get'])
+    def map_layers(self, request, **kwargs):
+        dataset: Dataset = self.get_object()
+        map_layers = list(dataset.get_map_layers().select_related('file_item__dataset'))
+
+        # Set serializer based on dataset type
+        if dataset.dataset_type == Dataset.DatasetType.RASTER:
+            serializer = uvdat_serializers.RasterMapLayerSerializer(map_layers, many=True)
+        elif dataset.dataset_type == Dataset.DatasetType.VECTOR:
+            # Inject tile extents
+            extents = dataset.get_map_layer_tile_extents()
+            for layer in map_layers:
+                layer.tile_extents = extents.pop(layer.id)
+
+            # Set serializer
+            serializer = uvdat_serializers.ExtendedVectorMapLayerSerializer(map_layers, many=True)
+        else:
+            raise NotImplementedError(f'Dataset Type {dataset.dataset_type}')
+
+        # Return response with rendered data
+        return Response(serializer.data, status=200)
+
     @action(detail=True, methods=['get'])
     def convert(self, request, **kwargs):
         dataset = self.get_object()