From ab193ee84138b989645fd506093f41db385392b2 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 8 Dec 2023 18:05:50 -0800 Subject: [PATCH 01/23] Add device, site and data source to Deployment responses (untested!) --- ami/main/api/serializers.py | 67 +++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 1e39ad9f3..b988724bb 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -83,10 +83,42 @@ class Meta: ] +class DeviceNestedSerializer(DefaultSerializer): + class Meta: + model = Device + fields = [ + "id", + "name", + "details", + ] + + +class SiteNestedSerializer(DefaultSerializer): + class Meta: + model = Site + fields = [ + "id", + "name", + "details", + ] + + +class StorageSourceNestedSerializer(DefaultSerializer): + class Meta: + model = S3StorageSource + fields = [ + "id", + "name", + "details", + ] + + class DeploymentListSerializer(DefaultSerializer): events = serializers.SerializerMethodField() occurrences = serializers.SerializerMethodField() project = ProjectNestedSerializer(read_only=True) + device = DeviceNestedSerializer(read_only=True) + site = SiteNestedSerializer(read_only=True) class Meta: model = Deployment @@ -108,6 +140,8 @@ class Meta: "longitude", "first_date", "last_date", + "device", + "site", ] def get_events(self, obj): @@ -285,24 +319,51 @@ class DeploymentSerializer(DeploymentListSerializer): events = DeploymentEventNestedSerializer(many=True, read_only=True) occurrences = serializers.SerializerMethodField() example_captures = DeploymentCaptureNestedSerializer(many=True, read_only=True) - data_source = serializers.SerializerMethodField(read_only=True) project_id = serializers.PrimaryKeyRelatedField( write_only=True, queryset=Project.objects.all(), source="project", ) + device_id = serializers.PrimaryKeyRelatedField( + write_only=True, + queryset=Device.objects.all(), + source="device", + ) + site_id = serializers.PrimaryKeyRelatedField( + write_only=True, + queryset=Site.objects.all(), + source="site", + ) + data_source = serializers.SerializerMethodField() + data_source_id = serializers.PrimaryKeyRelatedField( + write_only=True, + queryset=S3StorageSource.objects.all(), + source="data_source", + ) class Meta(DeploymentListSerializer.Meta): fields = DeploymentListSerializer.Meta.fields + [ "project_id", - "description", + "device_id", + "site_id", "data_source", + "data_source_id", + "description", "example_captures", # "capture_images", ] def get_data_source(self, obj): - return obj.data_source_uri() + """ + Add uri to nested serializer of the data source + + The data source is defined by both the StorageSource model + and the extra configuration in the Deployment model. + """ + + data = StorageSourceNestedSerializer(obj.data_source, context=self.context).data + data["uri"] = obj.data_source_uri() + return data def get_occurrences(self, obj): """ From 29265a0a6b009916d56c7d0055e9631bf9d995e4 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 8 Dec 2023 18:05:50 -0800 Subject: [PATCH 02/23] Add device, site and data source to Deployment responses (untested!) --- ami/main/api/serializers.py | 67 +++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 1e39ad9f3..b988724bb 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -83,10 +83,42 @@ class Meta: ] +class DeviceNestedSerializer(DefaultSerializer): + class Meta: + model = Device + fields = [ + "id", + "name", + "details", + ] + + +class SiteNestedSerializer(DefaultSerializer): + class Meta: + model = Site + fields = [ + "id", + "name", + "details", + ] + + +class StorageSourceNestedSerializer(DefaultSerializer): + class Meta: + model = S3StorageSource + fields = [ + "id", + "name", + "details", + ] + + class DeploymentListSerializer(DefaultSerializer): events = serializers.SerializerMethodField() occurrences = serializers.SerializerMethodField() project = ProjectNestedSerializer(read_only=True) + device = DeviceNestedSerializer(read_only=True) + site = SiteNestedSerializer(read_only=True) class Meta: model = Deployment @@ -108,6 +140,8 @@ class Meta: "longitude", "first_date", "last_date", + "device", + "site", ] def get_events(self, obj): @@ -285,24 +319,51 @@ class DeploymentSerializer(DeploymentListSerializer): events = DeploymentEventNestedSerializer(many=True, read_only=True) occurrences = serializers.SerializerMethodField() example_captures = DeploymentCaptureNestedSerializer(many=True, read_only=True) - data_source = serializers.SerializerMethodField(read_only=True) project_id = serializers.PrimaryKeyRelatedField( write_only=True, queryset=Project.objects.all(), source="project", ) + device_id = serializers.PrimaryKeyRelatedField( + write_only=True, + queryset=Device.objects.all(), + source="device", + ) + site_id = serializers.PrimaryKeyRelatedField( + write_only=True, + queryset=Site.objects.all(), + source="site", + ) + data_source = serializers.SerializerMethodField() + data_source_id = serializers.PrimaryKeyRelatedField( + write_only=True, + queryset=S3StorageSource.objects.all(), + source="data_source", + ) class Meta(DeploymentListSerializer.Meta): fields = DeploymentListSerializer.Meta.fields + [ "project_id", - "description", + "device_id", + "site_id", "data_source", + "data_source_id", + "description", "example_captures", # "capture_images", ] def get_data_source(self, obj): - return obj.data_source_uri() + """ + Add uri to nested serializer of the data source + + The data source is defined by both the StorageSource model + and the extra configuration in the Deployment model. + """ + + data = StorageSourceNestedSerializer(obj.data_source, context=self.context).data + data["uri"] = obj.data_source_uri() + return data def get_occurrences(self, obj): """ From cf6bd1ebd525ddc111ded3f99e53a7a917adbc1c Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 11 Dec 2023 07:45:23 +0000 Subject: [PATCH 03/23] Add forgotten migration --- .../migrations/0028_alter_occurrence_options.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ami/main/migrations/0028_alter_occurrence_options.py diff --git a/ami/main/migrations/0028_alter_occurrence_options.py b/ami/main/migrations/0028_alter_occurrence_options.py new file mode 100644 index 000000000..5342bed74 --- /dev/null +++ b/ami/main/migrations/0028_alter_occurrence_options.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.2 on 2023-12-11 02:44 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0027_update_occurrence_scores"), + ] + + operations = [ + migrations.AlterModelOptions( + name="occurrence", + options={"ordering": ["-determination_score"]}, + ), + ] From 3c9a0424aa86cba975b76def99cf1cfe581aaff8 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 11 Dec 2023 08:01:19 +0000 Subject: [PATCH 04/23] Fix naming of research site relation --- ami/main/api/serializers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index b988724bb..1602fac96 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -118,7 +118,7 @@ class DeploymentListSerializer(DefaultSerializer): occurrences = serializers.SerializerMethodField() project = ProjectNestedSerializer(read_only=True) device = DeviceNestedSerializer(read_only=True) - site = SiteNestedSerializer(read_only=True) + research_site = SiteNestedSerializer(read_only=True) class Meta: model = Deployment @@ -141,7 +141,7 @@ class Meta: "first_date", "last_date", "device", - "site", + "research_site", ] def get_events(self, obj): @@ -329,10 +329,10 @@ class DeploymentSerializer(DeploymentListSerializer): queryset=Device.objects.all(), source="device", ) - site_id = serializers.PrimaryKeyRelatedField( + research_site_id = serializers.PrimaryKeyRelatedField( write_only=True, queryset=Site.objects.all(), - source="site", + source="research_site", ) data_source = serializers.SerializerMethodField() data_source_id = serializers.PrimaryKeyRelatedField( @@ -345,7 +345,7 @@ class Meta(DeploymentListSerializer.Meta): fields = DeploymentListSerializer.Meta.fields + [ "project_id", "device_id", - "site_id", + "research_site_id", "data_source", "data_source_id", "description", From a5529073285bac829a6cc0d5422efd5d3b6abab3 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 11 Dec 2023 08:02:53 +0000 Subject: [PATCH 05/23] Update select related for Deployments --- ami/main/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index b4259f5fc..8b24e051c 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -121,7 +121,7 @@ class DeploymentViewSet(DefaultViewSet): for the list and detail views. """ - queryset = Deployment.objects.select_related("project") + queryset = Deployment.objects.select_related("project", "device", "research_site") filterset_fields = ["project"] ordering_fields = [ "created_at", From 23a8b11206af6b668fa0a8bd5742996ed4c80a02 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 8 Dec 2023 18:05:50 -0800 Subject: [PATCH 06/23] Add device, site and data source to Deployment responses (untested!) --- ami/main/api/serializers.py | 67 +++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 1e39ad9f3..b988724bb 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -83,10 +83,42 @@ class Meta: ] +class DeviceNestedSerializer(DefaultSerializer): + class Meta: + model = Device + fields = [ + "id", + "name", + "details", + ] + + +class SiteNestedSerializer(DefaultSerializer): + class Meta: + model = Site + fields = [ + "id", + "name", + "details", + ] + + +class StorageSourceNestedSerializer(DefaultSerializer): + class Meta: + model = S3StorageSource + fields = [ + "id", + "name", + "details", + ] + + class DeploymentListSerializer(DefaultSerializer): events = serializers.SerializerMethodField() occurrences = serializers.SerializerMethodField() project = ProjectNestedSerializer(read_only=True) + device = DeviceNestedSerializer(read_only=True) + site = SiteNestedSerializer(read_only=True) class Meta: model = Deployment @@ -108,6 +140,8 @@ class Meta: "longitude", "first_date", "last_date", + "device", + "site", ] def get_events(self, obj): @@ -285,24 +319,51 @@ class DeploymentSerializer(DeploymentListSerializer): events = DeploymentEventNestedSerializer(many=True, read_only=True) occurrences = serializers.SerializerMethodField() example_captures = DeploymentCaptureNestedSerializer(many=True, read_only=True) - data_source = serializers.SerializerMethodField(read_only=True) project_id = serializers.PrimaryKeyRelatedField( write_only=True, queryset=Project.objects.all(), source="project", ) + device_id = serializers.PrimaryKeyRelatedField( + write_only=True, + queryset=Device.objects.all(), + source="device", + ) + site_id = serializers.PrimaryKeyRelatedField( + write_only=True, + queryset=Site.objects.all(), + source="site", + ) + data_source = serializers.SerializerMethodField() + data_source_id = serializers.PrimaryKeyRelatedField( + write_only=True, + queryset=S3StorageSource.objects.all(), + source="data_source", + ) class Meta(DeploymentListSerializer.Meta): fields = DeploymentListSerializer.Meta.fields + [ "project_id", - "description", + "device_id", + "site_id", "data_source", + "data_source_id", + "description", "example_captures", # "capture_images", ] def get_data_source(self, obj): - return obj.data_source_uri() + """ + Add uri to nested serializer of the data source + + The data source is defined by both the StorageSource model + and the extra configuration in the Deployment model. + """ + + data = StorageSourceNestedSerializer(obj.data_source, context=self.context).data + data["uri"] = obj.data_source_uri() + return data def get_occurrences(self, obj): """ From 621e2e8ad03466e68885ba4f4abf428c7ce13b54 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 11 Dec 2023 07:45:23 +0000 Subject: [PATCH 07/23] Add forgotten migration --- .../migrations/0028_alter_occurrence_options.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ami/main/migrations/0028_alter_occurrence_options.py diff --git a/ami/main/migrations/0028_alter_occurrence_options.py b/ami/main/migrations/0028_alter_occurrence_options.py new file mode 100644 index 000000000..5342bed74 --- /dev/null +++ b/ami/main/migrations/0028_alter_occurrence_options.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.2 on 2023-12-11 02:44 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0027_update_occurrence_scores"), + ] + + operations = [ + migrations.AlterModelOptions( + name="occurrence", + options={"ordering": ["-determination_score"]}, + ), + ] From 64dfd161132d5ed6f630a19b4a78f798f8e511f0 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 11 Dec 2023 08:01:19 +0000 Subject: [PATCH 08/23] Fix naming of research site relation --- ami/main/api/serializers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index b988724bb..1602fac96 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -118,7 +118,7 @@ class DeploymentListSerializer(DefaultSerializer): occurrences = serializers.SerializerMethodField() project = ProjectNestedSerializer(read_only=True) device = DeviceNestedSerializer(read_only=True) - site = SiteNestedSerializer(read_only=True) + research_site = SiteNestedSerializer(read_only=True) class Meta: model = Deployment @@ -141,7 +141,7 @@ class Meta: "first_date", "last_date", "device", - "site", + "research_site", ] def get_events(self, obj): @@ -329,10 +329,10 @@ class DeploymentSerializer(DeploymentListSerializer): queryset=Device.objects.all(), source="device", ) - site_id = serializers.PrimaryKeyRelatedField( + research_site_id = serializers.PrimaryKeyRelatedField( write_only=True, queryset=Site.objects.all(), - source="site", + source="research_site", ) data_source = serializers.SerializerMethodField() data_source_id = serializers.PrimaryKeyRelatedField( @@ -345,7 +345,7 @@ class Meta(DeploymentListSerializer.Meta): fields = DeploymentListSerializer.Meta.fields + [ "project_id", "device_id", - "site_id", + "research_site_id", "data_source", "data_source_id", "description", From d654a94f38ebc9aae9c8f2efc9860eebf5c01222 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 11 Dec 2023 08:02:53 +0000 Subject: [PATCH 09/23] Update select related for Deployments --- ami/main/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index b4259f5fc..8b24e051c 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -121,7 +121,7 @@ class DeploymentViewSet(DefaultViewSet): for the list and detail views. """ - queryset = Deployment.objects.select_related("project") + queryset = Deployment.objects.select_related("project", "device", "research_site") filterset_fields = ["project"] ordering_fields = [ "created_at", From adf449f7e5e004a249a72aec8957783ac3be1b2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 18:24:22 -0800 Subject: [PATCH 10/23] Bump plotly.js from 2.20.0 to 2.25.2 in /ui (#341) Bumps [plotly.js](https://github.com/plotly/plotly.js) from 2.20.0 to 2.25.2. - [Release notes](https://github.com/plotly/plotly.js/releases) - [Changelog](https://github.com/plotly/plotly.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.js/compare/v2.20.0...v2.25.2) --- updated-dependencies: - dependency-name: plotly.js dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/package.json | 2 +- ui/yarn.lock | 112 ++++++++++++++++++++++++++++++------------------ 2 files changed, 72 insertions(+), 42 deletions(-) diff --git a/ui/package.json b/ui/package.json index ed3112a1a..1e788decb 100644 --- a/ui/package.json +++ b/ui/package.json @@ -26,7 +26,7 @@ "cmdk": "^0.2.0", "leaflet": "^1.9.3", "lodash": "^4.17.21", - "plotly.js": "^2.20.0", + "plotly.js": "^2.25.2", "react": "^18.2.0", "react-dom": "^18.2.0", "react-error-boundary": "^4.0.11", diff --git a/ui/yarn.lock b/ui/yarn.lock index 32c4b2f7c..289a4ffe4 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -2465,7 +2465,7 @@ __metadata: languageName: node linkType: hard -"@mapbox/geojson-rewind@npm:^0.5.0": +"@mapbox/geojson-rewind@npm:^0.5.2": version: 0.5.2 resolution: "@mapbox/geojson-rewind@npm:0.5.2" dependencies: @@ -2646,6 +2646,36 @@ __metadata: languageName: node linkType: hard +"@plotly/mapbox-gl@npm:v1.13.4": + version: 1.13.4 + resolution: "@plotly/mapbox-gl@npm:1.13.4" + dependencies: + "@mapbox/geojson-rewind": "npm:^0.5.2" + "@mapbox/geojson-types": "npm:^1.0.2" + "@mapbox/jsonlint-lines-primitives": "npm:^2.0.2" + "@mapbox/mapbox-gl-supported": "npm:^1.5.0" + "@mapbox/point-geometry": "npm:^0.1.0" + "@mapbox/tiny-sdf": "npm:^1.1.1" + "@mapbox/unitbezier": "npm:^0.0.0" + "@mapbox/vector-tile": "npm:^1.3.1" + "@mapbox/whoots-js": "npm:^3.1.0" + csscolorparser: "npm:~1.0.3" + earcut: "npm:^2.2.2" + geojson-vt: "npm:^3.2.1" + gl-matrix: "npm:^3.2.1" + grid-index: "npm:^1.1.0" + murmurhash-js: "npm:^1.0.0" + pbf: "npm:^3.2.1" + potpack: "npm:^1.0.1" + quickselect: "npm:^2.0.0" + rw: "npm:^1.3.3" + supercluster: "npm:^7.1.0" + tinyqueue: "npm:^2.0.3" + vt-pbf: "npm:^3.1.1" + checksum: e528983335ef8d8ef564e2c0b21a98888b2220750d069679a334d91a43487d12dbbaaf6c798e47ba42f8a5d17dd996207cb1f2ed081d00705631d8cbfcba0a44 + languageName: node + linkType: hard + "@plotly/point-cluster@npm:^3.1.9": version: 3.1.9 resolution: "@plotly/point-cluster@npm:3.1.9" @@ -6568,7 +6598,7 @@ __metadata: jest-environment-jsdom: "npm:^29.7.0" leaflet: "npm:^1.9.3" lodash: "npm:^4.17.21" - plotly.js: "npm:^2.20.0" + plotly.js: "npm:^2.25.2" prettier: "npm:2.8.4" react: "npm:^18.2.0" react-dom: "npm:^18.2.0" @@ -6957,6 +6987,13 @@ __metadata: languageName: node linkType: hard +"base64-arraybuffer@npm:^1.0.2": + version: 1.0.2 + resolution: "base64-arraybuffer@npm:1.0.2" + checksum: 3acac95c70f9406e87a41073558ba85b6be9dbffb013a3d2a710e3f2d534d506c911847d5d9be4de458af6362c676de0a5c4c2d7bdf4def502d00b313368e72f + languageName: node + linkType: hard + "base64-js@npm:^1.3.1": version: 1.5.1 resolution: "base64-js@npm:1.5.1" @@ -11967,37 +12004,6 @@ __metadata: languageName: node linkType: hard -"mapbox-gl@npm:1.10.1": - version: 1.10.1 - resolution: "mapbox-gl@npm:1.10.1" - dependencies: - "@mapbox/geojson-rewind": "npm:^0.5.0" - "@mapbox/geojson-types": "npm:^1.0.2" - "@mapbox/jsonlint-lines-primitives": "npm:^2.0.2" - "@mapbox/mapbox-gl-supported": "npm:^1.5.0" - "@mapbox/point-geometry": "npm:^0.1.0" - "@mapbox/tiny-sdf": "npm:^1.1.1" - "@mapbox/unitbezier": "npm:^0.0.0" - "@mapbox/vector-tile": "npm:^1.3.1" - "@mapbox/whoots-js": "npm:^3.1.0" - csscolorparser: "npm:~1.0.3" - earcut: "npm:^2.2.2" - geojson-vt: "npm:^3.2.1" - gl-matrix: "npm:^3.2.1" - grid-index: "npm:^1.1.0" - minimist: "npm:^1.2.5" - murmurhash-js: "npm:^1.0.0" - pbf: "npm:^3.2.1" - potpack: "npm:^1.0.1" - quickselect: "npm:^2.0.0" - rw: "npm:^1.3.3" - supercluster: "npm:^7.0.0" - tinyqueue: "npm:^2.0.3" - vt-pbf: "npm:^3.1.1" - checksum: 35d797c69407ac0c3fa6d8a57d337eda61f23476cac72b55c5a2dd1215d19761e02083d51b0e1874d92fcd431e569c2c67a1e75e1619469ef69355f15a6babdd - languageName: node - linkType: hard - "markdown-to-jsx@npm:^7.1.8": version: 7.3.2 resolution: "markdown-to-jsx@npm:7.3.2" @@ -13006,16 +13012,18 @@ __metadata: languageName: node linkType: hard -"plotly.js@npm:^2.20.0": - version: 2.20.0 - resolution: "plotly.js@npm:2.20.0" +"plotly.js@npm:^2.25.2": + version: 2.28.0 + resolution: "plotly.js@npm:2.28.0" dependencies: "@plotly/d3": "npm:3.8.1" "@plotly/d3-sankey": "npm:0.7.2" "@plotly/d3-sankey-circular": "npm:0.33.1" + "@plotly/mapbox-gl": "npm:v1.13.4" "@turf/area": "npm:^6.4.0" "@turf/bbox": "npm:^6.4.0" "@turf/centroid": "npm:^6.0.2" + base64-arraybuffer: "npm:^1.0.2" canvas-fit: "npm:^1.5.0" color-alpha: "npm:1.0.4" color-normalize: "npm:1.5.0" @@ -13037,7 +13045,6 @@ __metadata: has-hover: "npm:^1.0.1" has-passive-events: "npm:^1.0.0" is-mobile: "npm:^4.0.0" - mapbox-gl: "npm:1.10.1" mouse-change: "npm:^1.4.0" mouse-event-offset: "npm:^3.0.2" mouse-wheel: "npm:^1.2.0" @@ -13049,7 +13056,7 @@ __metadata: regl: "npm:@plotly/regl@^2.1.2" regl-error2d: "npm:^2.0.12" regl-line2d: "npm:^3.1.2" - regl-scatter2d: "npm:^3.2.8" + regl-scatter2d: "npm:^3.3.1" regl-splom: "npm:^1.0.14" strongly-connected-components: "npm:^1.0.1" superscript-text: "npm:^1.0.0" @@ -13059,7 +13066,7 @@ __metadata: topojson-client: "npm:^3.1.0" webgl-context: "npm:^2.2.0" world-calendars: "npm:^1.0.3" - checksum: f95918a3ddb0897b741cc90f082d636ecc648a855cdeefd9f175c1ff858d058ea22511624ee874e4cbb99398086f1ce952c7c3633a8b04ea0598b86d979108a9 + checksum: 9ebf5cae16230296f4c973e2727dd96158eec6e5b17ce74675864ceb092804e9783b71e5f766714e82e44e01a4af91927971710d523011026a9a665c6ca45c44 languageName: node linkType: hard @@ -13920,7 +13927,7 @@ __metadata: languageName: node linkType: hard -"regl-scatter2d@npm:^3.2.3, regl-scatter2d@npm:^3.2.8": +"regl-scatter2d@npm:^3.2.3": version: 3.2.8 resolution: "regl-scatter2d@npm:3.2.8" dependencies: @@ -13944,6 +13951,29 @@ __metadata: languageName: node linkType: hard +"regl-scatter2d@npm:^3.3.1": + version: 3.3.1 + resolution: "regl-scatter2d@npm:3.3.1" + dependencies: + "@plotly/point-cluster": "npm:^3.1.9" + array-range: "npm:^1.0.1" + array-rearrange: "npm:^2.2.2" + clamp: "npm:^1.0.1" + color-id: "npm:^1.1.0" + color-normalize: "npm:^1.5.0" + color-rgba: "npm:^2.1.1" + flatten-vertex-data: "npm:^1.0.2" + glslify: "npm:^7.0.0" + is-iexplorer: "npm:^1.0.0" + object-assign: "npm:^4.1.1" + parse-rect: "npm:^1.2.0" + pick-by-alias: "npm:^1.2.0" + to-float32: "npm:^1.1.0" + update-diff: "npm:^1.1.0" + checksum: 9a7f69d2039aeb1cca125d0d101cb122234b9e4ac56e190c9f434198ecd27411d6d4b70e2f2aee5a97471060aef290b74b9cac434398166ac330c47901e99283 + languageName: node + linkType: hard + "regl-splom@npm:^1.0.14": version: 1.0.14 resolution: "regl-splom@npm:1.0.14" @@ -14802,7 +14832,7 @@ __metadata: languageName: node linkType: hard -"supercluster@npm:^7.0.0": +"supercluster@npm:^7.1.0": version: 7.1.5 resolution: "supercluster@npm:7.1.5" dependencies: From cf139dca503b46008c1ec499966b19019c5c2860 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 18:27:16 -0800 Subject: [PATCH 11/23] Bump vite from 4.5.0 to 4.5.2 in /ui (#342) Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 4.5.0 to 4.5.2. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v4.5.2/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v4.5.2/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/package.json | 2 +- ui/yarn.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/package.json b/ui/package.json index 1e788decb..6d8d698bd 100644 --- a/ui/package.json +++ b/ui/package.json @@ -35,7 +35,7 @@ "react-plotly.js": "^2.6.0", "react-router-dom": "^6.8.2", "typescript": "^4.4.2", - "vite": "^4.5.0", + "vite": "^4.5.2", "vite-tsconfig-paths": "^4.2.1" }, "scripts": { diff --git a/ui/yarn.lock b/ui/yarn.lock index 289a4ffe4..352857dd3 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -6611,7 +6611,7 @@ __metadata: storybook: "npm:^7.5.3" ts-jest: "npm:^29.1.1" typescript: "npm:^4.4.2" - vite: "npm:^4.5.0" + vite: "npm:^4.5.2" vite-plugin-eslint: "npm:^1.8.1" vite-plugin-svgr: "npm:^4.1.0" vite-tsconfig-paths: "npm:^4.2.1" @@ -15953,9 +15953,9 @@ __metadata: languageName: node linkType: hard -"vite@npm:^4.5.0": - version: 4.5.0 - resolution: "vite@npm:4.5.0" +"vite@npm:^4.5.2": + version: 4.5.2 + resolution: "vite@npm:4.5.2" dependencies: esbuild: "npm:^0.18.10" fsevents: "npm:~2.3.2" @@ -15989,7 +15989,7 @@ __metadata: optional: true bin: vite: bin/vite.js - checksum: 7e21e9e4b80656ae5ee61e8c5edb5e8f589139c2b22c43e89d054c65a0194f1c1ef066fbc770204173c7eb244c798265042f988adda5880ad74337a053b28b7f + checksum: 68969ccf72ad2078aec7d9e023fce6de03746a4761f9308924212fff7bd42487145b270166cec66cddacfd7b1315ec5aa39ead174fbd7fcd463637a96ff4c9d1 languageName: node linkType: hard From e04ea5a3b776f556f4f4c0451d77ddab21512200 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Sat, 3 Feb 2024 02:27:53 +0000 Subject: [PATCH 12/23] Don't break on occurrences with disconnected events --- ami/main/api/serializers.py | 5 ++++- ami/main/api/views.py | 4 ++-- ami/main/charts.py | 7 +++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 1e39ad9f3..55c072965 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -507,7 +507,10 @@ def get_page_offset(self, obj) -> int: # @TODO this may not be correct. Test or remove if unnecessary. # the Occurrence to Session navigation in the UI will be using # another method. - return obj.event.captures.filter(timestamp__lt=obj.timestamp).count() + if not obj or not obj.event: + return 0 + else: + return obj.event.captures.filter(timestamp__lt=obj.timestamp).count() class TaxonOccurrenceNestedSerializer(DefaultSerializer): diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 9bc464f54..a7f4a8756 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -434,7 +434,7 @@ class OccurrenceViewSet(DefaultViewSet): """ queryset = ( - Occurrence.objects.exclude(detections=None) + Occurrence.objects.exclude(detections=None) # This must come before annotations .annotate( detections_count=models.Count("detections", distinct=True), duration=models.Max("detections__timestamp") - models.Min("detections__timestamp"), @@ -447,7 +447,7 @@ class OccurrenceViewSet(DefaultViewSet): ) .prefetch_related("detections") .order_by("-determination_score") - .all() + .exclude(event=None, first_appearance_time=None) # These must come after annotations ) serializer_class = OccurrenceSerializer filterset_fields = ["event", "deployment", "determination", "project"] diff --git a/ami/main/charts.py b/ami/main/charts.py index 9d309011b..42fb55497 100644 --- a/ami/main/charts.py +++ b/ami/main/charts.py @@ -175,6 +175,7 @@ def detections_per_hour(project_pk: int): .values("source_image__timestamp__hour") .annotate(num_detections=models.Count("id")) .order_by("source_image__timestamp__hour") + .exclude(source_image__timestamp=None) ) # hours, counts = list(zip(*detections_per_hour)) @@ -204,6 +205,7 @@ def occurrences_accumulated(project_pk: int): occurrences_per_day = ( Occurrence.objects.filter(project=project_pk) .values_list("event__start") + .exclude(event__start=None) .annotate(num_occurrences=models.Count("id")) .order_by("event__start") ) @@ -213,8 +215,8 @@ def occurrences_accumulated(project_pk: int): # Accumulate the counts counts = list(itertools.accumulate(counts)) # tickvals = [f"{d:%b %d}" for d in days] - tickvals = [f"{days[0]:%b %d}", f"{days[-1]:%b %d}"] - days = [f"{d:%b %d}" for d in days] + tickvals = [f"{days[0]:%b %d, %Y}", f"{days[-1]:%b %d, %Y}"] + days = [f"{d:%b %d, %Y}" for d in days] else: days, counts = [], [] tickvals = [] @@ -234,6 +236,7 @@ def event_detections_per_hour(event_pk: int): .values("source_image__timestamp__hour") .annotate(num_detections=models.Count("id")) .order_by("source_image__timestamp__hour") + .exclude(source_image__timestamp=None) ) # hours, counts = list(zip(*detections_per_hour)) From 9e6bb421f75a5d29755a00337913943a201273bb Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Sat, 3 Feb 2024 02:36:12 +0000 Subject: [PATCH 13/23] Fix filtering of occurrences without events --- ami/main/api/views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index a7f4a8756..87b4aa919 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -434,7 +434,8 @@ class OccurrenceViewSet(DefaultViewSet): """ queryset = ( - Occurrence.objects.exclude(detections=None) # This must come before annotations + Occurrence.objects.exclude(detections=None) + .exclude(event=None) # These must be independent exclude calls .annotate( detections_count=models.Count("detections", distinct=True), duration=models.Max("detections__timestamp") - models.Min("detections__timestamp"), @@ -447,7 +448,7 @@ class OccurrenceViewSet(DefaultViewSet): ) .prefetch_related("detections") .order_by("-determination_score") - .exclude(event=None, first_appearance_time=None) # These must come after annotations + .exclude(first_appearance_time=None) # This must come after annotations ) serializer_class = OccurrenceSerializer filterset_fields = ["event", "deployment", "determination", "project"] From f3191a89ffc0b51055deeb0055a5e4aeef35ee98 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Feb 2024 20:07:47 -0800 Subject: [PATCH 14/23] Move sentry to the base requirements --- requirements/base.txt | 2 +- requirements/production.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 8f728af9f..5b69e23e9 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -14,7 +14,7 @@ boto3==1.28 rich==13.5 pydantic<2.0 # Less than 2.0 because of django pydantic field django-pydantic-field==0.2.11 -sentry_sdk==1.39.2 +sentry-sdk==1.40.0 # https://github.com/getsentry/sentry-python # Django # ------------------------------------------------------------------------------ diff --git a/requirements/production.txt b/requirements/production.txt index 1db90db6c..30a49832c 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -4,7 +4,7 @@ gunicorn==20.1.0 # https://github.com/benoitc/gunicorn psycopg[c]==3.1.9 # https://github.com/psycopg/psycopg -sentry-sdk==1.26.0 # https://github.com/getsentry/sentry-python + # Django # ------------------------------------------------------------------------------ From a348b11d4b32e1f9ab3309be315b84408006a6e2 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Feb 2024 20:13:30 -0800 Subject: [PATCH 15/23] Merge migrations --- ami/main/migrations/0029_merge_20240202_2312.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ami/main/migrations/0029_merge_20240202_2312.py diff --git a/ami/main/migrations/0029_merge_20240202_2312.py b/ami/main/migrations/0029_merge_20240202_2312.py new file mode 100644 index 000000000..2b91e8da5 --- /dev/null +++ b/ami/main/migrations/0029_merge_20240202_2312.py @@ -0,0 +1,12 @@ +# Generated by Django 4.2.2 on 2024-02-02 23:12 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0028_alter_occurrence_options"), + ("main", "0028_alter_occurrence_options_alter_project_options_and_more"), + ] + + operations = [] From 8802aa45a28173b9d288c31baf22d50c4e7a3454 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Feb 2024 22:03:38 -0800 Subject: [PATCH 16/23] Fix migrations --- .../0025_update_deployment_aggregates.py | 7 ++++--- .../migrations/0027_update_occurrence_scores.py | 4 ++-- .../migrations/0028_alter_occurrence_options.py | 16 ---------------- ami/main/migrations/0029_merge_20240202_2312.py | 12 ------------ 4 files changed, 6 insertions(+), 33 deletions(-) delete mode 100644 ami/main/migrations/0028_alter_occurrence_options.py delete mode 100644 ami/main/migrations/0029_merge_20240202_2312.py diff --git a/ami/main/migrations/0025_update_deployment_aggregates.py b/ami/main/migrations/0025_update_deployment_aggregates.py index 4055be21e..2736f7cd3 100644 --- a/ami/main/migrations/0025_update_deployment_aggregates.py +++ b/ami/main/migrations/0025_update_deployment_aggregates.py @@ -8,12 +8,13 @@ # Save all Deployment objects to update their calculated fields. def update_deployment_aggregates(apps, schema_editor): - # Deployment = apps.get_model("main", "Deployment") - from ami.main.models import Deployment + Deployment = apps.get_model("main", "Deployment") + # from ami.main.models import Deployment for deployment in Deployment.objects.all(): logger.info(f"Updating deployment {deployment}") - deployment.save(update_calculated_fields=True) + # deployment.save(update_calculated_fields=True) + deployment.save() class Migration(migrations.Migration): diff --git a/ami/main/migrations/0027_update_occurrence_scores.py b/ami/main/migrations/0027_update_occurrence_scores.py index 59c2ee5d4..b2e6cf4d2 100644 --- a/ami/main/migrations/0027_update_occurrence_scores.py +++ b/ami/main/migrations/0027_update_occurrence_scores.py @@ -5,8 +5,8 @@ # Call save on all occurrences to update their scores def update_occurrence_scores(apps, schema_editor): - # Occurrence = apps.get_model("main", "Occurrence") - from ami.main.models import Occurrence + Occurrence = apps.get_model("main", "Occurrence") + # from ami.main.models import Occurrence for occurrence in Occurrence.objects.all(): occurrence.save() diff --git a/ami/main/migrations/0028_alter_occurrence_options.py b/ami/main/migrations/0028_alter_occurrence_options.py deleted file mode 100644 index 5342bed74..000000000 --- a/ami/main/migrations/0028_alter_occurrence_options.py +++ /dev/null @@ -1,16 +0,0 @@ -# Generated by Django 4.2.2 on 2023-12-11 02:44 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("main", "0027_update_occurrence_scores"), - ] - - operations = [ - migrations.AlterModelOptions( - name="occurrence", - options={"ordering": ["-determination_score"]}, - ), - ] diff --git a/ami/main/migrations/0029_merge_20240202_2312.py b/ami/main/migrations/0029_merge_20240202_2312.py deleted file mode 100644 index 2b91e8da5..000000000 --- a/ami/main/migrations/0029_merge_20240202_2312.py +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by Django 4.2.2 on 2024-02-02 23:12 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("main", "0028_alter_occurrence_options"), - ("main", "0028_alter_occurrence_options_alter_project_options_and_more"), - ] - - operations = [] From d398b2c66914e2302c2850ba488a91c3ca079ca5 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Feb 2024 22:07:13 -0800 Subject: [PATCH 17/23] Only filter taxa by threshold if a filter is active --- ami/main/api/views.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 2e747f3b9..7e2d92ed4 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -574,9 +574,25 @@ def filter_by_occurrence(self, queryset: QuerySet) -> tuple[QuerySet, bool]: event = Event.objects.get(id=event_id) queryset = super().get_queryset().filter(occurrences__event=event) + return queryset, filter_active + + def filter_by_classification_threshold(self, queryset: QuerySet) -> QuerySet: + """ + Filter taxa by their best determination score in occurrences. + + This is only applicable to list queries that are not filtered by occurrence, project, deployment, or event. + """ + # Look for a query param to filter by score + classification_threshold = self.request.query_params.get("classification_threshold") + + if classification_threshold is not None: + classification_threshold = FloatField(required=False).clean(classification_threshold) + else: + classification_threshold = DEFAULT_CONFIDENCE_THRESHOLD + queryset = ( queryset.annotate(best_determination_score=models.Max("occurrences__determination_score")) - .filter(best_determination_score__gte=DEFAULT_CONFIDENCE_THRESHOLD) + .filter(best_determination_score__gte=classification_threshold) .distinct() ) @@ -584,7 +600,7 @@ def filter_by_occurrence(self, queryset: QuerySet) -> tuple[QuerySet, bool]: if not self.request.query_params.get("ordering"): queryset = queryset.order_by("-best_determination_score") - return queryset, filter_active + return queryset def get_queryset(self) -> QuerySet: qs = super().get_queryset() @@ -594,9 +610,12 @@ def get_queryset(self) -> QuerySet: from rest_framework.exceptions import NotFound raise NotFound(detail=str(e)) + qs = qs.select_related("parent", "parent__parent") if filter_active: + qs = self.filter_by_classification_threshold(qs) + qs = qs.prefetch_related("occurrences") qs = qs.annotate( occurrences_count=models.Count("occurrences", distinct=True), From 827586935517f62dea183f81708600b5288b09b4 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Feb 2024 22:07:44 -0800 Subject: [PATCH 18/23] Ensure cached props are invalidated when updating determination --- ami/main/models.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ami/main/models.py b/ami/main/models.py index f724405fc..4dcbd9767 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -1323,6 +1323,7 @@ class Detection(BaseModel): # @TODO use structured data for bbox bbox = models.JSONField(null=True, blank=True) + # @TODO shouldn't this be automatically set by the source image? timestamp = models.DateTimeField(null=True, blank=True) # file = ( @@ -1577,7 +1578,10 @@ def save(self, update_determination=True, *args, **kwargs): # This may happen for legacy occurrences that were created # before the determination_score field was added self.determination_score = self.get_determination_score() - self.save(update_determination=False) + if not self.determination_score: + logger.warning(f"Could not determine score for {self}") + else: + self.save(update_determination=False) class Meta: ordering = ["-determination_score"] @@ -1601,6 +1605,12 @@ def update_occurrence_determination( """ needs_update = False + # Invalidate the cached properties so they will be re-calculated + if hasattr(occurrence, "best_identification"): + del occurrence.best_identification + if hasattr(occurrence, "best_prediction"): + del occurrence.best_prediction + current_determination = ( current_determination or Occurrence.objects.select_related("determination") From 42568076c1cd3a410308df03d24385f23c0beaa2 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Feb 2024 22:08:24 -0800 Subject: [PATCH 19/23] Update tests with latest taxa filter changes --- .github/workflows/test.backend.yml | 2 +- ami/main/tests.py | 60 ++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.backend.yml b/.github/workflows/test.backend.yml index 1357aca56..984b6ef83 100644 --- a/.github/workflows/test.backend.yml +++ b/.github/workflows/test.backend.yml @@ -47,7 +47,7 @@ jobs: run: docker-compose -f local.yml run --rm django python manage.py migrate - name: Run Django Tests - run: docker-compose -f local.yml run django pytest + run: docker-compose -f local.yml run --rm django python manage.py test - name: Tear down the Stack run: docker-compose -f local.yml down diff --git a/ami/main/tests.py b/ami/main/tests.py index 4a2b28c64..0740e4867 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -7,6 +7,7 @@ from ami.main.models import ( Deployment, + Detection, Event, Occurrence, Project, @@ -72,19 +73,42 @@ def create_taxa(project: Project) -> TaxaList: def create_occurrences( deployment: Deployment, - num: int = 12, + num: int = 6, ): event = Event.objects.filter(deployment=deployment).first() if not event: raise ValueError("No events found for deployment") for i in range(num): - Occurrence.objects.create( - project=deployment.project, - deployment=deployment, - determination=Taxon.objects.order_by("?").first(), - event=event, + # Every Occurrence requires a Detection + source_image = SourceImage.objects.filter(event=event).order_by("?").first() + if not source_image: + raise ValueError("No source images found for event") + taxon = Taxon.objects.filter(projects=deployment.project).order_by("?").first() + if not taxon: + raise ValueError("No taxa found for project") + detection = Detection.objects.create( + source_image=source_image, + timestamp=source_image.timestamp, # @TODO this should be automatically set to the source image timestamp ) + # Could speed this up by creating an Occurrence with a determined taxon directly + # but this tests more of the code. + detection.classifications.create( + taxon=taxon, + score=0.9, + timestamp=datetime.datetime.now(), + ) + occurrence = detection.associate_new_occurrence() + + # Assert that the occurrence was created and has a detection, event, first_appearance, + # and species determination + assert detection.occurrence is not None + assert detection.occurrence.event is not None + assert detection.occurrence.first_appearance is not None + assert occurrence.best_detection is not None + assert occurrence.best_prediction is not None + assert occurrence.determination is not None + assert occurrence.determination_score is not None class TestImageGrouping(TestCase): @@ -181,7 +205,9 @@ def setUp(self) -> None: create_captures(deployment=self.deployment) group_images_into_events(deployment=self.deployment) - create_occurrences(deployment=self.deployment) + create_taxa(project=self.project_one) + create_taxa(project=self.project_two) + create_occurrences(deployment=self.deployment, num=1) return super().setUp() @@ -190,6 +216,7 @@ def test_initial_project(self): assert self.deployment.captures.first().project == self.project_one assert self.deployment.events.first().project == self.project_one assert self.deployment.occurrences.first().project == self.project_one + assert self.deployment.occurrences.first().detections.first().source_image.project == self.project_one def test_change_project(self): self.deployment.project = self.project_two @@ -465,12 +492,19 @@ def setUp(self) -> None: create_captures(deployment=deployment_two) group_images_into_events(deployment=deployment_one) group_images_into_events(deployment=deployment_two) - create_occurrences(deployment=deployment_one, num=100) - create_occurrences(deployment=deployment_two, num=100) + create_occurrences(deployment=deployment_one, num=5) + create_occurrences(deployment=deployment_two, num=5) self.project_one = project_one self.project_two = project_two return super().setUp() + def test_occurrences_for_project(self): + # Test that occurrences are specific to each project + for project in [self.project_one, self.project_two]: + response = self.client.get(f"/api/v2/occurrences/?project={project.pk}") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], Occurrence.objects.filter(project=project).count()) + def test_taxa_list(self): from ami.main.models import Taxon @@ -487,7 +521,10 @@ def _test_taxa_for_project(self, project: Project): response = self.client.get(f"/api/v2/taxa/?project={project.pk}") self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["count"], Taxon.objects.filter(projects=project).count()) + project_occurred_taxa = Taxon.objects.filter(occurrences__project=project).distinct() + # project_any_taxa = Taxon.objects.filter(projects=project) + self.assertGreater(project_occurred_taxa.count(), 0) + self.assertEqual(response.json()["count"], project_occurred_taxa.count()) # Check counts for each taxon results = response.json()["results"] @@ -504,8 +541,9 @@ def test_taxa_for_project(self): def test_taxon_detail(self): from ami.main.models import Taxon - taxon = Taxon.objects.first() + taxon = Taxon.objects.last() assert taxon is not None + print("Testing taxon", taxon, taxon.pk) response = self.client.get(f"/api/v2/taxa/{taxon.pk}/") self.assertEqual(response.status_code, 200) self.assertEqual(response.json()["name"], taxon.name) From 3f9a86f0463338897a120aa2551223b161248e2e Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Feb 2024 22:27:43 -0800 Subject: [PATCH 20/23] Try switching back to GH actions v3 --- .github/workflows/test.backend.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.backend.yml b/.github/workflows/test.backend.yml index 984b6ef83..c6635ec4f 100644 --- a/.github/workflows/test.backend.yml +++ b/.github/workflows/test.backend.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code Repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 @@ -38,7 +38,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code Repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Build the Stack run: docker-compose -f local.yml build From 67b7bd03c2dc888886c37e808012b1644fb1576d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Sat, 3 Feb 2024 06:32:55 +0000 Subject: [PATCH 21/23] Try to match occurrence count in chart to other stats --- ami/main/charts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ami/main/charts.py b/ami/main/charts.py index 42fb55497..718569219 100644 --- a/ami/main/charts.py +++ b/ami/main/charts.py @@ -205,7 +205,9 @@ def occurrences_accumulated(project_pk: int): occurrences_per_day = ( Occurrence.objects.filter(project=project_pk) .values_list("event__start") + .exclude(event=None) .exclude(event__start=None) + .exclude(detections=None) .annotate(num_occurrences=models.Count("id")) .order_by("event__start") ) From b330e741cd313fe75e4e05358c18f32d53a33bea Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Sat, 3 Feb 2024 06:33:20 +0000 Subject: [PATCH 22/23] Don't regroup events every batch during image sync --- ami/main/models.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ami/main/models.py b/ami/main/models.py index f724405fc..01ad0645d 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -217,6 +217,7 @@ def _insert_or_update_batch_for_sync( total_files: int, total_size: int, sql_batch_size=500, + regroup_events_per_batch=False, ): logger.info(f"Bulk inserting or updating batch of {len(source_images)} SourceImages") try: @@ -236,9 +237,10 @@ def _insert_or_update_batch_for_sync( deployment.data_source_total_size = total_size deployment.data_source_last_checked = datetime.datetime.now() - events = group_images_into_events(deployment) - for event in events: - set_dimensions_for_collection(event) + if regroup_events_per_batch: + events = group_images_into_events(deployment) + for event in events: + set_dimensions_for_collection(event) deployment.save(update_calculated_fields=False) @@ -354,7 +356,7 @@ def data_source_uri(self) -> str | None: uri = None return uri - def sync_captures(self, batch_size=1000) -> int: + def sync_captures(self, batch_size=1000, regroup_events_per_batch=False) -> int: """Import images from the deployment's data source""" deployment = self @@ -379,17 +381,22 @@ def sync_captures(self, batch_size=1000) -> int: source_images.append(source_image) if len(source_images) >= django_batch_size: - _insert_or_update_batch_for_sync(deployment, source_images, total_files, total_size, sql_batch_size) + _insert_or_update_batch_for_sync( + deployment, source_images, total_files, total_size, sql_batch_size, regroup_events_per_batch + ) source_images = [] if source_images: # Insert/update the last batch - _insert_or_update_batch_for_sync(deployment, source_images, total_files, total_size, sql_batch_size) + _insert_or_update_batch_for_sync( + deployment, source_images, total_files, total_size, sql_batch_size, regroup_events_per_batch + ) _compare_totals_for_sync(deployment, total_files) # @TODO decide if we should delete SourceImages that are no longer in the data source self.save() + return total_files def update_children(self): From df81800933297b3afd391d6c4a24e136f727dc00 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Feb 2024 22:45:50 -0800 Subject: [PATCH 23/23] Docker compose continues to be broken in GH actions, testing manually --- .github/workflows/test.backend.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.backend.yml b/.github/workflows/test.backend.yml index c6635ec4f..c4467457c 100644 --- a/.github/workflows/test.backend.yml +++ b/.github/workflows/test.backend.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -34,20 +34,20 @@ jobs: uses: pre-commit/action@v3.0.0 # With no caching at all the entire ci process takes 4m 30s to complete! - test: - runs-on: ubuntu-latest - steps: - - name: Checkout Code Repository - uses: actions/checkout@v3 + # test: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout Code Repository + # uses: actions/checkout@v4 - - name: Build the Stack - run: docker-compose -f local.yml build + # - name: Build the Stack + # run: docker-compose -f local.yml build - - name: Run DB Migrations - run: docker-compose -f local.yml run --rm django python manage.py migrate + # - name: Run DB Migrations + # run: docker-compose -f local.yml run --rm django python manage.py migrate - - name: Run Django Tests - run: docker-compose -f local.yml run --rm django python manage.py test + # - name: Run Django Tests + # run: docker-compose -f local.yml run --rm django python manage.py test - - name: Tear down the Stack - run: docker-compose -f local.yml down + # - name: Tear down the Stack + # run: docker-compose -f local.yml down