diff --git a/ami/jobs/migrations/0010_job_limit_job_shuffle.py b/ami/jobs/migrations/0010_job_limit_job_shuffle.py new file mode 100644 index 000000000..690b9495a --- /dev/null +++ b/ami/jobs/migrations/0010_job_limit_job_shuffle.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.2 on 2023-11-30 16:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("jobs", "0009_remove_job_config"), + ] + + operations = [ + migrations.AddField( + model_name="job", + name="limit", + field=models.IntegerField( + blank=True, + default=100, + help_text="Limit the number of images to process", + null=True, + verbose_name="Limit", + ), + ), + migrations.AddField( + model_name="job", + name="shuffle", + field=models.BooleanField( + default=True, help_text="Process images in a random order", verbose_name="Shuffle" + ), + ), + ] diff --git a/ami/jobs/models.py b/ami/jobs/models.py index 3d1822f1b..67703cab4 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -320,10 +320,10 @@ def setup(self, save=True): if self.pipeline: collect_stage = self.progress.add_stage("Collect") self.progress.add_stage_param(collect_stage.key, "Total Images", "") - self.progress.add_stage_param(collect_stage.key, "Proccessed", "") - self.progress.add_stage_param(collect_stage.key, "Remaining", "") pipeline_stage = self.progress.add_stage("Process") + self.progress.add_stage_param(pipeline_stage.key, "Proccessed", "") + self.progress.add_stage_param(pipeline_stage.key, "Remaining", "") self.progress.add_stage_param(pipeline_stage.key, "Detections", "") self.progress.add_stage_param(pipeline_stage.key, "Classifications", "") @@ -390,8 +390,8 @@ def run(self): # shuffle=self.shuffle, ) ) - image_count = len(images) - image_count_label = f"{image_count}" + source_image_count = len(images) + self.progress.update_stage("collect", total_images=source_image_count) if self.shuffle: self.logger.info("Shuffling images") @@ -401,19 +401,18 @@ def run(self): TEMPORARY_LIMIT = 200 self.limit = self.limit or TEMPORARY_LIMIT - if self.limit and image_count > self.limit: - self.logger.warn(f"Limiting number of images to {self.limit} (out of {image_count})") + if self.limit and source_image_count > self.limit: + self.logger.warn(f"Limiting number of images to {self.limit} (out of {source_image_count})") images = images[: self.limit] image_count = len(images) - image_count_label = f"{image_count} (out of {len(images)})" + self.progress.add_stage_param("collect", "Limit", image_count) + else: + image_count = source_image_count self.progress.update_stage( "collect", status=JobState.SUCCESS, progress=1, - total=image_count_label, - proccessed=0, - remaining=image_count, ) total_detections = 0 @@ -433,6 +432,8 @@ def run(self): "process", status=JobState.STARTED, progress=(i + 1) / len(chunks), + proccessed=(i + 1) * CHUNK_SIZE, + remaining=image_count - (i + 1) * CHUNK_SIZE, detections=total_detections, classifications=total_classifications, ) @@ -444,23 +445,16 @@ def run(self): progress=(i + 1) / len(chunks), objects_created=len(objects), ) - self.progress.update_stage( - "collect", - proccessed=(i + 1) * CHUNK_SIZE, - remaining=image_count - (i + 1) * CHUNK_SIZE, - ) self.update_progress() self.save() self.progress.update_stage( "process", status=JobState.SUCCESS, - progress=1, ) self.progress.update_stage( "results", status=JobState.SUCCESS, - progress=1, ) self.update_status(JobState.SUCCESS)