Skip to content

Commit

Permalink
Clean up status logging
Browse files Browse the repository at this point in the history
  • Loading branch information
mihow committed Nov 30, 2023
1 parent 2297bad commit 805c926
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 17 deletions.
30 changes: 30 additions & 0 deletions ami/jobs/migrations/0010_job_limit_job_shuffle.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Generated by Django 4.2.2 on 2023-11-30 16:49

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("jobs", "0009_remove_job_config"),
]

operations = [
migrations.AddField(
model_name="job",
name="limit",
field=models.IntegerField(
blank=True,
default=100,
help_text="Limit the number of images to process",
null=True,
verbose_name="Limit",
),
),
migrations.AddField(
model_name="job",
name="shuffle",
field=models.BooleanField(
default=True, help_text="Process images in a random order", verbose_name="Shuffle"
),
),
]
28 changes: 11 additions & 17 deletions ami/jobs/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -320,10 +320,10 @@ def setup(self, save=True):
if self.pipeline:
collect_stage = self.progress.add_stage("Collect")
self.progress.add_stage_param(collect_stage.key, "Total Images", "")
self.progress.add_stage_param(collect_stage.key, "Proccessed", "")
self.progress.add_stage_param(collect_stage.key, "Remaining", "")

pipeline_stage = self.progress.add_stage("Process")
self.progress.add_stage_param(pipeline_stage.key, "Proccessed", "")
self.progress.add_stage_param(pipeline_stage.key, "Remaining", "")
self.progress.add_stage_param(pipeline_stage.key, "Detections", "")
self.progress.add_stage_param(pipeline_stage.key, "Classifications", "")

Expand Down Expand Up @@ -390,8 +390,8 @@ def run(self):
# shuffle=self.shuffle,
)
)
image_count = len(images)
image_count_label = f"{image_count}"
source_image_count = len(images)
self.progress.update_stage("collect", total_images=source_image_count)

if self.shuffle:
self.logger.info("Shuffling images")
Expand All @@ -401,19 +401,18 @@ def run(self):
TEMPORARY_LIMIT = 200
self.limit = self.limit or TEMPORARY_LIMIT

if self.limit and image_count > self.limit:
self.logger.warn(f"Limiting number of images to {self.limit} (out of {image_count})")
if self.limit and source_image_count > self.limit:
self.logger.warn(f"Limiting number of images to {self.limit} (out of {source_image_count})")
images = images[: self.limit]
image_count = len(images)
image_count_label = f"{image_count} (out of {len(images)})"
self.progress.add_stage_param("collect", "Limit", image_count)
else:
image_count = source_image_count

self.progress.update_stage(
"collect",
status=JobState.SUCCESS,
progress=1,
total=image_count_label,
proccessed=0,
remaining=image_count,
)

total_detections = 0
Expand All @@ -433,6 +432,8 @@ def run(self):
"process",
status=JobState.STARTED,
progress=(i + 1) / len(chunks),
proccessed=(i + 1) * CHUNK_SIZE,
remaining=image_count - (i + 1) * CHUNK_SIZE,
detections=total_detections,
classifications=total_classifications,
)
Expand All @@ -444,23 +445,16 @@ def run(self):
progress=(i + 1) / len(chunks),
objects_created=len(objects),
)
self.progress.update_stage(
"collect",
proccessed=(i + 1) * CHUNK_SIZE,
remaining=image_count - (i + 1) * CHUNK_SIZE,
)
self.update_progress()
self.save()

self.progress.update_stage(
"process",
status=JobState.SUCCESS,
progress=1,
)
self.progress.update_stage(
"results",
status=JobState.SUCCESS,
progress=1,
)

self.update_status(JobState.SUCCESS)
Expand Down

0 comments on commit 805c926

Please sign in to comment.