Refactor processors and add LPR postprocessing (#16722)

* recordings data pub/sub

* function to process recording stream frames

* model runner

* lpr model runner

* refactor to mixin class and use model runner

* separate out realtime and post processors

* move model and mixin folders

* basic postprocessor

* clean up

* docs

* postprocessing logic

* clean up

* return none if recordings are disabled

* run postprocessor handle_requests too

* tweak expansion

* add put endpoint

* postprocessor tweaks with endpoint

docs/docs/configuration/license_plate_recognition.md

@@ -41,6 +41,8 @@ lpr:
 
 Ensure that your camera is configured to detect objects of type `car`, and that a car is actually being detected by Frigate. Otherwise, LPR will not run.
 
+Like the other real-time processors in Frigate, license plate recognition runs on the camera stream defined by the `detect` role in your config. To ensure optimal performance, select a suitable resolution for this stream in your camera's firmware that fits your specific scene and requirements.
+
 ## Advanced Configuration
 
 Fine-tune the LPR feature using these optional parameters:
@@ -52,7 +54,7 @@ Fine-tune the LPR feature using these optional parameters:
   - Note: If you are using a Frigate+ model and you set the `threshold` in your objects config for `license_plate` higher than this value, recognition will never run. It's best to ensure these values match, or this `detection_threshold` is lower than your object config `threshold`.
 - **`min_area`**: Defines the minimum size (in pixels) a license plate must be before recognition runs.
   - Default: `1000` pixels.
-  - Depending on the resolution of your cameras, you can increase this value to ignore small or distant plates.
+  - Depending on the resolution of your camera's `detect` stream, you can increase this value to ignore small or distant plates.
 
 ### Recognition
 
@@ -114,7 +116,7 @@ lpr:
 Ensure that:
 
 - Your camera has a clear, well-lit view of the plate.
-- The plate is large enough in the image (try adjusting `min_area`).
+- The plate is large enough in the image (try adjusting `min_area`) or increasing the resolution of your camera's stream.
 - A `car` is detected first, as LPR only runs on recognized vehicles.
 
 If you are using a Frigate+ model or a custom model that detects license plates, ensure that `license_plate` is added to your list of objects to track.
@@ -143,7 +145,7 @@ Use `match_distance` to allow small character mismatches. Alternatively, define
 - View MQTT messages for `frigate/events` to verify detected plates.
 - Adjust `detection_threshold` and `recognition_threshold` settings.
 - If you are using a Frigate+ model or a model that detects license plates, watch the debug view (Settings --> Debug) to ensure that `license_plate` is being detected with a `car`.
-- Enable debug logs for LPR by adding `frigate.data_processing.real_time.license_plate_processor: debug` to your `logger` configuration. These logs are _very_ verbose, so only enable this when necessary.
+- Enable debug logs for LPR by adding `frigate.data_processing.common.license_plate: debug` to your `logger` configuration. These logs are _very_ verbose, so only enable this when necessary.
 
 ### Will LPR slow down my system?
 

frigate/api/classification.py

@@ -9,10 +9,13 @@
 from fastapi import APIRouter, Request, UploadFile
 from fastapi.responses import JSONResponse
 from pathvalidate import sanitize_filename
+from peewee import DoesNotExist
+from playhouse.shortcuts import model_to_dict
 
 from frigate.api.defs.tags import Tags
 from frigate.const import FACE_DIR
 from frigate.embeddings import EmbeddingsContext
+from frigate.models import Event
 
 logger = logging.getLogger(__name__)
 
@@ -176,3 +179,36 @@ def deregister_faces(request: Request, name: str, body: dict = None):
         content=({"success": True, "message": "Successfully deleted faces."}),
         status_code=200,
     )
+
+
+@router.put("/lpr/reprocess")
+def reprocess_license_plate(request: Request, event_id: str):
+    if not request.app.frigate_config.lpr.enabled:
+        message = "License plate recognition is not enabled."
+        logger.error(message)
+        return JSONResponse(
+            content=(
+                {
+                    "success": False,
+                    "message": message,
+                }
+            ),
+            status_code=400,
+        )
+
+    try:
+        event = Event.get(Event.id == event_id)
+    except DoesNotExist:
+        message = f"Event {event_id} not found"
+        logger.error(message)
+        return JSONResponse(
+            content=({"success": False, "message": message}), status_code=404
+        )
+
+    context: EmbeddingsContext = request.app.embeddings
+    response = context.reprocess_plate(model_to_dict(event))
+
+    return JSONResponse(
+        content=response,
+        status_code=200,
+    )

frigate/comms/embeddings_updater.py

@@ -15,6 +15,7 @@ class EmbeddingsRequestEnum(Enum):
     generate_search = "generate_search"
     register_face = "register_face"
     reprocess_face = "reprocess_face"
+    reprocess_plate = "reprocess_plate"
 
 
 class EmbeddingsResponder:

frigate/comms/recordings_updater.py

@@ -0,0 +1,36 @@
+"""Facilitates communication between processes."""
+
+import logging
+from enum import Enum
+
+from .zmq_proxy import Publisher, Subscriber
+
+logger = logging.getLogger(__name__)
+
+
+class RecordingsDataTypeEnum(str, Enum):
+    all = ""
+    recordings_available_through = "recordings_available_through"
+
+
+class RecordingsDataPublisher(Publisher):
+    """Publishes latest recording data."""
+
+    topic_base = "recordings/"
+
+    def __init__(self, topic: RecordingsDataTypeEnum) -> None:
+        topic = topic.value
+        super().__init__(topic)
+
+    def publish(self, payload: tuple[str, float]) -> None:
+        super().publish(payload)
+
+
+class RecordingsDataSubscriber(Subscriber):
+    """Receives latest recording data."""
+
+    topic_base = "recordings/"
+
+    def __init__(self, topic: RecordingsDataTypeEnum) -> None:
+        topic = topic.value
+        super().__init__(topic)

frigate/data_processing/real_time/license_plate_processor.py → frigate/data_processing/common/license_plate/mixin.py

@@ -13,34 +13,21 @@
 from pyclipper import ET_CLOSEDPOLYGON, JT_ROUND, PyclipperOffset
 from shapely.geometry import Polygon
 
-from frigate.comms.inter_process import InterProcessRequestor
-from frigate.config import FrigateConfig
 from frigate.const import FRIGATE_LOCALHOST
-from frigate.embeddings.onnx.lpr_embedding import (
-    LicensePlateDetector,
-    PaddleOCRClassification,
-    PaddleOCRDetection,
-    PaddleOCRRecognition,
-)
 from frigate.util.image import area
 
-from ..types import DataProcessorMetrics
-from .api import RealTimeProcessorApi
-
 logger = logging.getLogger(__name__)
 
 WRITE_DEBUG_IMAGES = False
 
 
-class LicensePlateProcessor(RealTimeProcessorApi):
-    def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics):
-        super().__init__(config, metrics)
-        self.requestor = InterProcessRequestor()
-        self.lpr_config = config.lpr
+class LicensePlateProcessingMixin:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
         self.requires_license_plate_detection = (
             "license_plate" not in self.config.objects.all_objects
         )
-        self.detected_license_plates: dict[str, dict[str, any]] = {}
 
         self.ctc_decoder = CTCDecoder()
 
@@ -52,42 +39,6 @@ def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics):
         self.box_thresh = 0.8
         self.mask_thresh = 0.8
 
-        self.lpr_detection_model = None
-        self.lpr_classification_model = None
-        self.lpr_recognition_model = None
-
-        if self.config.lpr.enabled:
-            self.detection_model = PaddleOCRDetection(
-                model_size="large",
-                requestor=self.requestor,
-                device="CPU",
-            )
-
-            self.classification_model = PaddleOCRClassification(
-                model_size="large",
-                requestor=self.requestor,
-                device="CPU",
-            )
-
-            self.recognition_model = PaddleOCRRecognition(
-                model_size="large",
-                requestor=self.requestor,
-                device="CPU",
-            )
-
-            self.yolov9_detection_model = LicensePlateDetector(
-                model_size="large",
-                requestor=self.requestor,
-                device="CPU",
-            )
-
-        if self.lpr_config.enabled:
-            # all models need to be loaded to run LPR
-            self.detection_model._load_model_and_utils()
-            self.classification_model._load_model_and_utils()
-            self.recognition_model._load_model_and_utils()
-            self.yolov9_detection_model._load_model_and_utils()
-
     def _detect(self, image: np.ndarray) -> List[np.ndarray]:
         """
         Detect possible license plates in the input image by first resizing and normalizing it,
@@ -114,7 +65,7 @@ def _detect(self, image: np.ndarray) -> List[np.ndarray]:
                 resized_image,
             )
 
-        outputs = self.detection_model([normalized_image])[0]
+        outputs = self.model_runner.detection_model([normalized_image])[0]
         outputs = outputs[0, :, :]
 
         boxes, _ = self._boxes_from_bitmap(outputs, outputs > self.mask_thresh, w, h)
@@ -143,7 +94,7 @@ def _classify(
                 norm_img = norm_img[np.newaxis, :]
                 norm_images.append(norm_img)
 
-        outputs = self.classification_model(norm_images)
+        outputs = self.model_runner.classification_model(norm_images)
 
         return self._process_classification_output(images, outputs)
 
@@ -183,7 +134,7 @@ def _recognize(
                 norm_image = norm_image[np.newaxis, :]
                 norm_images.append(norm_image)
 
-        outputs = self.recognition_model(norm_images)
+        outputs = self.model_runner.recognition_model(norm_images)
         return self.ctc_decoder(outputs)
 
     def _process_license_plate(
@@ -199,9 +150,9 @@ def _process_license_plate(
             Tuple[List[str], List[float], List[int]]: Detected license plate texts, confidence scores, and areas of the plates.
         """
         if (
-            self.detection_model.runner is None
-            or self.classification_model.runner is None
-            or self.recognition_model.runner is None
+            self.model_runner.detection_model.runner is None
+            or self.model_runner.classification_model.runner is None
+            or self.model_runner.recognition_model.runner is None
         ):
             # we might still be downloading the models
             logger.debug("Model runners not loaded")
@@ -665,7 +616,9 @@ def _preprocess_recognition_image(
         input_w = int(input_h * max_wh_ratio)
 
         # check for model-specific input width
-        model_input_w = self.recognition_model.runner.ort.get_inputs()[0].shape[3]
+        model_input_w = self.model_runner.recognition_model.runner.ort.get_inputs()[
+            0
+        ].shape[3]
         if isinstance(model_input_w, int) and model_input_w > 0:
             input_w = model_input_w
 
@@ -732,19 +685,13 @@ def _crop_license_plate(image: np.ndarray, points: np.ndarray) -> np.ndarray:
             image = np.rot90(image, k=3)
         return image
 
-    def __update_metrics(self, duration: float) -> None:
-        """
-        Update inference metrics.
-        """
-        self.metrics.alpr_pps.value = (self.metrics.alpr_pps.value * 9 + duration) / 10
-
     def _detect_license_plate(self, input: np.ndarray) -> tuple[int, int, int, int]:
         """
         Use a lightweight YOLOv9 model to detect license plates for users without Frigate+
 
         Return the dimensions of the detected plate as [x1, y1, x2, y2].
         """
-        predictions = self.yolov9_detection_model(input)
+        predictions = self.model_runner.yolov9_detection_model(input)
 
         confidence_threshold = self.lpr_config.detection_threshold
 
@@ -770,8 +717,8 @@ def _detect_license_plate(self, input: np.ndarray) -> tuple[int, int, int, int]:
 
         # Return the top scoring bounding box if found
         if top_box is not None:
-            # expand box by 15% to help with OCR
-            expansion = (top_box[2:] - top_box[:2]) * 0.1
+            # expand box by 30% to help with OCR
+            expansion = (top_box[2:] - top_box[:2]) * 0.30
 
             # Expand box
             expanded_box = np.array(
@@ -869,9 +816,8 @@ def _should_keep_previous_plate(
         # 5. Return True if we should keep the previous plate (i.e., if it scores higher)
         return prev_score > curr_score
 
-    def process_frame(self, obj_data: dict[str, any], frame: np.ndarray):
+    def lpr_process(self, obj_data: dict[str, any], frame: np.ndarray):
         """Look for license plates in image."""
-        start = datetime.datetime.now().timestamp()
 
         id = obj_data["id"]
 
@@ -934,7 +880,7 @@ def process_frame(self, obj_data: dict[str, any], frame: np.ndarray):
 
             # check that license plate is valid
             # double the value because we've doubled the size of the car
-            if license_plate_area < self.config.lpr.min_area * 2:
+            if license_plate_area < self.lpr_config.min_area * 2:
                 logger.debug("License plate is less than min_area")
                 return
 
@@ -972,7 +918,7 @@ def process_frame(self, obj_data: dict[str, any], frame: np.ndarray):
             # check that license plate is valid
             if (
                 not license_plate_box
-                or area(license_plate_box) < self.config.lpr.min_area
+                or area(license_plate_box) < self.lpr_config.min_area
             ):
                 logger.debug(f"Invalid license plate box {license_plate}")
                 return
@@ -1078,10 +1024,9 @@ def process_frame(self, obj_data: dict[str, any], frame: np.ndarray):
                 "plate": top_plate,
                 "char_confidences": top_char_confidences,
                 "area": top_area,
+                "obj_data": obj_data,
             }
 
-        self.__update_metrics(datetime.datetime.now().timestamp() - start)
-
     def handle_request(self, topic, request_data) -> dict[str, any] | None:
         return
 

frigate/data_processing/common/license_plate/model.py

@@ -0,0 +1,31 @@
+from frigate.embeddings.onnx.lpr_embedding import (
+    LicensePlateDetector,
+    PaddleOCRClassification,
+    PaddleOCRDetection,
+    PaddleOCRRecognition,
+)
+
+from ...types import DataProcessorModelRunner
+
+
+class LicensePlateModelRunner(DataProcessorModelRunner):
+    def __init__(self, requestor, device: str = "CPU", model_size: str = "large"):
+        super().__init__(requestor, device, model_size)
+        self.detection_model = PaddleOCRDetection(
+            model_size=model_size, requestor=requestor, device=device
+        )
+        self.classification_model = PaddleOCRClassification(
+            model_size=model_size, requestor=requestor, device=device
+        )
+        self.recognition_model = PaddleOCRRecognition(
+            model_size=model_size, requestor=requestor, device=device
+        )
+        self.yolov9_detection_model = LicensePlateDetector(
+            model_size=model_size, requestor=requestor, device=device
+        )
+
+        # Load all models once
+        self.detection_model._load_model_and_utils()
+        self.classification_model._load_model_and_utils()
+        self.recognition_model._load_model_and_utils()
+        self.yolov9_detection_model._load_model_and_utils()

frigate/data_processing/post/api.py

@@ -5,16 +5,22 @@
 
 from frigate.config import FrigateConfig
 
-from ..types import DataProcessorMetrics, PostProcessDataEnum
+from ..types import DataProcessorMetrics, DataProcessorModelRunner, PostProcessDataEnum
 
 logger = logging.getLogger(__name__)
 
 
 class PostProcessorApi(ABC):
     @abstractmethod
-    def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics) -> None:
+    def __init__(
+        self,
+        config: FrigateConfig,
+        metrics: DataProcessorMetrics,
+        model_runner: DataProcessorModelRunner,
+    ) -> None:
         self.config = config
         self.metrics = metrics
+        self.model_runner = model_runner
         pass
 
     @abstractmethod