diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3c81d18..cb1e8b9 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -3,7 +3,7 @@ name: Python application on: workflow_dispatch: pull_request: - types: [ labeled ] + types: [labeled] permissions: contents: read @@ -15,19 +15,26 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: 'pip' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 - - name: Lint with flake8 - run: | - flake8 . --count --show-source --statistics - - run: pip install -e . - - name: Test with unittest - run: | - python -m unittest discover tests \ No newline at end of file + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 + - name: Clone GuillaumeFalourd/poc-github-actions PUBLIC repository + uses: GuillaumeFalourd/clone-github-repo-action@v2.3 + with: + depth: 1 + branch: "master" + owner: "ITVRoC" + repository: "FlirImageExtractor" + - name: Lint with flake8 + run: | + flake8 . --count --show-source --statistics + - run: pip install -e . + - name: Test with unittest + run: | + python -m unittest discover tests diff --git a/tests/data/process_thermal/msx/IMG_0620.JPG b/tests/data/process_thermal/msx/IMG_0620.JPG new file mode 100644 index 0000000..ab973ac Binary files /dev/null and b/tests/data/process_thermal/msx/IMG_0620.JPG differ diff --git a/tests/data/process_thermal/msx/IMG_0622.JPG b/tests/data/process_thermal/msx/IMG_0622.JPG new file mode 100644 index 0000000..cf4163d Binary files /dev/null and b/tests/data/process_thermal/msx/IMG_0622.JPG differ diff --git a/tests/test_process_thermal.py b/tests/test_process_thermal.py new file mode 100644 index 0000000..be484a1 --- /dev/null +++ b/tests/test_process_thermal.py @@ -0,0 +1,70 @@ +import shutil +import unittest +from pathlib import Path + +from thermo_scenes.flir_thermal_images.custom_flir import CustomFlir + + +class TestThermalConversion(unittest.TestCase): + def test_process_thermal(self) -> None: + """ + Test of extracting RGB, thermal and csv files from the given msx images. + + 2 msx images will generate 2 RGB PNG files, 2 thermal PNG files, 2 csv files + and 1 temperature_bound.json file. + """ + msx_path = Path("tests/data/process_thermal/msx") + output_path = Path("tests/data/process_thermal/output") + + shutil.rmtree(output_path) + output_path.mkdir(exist_ok=True, parents=True) + + CustomFlir(path_to_msx_images=msx_path, path_to_output_folder=output_path) + + json_count = 0 + PNG_count = 0 + csv_count = 0 + + for folder in output_path.iterdir(): + if folder.suffix == ".json": + json_count += 1 + continue + for files in folder.iterdir(): + if files.suffix == ".PNG": + PNG_count += 1 + elif files.suffix == ".csv": + csv_count += 1 + + self.assertEqual(json_count, 1, "The number of the JSON file is not correct!") + self.assertEqual(PNG_count, 4, "The number of the PNG files is not correct!") + self.assertEqual(csv_count, 2, "The number of the CSV files is not correct!") + + def test_name_consistency(self) -> None: + """ + Test the name consistency of extracting RGB, thermal and csv files from the + given msx images. + + RGB image, thermal image and csv file should have the same stem as the extracted + msx image. + """ + msx_path = Path("tests/data/process_thermal/msx") + output_path = Path("tests/data/process_thermal/output") + + shutil.rmtree(output_path) + output_path.mkdir(exist_ok=True, parents=True) + + CustomFlir(path_to_msx_images=msx_path, path_to_output_folder=output_path) + + for folder in output_path.iterdir(): + if folder.suffix == ".json": + continue + namelist = [file.stem for file in msx_path.iterdir()] + for files in folder.iterdir(): + if files.stem in namelist: + namelist.remove(files.stem) + self.assertEqual( + len(namelist), + 0, + "Names of the extracted files are not consistent with the " + + "given msx images!", + ) diff --git a/thermo_scenes/docs/Collect_new_dataset.md b/thermo_scenes/docs/Collect_new_dataset.md index 1593261..e4354c5 100644 --- a/thermo_scenes/docs/Collect_new_dataset.md +++ b/thermo_scenes/docs/Collect_new_dataset.md @@ -47,10 +47,10 @@ sudo pip install numpy matplotlib pillow ### Extact raw data To create the CSV files, rgb images, and the greyscale raw temperature images, run -the following command, with `` the path to the MSX images, and ``, ``, and `` the paths where the temperature and rgb images, and csv files extracted will be saved. +the following command, with `` the path to the MSX images, and `` the path where the temperature and rgb images, and csv files extracted will be saved. ```bash -thermoscenes_preprocess_thermal.py --path-to-thermal-images --path-to-thermal-images-curated --path-to-rgb --path-to-csv-files +thermoscenes_preprocess_thermal.py --path-to-thermal-images --path-to-output-folder ``` > If your MSX images are in more than one folder, do it for all folders, and make sure that the temperature are rescaled to the same range in the next step. diff --git a/thermo_scenes/flir_thermal_images/custom_flir.py b/thermo_scenes/flir_thermal_images/custom_flir.py index e93378e..ed314da 100644 --- a/thermo_scenes/flir_thermal_images/custom_flir.py +++ b/thermo_scenes/flir_thermal_images/custom_flir.py @@ -1,4 +1,5 @@ import json +import shutil from pathlib import Path import numpy as np @@ -47,11 +48,41 @@ class CustomFlir(FlirImageExtractor): in the dataset. """ - def __init__(self) -> None: + def __init__(self, path_to_msx_images: Path, path_to_output_folder: Path) -> None: super().__init__() + + path_to_output_folder.mkdir(exist_ok=True, parents=True) + if path_to_msx_images != path_to_output_folder: + shutil.copytree( + path_to_msx_images, + Path(path_to_output_folder, "msx"), + dirs_exist_ok=True, + ) + + output_json_path = Path(path_to_output_folder, "temperature_bounds.json") + + output_rgb_folder = Path(path_to_output_folder, "rgb") + output_rgb_folder.mkdir(parents=True, exist_ok=True) + + output_thermal_folder = Path(path_to_output_folder, "thermal") + output_thermal_folder.mkdir(parents=True, exist_ok=True) + + output_csv_folder = Path(path_to_output_folder, "csv") + output_csv_folder.mkdir(parents=True, exist_ok=True) + self.absolute_max_temperature = None self.absolute_min_temperature = None + for img_path in path_to_msx_images.iterdir(): + self.process_image(Path(path_to_msx_images, img_path.name)) + self.export_thermal_to_csv(Path(output_csv_folder, img_path.stem + ".csv")) + self.save_rgb_images(output_rgb_folder) + + self.save_normalised_thermal_images( + str(output_thermal_folder), str(output_csv_folder) + ) + self.save_temperature_bounds(str(output_json_path)) + def save_rgb_images(self, path_to_rgb: Path) -> None: """ Save the extracted rgb images only in the `path_to_rgb` directory. @@ -60,9 +91,7 @@ def save_rgb_images(self, path_to_rgb: Path) -> None: img_visual = Image.fromarray(rgb_np) - fn_prefix = Path(self.flir_img_filename).name - - image_filename = Path(path_to_rgb, str(fn_prefix).split(".")[0] + ".PNG") + image_filename = Path(path_to_rgb, Path(self.flir_img_filename).stem + ".PNG") img_visual.save(image_filename) @@ -132,7 +161,7 @@ def save_normalised_coloured_thermal_images( ) thermal_filename = Path( - path_to_thermal_images_curated, str(filename).split(".")[0] + ".PNG" + path_to_thermal_images_curated, Path(filename).stem + ".PNG" ) img_thermal.save(str(thermal_filename)) diff --git a/thermo_scenes/scripts/preprocess_thermal.py b/thermo_scenes/scripts/preprocess_thermal.py index e3c100c..8326f4e 100644 --- a/thermo_scenes/scripts/preprocess_thermal.py +++ b/thermo_scenes/scripts/preprocess_thermal.py @@ -8,36 +8,19 @@ @dataclass class Paths: - path_to_thermal_images: str + path_to_msx_images: Path """Path to the thermal data extracted from Flir One App.""" - path_to_thermal_images_curated: str - """Path to the output thermal images created using raw thermal data.""" - path_to_rgb: str - """Path to the output rgb images""" - path_to_csv_files: str - """Path to the output csv files of temperature values""" - path_to_txt: str = "temperature_bounds.json" - """Path to the txt file with temperature bounds for the dataset""" + path_to_output_folder: Path + """Path to the output folder""" def main() -> None: paths = tyro.cli(Paths) - list_thermal = [] - list_filenames = [] - flir = CustomFlir() - for img_name in Path(paths.path_to_thermal_images).iterdir(): - flir.process_image(Path(paths.path_to_thermal_images, img_name.name)) - flir.export_thermal_to_csv( - Path(paths.path_to_csv_files, str(img_name.name).split(".")[0] + ".csv") - ) - flir.save_rgb_images(Path(paths.path_to_rgb)) - list_thermal.append(flir.get_thermal_np()) - list_filenames.append(str(img_name.name)) - - flir.save_normalised_thermal_images( - paths.path_to_thermal_images_curated, paths.path_to_csv_files + + CustomFlir( + path_to_msx_images=paths.path_to_msx_images, + path_to_output_folder=paths.path_to_output_folder, ) - flir.save_temperature_bounds(paths.path_to_txt) if __name__ == "__main__":