diff --git a/elodie/filesystem.py b/elodie/filesystem.py index 92d7807..4b96a40 100644 --- a/elodie/filesystem.py +++ b/elodie/filesystem.py @@ -487,9 +487,6 @@ def parse_mask_for_location(self, mask, location_parts, place_name): def process_checksum(self, _file, allow_duplicate): db = Db() checksum = db.checksum(_file) - if(checksum is None): - log.info('Could not get checksum for %s.' % _file) - return None # If duplicates are not allowed then we check if we've seen this file # before via checksum. We also check that the file exists at the diff --git a/elodie/localstorage.py b/elodie/localstorage.py index 17ac26c..7c8b05d 100644 --- a/elodie/localstorage.py +++ b/elodie/localstorage.py @@ -127,7 +127,6 @@ def checksum(self, file_path, blocksize=65536): hasher.update(buf) buf = f.read(blocksize) return hasher.hexdigest() - return None def get_hash(self, key): """Get the hash value for a given key. diff --git a/elodie/media/audio.py b/elodie/media/audio.py index 8409512..55131a0 100644 --- a/elodie/media/audio.py +++ b/elodie/media/audio.py @@ -20,7 +20,7 @@ class Audio(Video): __name__ = 'Audio' #: Valid extensions for audio files. - extensions = ('m4a',) + extensions = ('m4a', 'mp3', 'opus', 'ogg', 'flac') def __init__(self, source=None): super(Audio, self).__init__(source) diff --git a/elodie/media/photo.py b/elodie/media/photo.py index 07f3e04..2afcfa1 100644 --- a/elodie/media/photo.py +++ b/elodie/media/photo.py @@ -28,7 +28,7 @@ class Photo(Media): __name__ = 'Photo' #: Valid extensions for photo files. - extensions = ('arw', 'cr2', 'dng', 'gif', 'heic', 'jpeg', 'jpg', 'nef', 'png', 'rw2') + extensions = ('arw', 'cr2', 'dng', 'gif', 'heic', 'jpeg', 'jpg', 'nef', 'png', 'rw2', 'webp', 'svg') def __init__(self, source=None): super(Photo, self).__init__(source) diff --git a/elodie/media/video.py b/elodie/media/video.py index a53d238..c9cecb0 100644 --- a/elodie/media/video.py +++ b/elodie/media/video.py @@ -28,7 +28,7 @@ class Video(Media): __name__ = 'Video' #: Valid extensions for video files. - extensions = ('avi', 'm4v', 'mov', 'mp4', 'mpg', 'mpeg', '3gp', 'mts') + extensions = ('avi', 'm4v', 'mov', 'mp4', 'mpg', 'mpeg', '3gp', 'mts', 'mkv', 'webm') def __init__(self, source=None): super(Video, self).__init__(source) diff --git a/elodie/result.py b/elodie/result.py index 3fa7851..8006df0 100644 --- a/elodie/result.py +++ b/elodie/result.py @@ -8,17 +8,24 @@ def __init__(self): self.success = 0 self.error = 0 self.error_items = [] + self.duplicate = 0 + self.duplicate_items = [] def append(self, row): id, status = row + # status can only be True, False, or None if status: self.success += 1 + elif status is None: # status is only ever None if file checksum matched an existing file checksum and is therefore a duplicate file + self.duplicate += 1 + self.duplicate_items.append(id) else: self.error += 1 self.error_items.append(id) def write(self): + print("\n") if self.error > 0: error_headers = ["File"] error_result = [] @@ -29,10 +36,21 @@ def write(self): print(tabulate(error_result, headers=error_headers)) print("\n") + if self.duplicate > 0: + duplicate_headers = ["File"] + duplicate_result = [] + for id in self.duplicate_items: + duplicate_result.append([id]) + + print("****** DUPLICATE (NOT IMPORTED) DETAILS ******") + print(tabulate(duplicate_result, headers=duplicate_headers)) + print("\n") + headers = ["Metric", "Count"] result = [ ["Success", self.success], ["Error", self.error], + ["Duplicate, not imported", self.duplicate] ] print("****** SUMMARY ******") diff --git a/elodie/tests/elodie_test.py b/elodie/tests/elodie_test.py index a01bc30..333a941 100644 --- a/elodie/tests/elodie_test.py +++ b/elodie/tests/elodie_test.py @@ -300,8 +300,9 @@ def test_import_file_with_single_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--exclude-regex', origin_valid[0:5], '--allow-duplicates', origin_valid]) - assert 'Success 0' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 0' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output def test_import_file_with_multiple_exclude(): temporary_folder, folder = helper.create_working_folder() @@ -313,8 +314,9 @@ def test_import_file_with_multiple_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--exclude-regex', 'does not exist in path', '--exclude-regex', origin_valid[0:5], '--allow-duplicates', origin_valid]) - assert 'Success 0' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 0' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output def test_import_file_with_non_matching_exclude(): temporary_folder, folder = helper.create_working_folder() @@ -326,8 +328,10 @@ def test_import_file_with_non_matching_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--exclude-regex', 'does not exist in path', '--allow-duplicates', origin_valid]) - assert 'Success 1' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 1' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output + def test_import_directory_with_matching_exclude(): temporary_folder, folder = helper.create_working_folder() @@ -339,8 +343,9 @@ def test_import_directory_with_matching_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--source', folder, '--exclude-regex', folder[1:5], '--allow-duplicates']) - assert 'Success 0' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 0' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output def test_import_directory_with_non_matching_exclude(): temporary_folder, folder = helper.create_working_folder() @@ -352,8 +357,10 @@ def test_import_directory_with_non_matching_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--source', folder, '--exclude-regex', 'non-matching', '--allow-duplicates']) - assert 'Success 1' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 1' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output + @mock.patch('elodie.config.config_file', '%s/config.ini-import-file-with-single-config-exclude' % gettempdir()) def test_import_file_with_single_config_exclude(): @@ -379,8 +386,9 @@ def test_import_file_with_single_config_exclude(): if hasattr(load_config, 'config'): del load_config.config - assert 'Success 0' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 0' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output @mock.patch('elodie.config.config_file', '%s/config.ini-import-file-with-multiple-config-exclude' % gettempdir()) def test_import_file_with_multiple_config_exclude(): @@ -407,8 +415,9 @@ def test_import_file_with_multiple_config_exclude(): if hasattr(load_config, 'config'): del load_config.config - assert 'Success 0' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 0' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output def test_update_location_on_audio(): temporary_folder, folder = helper.create_working_folder() @@ -705,8 +714,9 @@ def test_verify_ok(): shutil.rmtree(folder) - assert 'Success 1' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 1' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output def test_verify_error(): temporary_folder, folder = helper.create_working_folder() @@ -725,7 +735,8 @@ def test_verify_error(): shutil.rmtree(folder) assert origin in result.output, result.output - assert 'Error 1' in result.output, result.output + assert 'Error 1' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output @mock.patch('elodie.config.config_file', '%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir()) def test_cli_batch_plugin_googlephotos(): diff --git a/elodie/tests/result_test.py b/elodie/tests/result_test.py index a373fd6..265714c 100644 --- a/elodie/tests/result_test.py +++ b/elodie/tests/result_test.py @@ -30,10 +30,11 @@ def call_result_and_assert(result, expected): def test_add_multiple_rows_with_success(): expected = """****** SUMMARY ****** -Metric Count --------- ------- -Success 2 -Error 0""" +Metric Count +----------------------- ------- +Success 2 +Error 0 +Duplicate, not imported 0""" result = Result() result.append(('id1', '/some/path/1')) result.append(('id2', '/some/path/2')) @@ -48,10 +49,11 @@ def test_add_multiple_rows_with_failure(): ****** SUMMARY ****** -Metric Count --------- ------- -Success 0 -Error 2""" +Metric Count +----------------------- ------- +Success 0 +Error 2 +Duplicate, not imported 0""" result = Result() result.append(('id1', False)) result.append(('id2', False)) @@ -65,10 +67,11 @@ def test_add_multiple_rows_with_failure_and_success(): ****** SUMMARY ****** -Metric Count --------- ------- -Success 1 -Error 1""" +Metric Count +----------------------- ------- +Success 1 +Error 1 +Duplicate, not imported 0""" result = Result() result.append(('id1', False)) result.append(('id2', '/some/path')) diff --git a/requirements.txt b/requirements.txt index 30d0e38..92bc1bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,8 @@ -click==6.6 -requests==2.20.0 -Send2Trash==1.3.0 -future==0.16.0 -configparser==3.5.0 -tabulate==0.7.7 -Pillow==6.2.2; python_version == '2.7' -Pillow==9.3; python_version >= '3.6' -six==1.9 +click==8.1.8 +requests==2.32.2 +Send2Trash==1.8.3 +future==1.0.0 +configparser==7.1.0 +tabulate==0.9.0 +Pillow==11.0.0 +six==1.17.0