diff --git a/anchorage/anchor.py b/anchorage/anchor.py index fe41077..be6bdc8 100644 --- a/anchorage/anchor.py +++ b/anchorage/anchor.py @@ -1,8 +1,11 @@ import os +import toml +import datetime from anchorage.bookmarks import bookmarks from anchorage.anchor_tools.local import create_archive, add as add_local, server from anchorage.anchor_tools.online import add as add_online +from anchorage.anchor_utils.system import home def anchor_online(collection, overwrite, @@ -14,6 +17,9 @@ def anchor_online(collection, overwrite, :param loglevel: 0 - Full log output 20 - Progress bar 50 - Suppress all output + + :return: Dictionary with the name, url and error message of all bookmarks + for which the archive process has failed. """ if isinstance(collection, bookmarks): @@ -27,7 +33,7 @@ def anchor_online(collection, overwrite, pb_label="ARCHIVING", ) - log_anchorage(err) + anchorage_error_log(err) def anchor_locally(collection, archive="./anchor", @@ -40,6 +46,9 @@ def anchor_locally(collection, archive="./anchor", :param loglevel: 0 - Full log output 20 - Progress bar 50 - Suppress all output + + :return: Dictionary with the name, url and error message of all bookmarks + for which the archive process has failed. """ create_archive(os.path.abspath(archive)) @@ -55,23 +64,31 @@ def anchor_locally(collection, archive="./anchor", pb_label="ARCHIVING", ) - log_anchorage(err) + anchorage_error_log(err) -def log_anchorage(error_log): +def anchorage_error_log(error_log): """ - TODO: Save error log to ~/anchorage/error_log.toml + Save _anchor_online_ or _anchor_locally_ error log dictionary to + + ~/.anchorage/error_log_YYYY_MM_DD.toml :param error_log: List of bookmarks for which the upload process has failed. """ - print(error_log) + anchorage_log = home() + f'/.anchorage/error_log_{datetime.datetime.now().strftime("%Y-%m-%d")}.toml' + with open(anchorage_log, 'w') as log_file: + toml.dump(error_log, log_file) -def anchor_log_locally(archive="anchor"): + +def anchor_error_log(where, archive="anchor"): """ TODO: Attempt anchor of all bookmarks in the current error log. - """ + :param where: + - 'online': archive online + - 'local': archive locally + """ create_archive(archive) pass diff --git a/anchorage/anchor_tools/online.py b/anchorage/anchor_tools/online.py index b2a9621..b16356a 100644 --- a/anchorage/anchor_tools/online.py +++ b/anchorage/anchor_tools/online.py @@ -1,12 +1,17 @@ from wayback import WaybackClient from archivenow import archivenow -from alexandria.shell import print_color, suppress_stdout -from alexandria.shell.color import colors +from alexandria.shell import suppress_stdout from anchorage.anchor_utils.aesthetic import str_log_info, str_log_error, str_log_success +class UploadException(Exception): + + def __init__(self, message): + super().__init__(message) + + def add(url, archive='ia', api_key=None, overwrite=False): """ Archive a website in one of the four archives supported by Archive Now (archivenow). @@ -27,25 +32,31 @@ def add(url, archive='ia', api_key=None, overwrite=False): """ def upload(url): - try: - if archive == 'cc': - with suppress_stdout(): - archive_url = archivenow.push(url, archive, api_key)[0] - else: - with suppress_stdout(): - archive_url = archivenow.push(url, archive)[0] + if archive == 'cc': + with suppress_stdout(): + archive_url = archivenow.push(url, archive, api_key)[0] + else: + with suppress_stdout(): + archive_url = archivenow.push(url, archive)[0] + + if "Error (The Internet Archive)" in archive_url: + print(str_log_error(url + " ->/ ->/ ->/ " + archive_url)) + raise UploadException(archive_url) + else: log = str_log_success(url + " -> -> -> " + archive_url) - return log - except BaseException as e: - print(str_log_error(url)) - print_color(e, "red") + return log try: archive_latest = next(WaybackClient().search(url)) # Search for URL using the WaybackMachine API if overwrite: - return upload(url) + log = upload(url) + print(log) else: - return str_log_info("SKIPPED", url + " => => => " + archive_latest[7]) - except: + log = str_log_info("SKIPPED", url + " => => => " + archive_latest[7]) + print(log) + except StopIteration: # If bookmark search yields "0" error (defined by Python _wayback_) - return upload(url) + log = upload(url) + print(log) + except UploadException as e: + return e diff --git a/anchorage/anchor_utils/aesthetic.py b/anchorage/anchor_utils/aesthetic.py index e1ca243..acdbeef 100644 --- a/anchorage/anchor_utils/aesthetic.py +++ b/anchorage/anchor_utils/aesthetic.py @@ -28,7 +28,7 @@ def str_log_progress(fraction, t_elapsed, t_remaining, c_fr="brightCyan", c_bg_fr="", c_te="", c_bg_te="", c_tr="brightCyan", c_bg_tr="", - l=11, m=15, n=20): + l=9, m=10, n=21): # Times def s_to_hms(_n): hours, remainder = divmod(_n, 3600) @@ -41,10 +41,10 @@ def s_to_hms(_n): # Set distances r = join_set_distance(fraction, "::", l) r = join_set_distance(r, t_elapsed, m) - r = join_set_distance(r, " ::", n) - r = join_set_distance(r, t_remaining, len(r)+2) + r = join_set_distance(r, "::", n) + r = join_set_distance(r, t_remaining, len(r)+1) - str_t, str_kind, str_msg = r.split(" :: ") + str_t, str_kind, str_msg = r.split("::") # Take care not to color whitespace for KIND string # It is assumed that all possible whitespace has @@ -57,16 +57,16 @@ def countstrip(s): n_ws = n0 - n1 return s, n_ws str_fraction, n_ws_fr = countstrip(fraction) - str_te, n_ws_te = countstrip(t_elapsed) - str_tr, n_ws_tr = countstrip(t_remaining) + str_te, n_ws_te = countstrip(t_elapsed) + str_tr, n_ws_tr = countstrip(t_remaining) # Color - c_str_fr = str_color(str_t, c_fr, c_bg_fr) + " " * n_ws_fr + c_str_fr = str_color(str_t, c_fr, c_bg_fr) + " " * n_ws_fr c_str_te = str_color(str_kind, c_te, c_bg_te) + " " * n_ws_te - c_str_tr = str_color(str_msg, c_tr, c_bg_tr) + " " * n_ws_tr + c_str_tr = str_color(str_msg, c_tr, c_bg_tr) + " " * n_ws_tr # Join in final string - r = " :: ".join([c_str_fr, c_str_te, c_str_tr]) + r = "::".join([c_str_fr, c_str_te, c_str_tr]) return r diff --git a/anchorage/bookmarks.py b/anchorage/bookmarks.py index f4716d5..3ba3074 100644 --- a/anchorage/bookmarks.py +++ b/anchorage/bookmarks.py @@ -333,9 +333,9 @@ def loop(self, f, :return: List with all [key, value] pairs for which `f` execution resulted in an error. """ - e = [] + e = {} t0 = datetime.datetime.now() - step = 0 + step = 1 total = len(self.bookmarks) if loglevel == 20: # Create tqdm progress bar if specified @@ -367,10 +367,10 @@ def loop(self, f, with suppress_stdout(): # Suppress function output if so specified f(key, value) else: - log = f(key, value) - print(log) - except: # Error: add entry to error list - e.append([key, value]) + f(key, value) + except BaseException as exception: # Error: add entry to error list + e[key] = {'url': value['url'], + 'Error message': str(exception)} if loglevel == 0: print("") diff --git a/anchorage/cli.py b/anchorage/cli.py index 73526ac..3422328 100644 --- a/anchorage/cli.py +++ b/anchorage/cli.py @@ -302,7 +302,7 @@ def main(): 'choices': [' - Full log output', ' - Progress bar', ' - Suppress all output'], 'filter': lambda choice: {'Full log output' : 0, 'Progress bar' : 20, - 'Suppress all output': 50}[choice] + 'Suppress all output': 50}[choice[4:]] }] loglevel = prompt(log_prompt, style=style)['loglevel'] newline() diff --git a/setup.py b/setup.py index 91975fc..319d86d 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="anchorage", - version="1.2.0.0", + version="1.2.1.0", author="Antonio Lopez Rivera", author_email="antonlopezr99@gmail.com", description="Anchor your little piece of internet.", diff --git a/tests/test_online.py b/tests/test_online.py index e307c3b..49ec27d 100644 --- a/tests/test_online.py +++ b/tests/test_online.py @@ -6,9 +6,13 @@ class OnlineArchiveTests(unittest.TestCase): def test_add(self): - print("\nONLINE ARCHIVE TESTS: ADD NEW SITE\n") - print(add("https://m.youtube.com/watch?v=XhExfYBfypI", "ia", overwrite=True)[0]) + print("\nONLINE ARCHIVE TESTS: ADD SITE\n") + assert 'SUCCESS' in add("https://m.youtube.com/watch?v=XhExfYBfypI", "ia", overwrite=True)[0] def test_existing(self): print("\nONLINE ARCHIVE TESTS: AVOID SAVING IF ALREADY ARCHIVED\n") - assert add("http://paulgraham.com/think.html", "ia") == "Bookmark already present in the Internet Archive" + assert 'SKIPPED' in add("http://paulgraham.com/think.html", "ia") + + def test_exception(self): + print("\nONLINE ARCHIVE TESTS: ATTEMPT SAVING FORBIDDEN BOOKMARK\n") + assert 'FAILURE' in add("https://github.com/alopezrivera/WB3168-Robot-Mechanics", "ia")