From 9ad1b5d7d1be7c90cd49e6ec4149ade3d05e3292 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Mon, 15 Jan 2024 22:23:17 -0500 Subject: [PATCH 01/32] way bigger... --- data/interfaces/carbon/css/style.css | 36 +- .../default/comicdetails_update.html | 4 +- data/interfaces/default/config.html | 72 +- data/interfaces/default/css/style.css | 36 +- data/interfaces/default/queue_management.html | 82 +- data/interfaces/default/searchresults.html | 6 +- lib/mega/__init__.py | 1 + lib/mega/crypto.py | 168 +++ lib/mega/errors.py | 62 + lib/mega/mega.py | 1090 +++++++++++++++++ mylar/PostProcessor.py | 27 +- mylar/__init__.py | 19 +- mylar/api.py | 79 +- mylar/carepackage.py | 4 +- mylar/cdh_mapping.py | 2 +- mylar/cmtagmylar.py | 2 +- mylar/config.py | 62 +- mylar/cv.py | 79 +- mylar/downloaders/__init__.py | 0 mylar/downloaders/external_server.py | 2 + mylar/downloaders/mediafire.py | 140 +++ mylar/downloaders/mega.py | 131 ++ mylar/downloaders/pixeldrain.py | 145 +++ mylar/filechecker.py | 103 +- mylar/getcomics.py | 847 ++++++++++--- mylar/helpers.py | 220 +++- mylar/importer.py | 2 +- mylar/req_test.py | 2 +- mylar/rsscheck.py | 2 +- mylar/sabnzbd.py | 6 +- mylar/search.py | 94 +- mylar/search_filer.py | 44 +- mylar/updater.py | 25 +- mylar/webserve.py | 88 +- requirements.txt | 2 +- 35 files changed, 3255 insertions(+), 429 deletions(-) create mode 100644 lib/mega/__init__.py create mode 100644 lib/mega/crypto.py create mode 100644 lib/mega/errors.py create mode 100644 lib/mega/mega.py create mode 100644 mylar/downloaders/__init__.py create mode 100644 mylar/downloaders/external_server.py create mode 100644 mylar/downloaders/mediafire.py create mode 100644 mylar/downloaders/mega.py create mode 100644 mylar/downloaders/pixeldrain.py diff --git a/data/interfaces/carbon/css/style.css b/data/interfaces/carbon/css/style.css index 8836d390..3dca4abc 100644 --- a/data/interfaces/carbon/css/style.css +++ b/data/interfaces/carbon/css/style.css @@ -1807,20 +1807,20 @@ div#artistheader h2 a { min-width: 95px; vertical-align: middle; } -#queue_table th#qcomicid { - max-width: 10px; - text-align: center; -} #queue_table th#qseries { - max-width: 475px; + max-width: 200px; text-align: center; } #queue_table th#qsize { - max-width: 35px; + max-width: 40px; + text-align: center; +} +#queue_table th#qlinktype { + max-width: 45px; text-align: center; } #queue_table th#qprogress { - max-width: 25px; + max-width: 20px; text-align: center; } #queue_table th#qstatus { @@ -1828,27 +1828,27 @@ div#artistheader h2 a { text-align: center; } #queue_table th#qdate { - max-width: 90px; + max-width: 75px; text-align: center; } #queue_table th#qoptions { - max-width: 160px; + max-width: 100px; text-align: center; } -#queue_table td#qcomicid { - max-width: 10px; - text-align: left; -} #queue_table td#qseries { - max-width: 475px; + max-width: 200px; text-align: left; } #queue_table td#qsize { - max-width: 35px; + max-width: 40px; + text-align: center; +} +#queue_table td#qlinktype { + max-width: 45px; text-align: center; } #queue_table td#qprogress { - max-width: 25px; + max-width: 20px; text-align: center; } #queue_table td#qstatus { @@ -1856,11 +1856,11 @@ div#artistheader h2 a { text-align: center; } #queue_table td#qdate { - min-width: 90px; + max-width: 75px; text-align: center; } #queue_table td#qoptions { - max-width: 160px; + max-width: 100px; text-align: center; } diff --git a/data/interfaces/default/comicdetails_update.html b/data/interfaces/default/comicdetails_update.html index 3c587bd8..3764bde4 100755 --- a/data/interfaces/default/comicdetails_update.html +++ b/data/interfaces/default/comicdetails_update.html @@ -1936,9 +1936,9 @@

settingsSettings + %if mylar.EXT_SERVER: +
+ +
+
+
+
+ + + (ie. http://private.server) +
+
+ + +
+
+ + +
+
+
+ %endif @@ -2020,14 +2042,16 @@

HISTORY

- + @@ -193,26 +193,69 @@

HISTORY

$('#queue_table').DataTable().ajax.reload(null, false); } else { $('#queue_table').DataTable( { + "preDrawCallback": function (settings){ + pageScrollPos = $('#queue_table div.datatables_scrollBody').scrollTop(); + }, "processing": true, "serverSide": true, "ajaxSource": 'queueManageIt', "paginationType": "full_numbers", "displayLength": 25, "sorting": [[5, 'desc']], - "stateSave": false, + "stateSave": true, "columnDefs": [ +// { +// "sortable": false, +// "targets": [ 0 ], +// "visible": false, +// }, { - "sortable": false, + "sortable": true, "targets": [ 0 ], - "visible": false, + "visible": true, + "data": "Series", + "render":function (data,type,full) { + return '' + full[0] + ''; + } }, { "sortable": true, "targets": [ 1 ], "visible": true, - "data": "Series", "render":function (data,type,full) { - return '' + full[1] + ''; + return full[1]; + } + }, + { + "sortable": true, + "targets": [ 2 ], + "visible": true, + "render":function (data,type,full) { + return full[8]; + } + }, + { + "sortable": true, + "targets": [ 3 ], + "visible": true, + "render":function (data,type,full) { + return full[2]; + } + }, + { + "sortable": true, + "targets": [ 4 ], + "visible": true, + "render":function (data,type,full) { + return full[3]; + } + }, + { + "sortable": true, + "targets": [ 5 ], + "visible": true, + "render":function (data,type,full) { + return full[4]; } }, { @@ -220,10 +263,10 @@

HISTORY

"targets": [ 6 ], "visible": true, "render":function (data,type,full) { - val = full[4] - var restartline = "('ddl_requeue?mode=restart&id="+String(full[6])+"',$(this));" - var resumeline = "('ddl_requeue?mode=resume&id="+String(full[6])+"',$(this));" - var removeline = "('ddl_requeue?mode=remove&id="+String(full[6])+"',$(this));" + val = full[3] + var restartline = "('ddl_requeue?mode=restart&id="+String(full[5])+"',$(this));" + var resumeline = "('ddl_requeue?mode=resume&id="+String(full[5])+"',$(this));" + var removeline = "('ddl_requeue?mode=remove&id="+String(full[5])+"',$(this));" if (val == 'Completed' || val == 'Failed' || val == 'Downloading'){ return 'RestartRemove'; } else if (val == 'Incomplete') { @@ -243,24 +286,25 @@

HISTORY

"infoFiltered":"(filtered from _MAX_ total items)" }, "rowCallback": function (nRow, aData, iDisplayIndex, iDisplayIndexFull) { - if (aData[4] === "Completed") { + if (aData[3] === "Completed") { $('td', nRow).closest('tr').addClass("gradeA"); - } else if (aData[4] === "Queued") { + } else if (aData[3] === "Queued") { $('td', nRow).closest('tr').addClass("gradeC"); - } else if (aData[4] === "Incomplete" || aData[4] == "Failed") { + } else if (aData[3] === "Incomplete" || aData[3] == "Failed") { $('td', nRow).closest('tr').addClass("gradeX"); } - nRow.children[0].id = 'qcomicid'; - nRow.children[1].id = 'qseries'; - nRow.children[2].id = 'qsize'; - nRow.children[3].id = 'qprogress'; - nRow.children[4].id = 'qstatus'; - nRow.children[5].id = 'qdate'; + //nRow.children[0].id = 'qcomicid'; + nRow.children[0].id = 'qseries'; + nRow.children[1].id = 'qsize'; + nRow.children[2].id = 'qprogress'; + nRow.children[3].id = 'qstatus'; + nRow.children[4].id = 'qdate'; return nRow; }, "drawCallback": function (o) { // Jump to top of page $('html,body').scrollTop(0); + $('#queue_table div.dataTables_scrollBody').scrollTop(pageScrollPos); }, "serverData": function ( sSource, aoData, fnCallback ) { /* Add some extra data to the sender */ diff --git a/data/interfaces/default/searchresults.html b/data/interfaces/default/searchresults.html index a8ff750b..c215c95b 100755 --- a/data/interfaces/default/searchresults.html +++ b/data/interfaces/default/searchresults.html @@ -217,11 +217,11 @@

Search results${ } } - var dropdown = ""+ "" + "" + "" + - "" + + "" + "" + "" + ""; @@ -255,7 +255,6 @@

Search results${ var bktype = document.getElementById("booktype"); var booktype = bktype.options[bktype.selectedIndex]; var query_id = retrieve_searchquery(); - console.log(location.value + ' --- '+ booktype.text); if (comlocation){ cloc = location.value; } else { @@ -275,6 +274,7 @@

Search results${ } if (btype) { bktype.text = dc.booktype; + location.value = dc.comlocation; } }); } diff --git a/lib/mega/__init__.py b/lib/mega/__init__.py new file mode 100644 index 00000000..48bd3665 --- /dev/null +++ b/lib/mega/__init__.py @@ -0,0 +1 @@ +from .mega import Mega # noqa diff --git a/lib/mega/crypto.py b/lib/mega/crypto.py new file mode 100644 index 00000000..61ddf157 --- /dev/null +++ b/lib/mega/crypto.py @@ -0,0 +1,168 @@ +from Crypto.Cipher import AES +import json +import base64 +import struct +import binascii +import random +import sys + +# Python3 compatibility +if sys.version_info < (3, ): + + def makebyte(x): + return x + + def makestring(x): + return x +else: + import codecs + + def makebyte(x): + return codecs.latin_1_encode(x)[0] + + def makestring(x): + return codecs.latin_1_decode(x)[0] + + +def aes_cbc_encrypt(data, key): + aes_cipher = AES.new(key, AES.MODE_CBC, makebyte('\0' * 16)) + return aes_cipher.encrypt(data) + + +def aes_cbc_decrypt(data, key): + aes_cipher = AES.new(key, AES.MODE_CBC, makebyte('\0' * 16)) + return aes_cipher.decrypt(data) + + +def aes_cbc_encrypt_a32(data, key): + return str_to_a32(aes_cbc_encrypt(a32_to_str(data), a32_to_str(key))) + + +def aes_cbc_decrypt_a32(data, key): + return str_to_a32(aes_cbc_decrypt(a32_to_str(data), a32_to_str(key))) + + +def stringhash(str, aeskey): + s32 = str_to_a32(str) + h32 = [0, 0, 0, 0] + for i in range(len(s32)): + h32[i % 4] ^= s32[i] + for r in range(0x4000): + h32 = aes_cbc_encrypt_a32(h32, aeskey) + return a32_to_base64((h32[0], h32[2])) + + +def prepare_key(arr): + pkey = [0x93C467E3, 0x7DB0C7A4, 0xD1BE3F81, 0x0152CB56] + for r in range(0x10000): + for j in range(0, len(arr), 4): + key = [0, 0, 0, 0] + for i in range(4): + if i + j < len(arr): + key[i] = arr[i + j] + pkey = aes_cbc_encrypt_a32(pkey, key) + return pkey + + +def encrypt_key(a, key): + return sum((aes_cbc_encrypt_a32(a[i:i + 4], key) + for i in range(0, len(a), 4)), ()) + + +def decrypt_key(a, key): + return sum((aes_cbc_decrypt_a32(a[i:i + 4], key) + for i in range(0, len(a), 4)), ()) + + +def encrypt_attr(attr, key): + attr = makebyte('MEGA' + json.dumps(attr)) + if len(attr) % 16: + attr += b'\0' * (16 - len(attr) % 16) + return aes_cbc_encrypt(attr, a32_to_str(key)) + + +def decrypt_attr(attr, key): + attr = aes_cbc_decrypt(attr, a32_to_str(key)) + attr = makestring(attr) + attr = attr.rstrip('\0') + return json.loads(attr[4:]) if attr[:6] == 'MEGA{"' else False + + +def a32_to_str(a): + return struct.pack('>%dI' % len(a), *a) + + +def str_to_a32(b): + if isinstance(b, str): + b = makebyte(b) + if len(b) % 4: + # pad to multiple of 4 + b += b'\0' * (4 - len(b) % 4) + return struct.unpack('>%dI' % (len(b) / 4), b) + + +def mpi_to_int(s): + """ + A Multi-precision integer is encoded as a series of bytes in big-endian + order. The first two bytes are a header which tell the number of bits in + the integer. The rest of the bytes are the integer. + """ + return int(binascii.hexlify(s[2:]), 16) + + +def extended_gcd(a, b): + if a == 0: + return (b, 0, 1) + else: + g, y, x = extended_gcd(b % a, a) + return (g, x - (b // a) * y, y) + + +def modular_inverse(a, m): + g, x, y = extended_gcd(a, m) + if g != 1: + raise Exception('modular inverse does not exist') + else: + return x % m + + +def base64_url_decode(data): + data += '=='[(2 - len(data) * 3) % 4:] + for search, replace in (('-', '+'), ('_', '/'), (',', '')): + data = data.replace(search, replace) + return base64.b64decode(data) + + +def base64_to_a32(s): + return str_to_a32(base64_url_decode(s)) + + +def base64_url_encode(data): + data = base64.b64encode(data) + data = makestring(data) + for search, replace in (('+', '-'), ('/', '_'), ('=', '')): + data = data.replace(search, replace) + return data + + +def a32_to_base64(a): + return base64_url_encode(a32_to_str(a)) + + +def get_chunks(size): + p = 0 + s = 0x20000 + while p + s < size: + yield (p, s) + p += s + if s < 0x100000: + s += 0x20000 + yield (p, size - p) + + +def make_id(length): + text = '' + possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + for i in range(length): + text += random.choice(possible) + return text diff --git a/lib/mega/errors.py b/lib/mega/errors.py new file mode 100644 index 00000000..00332ae8 --- /dev/null +++ b/lib/mega/errors.py @@ -0,0 +1,62 @@ +class ValidationError(Exception): + """ + Error in validation stage + """ + pass + + +_CODE_TO_DESCRIPTIONS = { + -1: ('EINTERNAL', + ('An internal error has occurred. Please submit a bug report, ' + 'detailing the exact circumstances in which this error occurred')), + -2: ('EARGS', 'You have passed invalid arguments to this command'), + -3: ('EAGAIN', + ('(always at the request level) A temporary congestion or server ' + 'malfunction prevented your request from being processed. ' + 'No data was altered. Retry. Retries must be spaced with ' + 'exponential backoff')), + -4: ('ERATELIMIT', + ('You have exceeded your command weight per time quota. Please ' + 'wait a few seconds, then try again (this should never happen ' + 'in sane real-life applications)')), + -5: ('EFAILED', 'The upload failed. Please restart it from scratch'), + -6: + ('ETOOMANY', + 'Too many concurrent IP addresses are accessing this upload target URL'), + -7: + ('ERANGE', ('The upload file packet is out of range or not starting and ' + 'ending on a chunk boundary')), + -8: ('EEXPIRED', + ('The upload target URL you are trying to access has expired. ' + 'Please request a fresh one')), + -9: ('ENOENT', 'Object (typically, node or user) not found'), + -10: ('ECIRCULAR', 'Circular linkage attempted'), + -11: ('EACCESS', + 'Access violation (e.g., trying to write to a read-only share)'), + -12: ('EEXIST', 'Trying to create an object that already exists'), + -13: ('EINCOMPLETE', 'Trying to access an incomplete resource'), + -14: ('EKEY', 'A decryption operation failed (never returned by the API)'), + -15: ('ESID', 'Invalid or expired user session, please relogin'), + -16: ('EBLOCKED', 'User blocked'), + -17: ('EOVERQUOTA', 'Request over quota'), + -18: ('ETEMPUNAVAIL', + 'Resource temporarily not available, please try again later'), + -19: ('ETOOMANYCONNECTIONS', 'many connections on this resource'), + -20: ('EWRITE', 'Write failed'), + -21: ('EREAD', 'Read failed'), + -22: ('EAPPKEY', 'Invalid application key; request not processed'), +} + + +class RequestError(Exception): + """ + Error in API request + """ + def __init__(self, message): + code = message + self.code = code + code_desc, long_desc = _CODE_TO_DESCRIPTIONS[code] + self.message = f'{code_desc}, {long_desc}' + + def __str__(self): + return self.message diff --git a/lib/mega/mega.py b/lib/mega/mega.py new file mode 100644 index 00000000..8dcbeeea --- /dev/null +++ b/lib/mega/mega.py @@ -0,0 +1,1090 @@ +import math +import re +import json +import logging +import secrets +from pathlib import Path +import hashlib +from Crypto.Cipher import AES +from Crypto.PublicKey import RSA +from Crypto.Util import Counter +import os +import random +import binascii +import tempfile +import shutil + +import requests +from tenacity import retry, wait_exponential, retry_if_exception_type + +from .errors import ValidationError, RequestError +from .crypto import (a32_to_base64, encrypt_key, base64_url_encode, + encrypt_attr, base64_to_a32, base64_url_decode, + decrypt_attr, a32_to_str, get_chunks, str_to_a32, + decrypt_key, mpi_to_int, stringhash, prepare_key, make_id, + makebyte, modular_inverse) + +logger = logging.getLogger(__name__) + + +class Mega: + def __init__(self, options=None): + self.schema = 'https' + self.domain = 'mega.co.nz' + self.timeout = 160 # max secs to wait for resp from api requests + self.sid = None + self.sequence_num = random.randint(0, 0xFFFFFFFF) + self.request_id = make_id(10) + self._trash_folder_node_id = None + + if options is None: + options = {} + self.options = options + + def login(self, email=None, password=None): + if email: + self._login_user(email, password) + else: + self.login_anonymous() + self._trash_folder_node_id = self.get_node_by_type(4)[0] + logger.info('Login complete') + return self + + def _login_user(self, email, password): + logger.info('Logging in user...') + email = email.lower() + get_user_salt_resp = self._api_request({'a': 'us0', 'user': email}) + user_salt = None + try: + user_salt = base64_to_a32(get_user_salt_resp['s']) + except KeyError: + # v1 user account + password_aes = prepare_key(str_to_a32(password)) + user_hash = stringhash(email, password_aes) + else: + # v2 user account + pbkdf2_key = hashlib.pbkdf2_hmac(hash_name='sha512', + password=password.encode(), + salt=a32_to_str(user_salt), + iterations=100000, + dklen=32) + password_aes = str_to_a32(pbkdf2_key[:16]) + user_hash = base64_url_encode(pbkdf2_key[-16:]) + resp = self._api_request({'a': 'us', 'user': email, 'uh': user_hash}) + if isinstance(resp, int): + raise RequestError(resp) + self._login_process(resp, password_aes) + + def login_anonymous(self): + logger.info('Logging in anonymous temporary user...') + master_key = [random.randint(0, 0xFFFFFFFF)] * 4 + password_key = [random.randint(0, 0xFFFFFFFF)] * 4 + session_self_challenge = [random.randint(0, 0xFFFFFFFF)] * 4 + + user = self._api_request({ + 'a': + 'up', + 'k': + a32_to_base64(encrypt_key(master_key, password_key)), + 'ts': + base64_url_encode( + a32_to_str(session_self_challenge) + + a32_to_str(encrypt_key(session_self_challenge, master_key))) + }) + + resp = self._api_request({'a': 'us', 'user': user}) + if isinstance(resp, int): + raise RequestError(resp) + self._login_process(resp, password_key) + + def _login_process(self, resp, password): + encrypted_master_key = base64_to_a32(resp['k']) + self.master_key = decrypt_key(encrypted_master_key, password) + if 'tsid' in resp: + tsid = base64_url_decode(resp['tsid']) + key_encrypted = a32_to_str( + encrypt_key(str_to_a32(tsid[:16]), self.master_key)) + if key_encrypted == tsid[-16:]: + self.sid = resp['tsid'] + elif 'csid' in resp: + encrypted_rsa_private_key = base64_to_a32(resp['privk']) + rsa_private_key = decrypt_key(encrypted_rsa_private_key, + self.master_key) + + private_key = a32_to_str(rsa_private_key) + # The private_key contains 4 MPI integers concatenated together. + rsa_private_key = [0, 0, 0, 0] + for i in range(4): + # An MPI integer has a 2-byte header which describes the number + # of bits in the integer. + bitlength = (private_key[0] * 256) + private_key[1] + bytelength = math.ceil(bitlength / 8) + # Add 2 bytes to accommodate the MPI header + bytelength += 2 + rsa_private_key[i] = mpi_to_int(private_key[:bytelength]) + private_key = private_key[bytelength:] + + first_factor_p = rsa_private_key[0] + second_factor_q = rsa_private_key[1] + private_exponent_d = rsa_private_key[2] + # In MEGA's webclient javascript, they assign [3] to a variable + # called u, but I do not see how it corresponds to pycryptodome's + # RSA.construct and it does not seem to be necessary. + rsa_modulus_n = first_factor_p * second_factor_q + phi = (first_factor_p - 1) * (second_factor_q - 1) + public_exponent_e = modular_inverse(private_exponent_d, phi) + + rsa_components = ( + rsa_modulus_n, + public_exponent_e, + private_exponent_d, + first_factor_p, + second_factor_q, + ) + rsa_decrypter = RSA.construct(rsa_components) + + encrypted_sid = mpi_to_int(base64_url_decode(resp['csid'])) + + sid = '%x' % rsa_decrypter._decrypt(encrypted_sid) + sid = binascii.unhexlify('0' + sid if len(sid) % 2 else sid) + self.sid = base64_url_encode(sid[:43]) + + @retry(retry=retry_if_exception_type(RuntimeError), + wait=wait_exponential(multiplier=2, min=2, max=60)) + def _api_request(self, data): + params = {'id': self.sequence_num} + self.sequence_num += 1 + + if self.sid: + params.update({'sid': self.sid}) + + # ensure input data is a list + if not isinstance(data, list): + data = [data] + + url = f'{self.schema}://g.api.{self.domain}/cs' + response = requests.post( + url, + params=params, + data=json.dumps(data), + timeout=self.timeout, + ) + json_resp = json.loads(response.text) + int_resp = None + try: + if isinstance(json_resp, list): + int_resp = json_resp[0] if isinstance(json_resp[0], + int) else None + elif isinstance(json_resp, int): + int_resp = json_resp + except IndexError: + int_resp = None + if int_resp is not None: + if int_resp == 0: + return int_resp + if int_resp == -3: + msg = 'Request failed, retrying' + logger.info(msg) + raise RuntimeError(msg) + raise RequestError(int_resp) + return json_resp[0] + + def _parse_url(self, url): + """Parse file id and key from url.""" + if 'getcomics' in url.lower(): + r = requests.head(url, verify=True) + if r.status_code == 302: + # 302 is redirect from CF so this is needed + url = r.headers['Location'] + if '/file/' in url: + # V2 URL structure + url = url.replace(' ', '') + file_id = re.findall(r'\W\w\w\w\w\w\w\w\w\W', url)[0][1:-1] + id_index = re.search(file_id, url).end() + key = url[id_index + 1:] + return f'{file_id}!{key}' + elif '!' in url: + # V1 URL structure + match = re.findall(r'/#!(.*)', url) + path = match[0] + return path + else: + raise RequestError('Url key missing') + + def _process_file(self, file, shared_keys): + if file['t'] == 0 or file['t'] == 1: + keys = dict( + keypart.split(':', 1) for keypart in file['k'].split('/') + if ':' in keypart) + uid = file['u'] + key = None + # my objects + if uid in keys: + key = decrypt_key(base64_to_a32(keys[uid]), self.master_key) + # shared folders + elif 'su' in file and 'sk' in file and ':' in file['k']: + shared_key = decrypt_key(base64_to_a32(file['sk']), + self.master_key) + key = decrypt_key(base64_to_a32(keys[file['h']]), shared_key) + if file['su'] not in shared_keys: + shared_keys[file['su']] = {} + shared_keys[file['su']][file['h']] = shared_key + # shared files + elif file['u'] and file['u'] in shared_keys: + for hkey in shared_keys[file['u']]: + shared_key = shared_keys[file['u']][hkey] + if hkey in keys: + key = keys[hkey] + key = decrypt_key(base64_to_a32(key), shared_key) + break + if file['h'] and file['h'] in shared_keys.get('EXP', ()): + shared_key = shared_keys['EXP'][file['h']] + encrypted_key = str_to_a32( + base64_url_decode(file['k'].split(':')[-1])) + key = decrypt_key(encrypted_key, shared_key) + file['shared_folder_key'] = shared_key + if key is not None: + # file + if file['t'] == 0: + k = (key[0] ^ key[4], key[1] ^ key[5], key[2] ^ key[6], + key[3] ^ key[7]) + file['iv'] = key[4:6] + (0, 0) + file['meta_mac'] = key[6:8] + # folder + else: + k = key + file['key'] = key + file['k'] = k + attributes = base64_url_decode(file['a']) + attributes = decrypt_attr(attributes, k) + file['a'] = attributes + # other => wrong object + elif file['k'] == '': + file['a'] = False + elif file['t'] == 2: + self.root_id = file['h'] + file['a'] = {'n': 'Cloud Drive'} + elif file['t'] == 3: + self.inbox_id = file['h'] + file['a'] = {'n': 'Inbox'} + elif file['t'] == 4: + self.trashbin_id = file['h'] + file['a'] = {'n': 'Rubbish Bin'} + return file + + def _init_shared_keys(self, files, shared_keys): + """ + Init shared key not associated with a user. + Seems to happen when a folder is shared, + some files are exchanged and then the + folder is un-shared. + Keys are stored in files['s'] and files['ok'] + """ + ok_dict = {} + for ok_item in files['ok']: + shared_key = decrypt_key(base64_to_a32(ok_item['k']), + self.master_key) + ok_dict[ok_item['h']] = shared_key + for s_item in files['s']: + if s_item['u'] not in shared_keys: + shared_keys[s_item['u']] = {} + if s_item['h'] in ok_dict: + shared_keys[s_item['u']][s_item['h']] = ok_dict[s_item['h']] + self.shared_keys = shared_keys + + def find_path_descriptor(self, path, files=()): + """ + Find descriptor of folder inside a path. i.e.: folder1/folder2/folder3 + Params: + path, string like folder1/folder2/folder3 + Return: + Descriptor (str) of folder3 if exists, None otherwise + """ + paths = path.split('/') + + files = files or self.get_files() + parent_desc = self.root_id + found = False + for foldername in paths: + if foldername != '': + for file in files.items(): + if (file[1]['a'] and file[1]['t'] + and file[1]['a']['n'] == foldername): + if parent_desc == file[1]['p']: + parent_desc = file[0] + found = True + if found: + found = False + else: + return None + return parent_desc + + def find(self, filename=None, handle=None, exclude_deleted=False): + """ + Return file object from given filename + """ + files = self.get_files() + if handle: + return files[handle] + path = Path(filename) + filename = path.name + parent_dir_name = path.parent.name + for file in list(files.items()): + parent_node_id = None + try: + if parent_dir_name: + parent_node_id = self.find_path_descriptor(parent_dir_name, + files=files) + if (filename and parent_node_id and file[1]['a'] + and file[1]['a']['n'] == filename + and parent_node_id == file[1]['p']): + if (exclude_deleted and self._trash_folder_node_id + == file[1]['p']): + continue + return file + elif (filename and file[1]['a'] + and file[1]['a']['n'] == filename): + if (exclude_deleted + and self._trash_folder_node_id == file[1]['p']): + continue + return file + except TypeError: + continue + + def get_files(self): + logger.info('Getting all files...') + files = self._api_request({'a': 'f', 'c': 1, 'r': 1}) + files_dict = {} + shared_keys = {} + self._init_shared_keys(files, shared_keys) + for file in files['f']: + processed_file = self._process_file(file, shared_keys) + # ensure each file has a name before returning + if processed_file['a']: + files_dict[file['h']] = processed_file + return files_dict + + def get_upload_link(self, file): + """ + Get a files public link inc. decrypted key + Requires upload() response as input + """ + if 'f' in file: + file = file['f'][0] + public_handle = self._api_request({'a': 'l', 'n': file['h']}) + file_key = file['k'][file['k'].index(':') + 1:] + decrypted_key = a32_to_base64( + decrypt_key(base64_to_a32(file_key), self.master_key)) + return (f'{self.schema}://{self.domain}' + f'/#!{public_handle}!{decrypted_key}') + else: + raise ValueError('''Upload() response required as input, + use get_link() for regular file input''') + + def get_link(self, file): + """ + Get a file public link from given file object + """ + file = file[1] + if 'h' in file and 'k' in file: + public_handle = self._api_request({'a': 'l', 'n': file['h']}) + if public_handle == -11: + raise RequestError("Can't get a public link from that file " + "(is this a shared file?)") + decrypted_key = a32_to_base64(file['key']) + return (f'{self.schema}://{self.domain}' + f'/#!{public_handle}!{decrypted_key}') + else: + raise ValidationError('File id and key must be present') + + def _node_data(self, node): + try: + return node[1] + except (IndexError, KeyError): + return node + + def get_folder_link(self, file): + try: + file = file[1] + except (IndexError, KeyError): + pass + if 'h' in file and 'k' in file: + public_handle = self._api_request({'a': 'l', 'n': file['h']}) + if public_handle == -11: + raise RequestError("Can't get a public link from that file " + "(is this a shared file?)") + decrypted_key = a32_to_base64(file['shared_folder_key']) + return (f'{self.schema}://{self.domain}' + f'/#F!{public_handle}!{decrypted_key}') + else: + raise ValidationError('File id and key must be present') + + def get_user(self): + user_data = self._api_request({'a': 'ug'}) + return user_data + + def get_node_by_type(self, type): + """ + Get a node by it's numeric type id, e.g: + 0: file + 1: dir + 2: special: root cloud drive + 3: special: inbox + 4: special trash bin + """ + nodes = self.get_files() + for node in list(nodes.items()): + if node[1]['t'] == type: + return node + + def get_files_in_node(self, target): + """ + Get all files in a given target, e.g. 4=trash + """ + if type(target) == int: + # convert special nodes (e.g. trash) + node_id = self.get_node_by_type(target) + else: + node_id = [target] + + files = self._api_request({'a': 'f', 'c': 1}) + files_dict = {} + shared_keys = {} + self._init_shared_keys(files, shared_keys) + for file in files['f']: + processed_file = self._process_file(file, shared_keys) + if processed_file['a'] and processed_file['p'] == node_id[0]: + files_dict[file['h']] = processed_file + return files_dict + + def get_id_from_public_handle(self, public_handle): + # get node data + node_data = self._api_request({'a': 'f', 'f': 1, 'p': public_handle}) + node_id = self.get_id_from_obj(node_data) + return node_id + + def get_id_from_obj(self, node_data): + """ + Get node id from a file object + """ + node_id = None + + for i in node_data['f']: + if i['h'] != '': + node_id = i['h'] + return node_id + + def get_quota(self): + """ + Get current remaining disk quota in MegaBytes + """ + json_resp = self._api_request({ + 'a': 'uq', + 'xfer': 1, + 'strg': 1, + 'v': 1 + }) + # convert bytes to megabyes + return json_resp['mstrg'] / 1048576 + + def get_storage_space(self, giga=False, mega=False, kilo=False): + """ + Get the current storage space. + Return a dict containing at least: + 'used' : the used space on the account + 'total' : the maximum space allowed with current plan + All storage space are in bytes unless asked differently. + """ + if sum(1 if x else 0 for x in (kilo, mega, giga)) > 1: + raise ValueError("Only one unit prefix can be specified") + unit_coef = 1 + if kilo: + unit_coef = 1024 + if mega: + unit_coef = 1048576 + if giga: + unit_coef = 1073741824 + json_resp = self._api_request({'a': 'uq', 'xfer': 1, 'strg': 1}) + return { + 'used': json_resp['cstrg'] / unit_coef, + 'total': json_resp['mstrg'] / unit_coef, + } + + def get_balance(self): + """ + Get account monetary balance, Pro accounts only + """ + user_data = self._api_request({"a": "uq", "pro": 1}) + if 'balance' in user_data: + return user_data['balance'] + + def delete(self, public_handle): + """ + Delete a file by its public handle + """ + return self.move(public_handle, 4) + + def delete_url(self, url): + """ + Delete a file by its url + """ + path = self._parse_url(url).split('!') + public_handle = path[0] + file_id = self.get_id_from_public_handle(public_handle) + return self.move(file_id, 4) + + def destroy(self, file_id): + """ + Destroy a file by its private id + """ + return self._api_request({ + 'a': 'd', + 'n': file_id, + 'i': self.request_id + }) + + def destroy_url(self, url): + """ + Destroy a file by its url + """ + path = self._parse_url(url).split('!') + public_handle = path[0] + file_id = self.get_id_from_public_handle(public_handle) + return self.destroy(file_id) + + def empty_trash(self): + # get list of files in rubbish out + files = self.get_files_in_node(4) + + # make a list of json + if files != {}: + post_list = [] + for file in files: + post_list.append({"a": "d", "n": file, "i": self.request_id}) + return self._api_request(post_list) + + def download(self, file, dest_path=None, dest_filename=None): + """ + Download a file by it's file object + """ + return self._download_file(file_handle=None, + file_key=None, + file=file[1], + dest_path=dest_path, + dest_filename=dest_filename, + is_public=False) + + def _export_file(self, node): + node_data = self._node_data(node) + self._api_request([{ + 'a': 'l', + 'n': node_data['h'], + 'i': self.request_id + }]) + return self.get_link(node) + + def export(self, path=None, node_id=None): + nodes = self.get_files() + if node_id: + node = nodes[node_id] + else: + node = self.find(path) + + node_data = self._node_data(node) + is_file_node = node_data['t'] == 0 + if is_file_node: + return self._export_file(node) + if node: + try: + # If already exported + return self.get_folder_link(node) + except (RequestError, KeyError): + pass + + master_key_cipher = AES.new(a32_to_str(self.master_key), AES.MODE_ECB) + ha = base64_url_encode( + master_key_cipher.encrypt(node_data['h'].encode("utf8") + + node_data['h'].encode("utf8"))) + + share_key = secrets.token_bytes(16) + ok = base64_url_encode(master_key_cipher.encrypt(share_key)) + + share_key_cipher = AES.new(share_key, AES.MODE_ECB) + node_key = node_data['k'] + encrypted_node_key = base64_url_encode( + share_key_cipher.encrypt(a32_to_str(node_key))) + + node_id = node_data['h'] + request_body = [{ + 'a': + 's2', + 'n': + node_id, + 's': [{ + 'u': 'EXP', + 'r': 0 + }], + 'i': + self.request_id, + 'ok': + ok, + 'ha': + ha, + 'cr': [[node_id], [node_id], [0, 0, encrypted_node_key]] + }] + self._api_request(request_body) + nodes = self.get_files() + return self.get_folder_link(nodes[node_id]) + + def download_url(self, url, dest_path=None, dest_filename=None, progress_hook=None, progress_args=None): + """ + Download a file by it's public url + """ + path = self._parse_url(url).split('!') + file_id = path[0] + file_key = path[1] + return self._download_file( + file_handle=file_id, + file_key=file_key, + dest_path=dest_path, + dest_filename=dest_filename, + is_public=True, + progress_hook=progress_hook, + progress_args=progress_args, + ) + + def _download_file(self, + file_handle, + file_key, + dest_path=None, + dest_filename=None, + is_public=False, + file=None, + progress_hook=None, + progress_args=None): + if file is None: + if is_public: + file_key = base64_to_a32(file_key) + file_data = self._api_request({ + 'a': 'g', + 'g': 1, + 'p': file_handle + }) + else: + file_data = self._api_request({ + 'a': 'g', + 'g': 1, + 'n': file_handle + }) + + k = (file_key[0] ^ file_key[4], file_key[1] ^ file_key[5], + file_key[2] ^ file_key[6], file_key[3] ^ file_key[7]) + iv = file_key[4:6] + (0, 0) + meta_mac = file_key[6:8] + else: + file_data = self._api_request({'a': 'g', 'g': 1, 'n': file['h']}) + k = file['k'] + iv = file['iv'] + meta_mac = file['meta_mac'] + + # Seems to happens sometime... When this occurs, files are + # inaccessible also in the official also in the official web app. + # Strangely, files can come back later. + if 'g' not in file_data: + raise RequestError('File not accessible anymore') + file_url = file_data['g'] + file_size = file_data['s'] + attribs = base64_url_decode(file_data['at']) + attribs = decrypt_attr(attribs, k) + + if dest_filename is not None: + file_name = dest_filename + else: + file_name = attribs['n'] + + input_file = requests.get(file_url, stream=True).raw + + if dest_path is None: + dest_path = '' + else: + dest_path += '/' + + with tempfile.NamedTemporaryFile(mode='w+b', + prefix='megapy_', + delete=False) as temp_output_file: + k_str = a32_to_str(k) + counter = Counter.new(128, + initial_value=((iv[0] << 32) + iv[1]) << 64) + aes = AES.new(k_str, AES.MODE_CTR, counter=counter) + + mac_str = '\0' * 16 + mac_encryptor = AES.new(k_str, AES.MODE_CBC, + mac_str.encode("utf8")) + iv_str = a32_to_str([iv[0], iv[1], iv[0], iv[1]]) + + for chunk_start, chunk_size in get_chunks(file_size): + chunk = input_file.read(chunk_size) + chunk = aes.decrypt(chunk) + temp_output_file.write(chunk) + + encryptor = AES.new(k_str, AES.MODE_CBC, iv_str) + for i in range(0, len(chunk) - 16, 16): + block = chunk[i:i + 16] + encryptor.encrypt(block) + + # fix for files under 16 bytes failing + if file_size > 16: + i += 16 + else: + i = 0 + + block = chunk[i:i + 16] + if len(block) % 16: + block += b'\0' * (16 - (len(block) % 16)) + mac_str = mac_encryptor.encrypt(encryptor.encrypt(block)) + + file_info = os.stat(temp_output_file.name) + logger.info('%s of %s downloaded', file_info.st_size, + file_size) + if progress_hook: + progress_hook({ + 'current': file_info.st_size, + 'total': file_size, + 'name': file_name, + 'tmp_filename': temp_output_file.name, + 'status': 'downloading', + 'args': progress_args + }) + file_mac = str_to_a32(mac_str) + # check mac integrity + if (file_mac[0] ^ file_mac[1], + file_mac[2] ^ file_mac[3]) != meta_mac: + raise ValueError('Mismatched mac') + output_path = Path(dest_path + file_name) + file_info = os.stat(temp_output_file.name) + temp_output_file.close() + shutil.move(temp_output_file.name, output_path) + if progress_hook: + progress_hook({ + 'current': file_info.st_size, + 'total': file_size, + 'name': file_name, + 'tmp_filename': temp_output_file.name, + 'status': 'finished', + 'args': progress_args + }) + return output_path + + def upload(self, filename, dest=None, dest_filename=None): + # determine storage node + if dest is None: + # if none set, upload to cloud drive node + if not hasattr(self, 'root_id'): + self.get_files() + dest = self.root_id + + # request upload url, call 'u' method + with open(filename, 'rb') as input_file: + file_size = os.path.getsize(filename) + ul_url = self._api_request({'a': 'u', 's': file_size})['p'] + + # generate random aes key (128) for file + ul_key = [random.randint(0, 0xFFFFFFFF) for _ in range(6)] + k_str = a32_to_str(ul_key[:4]) + count = Counter.new( + 128, initial_value=((ul_key[4] << 32) + ul_key[5]) << 64) + aes = AES.new(k_str, AES.MODE_CTR, counter=count) + + upload_progress = 0 + completion_file_handle = None + + mac_str = '\0' * 16 + mac_encryptor = AES.new(k_str, AES.MODE_CBC, + mac_str.encode("utf8")) + iv_str = a32_to_str([ul_key[4], ul_key[5], ul_key[4], ul_key[5]]) + if file_size > 0: + for chunk_start, chunk_size in get_chunks(file_size): + chunk = input_file.read(chunk_size) + upload_progress += len(chunk) + + encryptor = AES.new(k_str, AES.MODE_CBC, iv_str) + for i in range(0, len(chunk) - 16, 16): + block = chunk[i:i + 16] + encryptor.encrypt(block) + + # fix for files under 16 bytes failing + if file_size > 16: + i += 16 + else: + i = 0 + + block = chunk[i:i + 16] + if len(block) % 16: + block += makebyte('\0' * (16 - len(block) % 16)) + mac_str = mac_encryptor.encrypt(encryptor.encrypt(block)) + + # encrypt file and upload + chunk = aes.encrypt(chunk) + output_file = requests.post(ul_url + "/" + + str(chunk_start), + data=chunk, + timeout=self.timeout) + completion_file_handle = output_file.text + logger.info('%s of %s uploaded', upload_progress, + file_size) + else: + output_file = requests.post(ul_url + "/0", + data='', + timeout=self.timeout) + completion_file_handle = output_file.text + + logger.info('Chunks uploaded') + logger.info('Setting attributes to complete upload') + logger.info('Computing attributes') + file_mac = str_to_a32(mac_str) + + # determine meta mac + meta_mac = (file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]) + + dest_filename = dest_filename or os.path.basename(filename) + attribs = {'n': dest_filename} + + encrypt_attribs = base64_url_encode( + encrypt_attr(attribs, ul_key[:4])) + key = [ + ul_key[0] ^ ul_key[4], ul_key[1] ^ ul_key[5], + ul_key[2] ^ meta_mac[0], ul_key[3] ^ meta_mac[1], ul_key[4], + ul_key[5], meta_mac[0], meta_mac[1] + ] + encrypted_key = a32_to_base64(encrypt_key(key, self.master_key)) + logger.info('Sending request to update attributes') + # update attributes + data = self._api_request({ + 'a': + 'p', + 't': + dest, + 'i': + self.request_id, + 'n': [{ + 'h': completion_file_handle, + 't': 0, + 'a': encrypt_attribs, + 'k': encrypted_key + }] + }) + logger.info('Upload complete') + return data + + def _mkdir(self, name, parent_node_id): + # generate random aes key (128) for folder + ul_key = [random.randint(0, 0xFFFFFFFF) for _ in range(6)] + + # encrypt attribs + attribs = {'n': name} + encrypt_attribs = base64_url_encode(encrypt_attr(attribs, ul_key[:4])) + encrypted_key = a32_to_base64(encrypt_key(ul_key[:4], self.master_key)) + + # update attributes + data = self._api_request({ + 'a': + 'p', + 't': + parent_node_id, + 'n': [{ + 'h': 'xxxxxxxx', + 't': 1, + 'a': encrypt_attribs, + 'k': encrypted_key + }], + 'i': + self.request_id + }) + return data + + def _root_node_id(self): + if not hasattr(self, 'root_id'): + self.get_files() + return self.root_id + + def create_folder(self, name, dest=None): + dirs = tuple(dir_name for dir_name in str(name).split('/') if dir_name) + folder_node_ids = {} + for idx, directory_name in enumerate(dirs): + existing_node_id = self.find_path_descriptor(directory_name) + if existing_node_id: + folder_node_ids[idx] = existing_node_id + continue + if idx == 0: + if dest is None: + parent_node_id = self._root_node_id() + else: + parent_node_id = dest + else: + parent_node_id = folder_node_ids[idx - 1] + created_node = self._mkdir(name=directory_name, + parent_node_id=parent_node_id) + node_id = created_node['f'][0]['h'] + folder_node_ids[idx] = node_id + return dict(zip(dirs, folder_node_ids.values())) + + def rename(self, file, new_name): + file = file[1] + # create new attribs + attribs = {'n': new_name} + # encrypt attribs + encrypt_attribs = base64_url_encode(encrypt_attr(attribs, file['k'])) + encrypted_key = a32_to_base64(encrypt_key(file['key'], + self.master_key)) + # update attributes + return self._api_request([{ + 'a': 'a', + 'attr': encrypt_attribs, + 'key': encrypted_key, + 'n': file['h'], + 'i': self.request_id + }]) + + def move(self, file_id, target): + """ + Move a file to another parent node + params: + a : command + n : node we're moving + t : id of target parent node, moving to + i : request id + + targets + 2 : root + 3 : inbox + 4 : trash + + or... + target's id + or... + target's structure returned by find() + """ + + # determine target_node_id + if type(target) == int: + target_node_id = str(self.get_node_by_type(target)[0]) + elif type(target) in (str, ): + target_node_id = target + else: + file = target[1] + target_node_id = file['h'] + return self._api_request({ + 'a': 'm', + 'n': file_id, + 't': target_node_id, + 'i': self.request_id + }) + + def add_contact(self, email): + """ + Add another user to your mega contact list + """ + return self._edit_contact(email, True) + + def remove_contact(self, email): + """ + Remove a user to your mega contact list + """ + return self._edit_contact(email, False) + + def _edit_contact(self, email, add): + """ + Editing contacts + """ + if add is True: + l = '1' # add command + elif add is False: + l = '0' # remove command + else: + raise ValidationError('add parameter must be of type bool') + + if not re.match(r"[^@]+@[^@]+\.[^@]+", email): + ValidationError('add_contact requires a valid email address') + else: + return self._api_request({ + 'a': 'ur', + 'u': email, + 'l': l, + 'i': self.request_id + }) + + def get_public_url_info(self, url): + """ + Get size and name from a public url, dict returned + """ + file_handle, file_key = self._parse_url(url).split('!') + return self.get_public_file_info(file_handle, file_key) + + def import_public_url(self, url, dest_node=None, dest_name=None): + """ + Import the public url into user account + """ + file_handle, file_key = self._parse_url(url).split('!') + return self.import_public_file(file_handle, + file_key, + dest_node=dest_node, + dest_name=dest_name) + + def get_public_file_info(self, file_handle, file_key): + """ + Get size and name of a public file + """ + data = self._api_request({'a': 'g', 'p': file_handle, 'ssm': 1}) + if isinstance(data, int): + raise RequestError(data) + + if 'at' not in data or 's' not in data: + raise ValueError("Unexpected result", data) + + key = base64_to_a32(file_key) + k = (key[0] ^ key[4], key[1] ^ key[5], key[2] ^ key[6], + key[3] ^ key[7]) + + size = data['s'] + unencrypted_attrs = decrypt_attr(base64_url_decode(data['at']), k) + if not unencrypted_attrs: + return None + result = {'size': size, 'name': unencrypted_attrs['n']} + return result + + def import_public_file(self, + file_handle, + file_key, + dest_node=None, + dest_name=None): + """ + Import the public file into user account + """ + # Providing dest_node spare an API call to retrieve it. + if dest_node is None: + # Get '/Cloud Drive' folder no dest node specified + dest_node = self.get_node_by_type(2)[1] + + # Providing dest_name spares an API call to retrieve it. + if dest_name is None: + pl_info = self.get_public_file_info(file_handle, file_key) + dest_name = pl_info['name'] + + key = base64_to_a32(file_key) + k = (key[0] ^ key[4], key[1] ^ key[5], key[2] ^ key[6], + key[3] ^ key[7]) + + encrypted_key = a32_to_base64(encrypt_key(key, self.master_key)) + encrypted_name = base64_url_encode(encrypt_attr({'n': dest_name}, k)) + return self._api_request({ + 'a': + 'p', + 't': + dest_node['h'], + 'n': [{ + 'ph': file_handle, + 't': 0, + 'a': encrypted_name, + 'k': encrypted_key + }] + }) diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index 8a522b4d..3eca29f6 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -83,6 +83,9 @@ def __init__(self, nzb_name, nzb_folder, issueid=None, module=None, queue=None, self.valreturn = [] self.extensions = ('.cbr', '.cbz', '.pdf', '.cb7') + + self.extensions = tuple(x for x in self.extensions if x not in mylar.CONFIG.IGNORE_SEARCH_WORDS) + self.failed_files = 0 self.log = '' if issueid is not None: @@ -249,7 +252,7 @@ def tidyup(self, odir=None, del_nzbdir=False, sub_path=None, cacheonly=False, fi logger.fdebug('odir: %s [filename: %s][self.nzb_folder: %s]' % (odir, filename, self.nzb_folder)) logger.fdebug('sub_path: %s [cacheonly: %s][del_nzbdir: %s]' % (sub_path, cacheonly, del_nzbdir)) #if sub_path exists, then we need to use that in place of self.nzb_folder since the file was in a sub-directory within self.nzb_folder - if all([sub_path is not None, sub_path != self.nzb_folder]): #, self.issueid is not None]): + if all([sub_path is not None, sub_path != self.nzb_folder, sub_path != os.path.join(self.nzb_folder, 'mega')]): #, self.issueid is not None]): if self.issueid is None: logger.fdebug('Sub-directory detected during cleanup. Will attempt to remove if empty: %s' % sub_path) orig_folder = sub_path @@ -325,14 +328,17 @@ def tidyup(self, odir=None, del_nzbdir=False, sub_path=None, cacheonly=False, fi logger.warn('%s [%s] Unable to remove file : %s' % (self.module, e, os.path.join(tmp_folder, filename))) else: if not os.listdir(tmp_folder): - logger.fdebug('%s Tidying up. Deleting original folder location : %s' % (self.module, tmp_folder)) - try: - shutil.rmtree(tmp_folder) - except Exception as e: - logger.warn('%s [%s] Unable to delete original folder location: %s' % (self.module, e, tmp_folder)) + if os.path.join(mylar.CONFIG.DDL_LOCATION, 'mega') == tmp_folder: + logger.fdebug('%s Tidying up. %s sub-directory not being removed as is required for mega ddl' % (self.module, tmp_folder)) else: - logger.fdebug('%s Removed original folder location: %s' % (self.module, tmp_folder)) - self._log("Removed temporary directory : " + tmp_folder) + logger.fdebug('%s Tidying up. Deleting original folder location : %s' % (self.module, tmp_folder)) + try: + shutil.rmtree(tmp_folder) + except Exception as e: + logger.warn('%s [%s] Unable to delete original folder location: %s' % (self.module, e, tmp_folder)) + else: + logger.fdebug('%s Removed original folder location: %s' % (self.module, tmp_folder)) + self._log("Removed temporary directory : " + tmp_folder) else: self._log('Failed to remove temporary directory: ' + tmp_folder) logger.error('%s %s not empty. Skipping removal of directory - this will either be caught in further post-processing or it will have to be manually deleted.' % (self.module, tmp_folder)) @@ -646,7 +652,7 @@ def Process(self): self.nzb_folder = clocation # this is needed in order to delete after moving. else: try: - if all([pathlib.Path(tpath) != pathlib.Path(mylar.CACHE_DIR), pathlib.Path(tpath) != pathlib.Path(mylar.CONFIG.DDL_LOCATION)]): + if all([pathlib.Path(tpath) != pathlib.Path(mylar.CACHE_DIR), pathlib.Path(tpath) != pathlib.Path(mylar.CONFIG.DDL_LOCATION), pathlib.Path(tpath) != pathlib.Path(mylar.CONFIG.DDL_LOCATION).joinpath('mega')]): if pathlib.Path(tpath).is_file(): try: cloct = pathlib.Path(tpath).with_name(nfilename) @@ -3542,6 +3548,9 @@ def run(self): mylar.MONITOR_STATUS = 'Paused' helpers.job_management(write=True) else: + if mylar.API_LOCK is True: + logger.info('%s Unable to initiate folder monitor as another process is currently using it or using post-processing.' % self.module) + return helpers.job_management(write=True, job='Folder Monitor', current_run=helpers.utctimestamp(), status='Running') mylar.MONITOR_STATUS = 'Running' logger.info('%s Checking folder %s for newly snatched downloads' % (self.module, mylar.CONFIG.CHECK_FOLDER)) diff --git a/mylar/__init__.py b/mylar/__init__.py index dce799f0..58ec25b6 100644 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -152,6 +152,7 @@ REFRESH_QUEUE = queue.Queue() DDL_QUEUED = [] PACK_ISSUEIDS_DONT_QUEUE = {} +EXT_SERVER = False SEARCH_TIER_DATE = None COMICSORT = None PULLBYFILE = False @@ -245,7 +246,7 @@ def initialize(config_file): MONITOR_SCHEDULER, SEARCH_SCHEDULER, RSS_SCHEDULER, WEEKLY_SCHEDULER, VERSION_SCHEDULER, UPDATER_SCHEDULER, \ SCHED_RSS_LAST, SCHED_WEEKLY_LAST, SCHED_MONITOR_LAST, SCHED_SEARCH_LAST, SCHED_VERSION_LAST, SCHED_DBUPDATE_LAST, COMICINFO, SEARCH_TIER_DATE, \ BACKENDSTATUS_CV, BACKENDSTATUS_WS, PROVIDER_STATUS, EXT_IP, ISSUE_EXCEPTIONS, PROVIDER_START_ID, GLOBAL_MESSAGES, CHECK_FOLDER_CACHE, FOLDER_CACHE, SESSION_ID, \ - MAINTENANCE_UPDATE, MAINTENANCE_DB_COUNT, MAINTENANCE_DB_TOTAL, UPDATE_VALUE, REQS, IMPRINT_MAPPING, GC_URL, PACK_ISSUEIDS_DONT_QUEUE, DDL_QUEUED + MAINTENANCE_UPDATE, MAINTENANCE_DB_COUNT, MAINTENANCE_DB_TOTAL, UPDATE_VALUE, REQS, IMPRINT_MAPPING, GC_URL, PACK_ISSUEIDS_DONT_QUEUE, DDL_QUEUED, EXT_SERVER cc = mylar.config.Config(config_file) CONFIG = cc.read(startup=True) @@ -324,6 +325,10 @@ def initialize(config_file): str(random.getrandbits(256)).encode('utf-8') ).hexdigest()[0:32] + from mylar.downloaders import external_server as des + EXT_SERVER = des.EXT_SERVER + logger.info('[DDL] External server configuration available to be loaded: %s' % EXT_SERVER) + SESSION_ID = random.randint(10000,999999) CV_HEADERS = {'User-Agent': mylar.CONFIG.CV_USER_AGENT} @@ -807,7 +812,7 @@ def dbcheck(): c.execute('CREATE TABLE IF NOT EXISTS jobhistory (JobName TEXT, prev_run_datetime timestamp, prev_run_timestamp REAL, next_run_datetime timestamp, next_run_timestamp REAL, last_run_completed TEXT, successful_completions TEXT, failed_completions TEXT, status TEXT, last_date timestamp)') c.execute('CREATE TABLE IF NOT EXISTS manualresults (provider TEXT, id TEXT, kind TEXT, comicname TEXT, volume TEXT, oneoff TEXT, fullprov TEXT, issuenumber TEXT, modcomicname TEXT, name TEXT, link TEXT, size TEXT, pack_numbers TEXT, pack_issuelist TEXT, comicyear TEXT, issuedate TEXT, tmpprov TEXT, pack TEXT, issueid TEXT, comicid TEXT, sarc TEXT, issuearcid TEXT)') c.execute('CREATE TABLE IF NOT EXISTS storyarcs(StoryArcID TEXT, ComicName TEXT, IssueNumber TEXT, SeriesYear TEXT, IssueYEAR TEXT, StoryArc TEXT, TotalIssues TEXT, Status TEXT, inCacheDir TEXT, Location TEXT, IssueArcID TEXT, ReadingOrder INT, IssueID TEXT, ComicID TEXT, ReleaseDate TEXT, IssueDate TEXT, Publisher TEXT, IssuePublisher TEXT, IssueName TEXT, CV_ArcID TEXT, Int_IssueNumber INT, DynamicComicName TEXT, Volume TEXT, Manual TEXT, DateAdded TEXT, DigitalDate TEXT, Type TEXT, Aliases TEXT, ArcImage TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS ddl_info (ID TEXT UNIQUE, series TEXT, year TEXT, filename TEXT, size TEXT, issueid TEXT, comicid TEXT, link TEXT, status TEXT, remote_filesize TEXT, updated_date TEXT, mainlink TEXT, issues TEXT, site TEXT, submit_date TEXT, pack INTEGER)') + c.execute('CREATE TABLE IF NOT EXISTS ddl_info (ID TEXT UNIQUE, series TEXT, year TEXT, filename TEXT, size TEXT, issueid TEXT, comicid TEXT, link TEXT, status TEXT, remote_filesize TEXT, updated_date TEXT, mainlink TEXT, issues TEXT, site TEXT, submit_date TEXT, pack INTEGER, link_type TEXT, tmp_filename TEXT)') c.execute('CREATE TABLE IF NOT EXISTS exceptions_log(date TEXT UNIQUE, comicname TEXT, issuenumber TEXT, seriesyear TEXT, issueid TEXT, comicid TEXT, booktype TEXT, searchmode TEXT, error TEXT, error_text TEXT, filename TEXT, line_num TEXT, func_name TEXT, traceback TEXT)') c.execute('CREATE TABLE IF NOT EXISTS tmp_searches (query_id INTEGER, comicid INTEGER, comicname TEXT, publisher TEXT, publisherimprint TEXT, comicyear TEXT, issues TEXT, volume TEXT, deck TEXT, url TEXT, type TEXT, cvarcid TEXT, arclist TEXT, description TEXT, haveit TEXT, mode TEXT, searchtype TEXT, comicimage TEXT, thumbimage TEXT, PRIMARY KEY (query_id, comicid))') c.execute('CREATE TABLE IF NOT EXISTS notifs(session_id INT, date TEXT, event TEXT, comicid TEXT, comicname TEXT, issuenumber TEXT, seriesyear TEXT, status TEXT, message TEXT, PRIMARY KEY (session_id, date))') @@ -1524,6 +1529,16 @@ def dbcheck(): except sqlite3.OperationalError: c.execute('ALTER TABLE ddl_info ADD COLUMN pack INTEGER') + try: + c.execute('SELECT link_type from ddl_info') + except sqlite3.OperationalError: + c.execute('ALTER TABLE ddl_info ADD COLUMN link_type TEXT') + + try: + c.execute('SELECT tmp_filename from ddl_info') + except sqlite3.OperationalError: + c.execute('ALTER TABLE ddl_info ADD COLUMN tmp_filename TEXT') + ## -- provider_searches Table -- try: c.execute('SELECT id from provider_searches') diff --git a/mylar/api.py b/mylar/api.py index bb782393..34f6f60e 100644 --- a/mylar/api.py +++ b/mylar/api.py @@ -26,8 +26,8 @@ import shutil import queue import urllib.request, urllib.error, urllib.parse +from PIL import Image from . import cache -import imghdr from operator import itemgetter from cherrypy.lib.static import serve_file, serve_download import datetime @@ -40,7 +40,7 @@ 'getComicInfo', 'getIssueInfo', 'getArt', 'downloadIssue', 'regenerateCovers', 'refreshSeriesjson', 'seriesjsonListing', 'checkGlobalMessages', 'listProviders', 'changeProvider', 'addProvider', 'delProvider', - 'downloadNZB', 'getReadList', 'getStoryArc', 'addStoryArc'] + 'downloadNZB', 'getReadList', 'getStoryArc', 'addStoryArc', 'listAnnualSeries'] class Api(object): @@ -1032,7 +1032,8 @@ def _getArt(self, **kwargs): # Checks if its a valid path and file if os.path.isfile(image_path): # check if its a valid img - if imghdr.what(image_path): + imghdr = Image.open(image_path) + if imghdr.get_format_mimetype(): self.img = image_path return else: @@ -1050,7 +1051,8 @@ def _getArt(self, **kwargs): if img: # verify the img stream - if imghdr.what(None, img): + imghdr = Image.open(img) + if imghdr.get_format_mimetype(): with open(image_path, 'wb') as f: f.write(img) self.img = image_path @@ -1200,6 +1202,75 @@ def _addStoryArc(self, **kwargs): self.data = self._successResponse('Adding %s issue(s) to %s' % (issuecount, storyarcname)) return + def _listAnnualSeries(self, **kwargs): + # list_issues = true/false + # group_series = true/false + # show_downloaded_only = true/false + # - future: recreate as individual series (annual integration off after was enabled) = true/false + if all(['list_issues' not in kwargs, 'group_series' not in kwargs]): #, 'recreate' not in kwargs]): + self.data = self._failureResponse('Missing parameter(s): Must specify either `list_issues` or `group_series`') + return + else: + list_issues = True + group_series = True + show_downloaded = True + #recreate_from_annuals = True + try: + listissues = kwargs['list_issues'] + except Exception: + list_issues = False + try: + groupseries = kwargs['group_series'] + except Exception: + group_series = False + try: + showdownloaded = kwargs['show_downloaded'] + except Exception: + show_downloaded = False + + if group_series: + annual_listing = {} + else: + annual_listing = [] + + #try: + # recreatefromannuals = kwargs['recreate'] + #except Exception: + # recreate_from_annuals = False + + try: + myDB = db.DBConnection() + las = myDB.select('SELECT * from Annuals WHERE NOT Deleted') + except Exception as e: + self.data = self._failureResponse('Unable to query Annuals table - possibly no annuals have been detected as being integrated.') + return + + if las is None: + self.data = self._failureResponse('No annuals have been detected as ever being integrated.') + return + + annuals = {} + for lss in las: + if show_downloaded is False or all([show_downloaded is True, lss['Status'] == 'Downloaded']): + annuals = {'series': lss['ComicName'], + 'annualname': lss['ReleaseComicName'], + 'annualcomicid': lss['ReleaseComicID'], + 'issueid': int(lss['IssueID']), + 'filename': lss['Location'], + 'issuenumber': lss['Issue_Number']} + + if group_series is True: + if int(lss['ComicID']) not in annual_listing.keys(): + annual_listing[int(lss['comicid'])] = [annuals] + else: + annual_listing[int(lss['comicid'])] += [annuals] + else: + annuals['comicid'] = int(lss['ComicID']) + annual_listing.append(annuals) + + self.data = self._successResponse(annual_listing) + return + def _checkGlobalMessages(self, **kwargs): the_message = {'status': None, 'event': None, 'comicname': None, 'seriesyear': None, 'comicid': None, 'tables': None, 'message': None} if mylar.GLOBAL_MESSAGES is not None: diff --git a/mylar/carepackage.py b/mylar/carepackage.py index aa73b60f..bc45d0ce 100644 --- a/mylar/carepackage.py +++ b/mylar/carepackage.py @@ -176,8 +176,8 @@ def environment(self, vers_vals): f.close() def cleaned_config(self): - tmpconfig = configparser.SafeConfigParser() - tmpconfig.readfp(codecs.open(self.configpath, 'r', 'utf8')) + tmpconfig = configparser.ConfigParser() + tmpconfig.read_file(codecs.open(self.configpath, 'r', 'utf8')) if self.maintenance is True: self.log_dir = tmpconfig['Logs']['log_dir'] diff --git a/mylar/cdh_mapping.py b/mylar/cdh_mapping.py index d24ce00c..8ad92002 100644 --- a/mylar/cdh_mapping.py +++ b/mylar/cdh_mapping.py @@ -16,7 +16,7 @@ import requests import pathlib import re -from pkg_resources import parse_version +from packaging.version import parse as parse_version import mylar from mylar import logger diff --git a/mylar/cmtagmylar.py b/mylar/cmtagmylar.py index a86df773..604e5dc5 100644 --- a/mylar/cmtagmylar.py +++ b/mylar/cmtagmylar.py @@ -12,6 +12,7 @@ import time import zipfile import subprocess +from packaging.version import parse as parse_version from subprocess import CalledProcessError, check_output import mylar @@ -146,7 +147,6 @@ def run(dirName, nzbName=None, issueid=None, comversion=None, manual=None, filen logger.info('ct_check: %s' % ct_check) ctend = str(ct_check).find('[') ct_version = re.sub("[^0-9]", "", str(ct_check)[:ctend]) - from pkg_resources import parse_version if parse_version(ct_version) >= parse_version('1.3.1'): if any([mylar.CONFIG.COMICVINE_API == 'None', mylar.CONFIG.COMICVINE_API is None]): logger.fdebug('%s ComicTagger v.%s being used - no personal ComicVine API Key supplied. Take your chances.' % (module, ct_version)) diff --git a/mylar/config.py b/mylar/config.py index f2a79a40..d686a853 100644 --- a/mylar/config.py +++ b/mylar/config.py @@ -94,7 +94,7 @@ 'CREATE_FOLDERS': (bool, 'General', True), 'ALTERNATE_LATEST_SERIES_COVERS': (bool, 'General', False), 'SHOW_ICONS': (bool, 'General', False), - 'FORMAT_BOOKTYPE': (bool, 'General', False), + 'FORMAT_BOOKTYPE': (bool, 'General', True), 'CLEANUP_CACHE': (bool, 'General', False), 'SECURE_DIR': (str, 'General', None), 'ENCRYPT_PASSWORDS': (bool, 'General', False), @@ -358,11 +358,16 @@ 'ENABLE_DDL': (bool, 'DDL', False), 'ENABLE_GETCOMICS': (bool, 'DDL', False), + 'ENABLE_EXTERNAL_SERVER': (bool, 'DDL', False), + 'EXTERNAL_SERVER': (str, 'DDL', None), + 'EXTERNAL_USERNAME': (str, 'DDL', None), + 'EXTERNAL_APIKEY': (str, 'DDL', None), 'PACK_PRIORITY': (bool, 'DDL', False), 'DDL_QUERY_DELAY': (int, 'DDL', 15), 'DDL_LOCATION': (str, 'DDL', None), 'DDL_AUTORESUME': (bool, 'DDL', True), 'DDL_PREFER_UPSCALED': (bool, 'DDL', True), + 'DDL_PRIORITY_ORDER': (str, 'DDL', []), 'ENABLE_FLARESOLVERR': (bool, 'DDL', False), 'FLARESOLVERR_URL': (str, 'DDL', None), 'ENABLE_PROXY': (bool, 'DDL', False), @@ -1220,7 +1225,7 @@ def configure(self, update=False, startup=False): setattr(self, 'IGNORE_SEARCH_WORDS', [".exe", ".iso", "pdf-xpost", "pdf", "ebook"]) config.set('General', 'ignore_search_words', json.dumps(self.IGNORE_SEARCH_WORDS)) - logger.info('[IGNORE_SEARCH_WORDS] Words to flag search result as invalid: %s' % (self.IGNORE_SEARCH_WORDS,)) + logger.fdebug('[IGNORE_SEARCH_WORDS] Words to flag search result as invalid: %s' % (self.IGNORE_SEARCH_WORDS,)) if len(self.PROBLEM_DATES) > 0 and self.PROBLEM_DATES != '[]': if type(self.PROBLEM_DATES) != list: @@ -1319,19 +1324,54 @@ def configure(self, update=False, startup=False): config.set('General', 'folder_format', ann_remove) # need to recheck this cause of how enable_ddl and enable_getcomics are now - self.ENABLE_GETCOMICS = self.ENABLE_DDL - config.set('DDL', 'enable_getcomics', str(self.ENABLE_GETCOMICS)) + #self.ENABLE_GETCOMICS = self.ENABLE_DDL + #config.set('DDL', 'enable_getcomics', str(self.ENABLE_GETCOMICS)) if not self.DDL_LOCATION: self.DDL_LOCATION = self.CACHE_DIR if self.ENABLE_DDL is True: logger.info('Setting DDL Location set to : %s' % self.DDL_LOCATION) else: - dcreate = filechecker.validateAndCreateDirectory(self.DDL_LOCATION, create=True, dmode='ddl location') - if dcreate is False and self.ENABLE_DDL is True: + try: + os.makedirs(self.DDL_LOCATION) + #dcreate = filechecker.validateAndCreateDirectory(self.DDL_LOCATION, create=True, dmode='ddl location') + #if dcreate is False and self.ENABLE_DDL is True: + except Exception as e: logger.warn('Unable to create ddl_location specified in config: %s. Reverting to default cache location.' % self.DDL_LOCATION) self.DDL_LOCATION = self.CACHE_DIR + if self.ENABLE_DDL: + #make sure directory for mega downloads is created... + mega_ddl_path = os.path.join(self.DDL_LOCATION, 'mega') + if not os.path.isdir(mega_ddl_path): + try: + os.makedirs(mega_ddl_path) + #dcreate = filechecker.validateAndCreateDirectory(mega_ddl_path, create=True) + #if dcreate is False: + except Exception as e: + logger.error('Unable to create temp download directory [%s] for DDL-External. You will not be able to view the progress of the download.' % mega_ddl_path) + + if len(self.DDL_PRIORITY_ORDER) > 0 and self.DDL_PRIORITY_ORDER != '[]': + if type(self.DDL_PRIORITY_ORDER) != list: + try: + self.DDL_PRIORITY_ORDER = json.loads(self.DDL_PRIORITY_ORDER) + except Exception as e: + logger.warn('[DDL PRIORITY ORDER] Unable to load DDL priority order from setting to default') + setattr(self, 'DDL_PRIORITY_ORDER', ["mega", "mediafire", "pixeldrain", "main"]) + + # validate entries + ddl_pros = ['mega', 'mediafire', 'pixeldrain', 'main'] + for dpo in self.DDL_PRIORITY_ORDER: + if dpo.lower() not in ddl_pros: + logger.warn('[DDL PRIORITY ORDER] Invalid value detected - removing %s' % dpo) + self.DDL_PRIORITY_ORDER.pop(self.DDL_PRIORITY_ORDER.index(dpo)) + + else: + setattr(self, 'DDL_PRIORITY_ORDER', ["mega", "mediafire", "pixeldrain", "main"]) #default order + config.set('DDL', 'ddl_priority_order', json.dumps(self.DDL_PRIORITY_ORDER)) + + logger.info('[DDL PRIORITY ORDER] DDL will attempt to use the following 3rd party sites in this specific download order: %s' % self.DDL_PRIORITY_ORDER) + if self.SEARCH_TIER_CUTOFF is None: self.SEARCH_TIER_CUTOFF = 14 config.set('General', 'search_tier_cutoff', str(self.SEARCH_TIER_CUTOFF)) @@ -1583,8 +1623,11 @@ def provider_sequence(self): if self.ENABLE_GETCOMICS: PR.append('DDL(GetComics)') PR_NUM +=1 + if self.ENABLE_EXTERNAL_SERVER: + PR.append('DDL(External)') + PR_NUM +=1 - PPR = ['32p', 'nzb.su', 'dognzb', 'Experimental', 'DDL(GetComics)'] + PPR = ['32p', 'nzb.su', 'dognzb', 'Experimental', 'DDL(GetComics)', 'DDL(External)'] if self.NEWZNAB: for ens in self.EXTRA_NEWZNABS: if str(ens[5]) == '1': # if newznabs are enabled @@ -1750,6 +1793,8 @@ def write_out_provider_searches(self): # id of 0 means it hasn't been assigned - so we need to assign it before we build out the dict if 'DDL(GetComics)' in prov_t: t_id = 200 + if 'DDL(External)' in prov_t: + t_id = 201 elif any(['experimental' in prov_t, 'Experimental' in prov_t]): t_id = 101 elif 'dog' in prov_t: @@ -1785,6 +1830,9 @@ def write_out_provider_searches(self): if 'DDL(GetComics)' in tmp_prov: t_type = 'DDL' t_id = 200 + if 'DDL(External)' in tmp_prov: + t_type = 'DDL(External)' + t_id = 201 elif any(['experimental' in tmp_prov, 'Experimental' in tmp_prov]): tmp_prov = 'experimental' t_type = 'experimental' diff --git a/mylar/cv.py b/mylar/cv.py index cc7529bb..5ff3d09c 100755 --- a/mylar/cv.py +++ b/mylar/cv.py @@ -939,11 +939,11 @@ def GetSeriesYears(dom): #sometimes it's volume 5 and ocassionally it's fifth volume. if i == 0: vfind = comicDes[v_find:v_find +15] #if it's volume 5 format - basenums = {'zero': '0', 'one': '1', 'two': '2', 'three': '3', 'four': '4', 'five': '5', 'six': '6', 'seven': '7', 'eight': '8', 'nine': '9', 'ten': '10', 'i': '1', 'ii': '2', 'iii': '3', 'iv': '4', 'v': '5'} + basenums = basenum_mapping(ordinal=False) logger.fdebug('volume X format - %s: %s' % (i, vfind)) else: vfind = comicDes[:v_find] # if it's fifth volume format - basenums = {'zero': '0', 'first': '1', 'second': '2', 'third': '3', 'fourth': '4', 'fifth': '5', 'sixth': '6', 'seventh': '7', 'eighth': '8', 'nineth': '9', 'tenth': '10', 'i': '1', 'ii': '2', 'iii': '3', 'iv': '4', 'v': '5'} + basenums = basenum_mapping(ordinal=True) logger.fdebug('X volume format - %s: %s' % (i, vfind)) for nums in basenums: if nums in vfind.lower(): @@ -1263,7 +1263,7 @@ def get_imprint_volume_and_booktype(series, comicyear, publisher, firstissueid, #sometimes the deck has volume labels try: - comic_deck = deck + comic_deck = deck.strip() desdeck +=1 except: comic_deck = 'None' @@ -1357,17 +1357,20 @@ def get_imprint_volume_and_booktype(series, comicyear, publisher, firstissueid, # check to see if it matches the existing volume and if so replace it with any new # values since the incorrect volume is incorrect. incorrect_volume = comicDes[v_find:v_find+15] - if comicDes[v_find+7:comicDes.find(' ', v_find+7)].isdigit(): - comic['ComicVersion'] = re.sub("[^0-9]", "", comicDes[v_find+7:comicDes.find(' ', v_find+7)]).strip() + cbd = comicDes.find(' ', v_find+7) + if comicDes.find(' ', v_find+7) == -1: + cbd = len(comicDes) + if comicDes[v_find+7:cbd].isdigit(): + comic['ComicVersion'] = re.sub("[^0-9]", "", comicDes[v_find+7:cbd]).strip() break elif i == 0: vfind = comicDes[v_find:v_find +15] #if it's volume 5 format - basenums = {'zero': '0', 'one': '1', 'two': '2', 'three': '3', 'four': '4', 'five': '5', 'six': '6', 'seven': '7', 'eight': '8', 'nine': '9', 'ten': '10', 'i': '1', 'ii': '2', 'iii': '3', 'iv': '4', 'v': '5'} - logger.fdebug('volume X format - ' + str(i) + ': ' + vfind) + basenums = basenum_mapping(ordinal=False) + logger.fdebug('volume X format - %s: %s' % (i, vfind)) else: - vfind = comicDes[:v_find] # if it's fifth volume format - basenums = {'zero': '0', 'first': '1', 'second': '2', 'third': '3', 'fourth': '4', 'fifth': '5', 'sixth': '6', 'seventh': '7', 'eighth': '8', 'nineth': '9', 'tenth': '10', 'i': '1', 'ii': '2', 'iii': '3', 'iv': '4', 'v': '5'} - logger.fdebug('X volume format - ' + str(i) + ': ' + vfind) + vfind = comicDes[:v_find] # if it's fifth volume format + basenums = basenum_mapping(ordinal=True) + logger.fdebug('X volume format - %s: %s' % (i, vfind)) og_vfind = vfind for nums in basenums: if nums in vfind.lower(): @@ -1417,3 +1420,59 @@ def get_imprint_volume_and_booktype(series, comicyear, publisher, firstissueid, logger.info('comic_values: %s' % (comic,)) return comic + +def basenum_mapping(ordinal=False): + if not ordinal: + basenums = {'zero': '0', + 'one': '1', + 'two': '2', + 'three': '3', + 'four': '4', + 'five': '5', + 'six': '6', + 'seven': '7', + 'eight': '8', + 'nine': '9', + 'ten': '10', + 'eleven': '11', + 'twelve': '12', + 'thirteen': '13', + 'fourteen': '14', + 'fifteen': '15', + 'i': '1', + 'ii': '2', + 'iii': '3', + 'iv': '4', + 'v': '5', + 'vi': '6', + 'vii': '7', + 'viii': '8', + 'xi': '9'} + else: + basenums = {'zero': '0', + 'first': '1', + 'second': '2', + 'third': '3', + 'fourth': '4', + 'fifth': '5', + 'sixth': '6', + 'seventh': '7', + 'eighth': '8', + 'nineth': '9', + 'tenth': '10', + 'eleventh': '11', + 'twelfth': '12', + 'thirteenth': '13', + 'fourteenth': '14', + 'fifteenth': '15', + 'i': '1', + 'ii': '2', + 'iii': '3', + 'iv': '4', + 'v': '5', + 'vi': '6', + 'vii': '7', + 'viii': '8', + 'xi': '9'} + + return basenums diff --git a/mylar/downloaders/__init__.py b/mylar/downloaders/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mylar/downloaders/external_server.py b/mylar/downloaders/external_server.py new file mode 100644 index 00000000..2911cc8f --- /dev/null +++ b/mylar/downloaders/external_server.py @@ -0,0 +1,2 @@ +#placeholder +EXT_SERVER=False diff --git a/mylar/downloaders/mediafire.py b/mylar/downloaders/mediafire.py new file mode 100644 index 00000000..da4ad61c --- /dev/null +++ b/mylar/downloaders/mediafire.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- + +# This file is part of Mylar. +# +# Mylar is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Mylar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Mylar. If not, see . + +import os +import os.path as osp +import urllib +import re +import shutil +import sys +import requests +import mylar +from mylar import db, helpers, logger, search, search_filer + +class MediaFire(object): + + def __init__(self): + self.dl_location = os.path.join(mylar.CONFIG.DDL_LOCATION) + self.headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.5481.178 Safari/537.36" + } + self.session = requests.Session() + + def extractDownloadLink(self, contents): + for line in contents.splitlines(): + m = re.search(r'href="((http|https)://download[^"]+)', line) + if m: + return m.groups()[0] + + def ddl_download(self, url, id, issueid): + url_origin = url + + while True: + t = self.session.get( + url, + verify=True, + headers=self.headers, + stream=True, + timeout=(30,30) + ) + + if 'Content-Disposition' in t.headers: + # This is the file + break + + # Need to redirect with confiramtion + url = self.extractDownloadLink(t.text) + + if url is None: + #link no longer valid + return {"success": False, "filename": None, "path": None, "link_type_failure": 'GC-Media'} + + m = re.search( + 'filename="(.*)"', t.headers['Content-Disposition'] + ) + filename = m.groups()[0] + filename = filename.encode('iso8859').decode('utf-8') + + file, ext = os.path.splitext(filename) + filename = '%s[__%s__]%s' % (file, issueid, ext) + + try: + filesize = int(t.headers['Content-Length']) + except Exception: + filesize = 0 + + fileinfo = {'filename': filename, + 'filesize': filesize} + + logger.fdebug('Downloading...') + logger.fdebug('%s [%s bytes]' % (filename, filesize)) + logger.fdebug('From: %s' % url_origin) + logger.fdebug('To: %s' % os.path.join(self.dl_location, filename)) + + myDB = db.DBConnection() + ## write the filename to the db for tracking purposes... + logger.fdebug('[Writing to db: %s' % (filename)) + myDB.upsert( + 'ddl_info', + {'filename': str(filename), 'remote_filesize': str(filesize), 'size': helpers.human_size(filesize)}, + {'id': id}, + ) + return self.mediafire_dl(url, id, fileinfo, issueid) + + def mediafire_dl(self, url, id, fileinfo, issueid): + filepath = os.path.join(self.dl_location, fileinfo['filename']) + + myDB = db.DBConnection() + myDB.upsert( + 'ddl_info', + {'tmp_filename': fileinfo['filename']}, # tmp_filename should be all that's needed to be updated at this point... + {'id': id}, + ) + + try: + response = self.session.get( + url, + verify=True, + headers=self.headers, + stream=True, + timeout=(30,30) + ) + + logger.fdebug('[MediaFire] now writing....') + with open(filepath, 'wb') as f: + for chunk in response.iter_content(chunk_size=4096): + if chunk: + f.write(chunk) + f.flush() + + except Exception as e: + logger.fdebug('[MediaFire][ERROR] %s' % e) + if 'EBLOCKED' in str(e): + logger.fdebug('[MediaFire] Content has been removed - we should move on to the next one at this point.') + return {"success": False, "filename": None, "path": None, "link_type_failure": 'GC-Media'} + + logger.fdebug('[MediaFire] download completed - donwloaded %s / %s' % (os.stat(filepath).st_size, fileinfo['filesize'])) + + logger.fdebug('[MediaFire] ddl_linked - filename: %s' % fileinfo['filename']) + + file, ext = os.path.splitext(fileinfo['filename']) + if ext == '.zip': + ggc = mylar.getcomics.GC() + return ggc.zip_zip(id, str(filepath), fileinfo['filename']) + else: + return {"success": True, "filename": fileinfo['filename'], "path": str(filepath)} + diff --git a/mylar/downloaders/mega.py b/mylar/downloaders/mega.py new file mode 100644 index 00000000..f5630f01 --- /dev/null +++ b/mylar/downloaders/mega.py @@ -0,0 +1,131 @@ + +# This file is part of Mylar. +# +# Mylar is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Mylar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Mylar. If not, see . + +import requests +import sys +import os +import datetime +import time +from pathlib import Path +from mega import Mega +import mylar +from mylar import db, helpers, logger + +class MegaNZ(object): + + def __init__(self, query=None): + # if query is None, it's downloading not searching. + self.query = query + self.dl_location = os.path.join(mylar.CONFIG.DDL_LOCATION, 'mega') + self.wrote_tmp = False + + def ddl_download(self, link, filename, id, issueid=None, site=None): + self.id = id + if self.dl_location is not None and not os.path.isdir( + self.dl_location + ): + checkdirectory = mylar.filechecker.validateAndCreateDirectory( + self.dl_location, True + ) + if not checkdirectory: + logger.warn( + '[ABORTING] Error trying to validate/create DDL download' + ' directory: %s.' % self.dl_location + ) + return {"success": False, "filename": filename, "path": None} + + + logger.info('trying link: %s' % (link,)) + logger.info('dl_location: %s / filename: %s' % (self.dl_location, filename)) + + mega = Mega() + try: + m = mega.login() + # ddl -mega from gc filename isn't known until AFTER file begins downloading + # try to get it using the pulic_url_info endpoint + pui = m.get_public_url_info(link) + if pui: + filesize = pui['size'] + filename = pui['name'] + + myDB = db.DBConnection() + # write the filename to the db for tracking purposes... + logger.info('[get-public-url-info resolved] Writing to db: %s [%s]' % (filename, filesize)) + myDB.upsert( + 'ddl_info', + {'filename': str(filename), 'remote_filesize': str(filesize), 'size': helpers.human_size(str(filesize))}, + {'id': self.id}, + ) + + if filename is None: + # so null filename now, and it'll be assigned in self.testing + filename = m.download_url(link, self.dl_location, progress_hook=self.testing) + else: + filename = m.download_url(link, self.dl_location, filename, self.testing) + except Exception as e: + logger.warn('[MEGA][ERROR] %s' % e) + if 'EBLOCKED' in str(e): + logger.warn('Content has been removed - we should move on to the next one at this point.') + return {"success": False, "filename": filename, "path": None, "link_type_failure": site} + else: + og_filepath = os.path.join(self.dl_location, filename) # just default + if filename is not None: + og_filepath = filename + #filepath = filename.parent.absolute() + file, ext = os.path.splitext(os.path.basename( filename ) ) + filename = '%s[__%s__]%s' % (file, issueid, ext) + filepath = og_filepath.with_name(filename) + try: + filepath = og_filepath.replace(filepath) + except Exception as e: + logger.warn('unable to rename/replace %s with %s' % (og_filepath, filepath)) + else: + logger.info('ddl_linked - filename: %s' % filename) + if ext == '.zip': + ggc = mylar.getcomics.GC() + return ggc.zip_zip(id, str(filepath), filename) + else: + return {"success": True, "filename": filename, "path": str(filepath)} + else: + logger.warn('filename returned from download has a None value') + return {"success": False, "filename": filename, "path": None, "link_type_failure": site} + + def testing(self, data): + mth = ( data['current'] / data['total'] ) * 100 + + if data['tmp_filename'] is not None and self.wrote_tmp is False: + myDB = db.DBConnection() + # write the filename to the db for tracking purposes... + logger.info('writing to db: %s [%s][%s]' % (data['name'], data['total'], data['tmp_filename'])) + myDB.upsert( + 'ddl_info', + {'tmp_filename': str(data['tmp_filename'])}, # tmp_filename should be all that's needed to be updated at this point... + #{'filename': str(data['name']), 'tmp_filename': str(data['tmp_filename']), 'remote_filesize': str(data['total'])}, + {'id': self.id}, + ) + self.wrote_tmp = True + + #logger.info('%s%s' % (mth, '%')) + #logger.info('data: %s' % (data,)) + + if mth >= 100.0: + logger.info('status: %s' % (data['status'])) + logger.info('successfully downloaded %s [%s bytes]' % (data['name'], data['total'])) + +if __name__ == '__main__': + test = MegaNZ(sys.argv[1]) + test.ddl_search() + diff --git a/mylar/downloaders/pixeldrain.py b/mylar/downloaders/pixeldrain.py new file mode 100644 index 00000000..5b11015e --- /dev/null +++ b/mylar/downloaders/pixeldrain.py @@ -0,0 +1,145 @@ +# This file is part of Mylar. +# +# Mylar is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Mylar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Mylar. If not, see . + +import requests +import sys +import os +import time +from operator import itemgetter +from pathlib import Path +import urllib + +import mylar +from mylar import db, helpers, logger, search, search_filer + +class PixelDrain(object): + + def __init__(self): + self.dl_location = mylar.CONFIG.DDL_LOCATION + self.headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1', + 'Referer': 'https://pixeldrain.com', + } + self.session = requests.Session() + + def ddl_download(self, link, id, issueid): + self.id = id + self.url = link + if self.dl_location is not None and not os.path.isdir( + self.dl_location + ): + checkdirectory = mylar.filechecker.validateAndCreateDirectory( + self.dl_location, True + ) + if not checkdirectory: + logger.warn( + '[PixelDrain][ABORTING] Error trying to validate/create DDL download' + ' directory: %s.' % self.dl_location + ) + return {"success": False, "filename": filename, "path": None, "link_type_failure": 'GC-Pixel'} + + + t = self.session.get( + self.url, + verify=True, + headers=self.headers, + stream=True, + timeout=(30,30) + ) + + file_id = os.path.basename( + urllib.parse.unquote(t.url) + ) # .decode('utf-8')) + logger.fdebug(t.url) + logger.fdebug(t) + logger.fdebug('[PixelDrain] file_id: %s' % file_id) + + logger.fdebug('[PixelDrain] retrieving info for file_id: %s' % file_id) + f_info = self.session.get(f"https://pixeldrain.com/api/file/{file_id}/info", verify=True, headers=self.headers,stream=True) + if f_info.status_code == 200: + info = f_info.json() + logger.fdebug('[PixelDrain] pixeldrain_info_response: %s' % info) + file_info = {'filename': info['name'], + 'filesize': info['size'], + 'avail': info['availability'], + 'can_dl': info['can_download']} + else: + # should return null here - unobtainable link. + file_info = None + + myDB = db.DBConnection() + # write the filename to the db for tracking purposes... + logger.info('[PixelDrain] Writing to db: %s [%s]' % (file_info['filename'], file_info['filesize'])) + myDB.upsert( + 'ddl_info', + {'filename': str(file_info['filename']), 'remote_filesize': str(file_info['filesize']), 'size': helpers.human_size(file_info['filesize'])}, + {'id': self.id}, + ) + logger.fdebug(file_info) + + logger.fdebug('[PixelDrain] now threading the send') + return self.pixel_ddl(file_id, file_info, issueid) + + def pixel_ddl(self, file_id, fileinfo, issueid): + filename = fileinfo['filename'] + file, ext = os.path.splitext(os.path.basename( filename ) ) + filename = '%s[__%s__]%s' % (file, issueid, ext) + + filesize = fileinfo['filesize'] + filepath = os.path.join(self.dl_location, filename) + + myDB = db.DBConnection() + myDB.upsert( + 'ddl_info', + {'tmp_filename': filename}, # tmp_filename should be all that's needed to be updated at this point... + {'id': self.id}, + ) + + try: + response = self.session.get( + 'https://pixeldrain.com/api/file/'+file_id, + verify=True, + headers=self.headers, + stream=True, + timeout=(30,30) + ) + + logger.fdebug('[PixelDrain] now writing....') + with open(filepath, 'wb') as f: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + f.flush() + + except Exception as e: + logger.warn('[PixelDrain][ERROR] %s' % e) + if 'EBLOCKED' in str(e): + logger.warn('[PixelDrain] Content has been removed - we should move on to the next one at this point.') + return {"success": False, "filename": filename, "path": None, "link_type_failure": 'GC-Pixel'} + + logger.fdebug('[PixelDrain] download completed - donwloaded %s / %s' % (os.stat(filepath).st_size, filesize)) + + logger.info('[PixelDrain] ddl_linked - filename: %s' % filename) + + if ext == '.zip': + ggc = mylar.getcomics.GC() + return ggc.zip_zip(self.id, str(filepath), filename) + else: + return {"success": True, "filename": filename, "path": str(filepath)} + +if __name__ == '__main__': + test = PixelDrain() + test.ddl_down() + diff --git a/mylar/filechecker.py b/mylar/filechecker.py index 5711f320..d57ccf58 100755 --- a/mylar/filechecker.py +++ b/mylar/filechecker.py @@ -258,6 +258,7 @@ def parseit(self, path, filename, subpath=None): #parse out the extension for type comic_ext = ('.cbr','.cbz','.cb7','.pdf') + comic_ext = tuple(x for x in comic_ext if x not in mylar.CONFIG.IGNORE_SEARCH_WORDS) if os.path.splitext(filename)[1].endswith(comic_ext): filetype = os.path.splitext(filename)[1] else: @@ -729,7 +730,16 @@ def parseit(self, path, filename, subpath=None): volumeprior_label = sf sep_volume = True logger.fdebug('volume label detected, but vol. number is not adjacent, adjusting scope to include number.') - elif 'volume' in sf.lower() or all(['part' in sf.lower(), len(sf) == 4]): + elif 'volume' in sf.lower(): + volume = re.sub("[^0-9]", "", sf) + if volume.isdigit(): + volume_found['volume'] = volume + volume_found['position'] = split_file.index(sf) + else: + volumeprior = True + volumeprior_label = sf + sep_volume = True + elif all(['part' in sf.lower(), len(sf) == 4]): if self.watchcomic is not None and 'part' not in self.watchcomic.lower(): volume = re.sub("[^0-9]", "", sf) if volume.isdigit(): @@ -896,6 +906,7 @@ def parseit(self, path, filename, subpath=None): logger.fdebug('yearposition[%s] -- dc[position][%s]' % (yearposition, x['yearposition'])) if yearposition < x['yearposition']: if all([len(possible_issuenumbers) == 1, possible_issuenumbers[0]['number'] == x['year'], x['yearposition'] != possible_issuenumbers[0]['position']]): + logger.fdebug('issue2year is true') issue2year = True highest_series_pos = x['yearposition'] yearposition = x['yearposition'] @@ -1113,6 +1124,7 @@ def parseit(self, path, filename, subpath=None): splitvalue = None alt_series = None alt_issue = None + onefortheleader = False try: if yearposition is not None: try: @@ -1127,15 +1139,28 @@ def parseit(self, path, filename, subpath=None): except: pass else: - if tmpval > 2: - logger.fdebug('There are %s extra words between the issue # and the year position. Deciphering if issue title or part of series title.' % tmpval) - tmpval1 = ' '.join(split_file[issue_number_position:yearposition]) - if split_file[issue_number_position+1] == '-': - usevalue = ' '.join(split_file[issue_number_position+2:yearposition]) - splitv = split_file[issue_number_position+2:yearposition] + if tmpval >= 2: + #logger.fdebug('There are %s extra words between the issue # and the year position. Deciphering if issue title or part of series title.' % tmpval) + #logger.fdebug('split_file[issue_number_position]: [%s] --> %s' % (issue_number_position, split_file[issue_number_position])) + #logger.fdebug('split_file[yearposition]: [%s] --> %s' % (yearposition, split_file[yearposition])) + #2024-01-07 - new for one-shot where no issue number in filename, but year is present in title + if split_file[issue_number_position] == re.sub(r'[\)\(]', '', split_file[yearposition]).strip(): + logger.fdebug('issue number is the same as year - assuming issue number is actually part of the title and this is a one-shot-type of book') + onefortheleader = True + if [True for x in split_file if x.lower() == 'annual']: + booktype = 'issue' + else: + booktype = 'TPB/GN/HC/One-Shot' + issue_number = None else: - splitv = split_file[issue_number_position:yearposition] - splitvalue = ' '.join(splitv) + tmpval1 = ' '.join(split_file[issue_number_position:yearposition]) + if split_file[issue_number_position+1] == '-': + usevalue = ' '.join(split_file[issue_number_position+2:yearposition]) + splitv = split_file[issue_number_position+2:yearposition] + else: + splitv = split_file[issue_number_position:yearposition] + splitvalue = ' '.join(splitv) + #end 2024-01-07 else: #store alternate naming of title just in case if '-' not in split_file[0]: @@ -1173,7 +1198,7 @@ def parseit(self, path, filename, subpath=None): #logger.info('volume_found: ' + str(volume_found)) #2017-10-21 - if highest_series_pos > issue_number_position: + if highest_series_pos > issue_number_position and not onefortheleader: highest_series_pos = issue_number_position #if volume_found['position'] >= issue_number_position: # highest_series_pos = issue_number_position @@ -1279,7 +1304,7 @@ def parseit(self, path, filename, subpath=None): series_name_decoded = re.sub('special', '', series_name_decoded, flags=re.I).strip() if (any([issue_number is None, series_name is None]) and booktype == 'issue'): - + bythepass = False if all([issue_number is None, booktype == 'issue', issue_volume is not None]): if ignore_mod_position != -1: logger.fdebug('Possible Annual detected - no identifying issue number present, no clarification in filename - assuming year (%s) as issue number' % issue_year) @@ -1288,30 +1313,39 @@ def parseit(self, path, filename, subpath=None): logger.fdebug('Possible UNKNOWN TPB/GN/HC detected - no issue number present, no clarification in filename, but volume present with series title') booktype = 'TPB/GN/HC/One-Shot' else: - logger.fdebug('Cannot parse the filename properly. I\'m going to make note of this filename so that my evil ruler can make it work.') - - if series_name is not None: - dreplace = self.dynamic_replace(series_name)['mod_seriesname'] + if all([issue_number is None, issue_volume is None, 'annual' in series_name.lower(), booktype == 'issue']): + if ignore_mod_position != -1: + logger.fdebug('Possible Annual detected - no identifying issue number present, no clarification in filename - assuming year (%s) as issue number' % issue_year) + issue_number = issue_year + bythepass = True + else: + logger.fdebug('Cannot parse the filename properly. I\'m going to make note of this filename so that my evil ruler can make it work.') else: - dreplace = None - return {'parse_status': 'failure', - 'sub': path_list, - 'comicfilename': filename, - 'comiclocation': self.dir, - 'series_name': series_name, - 'series_name_decoded': series_name_decoded, - 'issueid': issueid, - 'alt_series': alt_series, - 'alt_issue': alt_issue, - 'dynamic_name': dreplace, - 'issue_number': issue_number, - 'justthedigits': issue_number, #redundant but it's needed atm - 'series_volume': issue_volume, - 'issue_year': issue_year, - 'annual_comicid': None, - 'scangroup': scangroup, - 'booktype': booktype, - 'reading_order': None} + logger.fdebug('Cannot parse the filename properly. I\'m going to make note of this filename so that my evil ruler can make it work.') + + if not bythepass: + if series_name is not None: + dreplace = self.dynamic_replace(series_name)['mod_seriesname'] + else: + dreplace = None + return {'parse_status': 'failure', + 'sub': path_list, + 'comicfilename': filename, + 'comiclocation': self.dir, + 'series_name': series_name, + 'series_name_decoded': series_name_decoded, + 'issueid': issueid, + 'alt_series': alt_series, + 'alt_issue': alt_issue, + 'dynamic_name': dreplace, + 'issue_number': issue_number, + 'justthedigits': issue_number, #redundant but it's needed atm + 'series_volume': issue_volume, + 'issue_year': issue_year, + 'annual_comicid': None, + 'scangroup': scangroup, + 'booktype': booktype, + 'reading_order': None} if self.justparse: return {'parse_status': 'success', @@ -1586,6 +1620,7 @@ def char_file_position(self, file, findchar, lastpos): def traverse_directories(self, dir): filelist = [] comic_ext = ('.cbr','.cbz','.cb7','.pdf') + comic_ext = tuple(x for x in comic_ext if x not in mylar.CONFIG.IGNORE_SEARCH_WORDS) if all([mylar.CONFIG.ENABLE_TORRENTS is True, self.pp_mode is True]): from mylar import db diff --git a/mylar/getcomics.py b/mylar/getcomics.py index d9420484..de1deca4 100644 --- a/mylar/getcomics.py +++ b/mylar/getcomics.py @@ -138,7 +138,7 @@ def __init__(self, query=None, issueid=None, comicid=None, oneoff=False, session if mylar.CONFIG.ENABLE_PROXY: self.session.proxies.update({ 'http': mylar.CONFIG.HTTP_PROXY, - 'https': mylar.CONFIG.HTTPS_PROXY + 'https': mylar.CONFIG.HTTPS_PROXY }) self.session_path = session_path if session_path is not None else os.path.join(mylar.CONFIG.SECURE_DIR, ".gc_cookies.dat") @@ -155,6 +155,8 @@ def __init__(self, query=None, issueid=None, comicid=None, oneoff=False, session self.search_format = ['"%s #%s (%s)"', '%s #%s (%s)', '%s #%s', '%s %s'] + self.pack_receipts = ['+ TPBs', '+TPBs', '+ TPB', '+TPB', 'TPB', '+ Deluxe Books', '+ Annuals', '+Annuals', ' & '] + self.provider_stat = provider_stat def search(self,is_info=None): @@ -313,7 +315,7 @@ def loadsite(self, id, link): verify=True, headers=self.headers, stream=True, - timeout=(30,10) + timeout=(30,30) ) with open(title + '.html', 'wb') as f: @@ -339,7 +341,7 @@ def perform_search_queries(self, queryline): params={'s': queryline}, verify=True, headers=self.headers, - timeout=(30,10) + timeout=(30,30) ).text write_time = time.time() @@ -347,14 +349,23 @@ def perform_search_queries(self, queryline): self.provider_stat['lastrun'] = write_time page_results, next_url = self.parse_search_result(page_html) + #logger.fdebug('page_results: %s' % page_results) + possible_choices = [] for result in page_results: if 'Weekly' not in self.query.get('comicname', "") and 'Weekly' in result.get('title', ""): continue if result["link"] in seen_urls: continue seen_urls.add(result["link"]) + #if not mylar.CONFIG.PACK_PRIORITY: + # possible_choices.append(result) + #else: yield result + #if not mylar.CONFIG.PACK_PRIORITY and possible_choices: + # logger.info('[pack-last choices] possible choices to check before page-loading next page..: %s' % possible_choices) + # yield possible_choices + def parse_search_result(self, page_html): resultlist = [] soup = BeautifulSoup(page_html, 'html.parser') @@ -380,72 +391,16 @@ def parse_search_result(self, page_html): titlefind = f.find("h1", {"class": "post-title"}) title = titlefind.get_text(strip=True) title = re.sub('\u2013', '-', title).strip() - filename = title - issues = None - pack = False - # see if it's a pack type - issfind_st = title.find('#') - issfind_en = title.find('-', issfind_st) - if issfind_en != -1: - if all([title[issfind_en + 1] == ' ', title[issfind_en + 2].isdigit()]): - iss_en = title.find(' ', issfind_en + 2) - if iss_en != -1: - issues = title[issfind_st + 1 : iss_en] - pack = True - if title[issfind_en + 1].isdigit(): - iss_en = title.find(' ', issfind_en + 1) - if iss_en != -1: - issues = title[issfind_st + 1 : iss_en] - pack = True - - # to handle packs that are denoted without a # sign being present. - # if there's a dash, check to see if both sides of the dash are numeric. - if pack is False and title.find('-') != -1: - issfind_en = title.find('-') - if all( - [ - title[issfind_en + 1] == ' ', - title[issfind_en + 2].isdigit(), - ] - ) and all( - [ - title[issfind_en -1] == ' ', - ] - ): - spaces = [m.start() for m in re.finditer(' ', title)] - dashfind = title.find('-') - space_beforedash = title.find(' ', dashfind - 1) - space_afterdash = title.find(' ', dashfind + 1) - if not title[space_afterdash+1].isdigit(): - pass - else: - iss_end = title.find(' ', space_afterdash + 1) - if iss_end == -1: - iss_end = len(title) - set_sp = None - for sp in spaces: - if sp < space_beforedash: - prior_sp = sp - else: - set_sp = prior_sp - break - if title[set_sp:space_beforedash].strip().isdigit(): - issues = title[set_sp:iss_end].strip() - pack = True - - # if it's a pack - remove the issue-range and the possible issue years - # (cause it most likely will span) and pass thru as separate items - if pack is True: - f_iss = title.find('#') - if f_iss != -1: - title = '%s%s'.strip() % (title[:f_iss-1], title[f_iss+1:]) - title = re.sub(issues, '', title).strip() - # kill any brackets in the issue line here. - issues = re.sub(r'[\(\)\[\]]', '', issues).strip() - if title.endswith('#'): - title = title[:-1].strip() - title += '#1' # we add this dummy value back in so the parser won't choke as we have the issue range stored already + pack_checker = self.check_for_pack(title, issue_in_pack=self.query['issue']) + if pack_checker: + issues = pack_checker['issues'] + gc_booktype = pack_checker['gc_booktype'] + pack = pack_checker['pack'] + series = pack_checker['series'] + title = pack_checker['title'] + filename = pack_checker['filename'] + year = pack_checker['year'] else: if any( [ @@ -456,6 +411,10 @@ def parse_search_result(self, page_html): ] ): continue + else: + pack = False + issues = None + filename = series = title = None needle_style = "text-align: center;" option_find = f.find("p", {"style": needle_style}) @@ -518,6 +477,8 @@ def parse_search_result(self, page_html): "filename": filename, "size": re.sub(' ', '', size).strip(), "pack": pack, + "series": series, + "gc_booktype": gc_booktype, "issues": issues, "link": link, "year": year, @@ -525,8 +486,13 @@ def parse_search_result(self, page_html): "site": 'DDL(GetComics)', } ) - - logger.fdebug('%s [%s]' % (title, size)) + if pack: + pck = 'yes' + fname = filename + else: + pck = 'no' + fname = title + logger.fdebug('%s [%s] [PACK: %s]' % (fname, size, pck)) older_posts_a = soup.find("a", class_="pagination-older") next_page = None @@ -535,24 +501,33 @@ def parse_search_result(self, page_html): return resultlist, next_page - def parse_downloadresults(self, id, mainlink, comicinfo=None): + def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, link_type_failure=None): try: booktype = comicinfo[0]['booktype'] except Exception: booktype = None - try: - pack = comicinfo[0]['pack'] - except Exception: - pack = False + pack = pack_numbers = pack_issuelist = None + logger.fdebug('packinfo: %s' % (packinfo,)) - myDB = db.DBConnection() + if packinfo is not None: + pack = packinfo['pack'] + pack_numbers = packinfo['pack_numbers'] + pack_issuelist = packinfo['pack_issuelist'] + else: + try: + pack = comicinfo[0]['pack'] + except Exception as e: + pack = False + myDB = db.DBConnection() series = None year = None size = None - title = os.path.join(mylar.CONFIG.CACHE_DIR, 'getcomics-' + id) + if not os.path.exists(title): + logger.fdebug('Unable to locate local cached html file - attempting to retrieve page results again..') + self.loadsite(id, mainlink) soup = BeautifulSoup(open(title + '.html', encoding='utf-8'), 'html.parser') i = 0 @@ -564,6 +539,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): looped_thru_once = True beeswax = soup.findAll("p", {"style": "text-align: center;"}) + logger.info('[DDL-GATHERER-OF-LINKAGE] Now compiling release information & available links...') while True: #logger.fdebug('count_bees: %s' % count_bees) f = beeswax[count_bees] @@ -573,9 +549,9 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): if not linkage: linkage_test = f.text.strip() if 'support and donation' in linkage_test: - logger.fdebug('detected end of links - breaking out here...') if looped_thru_once is False: valid_links[multiple_links].update({'links': gather_links}) + #logger.fdebug('detected end of links - breaking out here...') break if looped_thru_once and all( @@ -585,7 +561,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): 'Size' in linkage_test, ] ): - logger.fdebug('detected headers of title - ignoring this portion...') + #logger.fdebug('detected headers of title - ignoring this portion...') while True: prev_option = option_find option_find = option_find.findNext(text=True) @@ -648,17 +624,25 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): i = 0 series = None else: - logger.info('[DDL-GATHERER-OF-LINKAGE] Now compiling available links...') - gather_links.append({ - "series": series, - "site": lk['title'], - "year": year, - "issues": None, - "size": size, - "links": lk['href'], - "pack": comicinfo[0]['pack'] - }) - #logger.fdebug('gather_links so far: %s' % gather_links) + t_site = re.sub('link', '', lk['title'].lower()).strip() + ltf = False + if link_type_failure is not None: + logger.fdebug('link_type_failure: %s' % link_type_failure) + if [True for tst in link_type_failure if t_site.lower() in tst.lower()]: + logger.fdebug('[REDO-FAILURE-DETECTION] detected previous invalid link for %s - ignoring this result' + ' and seeing if anything else can be downloaded.' % t_site) + ltf = True + if not ltf: + gather_links.append({ + "series": series, + "site": t_site, + "year": year, + "issues": None, + "size": size, + "links": lk['href'], + "pack": comicinfo[0]['pack'] + }) + #logger.fdebug('gather_links so far: %s' % gather_links) count_bees +=1 #logger.fdebug('final valid_links: %s' % (valid_links)) @@ -672,59 +656,211 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): for a in y['links']: if k != 'normal': # if it's HD-Upscaled / SD-Digital it needs to be handled differently than a straight DL link - if any([a['site'].lower() == 'download now', a['site'].lower() == 'mirror download']): # 'MEGA' in a['site'], 'PIXEL' in a['site']]): - tmp_links.append(a) - tmp_sites.append(k) - site_position[k] = cntr - #logger.fdebug('%s -- %s' % (k, a['series'])) + if any([a['site'].lower() == 'download now', a['site'].lower() == 'mirror download']): + d_site = '%s:%s' % (k, a['site'].lower()) + tmp_a = a + tmp_a['site_type'] = d_site + tmp_links.append(tmp_a) + tmp_sites.append(d_site) + site_position[d_site] = cntr + logger.fdebug('%s -- %s' % (d_site, a['series'])) + cntr +=1 + elif any(['mega' in a['site'].lower(), 'pixel' in a['site'].lower(), 'mediafire' in a['site'].lower()]): + if 'mega' in a['site'].lower(): + d_site = '%s:%s' % (k, 'mega') + elif 'pixel' in a['site'].lower(): + d_site = '%s:%s' % (k, 'pixeldrain') + else: + d_site = '%s:%s' % (k, 'mediafire') + tmp_a = a + tmp_a['site_type'] = d_site + tmp_links.append(tmp_a) + tmp_sites.append(d_site) + site_position[d_site] = cntr + logger.fdebug('%s -- %s' % (d_site, a['series'])) cntr +=1 else: - if any([a['site'].lower() == 'download now', a['site'].lower() == 'mirror download']): - tmp_links.append(a) - tmp_sites.append(a['site'].lower()) - site_position[a['site'].lower()] = cntr - #logger.fdebug('%s -- %s' % (a['site'], a['series'])) + if any( + [ + a['site'].lower() == 'download now', + a['site'].lower() == 'mirror download', + 'mega' in a['site'].lower(), + 'pixel' in a['site'].lower(), + 'mediafire' in a['site'].lower() + ] + ): + t_site = a['site'].lower() + if 'mega' in a['site'].lower(): + t_site = 'mega' + elif 'pixel' in a['site'].lower(): + t_site = 'pixeldrain' + elif 'mediafire' in a['site'].lower(): + t_site = 'mediafire' + d_site = '%s:%s' % (k, t_site) + tmp_a = a + tmp_a['site_type'] = d_site + tmp_links.append(tmp_a) + tmp_sites.append(d_site) + site_position[d_site] = cntr + logger.fdebug('%s -- %s' % (d_site, a['series'])) cntr +=1 + #logger.fdebug('tmp_links: %s' % (tmp_links)) - #logger.fdebug('tmp_sites: %s' % (tmp_sites)) + logger.fdebug('tmp_sites: %s' % (tmp_sites)) + logger.fdebug('site_position: %s' % (site_position)) + link_types = ('HD-Upscaled', 'SD-Digital', 'HD-Digital') + link_matched = False if len(tmp_links) == 1: - logger.info('only one available item that can be downloaded via main server - %s. Let\'s do this..' % tmp_links[0]['series']) link = tmp_links[0] series = link['series'] + logger.info('only one available item that can be downloaded via %s - %s. Let\'s do this..' % (link['site'], series)) + link_matched = True elif len(tmp_links) > 1: logger.info('Multiple available download options (%s) - checking configuration to see which to grab...' % (" ,".join(tmp_sites))) - if any([('HD-Upscaled', 'SD-Digital', 'HD-Digital') in tmp_sites]): - if mylar.CONFIG.DDL_PREFER_UPSCALED: - kk = tmp_links[site_position['HD-Upscaled']] - if not kk: - kk = tmp_links[site_position['HD-Digital']] - logger.info('HD-Digital preference detected...attempting %s' % kk['series']) + site_check = [y for x in link_types for y in tmp_sites if x in y] + for ddlp in mylar.CONFIG.DDL_PRIORITY_ORDER: + force_title = False + site_lp = ddlp + logger.fdebug('priority ddl enabled - checking %s' % site_lp) + if site_check: #any([('HD-Upscaled', 'SD-Digital', 'HD-Digital') in tmp_sites]): + if mylar.CONFIG.DDL_PREFER_UPSCALED: + if site_lp == 'mega': + sub_site_chk = [y for y in tmp_sites if 'mega' in y] + if sub_site_chk: + if any('HD-Upscaled' in ssc for ssc in sub_site_chk): + kk = tmp_links[site_position['HD-Upscaled:mega']] + logger.info('[MEGA] HD-Upscaled preference detected...attempting %s' % kk['series']) + link_matched = True + elif any('HD-Digital' in ssc for ssc in sub_site_chk): + kk = tmp_links[site_position['HD-Digital:mega']] + logger.info('[MEGA] HD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + + elif not link_matched and site_lp == 'pixeldrain': + sub_site_chk = [y for y in tmp_sites if 'pixel' in y] + if sub_site_chk: + if any('HD-Upscaled' in ssc for ssc in sub_site_chk): + kk = tmp_links[site_position['HD-Upscaled:pixeldrain']] + logger.info('[PixelDrain] HD-Upscaled preference detected...attempting %s' % kk['series']) + link_matched = True + elif any('HD-Digital' in ssc for ssc in sub_site_chk): + kk = tmp_links[site_position['HD-Digital:pixeldrain']] + logger.info('[PixelDrain] HD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + + elif not link_matched and site_lp == 'mediafire': + sub_site_chk = [y for y in tmp_sites if 'mediafire' in y] + if sub_site_chk: + if any('HD-Upscaled' in ssc for ssc in sub_site_chk): + kk = tmp_links[site_position['HD-Upscaled:mediafire']] + logger.info('[mediafire] HD-Upscaled preference detected...attempting %s' % kk['series']) + link_matched = True + elif any('HD-Digital' in ssc for ssc in sub_site_chk): + kk = tmp_links[site_position['HD-Digital:mediafire']] + logger.info('[mediafire] HD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + + elif not link_matched and site_lp == 'main': + sub_site_chk = [y for y in tmp_sites if 'download now' in y] + if any('HD-Upscaled' in ssc for ssc in sub_site_chk): + kk = tmp_links[site_position['HD-Upscaled:download now']] + logger.info('[MAIN-SERVER] HD-Upscaled preference detected...attempting %s' % kk['series']) + link_matched = True + elif any('HD-Digital' in ssc for ssc in sub_site_chk): + kk = tmp_links[site_position['HD-Digital:download now']] + logger.info('[MAIN-SERVER] HD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True else: - logger.info('HD-Upscaled preference detected...attempting %s' % kk['series']) + if not link_matched and site_lp == 'mega': + sub_site_chk = [y for y in tmp_sites if 'mega' in y] + if sub_site_chk: + kk = tmp_links[site_position['SD-Digtal:mega']] + logger.info('[MEGA] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + + elif not link_matched and site_lp == 'pixeldrain': + sub_site_chk = [y for y in tmp_sites if 'pixel' in y] + if sub_site_chk: + kk = tmp_links[site_position['SD-Digtal:pixeldrain']] + logger.info('[PixelDrain] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + + elif not link_matched and site_lp == 'mediafire': + sub_site_chk = [y for y in tmp_sites if 'mediafire' in y] + if sub_site_chk: + kk = tmp_links[site_position['SD-Digtal:mediafire']] + logger.info('[mediafire] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + + elif not link_matched and site_lp == 'main': + try: + kk = tmp_links[site_position['SD-Digital:download now']] + logger.info('[MAIN-SERVER] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + except Exception as e: + kk = tmp_links[site_position['SD-Digital:mirror download']] + logger.info('[MIRROR-SERVER] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + + link = kk + series = link['series'] + #logger.fdebug('link: %s' % link) else: - kk = tmp_links[site_position['SD-Digital']] - logger.info('SD-Digital preference detected...attempting %s' % kk['series']) - link = kk - series = link['series'] - #logger.fdebug('link: %s' % link) - else: - if 'download now' in tmp_sites: - link = tmp_links[site_position['download now']] - elif 'mirror download' in tmp_sites: - link = tmp_links[site_position['mirror download']] - else: - link = tmp_links[0] - series = link['series'] + if not link_matched and site_lp == 'mega': + sub_site_chk = [y for y in tmp_sites if 'mega' in y] + if sub_site_chk: + try: + link = tmp_links[site_position['normal:mega']] + link_matched = True + except Exception as e: + link = tmp_links[site_position['normal:mega link']] + link_matched = True + elif not link_matched and site_lp == 'pixeldrain': + sub_site_chk = [y for y in tmp_sites if 'pixel' in y] + if sub_site_chk: + try: + link = tmp_links[site_position['normal:pixeldrain']] + link_matched = True + except Exception as e: + logger.info('[PIXELDRAIN] Unable to attain proper link...') + link_matched = False + elif not link_matched and site_lp == 'mediafire': + sub_site_chk = [y for y in tmp_sites if 'mediafire' in y] + if sub_site_chk: + try: + link = tmp_links[site_position['normal:mediafire']] + link_matched = True + except Exception as e: + logger.info('[mediafire] Unable to attain proper link...') + link_matched = False + elif not link_matched and site_lp == 'main': + if 'download now' in tmp_sites: + link = tmp_links[site_position['normal:download now']] + elif 'mirror download' in tmp_sites: + link = tmp_links[site_position['normal:mirror download']] + else: + link = tmp_links[0] + force_title = True + if 'sh.st' in link: + logger.fdebug('[Paywall-link detected] this is not a valid link') + else: + if force_title: + series = link['series'] + link_matched = True + + dl_selection = link['site'] else: logger.info('No valid items available that I am able to download from. Not downloading...') - return {'success': False} + return {'success': False, 'links_exhausted': link_type_failure} logger.fdebug( - 'Now downloading: %s [%s] / %s ... this can take a while' - ' (go get some take-out)...' % (series, year, size) + '[%s] Now downloading: %s [%s] / %s ... this can take a while' + ' (go get some take-out)...' % (dl_selection, series, year, size) ) + tmp_filename = '%s (%s)' % (series, year) + links = [] if link is None and possible_more.name == 'ul': @@ -739,12 +875,17 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): tmp = linkline['href'] except Exception: continue + + site = linkline.findNext(text=True) + #logger.fdebug('servertype: %s' % (site)) + if tmp: if any( [ 'run.php' in linkline['href'], 'go.php' in linkline['href'], 'comicfiles.ru' in linkline['href'], + ('links.php' in linkline['href'] and site == 'Main Server'), ] ): volume = x.findNext(text=True) @@ -759,6 +900,13 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): else: series = volume[:issues_st].strip() issues = volume[issues_st + 1 : series_st].strip() + ver_check = self.pack_check(issues, packinfo) + if ver_check is False: + #logger.fdebug('ver_check is False - ignoring') + continue + + if issues is None and any([booktype == 'Print', booktype is None, booktype == 'Digital']): + continue year_end = volume.find(')', series_st + 1) year = re.sub( r'[\(\)]', '', volume[series_st + 1 : year_end] @@ -768,7 +916,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): r'[\(\)]', '', volume[year_end + 1 : size_end] ).strip() linked = linkline['href'] - site = linkline.findNext(text=True) + #site = linkline.findNext(text=True) if site == 'Main Server': links.append( { @@ -783,7 +931,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): ) else: if booktype != 'TPB' and pack is False: - logger.fdebug('TPB links detected, but booktype set to %s' % booktype) + logger.fdebug('Extra links detected, possibly different booktypes - but booktype set to %s' % booktype) else: check_extras = soup.findAll("h3") for sb in check_extras: @@ -829,7 +977,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): 'Unable to retrieve any valid immediate download links.' ' They might not exist.' ) - return {'success': False} + return {'success': False, 'links_exhausted': link_type_failure} if all([link is not None, len(links) == 0]): logger.info( 'Only one item discovered, changing queue length to accomodate: %s [%s]' @@ -854,10 +1002,28 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): mod_id = id else: mod_id = id + '-' + str(cnt) - # logger.fdebug('[%s] %s (%s) %s [%s][%s]' - # % (x['site'], x['series'], x['year'], x['issues'], - # x['size'], x['link']) - # ) + + lt_site = x['site'].lower() + if any([lt_site == 'main server', lt_site == 'download now']): + link_type = 'GC-Main' + elif lt_site == 'mirror download': + link_type = 'GC-Mirror' + elif lt_site == 'mega': + link_type = 'GC-Mega' + elif lt_site == 'mediafire': + link_type = 'GC-Media' + elif lt_site == 'pixeldrain': + link_type = 'GC-Pixel' + else: + logger.warn('[GC-Site-Unknown] Unknown site detected...%s' % lt_site) + link_type = 'Unknown' + + if self.issueid is None: + self.issueid = comicinfo[0]['IssueID'] + if self.comicid is None: + self.comicid = comicinfo[0]['ComicID'] + if self.oneoff is None: + self.oneoff = comicinfo[0]['oneoff'] ctrlval = {'id': mod_id} vals = { @@ -871,11 +1037,18 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): 'mainlink': mainlink, 'site': 'DDL(GetComics)', 'pack': x['pack'], + 'link_type': link_type, 'updated_date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), 'status': 'Queued', } myDB.upsert('ddl_info', vals, ctrlval) + #tmp_filename = None + #if any([link_type == 'Mega', link_type == 'Mega Link']): + # this is needed so that we assign some tmp filename + # (it will get renamed upon completion anyways) + #tmp_filename = comicinfo[0]['nzbtitle'] + mylar.DDL_QUEUE.put( { 'link': x['links'], @@ -887,6 +1060,10 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): 'issueid': self.issueid, 'oneoff': self.oneoff, 'id': mod_id, + 'link_type': link_type, + 'filename': tmp_filename, + 'comicinfo': comicinfo, + 'packinfo': packinfo, 'site': 'DDL(GetComics)', 'remote_filesize': 0, 'resume': None, @@ -894,10 +1071,16 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None): ) cnt += 1 - return {'success': True} + return {'success': True, 'site': link_type} + + def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_filesize=0, link_type=None): + #logger.fdebug('[%s] %s -- mainlink: %s' % (id, link, mainlink)) + if 'sh.st' in link: + logger.fdebug('[Paywall-link detected] This is not a valid link, this should be requeued to search to gather all available links') + return { + "success": False, + "link_type": link_type} - def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_filesize=0): - # logger.info('[%s] %s -- mainlink: %s' % (id, link, mainlink)) if mylar.DDL_LOCK is True: logger.fdebug( '[DDL] Another item is currently downloading via DDL. Only one item can' @@ -908,9 +1091,9 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files mylar.DDL_LOCK = True myDB = db.DBConnection() + mylar.DDL_QUEUED.append(id) filename = None self.cookie_receipt() - #self.headers['Accept-encoding'] = 'gzip' try: with requests.Session() as s: if resume is not None: @@ -919,7 +1102,6 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files ) self.headers['Range'] = 'bytes=%d-' % resume - #logger.fdebug('session cookies: %s' % (self.session.cookies,)) t = self.session.get( link, verify=True, @@ -930,7 +1112,7 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files filename = os.path.basename( urllib.parse.unquote(t.url) - ) # .decode('utf-8')) + ) if 'GetComics.INFO' in filename: filename = re.sub('GetComics.INFO', '', filename, re.I).strip() @@ -948,18 +1130,16 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files if 'run.php-urls' not in link: link = re.sub('run.php-url=', 'run.php-urls', link) link = re.sub('go.php-url=', 'run.php-urls', link) - #logger.fdebug('session cookies: %s' % (self.session.cookies,)) t = self.session.get( link, verify=True, headers=self.headers, stream=True, - timeout=(30,10) + timeout=(30,30) ) - t.headers['Accept-encoding'] = 'gzip' filename = os.path.basename( urllib.parse.unquote(t.url) - ) # .decode('utf-8')) + ) if 'GetComics.INFO' in filename: filename = re.sub( 'GetComics.INFO', '', filename, re.I @@ -983,6 +1163,7 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files "success": False, "filename": filename, "path": None, + "link_type": link_type, } else: @@ -997,7 +1178,11 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files ) remote_filesize = 0 mylar.DDL_LOCK = False - return {"success": False, "filename": filename, "path": None} + return { + "success": False, + "filename": filename, + "path": None, + "link_type": link_type} # write the filename to the db for tracking purposes... myDB.upsert( @@ -1017,14 +1202,15 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files '[ABORTING] Error trying to validate/create DDL download' ' directory: %s.' % mylar.CONFIG.DDL_LOCATION ) - return {"success": False, "filename": filename, "path": None} + return { + "success": False, + "filename": filename, + "path": None, + "link_type": link_type} dst_path = os.path.join(mylar.CONFIG.DDL_LOCATION, filename) - # if t.headers.get('content-encoding') == 'gzip': - # buf = StringIO(t.content) - # f = gzip.GzipFile(fileobj=buf) - + t.headers['Accept-encoding'] = 'gzip' if resume is not None: with open(dst_path, 'ab') as f: for chunk in t.iter_content(chunk_size=1024): @@ -1055,53 +1241,358 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files except requests.exceptions.Timeout as e: logger.error('[ERROR] download has timed out due to inactivity...: %s', e) mylar.DDL_LOCK = False - return {"success": False, "filename": filename, "path": None} + return { + "success": False, + "filename": filename, + "path": None, + "link_type": link_type} except Exception as e: logger.error('[ERROR] %s' % e) mylar.DDL_LOCK = False - return {"success": False, "filename": filename, "path": None} - + return { + "success": False, + "filename": filename, + "path": None, + "link_type": link_type} else: mylar.DDL_LOCK = False - if os.path.isfile(dst_path): - if dst_path.endswith('.zip'): - new_path = os.path.join( - mylar.CONFIG.DDL_LOCATION, re.sub('.zip', '', filename).strip() - ) - logger.info( - 'Zip file detected.' - ' Unzipping into new modified path location: %s' % new_path + return self.zip_zip(id, dst_path, filename) + + + def zip_zip(self, id, dst_path, filename): + if os.path.isfile(dst_path): + if dst_path.endswith('.zip'): + new_path = os.path.join( + mylar.CONFIG.DDL_LOCATION, re.sub('.zip', '', filename).strip() + ) + logger.info( + 'Zip file detected.' + ' Unzipping into new modified path location: %s' % new_path + ) + try: + zip_f = zipfile.ZipFile(dst_path, 'r') + zip_f.extractall(new_path) + zip_f.close() + except Exception as e: + logger.warn( + '[ERROR: %s] Unable to extract zip file: %s' % (e, new_path) ) + return {"success": False, "filename": filename, "path": None} + else: try: - zip_f = zipfile.ZipFile(dst_path, 'r') - zip_f.extractall(new_path) - zip_f.close() + os.remove(dst_path) except Exception as e: logger.warn( - '[ERROR: %s] Unable to extract zip file: %s' % (e, new_path) + '[ERROR: %s] Unable to remove zip file from %s after' + ' extraction.' % (e, dst_path) ) - return {"success": False, "filename": filename, "path": None} - else: - try: - os.remove(dst_path) - except Exception as e: - logger.warn( - '[ERROR: %s] Unable to remove zip file from %s after' - ' extraction.' % (e, dst_path) - ) - filename = None - else: - new_path = dst_path - return {"success": True, "filename": filename, "path": new_path} + filename = None + else: + new_path = dst_path + return {"success": True, "filename": filename, "path": new_path} mylar.DDL_LOCK = False return {"success": False, "filename": filename, "path": None} + def check_for_pack(self, title, issue_in_pack=None): + + og_title = title + + volume_label = None + annuals = False + issues = None + pack = False + + filename = series = title + + #logger.fdebug('pack: %s' % pack) + tpb = False + gc_booktype = None + # see if it's a pack type + + volume_issues = None + title_length = len(title) + + # find the year + year_check = re.search(r'(\d{4}-\d{4})', title, flags=re.I) + if not year_check: + year_check = re.findall(r'(\d{4})', title, flags=re.I) + if year_check: + for yc in year_check: + if title[title.find(yc)-1] != '#': + if yc.startswith('19') or yc.startswith('20'): + year = yc + logger.fdebug('year: %s' % year) + break + else: + #logger.fdebug('no year found within name...') + year = None + else: + year = year_check.group() + #logger.fdebug('year range: %s' % year) + + if year is not None: + title = re.sub(year, '', title).strip() + title = re.sub('\(\)', '', title).strip() + + issfind_st = title.find('#') + #logger.fdebug('issfind_st: %s' % issfind_st) + if issfind_st != -1: + issfind_en = title.find('-', issfind_st) + else: + # if it's -1, odds are it's a range, so need to work back + issfind_en = title.find('-') + #logger.fdebug('issfind_en: %s' % issfind_en) + #logger.fdebug('issfind_en-2: %s' % issfind_en -2) + if title[issfind_en -2].isdigit(): + issfind_st = issfind_en -2 + #logger.fdebug('issfind_st: %s' % issfind_st) + #logger.fdebug('rfind: %s' % title.lower().rfind('vol')) + if title.lower().rfind('vol') != -1: + #logger.fdebug('yes') + vol_find = title.lower().rfind('vol') + logger.fdebug('vol_find: %s' % vol_find) + if vol_find+2 == issfind_st: #vol 1 - 5 + issfind_st = vol_find+3 + if vol_find+3 == issfind_st: #vol. 1 - 5 + issfind_st = vol_find+4 + + #logger.fdebug('issfind_en: %s' % issfind_en) + if issfind_en != -1: + if all([title[issfind_en + 1] == ' ', title[issfind_en + 2].isdigit()]): + iss_en = title.find(' ', issfind_en + 2) + #logger.info('iss_en: %s [%s]' % (iss_en, title[iss_en +2])) + if iss_en == -1: + iss_en = len(title) + if iss_en != -1: + #logger.info('issfind_st: %s' % issfind_st) + issues = title[issfind_st : iss_en] + if title.lower().rfind('vol') == issfind_st -5 or title.lower().rfind('vol') == issfind_st -4: + series = '%s %s' % (title[:title.lower().rfind('vol')].strip(), title[iss_en:].strip()) + logger.info('new series: %s' % series) + volume_issues = issues + if len(title) - 6 > title.lower().rfind('tpb') > 1: + gc_booktype = 'TPB' + elif len(title) - 6 > title.lower().rfind('gn') > 1: + gc_booktype = 'GN' + elif len(title) - 6 > title.lower().rfind('hc') > 1: + gc_booktype = 'HC' + elif len(title) - 6 > title.lower().rfind('one-shot') > 1: + gc_booktype = 'One-Shot' + else: + gc_booktype = 'TPB/GN/HC/One-Shot' + tpb = True + #else: + t1 = title.lower().rfind('volume') + vcheck = 'volume' + if t1 == -1: + t1 = title.lower().rfind('vol.') + vcheck = 'vol.' + if t1 == -1: + t1 = title.lower().rfind('vol') + vcheck = 'vol' + if t1 != -1: + #logger.fdebug('vcheck: %s' % (len(vcheck))) + #logger.fdebug('title.find: %s' % title.lower().find(' ', t1)) + vv = title.lower().find(' ', title.lower().find(' ', t1)+1) + #logger.fdebug('vv: %s' % vv) + if tpb: + volume_label = title[t1:t1+len(vcheck)].strip() + else: + volume_label = title[t1:vv].strip() + logger.fdebug('volume discovered: %s' % volume_label) + + pack = True + logger.fdebug('issues: %s' % issues) + elif title[issfind_en + 1].isdigit(): + iss_en = title.find(' ', issfind_en + 1) + if iss_en != -1: + issues = title[issfind_st + 1 : iss_en] + pack = True + + + # to handle packs that are denoted without a # sign being present. + # if there's a dash, check to see if both sides of the dash are numeric. + logger.fdebug('pack: [%s] %s' % (type(pack),pack)) + if not pack and title.find('-') != -1: + #logger.fdebug('title: %s' % title) + #logger.fdebug('title[issfind_en+1]: %s' % title[issfind_en +1]) + #logger.fdebug('title[issfind_en+2]: %s' % title[issfind_en +2]) + #logger.fdebug('title[issfind_en-1]: %s' % title[issfind_en -1]) + if all( + [ + title[issfind_en + 1] == ' ', + title[issfind_en + 2].isdigit(), + ] + ) and all( + [ + title[issfind_en -1] == ' ', + ] + ): + spaces = [m.start() for m in re.finditer(' ', title)] + dashfind = title.find('–') + space_beforedash = title.find(' ', dashfind - 1) + space_afterdash = title.find(' ', dashfind + 1) + if not title[space_afterdash+1].isdigit(): + pass + else: + iss_end = title.find(' ', space_afterdash + 1) + if iss_end == -1: + iss_end = len(title) + set_sp = None + for sp in spaces: + if sp < space_beforedash: + prior_sp = sp + else: + set_sp = prior_sp + break + if title[set_sp:space_beforedash].strip().isdigit(): + issues = title[set_sp:iss_end].strip() + pack = True + # if it's a pack - remove the issue-range and the possible issue years + # (cause it most likely will span) and pass thru as separate items + if series is None: + series = title + if pack is True or pack is False: + #f_iss = series.find('#') + #if f_iss != -1: + # series = '%s%s'.strip() % (title[:f_iss-1], title[f_iss+1:]) + #series = re.sub(issues, '', series).strip() + # kill any brackets in the issue line here. + #issgggggggues = re.sub(r'[\(\)\[\]]', '', issues).strip() + #if series.endswith('#'): + # series = series[:-1].strip() + #title += ' #1' # we add this dummy value back in so the parser won't choke as we have the issue range stored already + + #if year is not None: + # title += ' (%s)' % year + logger.fdebug('pack_check: %s/ title: %s' % (pack, title)) + og_series = series + if pack is False: + f_iss = title.find('#') + #logger.fdebug('f_iss: %s' % f_iss) + if f_iss != -1: + series = '%s'.strip() % (title[:f_iss-1]) + #logger.fdebug('changed_title: %s' % series) + #logger.fdebug('title: %s' % title) + issues = r'%s' % issues + title = re.sub(issues, '', title).strip() + # kill any brackets in the issue line here. + #logger.fdebug('issues-before: %s' % issues) + issues = re.sub(r'[\(\)\[\]]', '', issues).strip() + #logger.fdebug('issues-after: %s' % issues) + if series.endswith('#'): + series = series[:-1].strip() + + og_series = series + + crap = re.findall(r"\(.*?\)", title) + for c in crap: + title = re.sub(c, '', title).strip() + if crap: + title= re.sub(r'[\(\)\[\]]', '', title).strip() + + pr = [x for x in self.pack_receipts if x in title] + nott = title + for x in pr: + #logger.fdebug('removing %s from %s' % (x, title)) + try: + if x == 'TPB': + if gc_booktype is None: + gc_booktype = 'TPB' + tpb = True + # may have to put HC/GN/One_shot in here as well... + if 'Annual' in x: + annuals = True + if ' &' in x and title.rfind(x) <= len(title) + 3: + x = title[title.rfind(x):] + nt = nott.replace(x, '').strip() + #logger.fdebug('[%s] new nott: %s' % (x, nott)) + except Exception as e: + #logger.warn('error: %s' % e) + pass + else: + nott = nt + + #logger.fdebug('title: %s' % title) + #logger.fdebug('final nott: %s' % nott) + if nott != title: + series = re.sub('\s+', ' ', nott).strip() + series = re.sub(r'[\(\)\[\]]', '', series).strip() + title = re.sub('\s+', ' ', nott).strip() + title = re.sub(r'[\(\)\[\]]', '', title).strip() + else: + series = re.sub('\s+', ' ', nott).strip() + + if pack is True: + if year is not None: + title += ' (%s)' % year + else: + title = '%s #%s (%s)' % (title, self.query['issue'], self.query['year']) + else: + title = series = filename + #if all([year is not None, year not in title]): + # title += ' (%s)' % year + #logger.fdebug('final title: %s' % (title)) + + if volume_label: + series = re.sub(volume_label, '', series, flags=re.I).strip() + volume_label = re.sub('[^0-9]', '', volume_label).strip() + + if tpb and gc_booktype is None: + gc_booktype = 'TPB' + else: + if gc_booktype is None: + gc_booktype = 'issue' + + logger.fdebug('title: %s' % title) + logger.fdebug('series: %s' % series) + logger.fdebug('filename: %s' % filename) + logger.fdebug('year: %s' % year) + logger.fdebug('pack: %s' % pack) + logger.fdebug('tpb/gn/hc: %s' % tpb) + if all([pack, tpb]): + logger.fdebug('volumes: %s' % volume_issues) + else: + if volume_label: + logger.fdebug('volume: %s' % volume_label) + if annuals: + logger.fdebug('annuals: %s' % annuals) + logger.fdebug('issues: %s' % issues) + + return {'title': title, + 'filename': filename, + 'series': series, + 'year': year, + 'pack': pack, + 'volume': volume_label, + 'annuals': annuals, + 'gc_booktype': gc_booktype, + 'issues': issues} + else: + return None + + def pack_check(self, issues, packinfo): + try: + pack_numbers = packinfo['pack_numbers'] + issue_range = packinfo['pack_issuelist']['issue_range'] + ist = re.sub('#', '', issues).strip() + iss = ist.find('-') + first_iss = issues[:iss].strip() + last_iss = issues[iss+1:].strip() + if all([int(first_iss) in issue_range, int(last_iss) in issue_range]): + logger.fdebug('first issue(%s) and last issue (%s) of ddl link fall within pack range of %s' % (first_iss, last_iss, pack_numbers)) + return True + except Exception as e: + pass + + return False + def issue_list(self, pack): # packlist = [x.strip() for x in pack.split(',)] packlist = pack.replace('+', ' ').replace(',', ' ').split() - print(packlist) + #logger.fdebug(packlist) plist = [] pack_issues = [] for pl in packlist: @@ -1125,7 +1616,7 @@ def issue_list(self, pack): pack_issues.append(pi) pack_issues.sort() - print("pack_issues: %s" % pack_issues) + logger.fdebug("pack_issues: %s" % pack_issues) # if __name__ == '__main__': diff --git a/mylar/helpers.py b/mylar/helpers.py index 04f76beb..4182e060 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -43,6 +43,7 @@ import mylar from . import logger from mylar import db, sabnzbd, nzbget, process, getcomics, getimage +from mylar.downloaders import mega, pixeldrain, mediafire def multikeysort(items, columns): @@ -1460,7 +1461,7 @@ def havetotals(refreshit=None): # totalissues += comic['TotalAnnuals'] haveissues = comic['Have'] except TypeError: - logger.warning('[Warning] ComicID: ' + str(comic['ComicID']) + ' is incomplete - Removing from DB. You should try to re-add the series.') + logger.warn('[Warning] ComicID: ' + str(comic['ComicID']) + ' is incomplete - Removing from DB. You should try to re-add the series.') myDB.action("DELETE from COMICS WHERE ComicID=? AND ComicName LIKE 'Comic ID%'", [comic['ComicID']]) myDB.action("DELETE from ISSUES WHERE ComicID=? AND ComicName LIKE 'Comic ID%'", [comic['ComicID']]) continue @@ -2228,7 +2229,7 @@ def duplicate_filecheck(filename, ComicID=None, IssueID=None, StoryArcID=None, r series = myDB.selectone("SELECT * FROM comics WHERE ComicID=?", [dupchk['ComicID']]).fetchone() #if it's a retry and the file was already snatched, the status is Snatched and won't hit the dupecheck. - #rtnval will be one of 3: + #rtnval will be one of 3: #'write' - write new file #'dupe_file' - do not write new file as existing file is better quality #'dupe_src' - write new file, as existing file is a lesser quality (dupe) @@ -2273,8 +2274,34 @@ def duplicate_filecheck(filename, ComicID=None, IssueID=None, StoryArcID=None, r #keywords to force keep / delete #this will be eventually user-controlled via the GUI once the options are enabled. + fixed = False + fixed_file = re.findall(r'[(]f\d{1}[)]', filename.lower()) + fixed_db_file = re.findall(r'[(]f\d{1}[)]', dupchk['Location'].lower()) + if all([fixed_file, not fixed_db_file]): + logger.info('[DUPECHECK] %s is a "Fixed" version that should be retained over existing version. Bypassing filesize/filetype check.' % filename) + fixed = True + rtnval = {'action': "dupe_src", + 'to_dupe': os.path.join(series['ComicLocation'], dupchk['Location'])} + elif all([fixed_db_file, not fixed_file]): + logger.info('[DUPECHECK] %s is a "Fixed" version that should be retained over newly aquired version. Bypassing filesize/filetype check.' % filename) + fixed = True + rtnval = {'action': "dupe_file", + 'to_dupe': filename} + elif all([fixed_file, fixed_db_file]): + ff_int = int(re.sub('[^0-9]', '', fixed_file).strip()) + fdf_int = int(re.sub('[^0-9]', '', fixed_db_file).strip()) + if ff_int > fdf_int: + logger.info('[DUPECHECK] %s is a higher "Fixed" version (%s) that should be retained over existing version(%s). Bypassing filesize/filetype check.' % (fixed_file, fixed_db_file, filename)) + fixed = True + rtnval = {'action': "dupe_src", + 'to_dupe': os.path.join(series['ComicLocation'], dupchk['Location'])} + else: + logger.info('[DUPECHECK] %s is a higher "Fixed" version (%s) that should be retained over existing version(%s). Bypassing filesize/filetype check.' % (fixed_db_file, fixed_file, os.path.join(series['ComicLocation'], dupehk['Location']))) + fixed = True + rtnval = {'action': "dupe_file", + 'to_dupe': filename} - if int(dupsize) == 0: + elif int(dupsize) == 0: logger.info('[DUPECHECK] Existing filesize is 0 as I cannot locate the original entry.') if dupchk['Status'] == 'Archived': logger.info('[DUPECHECK] Assuming issue is Archived.') @@ -2288,7 +2315,7 @@ def duplicate_filecheck(filename, ComicID=None, IssueID=None, StoryArcID=None, r tmp_dupeconstraint = mylar.CONFIG.DUPECONSTRAINT - if any(['cbr' in mylar.CONFIG.DUPECONSTRAINT, 'cbz' in mylar.CONFIG.DUPECONSTRAINT]): + if any(['cbr' in mylar.CONFIG.DUPECONSTRAINT, 'cbz' in mylar.CONFIG.DUPECONSTRAINT]) and not fixed: if 'cbr' in mylar.CONFIG.DUPECONSTRAINT: if filename.endswith('.cbr'): #this has to be configured in config - either retain cbr or cbz. @@ -2330,7 +2357,7 @@ def duplicate_filecheck(filename, ComicID=None, IssueID=None, StoryArcID=None, r rtnval = {'action': "dupe_src", 'to_dupe': os.path.join(series['ComicLocation'], dupchk['Location'])} - if mylar.CONFIG.DUPECONSTRAINT == 'filesize' or tmp_dupeconstraint == 'filesize': + if not fixed and (mylar.CONFIG.DUPECONSTRAINT == 'filesize' or tmp_dupeconstraint == 'filesize'): if filesz <= int(dupsize) and int(dupsize) != 0: logger.info('[DUPECHECK-FILESIZE PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining currently scanned in filename : ' + dupchk['Location']) rtnval = {'action': "dupe_file", @@ -2519,25 +2546,39 @@ def crc(filename): except UnicodeEncodeError: filename = "invalid" filename = filename.encode(mylar.SYS_ENCODING) - + return hashlib.md5(filename).hexdigest() -def issue_find_ids(ComicName, ComicID, pack, IssueNumber): - #import db +def issue_find_ids(ComicName, ComicID, pack, IssueNumber, pack_id): + #logger.fdebug('pack: %s' % pack) myDB = db.DBConnection() issuelist = myDB.select("SELECT * FROM issues WHERE ComicID=?", [ComicID]) if 'Annual' not in pack: - packlist = [x.strip() for x in pack.split(',')] + if ',' not in pack: + packlist = pack.split(' ') + pack = re.sub('#', '', pack).strip() + else: + packlist = [x.strip() for x in pack.split(',')] plist = [] pack_issues = [] + logger.fdebug('packlist: %s' % packlist) for pl in packlist: + pl = re.sub('#', '', pl).strip() if '-' in pl: - plist.append(list(range(int(pl[:pl.find('-')]),int(pl[pl.find('-')+1:])+1))) + le_range = list(range(int(pack[:pack.find('-')]),int(pack[pack.find('-')+1:])+1)) + for x in le_range: + if not [y for y in plist if y == x]: + plist.append(int(x)) + #logger.fdebug('plist: %s' % plist) else: - plist.append(int(pl)) + #logger.fdebug('starting single: %s' % pl) + if not [x for x in plist if x == int(pl)]: + #logger.fdebug('single not present') + plist.append(int(pl)) + #logger.fdebug('plist:%s' % plist) for pi in plist: if type(pi) == list: @@ -2552,43 +2593,63 @@ def issue_find_ids(ComicName, ComicID, pack, IssueNumber): #remove the annuals wording tmp_annuals = pack[pack.find('Annual'):] tmp_ann = re.sub('[annual/annuals/+]', '', tmp_annuals.lower()).strip() - tmp_pack = re.sub('[annual/annuals/+]', '', pack.lower()).strip() + tmp_pack = re.sub('[annual/annuals/+]', '', pack.lower()).strip() pack_issues_numbers = re.findall(r'\d+', tmp_pack) pack_issues = list(range(int(pack_issues_numbers[0]),int(pack_issues_numbers[1])+1)) annualize = True issues = {} issueinfo = [] + write_valids = [] # to keep track of snatched packs already downloading so we don't re-queue/download again Int_IssueNumber = issuedigits(IssueNumber) valid = False for iss in pack_issues: - int_iss = issuedigits(iss) + int_iss = issuedigits(str(iss)) for xb in issuelist: if xb['Status'] != 'Downloaded': if xb['Int_IssueNumber'] == int_iss: + if Int_IssueNumber == xb['Int_IssueNumber']: + valid = True issueinfo.append({'issueid': xb['IssueID'], 'int_iss': int_iss, 'issuenumber': xb['Issue_Number']}) - break - for x in issueinfo: - if Int_IssueNumber == x['int_iss']: - valid = True - break + write_valids.append({'issueid': xb['IssueID'], + 'pack_id': pack_id}) + break + else: + logger.info('issue #%s exists in the pack and is already in a Downloaded state. Mark the issue as anything' + 'other than Wanted if you want the pack to be downloaded.' % iss) + if valid: + for wv in write_valids: + mylar.PACK_ISSUEIDS_DONT_QUEUE[wv['issueid']] = wv['pack_id'] issues['issues'] = issueinfo + logger.fdebug('pack_issueids_dont_queue: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) if len(issues['issues']) == len(pack_issues): - logger.info('Complete issue count of ' + str(len(pack_issues)) + ' issues are available within this pack for ' + ComicName) + logger.fdebug('Complete issue count of %s issues are available within this pack for %s' % (len(pack_issues), ComicName)) else: - logger.info('Issue counts are not complete (not a COMPLETE pack) for ' + ComicName) + logger.fdebug('Issue counts are not complete (not a COMPLETE pack) for %s' % ComicName) issues['issue_range'] = pack_issues issues['valid'] = valid return issues +def reverse_the_pack_snatch(pack_id, comicid): + logger.info('[REVERSE UNO] Reversal of issues marked as Snatched via pack download reversing due to invalid link retrieval..') + #logger.fdebug(mylar.PACK_ISSUEIDS_DONT_QUEUE) + reverselist = [issueid for issueid, packid in mylar.PACK_ISSUEIDS_DONT_QUEUE.items() if pack_id == packid] + myDB = db.DBConnection() + for x in reverselist: + myDB.upsert("issues", {"Status": "Skipped"}, {"IssueID": x}) + if reverselist: + logger.info('[REVERSE UNO] Reversal completed for %s issues' % len(reverselist)) + mylar.GLOBAL_MESSAGES = {'status': 'success', 'comicid': comicid, 'tables': 'both', 'message': 'Successfully changed status of %s issues to %s' % (len(reverselist), 'Skipped')} + + def conversion(value): if type(value) == str: try: @@ -3220,9 +3281,19 @@ def ddl_downloader(queue): elif mylar.DDL_LOCK is False and queue.qsize() >= 1: item = queue.get(True) + if item == 'exit': logger.info('Cleaning up workers for shutdown') break + + if item['id'] not in mylar.DDL_QUEUED: + mylar.DDL_QUEUED.append(item['id']) + + try: + link_type_failure = item['link_type_failure'] + except Exception as e: + link_type_failure = [] + logger.info('Now loading request from DDL queue: %s' % item['series']) #write this to the table so we have a record of what's going on. @@ -3239,8 +3310,23 @@ def ddl_downloader(queue): remote_filesize = helpers.human2bytes(re.sub('/s', '', item['size'][:-1]).strip()) except Exception: remote_filesize = 0 - ddz = getcomics.GC() - ddzstat = ddz.downloadit(item['id'], item['link'], item['mainlink'], item['resume'], item['issueid'], remote_filesize) + + if any([item['link_type'] == 'GC-Main', item['link_type'] == 'GC_Mirror']): + ddz = getcomics.GC() + ddzstat = ddz.downloadit(item['id'], item['link'], item['mainlink'], item['resume'], item['issueid'], remote_filesize) + elif item['link_type'] == 'GC-Mega': + meganz = mega.MegaNZ() + ddzstat = meganz.ddl_download(item['link'], None, item['id'], item['issueid'], item['link_type']) #item['filename'], item['id']) + elif item['link_type'] == 'GC-Media': + mediaf = mediafire.MediaFire() + ddzstat = mediaf.ddl_download(item['link'], item['id'], item['issueid']) #item['filename'], item['id']) + elif item['link_type'] == 'GC-Pixel': + pdrain = pixeldrain.PixelDrain() + ddzstat = pdrain.ddl_download(item['link'], item['id'], item['issueid']) #item['filename'], item['id']) + + elif item['site'] == 'DDL(External)': + meganz = mega.MegaNZ() + ddzstat = meganz.ddl_download(item['link'], item['filename'], item['id'], item['issueid'], item['link_type']) if ddzstat['success'] is True: tdnow = datetime.datetime.now() @@ -3272,16 +3358,41 @@ def ddl_downloader(queue): 'download_info': {'provider': 'DDL', 'id': item['id']}}) except Exception as e: logger.error('process error: %s [%s]' %(e, ddzstat)) + + #logger.fdebug('mylar.ddl_queued: %s' % mylar.DDL_QUEUED) + mylar.DDL_QUEUED.remove(item['id']) + #logger.fdebug('before-pack_issueids: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) + pck_cnt = 0 + if item['comicinfo'][0]['pack'] is True: + logger.fdebug('[PACK DETECTION] Attempting to remove issueids from the pack dont-queue list') + for x,y in dict(mylar.PACK_ISSUEIDS_DONT_QUEUE).items(): + if y == item['id']: + pck_cnt +=1 + del mylar.PACK_ISSUEIDS_DONT_QUEUE[x] + + #logger.fdebug('after-pack_issueids: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) + logger.fdebug('Successfully removed %s issueids from pack queue list as download is completed.' % pck_cnt) + elif all([ddzstat['success'] is True, mylar.CONFIG.POST_PROCESSING is False]): path = ddzstat['path'] if ddzstat['filename'] is not None: path = os.path.join(path, ddzstat['filename']) logger.info('File successfully downloaded. Post Processing is not enabled - item retained here: %s' % (path,)) else: - logger.info('[Status: %s] Failed to download: %s ' % (ddzstat['success'], ddzstat)) - nval = {'status': 'Failed', - 'updated_date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M')} - myDB.upsert('ddl_info', nval, ctrlval) + try: + ltf = ddzstat['links_exhausted'] + except Exception as e: + logger.info('[Status: %s] Failed to download item from %s : %s ' % (ddzstat['success'], item['link_type'], ddzstat)) + link_type_failure.append(item['link_type']) + ggc = getcomics.GC() + ggc.parse_downloadresults(item['id'], item['mainlink'], item['comicinfo'], item['packinfo'], link_type_failure) + else: + logger.info('[REDO] Exhausted all available links [%s] for issueid %s and was not able to download anything' % (link_type_failure, item['issueid'])) + nval = {'status': 'Failed', + 'updated_date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M')} + myDB.upsert('ddl_info', nval, ctrlval) + #undo all snatched items, to previous status via item['id'] - this will be set to Skipped currently regardless of previous status + reverse_the_pack_snatch(item['id'], item['comicid']) else: time.sleep(5) @@ -3329,30 +3440,39 @@ def search_queue(queue): logger.info('[SEARCH-QUEUE] Cleaning up workers for shutdown') break - logger.info('[SEARCH-QUEUE] Now loading item from search queue: %s' % item) - if mylar.SEARCHLOCK is False: - arcid = None - comicid = item['comicid'] - issueid = item['issueid'] - if issueid is not None: - if '_' in issueid: - arcid = issueid - comicid = None # required for storyarcs to work - issueid = None # required for storyarcs to work - mofo = mylar.filers.FileHandlers(ComicID=comicid, IssueID=issueid, arcID=arcid) - local_check = mofo.walk_the_walk() - if local_check['status'] is True: - mylar.PP_QUEUE.put({'nzb_name': local_check['filename'], - 'nzb_folder': local_check['filepath'], - 'failed': False, - 'issueid': item['issueid'], - 'comicid': item['comicid'], - 'apicall': True, - 'ddl': False, - 'download_info': None}) - else: - ss_queue = mylar.search.searchforissue(item['issueid']) - time.sleep(5) #arbitrary sleep to let the process attempt to finish pp'ing + gumbo_line = True + #logger.fdebug('pack_issueids_dont_queue: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) + #logger.fdebug('ddl_queued: %s' % mylar.DDL_QUEUED) + if item['issueid'] in mylar.PACK_ISSUEIDS_DONT_QUEUE: + if mylar.PACK_ISSUEIDS_DONT_QUEUE[item['issueid']] in mylar.DDL_QUEUED: + logger.fdebug('[SEARCH-QUEUE-PACK-DETECTION] %s already queued to download via pack...Ignoring' % item['issueid']) + gumbo_line = False + + if gumbo_line: + logger.fdebug('[SEARCH-QUEUE] Now loading item from search queue: %s' % item) + if mylar.SEARCHLOCK is False: + arcid = None + comicid = item['comicid'] + issueid = item['issueid'] + if issueid is not None: + if '_' in issueid: + arcid = issueid + comicid = None # required for storyarcs to work + issueid = None # required for storyarcs to work + mofo = mylar.filers.FileHandlers(ComicID=comicid, IssueID=issueid, arcID=arcid) + local_check = mofo.walk_the_walk() + if local_check['status'] is True: + mylar.PP_QUEUE.put({'nzb_name': local_check['filename'], + 'nzb_folder': local_check['filepath'], + 'failed': False, + 'issueid': item['issueid'], + 'comicid': item['comicid'], + 'apicall': True, + 'ddl': False, + 'download_info': None}) + else: + ss_queue = mylar.search.searchforissue(item['issueid']) + time.sleep(5) #arbitrary sleep to let the process attempt to finish pp'ing if mylar.SEARCHLOCK is True: logger.fdebug('[SEARCH-QUEUE] Another item is currently being searched....') diff --git a/mylar/importer.py b/mylar/importer.py index 55714c77..e3350cc1 100644 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -1096,7 +1096,7 @@ def issue_collection(issuedata, nostatus, serieslast_updated=None): #logger.fdebug("Not changing the status at this time - reverting to previous module after to re-append existing status") pass #newValueDict['Status'] = "Skipped" - logger.info('issue_collection results: [%s] %s' % (controlValueDict, newValueDict)) + #logger.fdebug('issue_collection results: [%s] %s' % (controlValueDict, newValueDict)) try: myDB.upsert(dbwrite, newValueDict, controlValueDict) except sqlite3.InterfaceError as e: diff --git a/mylar/req_test.py b/mylar/req_test.py index 83ae3c5e..f3029fb6 100644 --- a/mylar/req_test.py +++ b/mylar/req_test.py @@ -21,7 +21,7 @@ import codecs import configparser import platform -from pkg_resources import parse_version +from packaging.version import parse as parse_version import mylar from mylar import logger diff --git a/mylar/rsscheck.py b/mylar/rsscheck.py index b74727b0..bf6a48dc 100755 --- a/mylar/rsscheck.py +++ b/mylar/rsscheck.py @@ -26,7 +26,7 @@ import random from bs4 import BeautifulSoup from io import StringIO -from pkg_resources import parse_version +from packaging.version import parse as parse_version import mylar from mylar import db, logger, ftpsshup, helpers, auth32p, utorrent, helpers, filechecker diff --git a/mylar/sabnzbd.py b/mylar/sabnzbd.py index fbc45dd1..9071cd92 100644 --- a/mylar/sabnzbd.py +++ b/mylar/sabnzbd.py @@ -22,7 +22,7 @@ import sys import re import time -from pkg_resources import parse_version +from packaging.version import parse as parse_version import mylar from mylar import logger, cdh_mapping @@ -166,10 +166,6 @@ def historycheck(self, nzbinfo, roundtwo=False): try: min_sab = '3.2.0' sab_vers = mylar.CONFIG.SAB_VERSION - if 'beta' in sab_vers: - sab_vers = re.sub('[^0-9]', '', sab_vers) - if len(sab_vers) > 3: - sab_vers = sab_vers[:-1] # remove beta value entirely... if parse_version(sab_vers) >= parse_version(min_sab): logger.fdebug('SABnzbd version is higher than 3.2.0. Querying history based on nzo_id directly.') hist_params['nzo_ids'] = sendresponse diff --git a/mylar/search.py b/mylar/search.py index 24d32549..7537993e 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -30,7 +30,9 @@ nzbget, search_filer, getcomics, + downloaders, ) +from mylar.downloaders import external_server as exs import feedparser import requests @@ -271,6 +273,11 @@ def search_init( searchprov['DDL(GetComics)'] = ({'id': 200, 'type': 'DDL', 'lastrun': 0, 'active': True, 'hits': 0}) else: searchprov['DDL(GetComics)']['active'] = True + elif prov_order[prov_count] == 'DDL(External)' and not provider_blocked and 'DDL(External)' not in checked_once: + if 'DDL(External)' not in searchprov.keys(): + searchprov['DDL(External)'] = ({'id': 201, 'type': 'DDL(External)', 'lastrun': 0, 'active': True, 'hits': 0}) + else: + searchprov['DDL(External)']['active'] = True elif prov_order[prov_count] == '32p' and not provider_blocked: searchprov['32P'] = ({'type': 'torrent', 'lastrun': 0, 'active': True, 'hits': 0}) elif prov_order[prov_count].lower() == 'experimental' and not provider_blocked and 'experimental' not in checked_once: @@ -447,6 +454,7 @@ def search_init( ) and ''.join(current_prov.keys()) in ( '32P', 'DDL(GetComics)', + 'DDL(External)', 'Public Torrents', 'experimental', ): @@ -472,6 +480,10 @@ def search_init( issuedisplay = None else: issuedisplay = StoreDate[5:] + if 'annual' in ComicName.lower(): + if re.findall('(?:19|20)\d{2}', ComicName): + issuedisplay = None + if issuedisplay is None: logger.info( 'Could not find %s (%s) using %s [%s]' @@ -618,6 +630,15 @@ def provider_order(initial_run=False): ddlprovider.append('DDL(GetComics)') ddls+=1 + if all( + [ + mylar.CONFIG.ENABLE_EXTERNAL_SERVER is True, + not helpers.block_provider_check('DDL(External)'), + ] + ): + ddlprovider.append('DDL(External)') + ddls+=1 + if initial_run: logger.fdebug('nzbprovider(s): %s' % nzbprovider) # -------- @@ -962,9 +983,12 @@ def NZB_SEARCH( 'year': comyear} b = getcomics.GC(query=fline, provider_stat=provider_stat) verified_matches = b.search(is_info=is_info) + elif nzbprov == 'DDL(External)': + b = exs.MegaNZ(query='%s' % ComicName, provider_stat=provider_stat) + verified_matches = b.ddl_search(is_info=is_info) #logger.fdebug('bb returned from %s: %s' % (nzbprov, verified_matches)) - elif RSS == "yes": + elif RSS == "yes" and 'DDL(External)' not in nzbprov: if 'DDL(GetComics)' in nzbprov: #only GC has an available RSS Feed logger.fdebug( @@ -1412,8 +1436,9 @@ def verification(verified_matches, is_info): if is_info['foundc']['status'] is True: #foundcomic.append("yes") - logger.fdebug('mylar.COMICINFO: %s' % verified_matches) - logger.fdebug('verified_index: %s' % verified_index) + #logger.fdebug('mylar.COMICINFO: %s' % verified_matches) + #logger.fdebug('verified_index: %s' % verified_index) + #logger.fdebug('isinfo: %s' % is_info) if verified_matches[verified_index]['pack'] is True: try: issinfo = verified_matches[verified_index]['pack_issuelist'] @@ -1450,10 +1475,10 @@ def verification(verified_matches, is_info): ) notify_snatch( sent_to, - is_info['ComicName'], - is_info['ComicYear'], + verified_matches[verified_index]['entry']['series'], #is_info['ComicName'], + verified_matches[verified_index]['entry']['year'], #is_info['ComicYear'], verified_matches[verified_index]['pack_numbers'], - is_info['nzbprov'], + verified_matches[verified_index]['nzbprov'], True, ) else: @@ -1557,6 +1582,7 @@ def searchforissue(issueid=None, new=False, rsschecker=None, manual=False): and any( [ mylar.CONFIG.ENABLE_GETCOMICS is True, + mylar.CONFIG.ENABLE_EXTERNAL_SERVER is True, ] )) or any( @@ -2027,8 +2053,8 @@ def searchforissue(issueid=None, new=False, rsschecker=None, manual=False): mylar.CONFIG.NZBSU is True, mylar.CONFIG.DOGNZB is True, mylar.CONFIG.EXPERIMENTAL is True, - mylar.CONFIG.ENABLE_DDL is True, - #mylar.CONFIG.ENABLE_GETCOMICS is True, + mylar.CONFIG.ENABLE_GETCOMICS is True, + mylar.CONFIG.ENABLE_EXTERNAL_SERVER is True, ] ) or all([mylar.CONFIG.NEWZNAB is True, len(ens) > 0]) @@ -2497,6 +2523,7 @@ def searchIssueIDList(issuelist): and any( [ mylar.CONFIG.ENABLE_GETCOMICS is True, + mylar.CONFIG.ENABLE_EXTERNAL_SERVER is True ] ) ) or any( @@ -3129,22 +3156,45 @@ def searcher( tmp_issueid = IssueArcID else: tmp_issueid = IssueID + + # we need to pass in if it's a pack and what issues are present therein + pack_info = {'pack': comicinfo[0]['pack'], + 'pack_numbers': comicinfo[0]['pack_numbers'], + 'pack_issuelist': comicinfo[0]['pack_issuelist']} + if nzbprov == 'DDL(GetComics)': #GC requires an extra step - do it now. ggc = getcomics.GC(issueid=tmp_issueid, comicid=ComicID) ggc.loadsite(nzbid, link) - ddl_it = ggc.parse_downloadresults(nzbid, link, comicinfo) + ddl_it = ggc.parse_downloadresults(nzbid, link, comicinfo, pack_info) + tnzbprov = ddl_it['site'] + else: + cinfo = {'id': nzbid, + 'series': comicinfo[0]['ComicName'], + 'year': comicinfo[0]['comyear'], + 'size': comicinfo[0]['size'], + 'issues': comicinfo[0]['IssueNumber'], + 'issueid': comicinfo[0]['IssueID'], + 'comicid': comicinfo[0]['ComicID'], + 'filename': comicinfo[0]['nzbtitle'], + 'oneoff': comicinfo[0]['oneoff'], + 'link': link, + 'site': nzbprov} + + meganz = exs.MegaNZ(provider_stat=provider_stat) + ddl_it = meganz.queue_the_download(cinfo, comicinfo, pack_info) + tnzbprov = 'Mega' if ddl_it['success'] is True: logger.info( - 'Successfully snatched %s from DDL site. It is currently being queued' - ' to download in position %s' % (nzbname, mylar.DDL_QUEUE.qsize()) + '[%s] Successfully snatched %s from DDL site. It is currently being queued' + ' to download in position %s' % (tnzbprov, nzbname, mylar.DDL_QUEUE.qsize()) ) else: - logger.info('Failed to retrieve %s from the DDL site.' % nzbname) + logger.info('[%s] Failed to retrieve %s from the DDL site.' % (tnzbprov, nzbname)) return "ddl-fail" - sent_to = "is downloading it directly via %s" % nzbprov + sent_to = "is downloading it directly via %s" % tnzbprov elif mylar.USE_BLACKHOLE and all( [nzbprov != '32P', nzbprov != 'WWT', nzbprov != 'DEM', provider_stat['type'] != 'torznab'] @@ -3681,15 +3731,25 @@ def searcher( def notify_snatch(sent_to, comicname, comyear, IssueNumber, nzbprov, pack): + # pack = {"pack": True, "issues": '#1 - 60', "years": "(1997-2002"} + #logger.fdebug('sent_to: %s' % sent_to) + #logger.fdebug('pack: %s' % pack) + #logger.fdebug('Issue: %s' % IssueNumber) + #logger.fdebug('nzbprov: %s' % nzbprov) + #logger.fdebug('comyear: %s' % comyear) + #logger.fdebug('comicname: %s' % comicname) + if pack is False: snline = 'Issue snatched!' + if IssueNumber is not None: + snatched_name = '%s (%s) #%s' % (comicname, comyear, IssueNumber) + else: + snatched_name = '%s (%s)' % (comicname, comyear) else: snline = 'Pack snatched!' + snatched_name = '%s %s (%s)' % (comicname, IssueNumber, comyear) - if IssueNumber is not None: - snatched_name = '%s (%s) #%s' % (comicname, comyear, IssueNumber) - else: - snatched_name = '%s (%s)' % (comicname, comyear) + #logger.fdebug('snatched_name: %s' % snatched_name) nzbprov = re.sub(r'\(newznab\)', '', nzbprov).strip() nzbprov = re.sub(r'\(torznab\)', '', nzbprov).strip() diff --git a/mylar/search_filer.py b/mylar/search_filer.py index cc822d7a..25e873b3 100644 --- a/mylar/search_filer.py +++ b/mylar/search_filer.py @@ -139,6 +139,8 @@ def _process_entry(self, entry, is_info): comsize_b = None else: comsize_b = helpers.human2bytes(entry['size']) + elif entry['site'] == 'DDL(External)': + comsize_b = '0' #External links ! filesize except Exception: tmpsz = entry.enclosures[0] comsize_b = tmpsz['length'] @@ -417,10 +419,34 @@ def _process_entry(self, entry, is_info): 'removed extra information after issue # that' ' is not necessary: %s' % cleantitle ) + # only send it to parser if it's not a DDL + pack (already parsed) + if entry['pack'] is True and 'DDL' in entry['site']: + logger.fdebug('parsing pack...') + ffc = filechecker.FileChecker() + dnr = ffc.dynamic_replace(entry['series']) + parsed_comic = {'booktype': entry['gc_booktype'], + 'comicfilename': entry['filename'], + 'series_name': entry['series'], + 'series_name_decoded': entry['series'], + 'issueid': None, + 'dynamic_name': dnr['mod_seriesname'], + 'issues': entry['issues'], + 'series_volume': None, + 'alt_series': None, + 'alt_issue': None, + 'issue_year': entry['year'], + 'issue_number': None, + 'scangroup': None, + 'reading_order': None, + 'sub': None, + 'comiclocation': None, + 'parse_status': 'success'} + # send it to the parser here. - p_comic = filechecker.FileChecker(file=ComicTitle, watchcomic=ComicName) - parsed_comic = p_comic.listFiles() + else: + p_comic = filechecker.FileChecker(file=ComicTitle, watchcomic=ComicName) + parsed_comic = p_comic.listFiles() logger.fdebug('parsed_info: %s' % parsed_comic) logger.fdebug( @@ -644,7 +670,7 @@ def _process_entry(self, entry, is_info): elif UseFuzzy == "1": yearmatch = True - if yearmatch is False: + if yearmatch is False and entry['pack'] is False: return None annualize = False @@ -711,6 +737,7 @@ def _process_entry(self, entry, is_info): booktype != 'TPB', booktype != 'HC', booktype != 'GN', + booktype != 'TPB/GN/HC/One-Shot', ] ) and ( int(F_ComicVersion) == int(D_ComicVersion) @@ -723,6 +750,7 @@ def _process_entry(self, entry, is_info): booktype == 'TPB', booktype == 'HC', booktype == 'GN', + booktype == 'TPB/GN/HC/One-Shot', ] ) and any([ all( @@ -747,6 +775,7 @@ def _process_entry(self, entry, is_info): booktype == 'TPB', booktype == 'HC', booktype == 'GN', + booktype == 'TPB/GN/HC/One-Shot', ] ) and all( [ @@ -771,7 +800,6 @@ def _process_entry(self, entry, is_info): except Exception: pack_test = False - if all(['DDL' in nzbprov, pack_test is True]): logger.fdebug( '[PACK-QUEUE] %s Pack detected for %s.' @@ -785,7 +813,7 @@ def _process_entry(self, entry, is_info): if not entry['title'].startswith('0-Day Comics Pack'): pack_issuelist = entry['issues'] issueid_info = helpers.issue_find_ids( - ComicName, ComicID, pack_issuelist, IssueNumber + ComicName, ComicID, pack_issuelist, IssueNumber, entry['id'] ) if issueid_info['valid'] is True: logger.info( @@ -943,7 +971,7 @@ def _process_entry(self, entry, is_info): filecomic['booktype'] == 'TPB', filecomic['booktype'] == 'GN', filecomic['booktype'] == 'HC', - filecomic['booktype'] == 'TPB/GN/HC', + filecomic['booktype'] == 'TPB/GN/HC/One-Shot', ] ) and all( [ @@ -957,7 +985,7 @@ def _process_entry(self, entry, is_info): filecomic['booktype'] == 'TPB', filecomic['booktype'] == 'GN', filecomic['booktype'] == 'HC', - filecomic['booktype'] == 'TPB/GN/HC', + filecomic['booktype'] == 'TPB/GN/HC/One-Shot', ] ) and all( [ @@ -1111,6 +1139,7 @@ def check_for_first_result(self, entries, is_info, prefer_pack=False): candidate = None for entry in entries: maybe_value = self._process_entry(entry, is_info) + #logger.fdebug('maybe_value: %s' % maybe_value) if maybe_value is not None: # If we have a value which matches our pack/not-pack # preference, return it: otherwise, store it for return if we @@ -1120,4 +1149,5 @@ def check_for_first_result(self, entries, is_info, prefer_pack=False): # (This reduces to prefer_pack == is_pack, but that's harder to grok) return maybe_value candidate = maybe_value + #logger.fdebug('candidate: %s' % candidate) return candidate diff --git a/mylar/updater.py b/mylar/updater.py index b2741b0f..355603bd 100755 --- a/mylar/updater.py +++ b/mylar/updater.py @@ -1205,6 +1205,8 @@ def forceRescan(ComicID, archive=None, module=None, recheck=False): if all([booktype == 'TPB', iscnt > 1]) or all([booktype == 'GN', iscnt > 1]) or all([booktype =='HC', iscnt > 1]) or all([booktype == 'One-Shot', iscnt == 1, cla['JusttheDigits'] is None]): if cla['SeriesVolume'] is not None: just_the_digits = re.sub('[^0-9]', '', cla['SeriesVolume']).strip() + elif cla['JusttheDigits'] is None: + just_the_digits = None else: just_the_digits = re.sub('[^0-9]', '', cla['JusttheDigits']).strip() else: @@ -1264,7 +1266,7 @@ def forceRescan(ComicID, archive=None, module=None, recheck=False): logger.fdebug('[ANNUAL-CHK] No annuals with identical issue numbering across annual volumes were detected for this series') mc_annualnumber = None else: - logger.fdebug('[ANNUAL-CHK] Multiple issues with identical numbering were detected across multiple annual volumes. Attempting to accomodate.') + logger.fdebug('[ANNUAL-CHK] Multiple annuals with identical numbering were detected across multiple annual volumes. Attempting to accomodate.') for mc in mult_ann_check: mc_annualnumber.append({"Int_IssueNumber": mc['Int_IssueNumber']}) @@ -1468,8 +1470,8 @@ def forceRescan(ComicID, archive=None, module=None, recheck=False): isslocation = tmpfc['ComicFilename'] #helpers.conversion(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug(module + ' .......filename: ' + isslocation) - logger.fdebug(module + ' .......filesize: ' + str(tmpfc['ComicSize'])) - # to avoid duplicate issues which screws up the count...let's store the filename issues then + logger.fdebug(module + ' .......filesize: ' + str(tmpfc['ComicSize'])) + # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... issuedupechk.append({'fcdigit': fcdigit, 'filename': tmpfc['ComicFilename'], @@ -1513,17 +1515,16 @@ def forceRescan(ComicID, archive=None, module=None, recheck=False): break int_iss = helpers.issuedigits(reann['Issue_Number']) #logger.fdebug(module + ' int_iss:' + str(int_iss)) - issyear = reann['IssueDate'][:4] old_status = reann['Status'] - year_check = re.findall(r'(/d{4})(?=[\s]|annual\b|$)', temploc, flags=re.I) + year_check = re.findall(r'(\d{4})(?=[\s]|annual\b|$)', temploc, flags=re.I) if year_check: ann_line = '%s annual' % year_check[0] logger.fdebug('ann_line: %s' % ann_line) - fcdigit = helpers.issuedigits(re.sub(ann_line, '', temploc.lower()).strip()) - #fcdigit = helpers.issuedigits(re.sub('2021 annual', '', temploc.lower()).strip()) - fcdigit = helpers.issuedigits(re.sub('annual', '', temploc.lower()).strip()) + fcdigit = helpers.issuedigits('1') + else: + fcdigit = helpers.issuedigits(re.sub('annual', '', temploc.lower()).strip()) if fcdigit == 999999999999999: fcdigit = helpers.issuedigits(re.sub('special', '', temploc.lower()).strip()) @@ -1802,11 +1803,13 @@ def forceRescan(ComicID, archive=None, module=None, recheck=False): logger.fdebug('[nothaves] issue_status_generation_time_taken: %s' % (datetime.datetime.now() - u_start)) - if len(update_iss) > 0: + if any([len(update_iss) > 0, len(update_ann) > 0]): r_start = datetime.datetime.now() try: - myDB.action("UPDATE issues SET Status=? WHERE IssueID=?", update_iss, executemany=True) - myDB.action("UPDATE annuals SET Status=? WHERE IssueID=?", update_ann, executemany=True) + if len(update_iss) > 0: + myDB.action("UPDATE issues SET Status=? WHERE IssueID=?", update_iss, executemany=True) + if len(update_ann) > 0: + myDB.action("UPDATE annuals SET Status=? WHERE IssueID=?", update_ann, executemany=True) except Exception as e: logger.warn('Error updating: %s' % e) logger.fdebug('[nothaves] issue_status_writing took %s' % (datetime.datetime.now() - r_start)) diff --git a/mylar/webserve.py b/mylar/webserve.py index eb648796..4d06f9fd 100644 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -29,7 +29,7 @@ import stat import ntpath from pathlib import Path -from pkg_resources import parse_version +from packaging.version import parse as parse_version from mako.template import Template from mako.lookup import TemplateLookup @@ -1881,7 +1881,7 @@ def deleteSeries(self, ComicID, delete_dir=None): else: logger.warn('Unable to remove directory as it does not exist in : ' + seriesdir) else: - logger.warn('Unable to remove directory as it does not exist in : ' + seriesdir) + logger.warn('Unable to remove directory as it does not exist.') if seriesvol is None: dspline = '%s (%s)' % (ComicName, seriesyear) else: @@ -3535,6 +3535,10 @@ def ddl_requeue(self, mode, id=None, issueid=None): 'status': x['status'], 'year': x['year'], 'size': x['size'], + 'link_type': x['link_type'], + 'pack': x['pack'], + 'oneoff': x['oneoff'], + 'filename': x['filename'], 'remote_filesize': x['remote_filesize'], 'comicid': x['comicid'], 'issueid': x['issueid'], @@ -3566,11 +3570,16 @@ def ddl_requeue(self, mode, id=None, issueid=None): 'series': item['series'], 'year': item['year'], 'size': item['size'], - 'remote_filesize': item['remote_filesize'], 'comicid': item['comicid'], 'issueid': item['issueid'], - 'site': item['site'], + 'oneoff': item['oneoff'], 'id': item['id'], + 'link_type': item['link_type'], + 'filename': item['filename'], + 'comicinfo': None, + 'packinfo': None, + 'site': item['site'], + 'remote_filesize': item['remote_filesize'], 'resume': resume}) linemessage = '%s successful for %s' % (mode, item['series']) @@ -3599,8 +3608,8 @@ def queueManage(self): # **args): myDB = db.DBConnection() resultlist = 'There are currently no items waiting in the Direct Download (DDL) Queue for processing.' - s_info = myDB.select("SELECT a.ComicName, a.ComicVersion, a.ComicID, a.ComicYear, b.Issue_Number, b.IssueID, c.size, c.status, c.id, c.updated_date, c.issues, c.year FROM comics as a INNER JOIN issues as b ON a.ComicID = b.ComicID INNER JOIN ddl_info as c ON b.IssueID = c.IssueID") # WHERE c.status != 'Downloading'") - o_info = myDB.select("Select a.ComicName, b.Issue_Number, a.IssueID, a.ComicID, c.size, c.status, c.id, c.updated_date, c.issues, c.year from oneoffhistory a join snatched b on a.issueid=b.issueid join ddl_info c on b.issueid=c.issueid where b.provider like 'DDL%'") + s_info = myDB.select("SELECT a.ComicName, a.ComicVersion, a.ComicID, a.ComicYear, b.Issue_Number, b.IssueID, c.series as filename, c.size, c.status, c.id, c.updated_date, c.issues, c.year, c.pack FROM comics as a INNER JOIN issues as b ON a.ComicID = b.ComicID INNER JOIN ddl_info as c ON b.IssueID = c.IssueID") # WHERE c.status != 'Downloading'") + o_info = myDB.select("Select a.ComicName, b.Issue_Number, a.IssueID, a.ComicID, c.series as filename, c.size, c.status, c.id, c.updated_date, c.issues, c.year, c.pack from oneoffhistory a join snatched b on a.issueid=b.issueid join ddl_info c on b.issueid=c.issueid where b.provider like 'DDL%'") tmp_list = {} if s_info: resultlist = [] @@ -3614,11 +3623,16 @@ def queueManage(self): # **args): year = si['year'] issue = '#%s' % si['issues'] + if si['pack']: + series = si['filename'] + else: + series = si['ComicName'] + if si['status'] == 'Completed': si_status = '100%' else: si_status = '' - resultlist.append({'series': si['ComicName'], + resultlist.append({'series': series, 'issue': issue, 'id': si['id'], 'volume': si['ComicVersion'], @@ -3634,6 +3648,7 @@ def queueManage(self): # **args): 'issueid': si['issueid'], 'updated_date': si['updated_date'] } + #logger.info('s_info: %s' % (resultlist)) if o_info: if type(resultlist) is str: resultlist = [] @@ -3651,12 +3666,17 @@ def queueManage(self): # **args): year = oi['year'] issue = '#%s' % oi['issues'] + if oi['pack']: + series = oi['filename'] + else: + series = oi['ComicName'] + if oi['status'] == 'Completed': oi_status = '100%' else: oi_status = '' - resultlist.append({'series': oi['ComicName'], + resultlist.append({'series': series, 'issue': issue, 'id': oi['id'], 'volume': None, @@ -3668,6 +3688,7 @@ def queueManage(self): # **args): 'updated_date': oi['updated_date'], 'progress': oi_status}) + #logger.info('o_info: %s' % (resultlist)) return serve_template(templatename="queue_management.html", title="Queue Management", resultlist=resultlist) #activelist=activelist, resultlist=resultlist) queueManage.exposed = True @@ -3679,8 +3700,8 @@ def queueManageIt(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSort myDB = db.DBConnection() resultlist = 'There are currently no items waiting in the Direct Download (DDL) Queue for processing.' - s_info = myDB.select("SELECT a.ComicName, a.ComicVersion, a.ComicID, a.ComicYear, b.Issue_Number, b.IssueID, c.series as filename, c.size, c.status, c.id, c.updated_date, c.issues, c.year, c.pack FROM comics as a INNER JOIN issues as b ON a.ComicID = b.ComicID INNER JOIN ddl_info as c ON b.IssueID = c.IssueID") # WHERE c.status != 'Downloading'") - o_info = myDB.select("Select a.ComicName, b.Issue_Number, a.IssueID, a.ComicID, c.series as filename, c.size, c.status, c.id, c.updated_date, c.issues, c.year, c.pack from oneoffhistory a join snatched b on a.issueid=b.issueid join ddl_info c on b.issueid=c.issueid where b.provider like 'DDL%'") + s_info = myDB.select("SELECT a.ComicName, a.ComicVersion, a.ComicID, a.ComicYear, b.Issue_Number, b.IssueID, c.series as filename, c.size, c.status, c.id, c.updated_date, c.issues, c.year, c.pack, c.link_type FROM comics as a INNER JOIN issues as b ON a.ComicID = b.ComicID INNER JOIN ddl_info as c ON b.IssueID = c.IssueID") # WHERE c.status != 'Downloading'") + o_info = myDB.select("Select a.ComicName, b.Issue_Number, a.IssueID, a.ComicID, c.series as filename, c.size, c.status, c.id, c.updated_date, c.issues, c.year, c.pack, c.link_type from oneoffhistory a join snatched b on a.issueid=b.issueid join ddl_info c on b.issueid=c.issueid where b.provider like 'DDL%'") tmp_list = {} if s_info: resultlist = [] @@ -3700,7 +3721,10 @@ def queueManageIt(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSort si_status = '' if si['pack']: - series = si['filename'] + if si['year'] not in si['filename']: + series = '%s (%s)' % (si['filename'], si['year']) + else: + series = si['filename'] else: if issue is not None: if si['ComicVersion'] is not None: @@ -3716,6 +3740,7 @@ def queueManageIt(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSort resultlist.append({'series': series, #i['ComicName'], 'issue': issue, 'queueid': si['id'], + 'linktype': si['link_type'], 'volume': si['ComicVersion'], 'year': year, 'size': si['size'].strip(), @@ -3729,6 +3754,7 @@ def queueManageIt(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSort 'issueid': si['issueid'], 'updated_date': si['updated_date'] } + #logger.info('s_info: %s' % (resultlist)) if o_info: if type(resultlist) is str: resultlist = [] @@ -3753,7 +3779,10 @@ def queueManageIt(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSort oi_status = '' if oi['pack']: - series = oi['filename'] + if oi['year'] not in oi['filename']: + series = '%s (%s)' % (oi['filename'], oi['year']) + else: + series = oi['filename'] else: if issue is not None: series = '%s %s (%s)' % (oi['ComicName'], issue, year) @@ -3764,6 +3793,7 @@ def queueManageIt(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSort 'issue': issue, 'queueid': oi['id'], 'volume': None, + 'linktype': oi['link_type'], 'year': year, 'size': oi['size'].strip(), 'comicid': oi['ComicID'], @@ -3772,6 +3802,7 @@ def queueManageIt(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSort 'updated_date': oi['updated_date'], 'progress': oi_status}) + #logger.info('o_info: %s' % (resultlist)) if sSearch == "" or sSearch == None: filtered = resultlist[::] @@ -3793,7 +3824,7 @@ def queueManageIt(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSort rows = filtered[iDisplayStart:(iDisplayStart + iDisplayLength)] else: rows = filtered - rows = [[row['comicid'], row['series'], row['size'], row['progress'], row['status'], row['updated_date'], row['queueid'], row['issueid']] for row in rows] + rows = [[row['series'], row['size'], row['progress'], row['status'], row['updated_date'], row['queueid'], row['issueid'], row['comicid'], row['linktype']] for row in rows] #rows = [{'comicid': row['comicid'], 'series': row['series'], 'size': row['size'], 'progress': row['progress'], 'status': row['status'], 'updated_date': row['updated_date']} for row in rows] #logger.info('rows: %s' % rows) return json.dumps({ @@ -6592,6 +6623,10 @@ def config(self): "enable_ddl": helpers.checked(mylar.CONFIG.ENABLE_DDL), "enable_getcomics": helpers.checked(mylar.CONFIG.ENABLE_GETCOMICS), "ddl_prefer_upscaled": helpers.checked(mylar.CONFIG.DDL_PREFER_UPSCALED), + "enable_external_server": helpers.checked(mylar.CONFIG.ENABLE_EXTERNAL_SERVER), + "external_server": mylar.CONFIG.EXTERNAL_SERVER, + "external_username": mylar.CONFIG.EXTERNAL_USERNAME, + "external_apikey": mylar.CONFIG.EXTERNAL_APIKEY, "enable_rss": helpers.checked(mylar.CONFIG.ENABLE_RSS), "rss_checkinterval": mylar.CONFIG.RSS_CHECKINTERVAL, "rss_last": rss_sclast, @@ -7079,7 +7114,7 @@ def configUpdate(self, **kwargs): 'prowl_enabled', 'prowl_onsnatch', 'pushover_enabled', 'pushover_onsnatch', 'pushover_image', 'mattermost_enabled', 'mattermost_onsnatch', 'boxcar_enabled', 'boxcar_onsnatch', 'pushbullet_enabled', 'pushbullet_onsnatch', 'telegram_enabled', 'telegram_onsnatch', 'telegram_image', 'discord_enabled', 'discord_onsnatch', 'slack_enabled', 'slack_onsnatch', 'email_enabled', 'email_enc', 'email_ongrab', 'email_onpost', 'gotify_enabled', 'gotify_server_url', 'gotify_token', 'gotify_onsnatch', 'opds_enable', 'opds_authentication', 'opds_metainfo', 'opds_pagesize', 'enable_ddl', - 'enable_getcomics', 'ddl_prefer_upscaled', 'deluge_pause'] #enable_public + 'enable_getcomics', 'enable_external_server', 'ddl_prefer_upscaled', 'deluge_pause'] #enable_public for checked_config in checked_configs: if checked_config not in kwargs: @@ -7200,8 +7235,11 @@ def SABtest(self, sabhost=None, sabusername=None, sabpassword=None, sabapikey=No try: v = requests.get(querysab, params={'mode': 'version'}, verify=verify) if str(v.status_code) == '200': - logger.fdebug('sabnzbd version: %s' % v.content) - version = v.text + try: + version = v.json()['version'] + except Exception as e: + version = v.text.strip() + logger.fdebug('sabnzbd version: %s' % version) r = requests.get(querysab, params=payload, verify=verify) except Exception as e: logger.warn('Error fetching data from %s: %s' % (querysab, e)) @@ -7218,8 +7256,11 @@ def SABtest(self, sabhost=None, sabusername=None, sabpassword=None, sabapikey=No try: v = requests.get(querysab, params={'mode': 'version'}, verify=verify) if str(v.status_code) == '200': - logger.fdebug('sabnzbd version: %s' % v.text) - version = v.text + try: + version = v.json()['version'] + except Exception as e: + version = v.text.strip() + logger.fdebug('sabnzbd version: %s' % version) r = requests.get(querysab, params=payload, verify=verify) except Exception as e: logger.warn('Error fetching data from %s: %s' % (sabhost, e)) @@ -8340,7 +8381,14 @@ def check_ActiveDDL(self): if active['filename'] is not None: # if this is resumed, we need to use the resume value which holds the filesize of the resume filelocation = os.path.join(mylar.CONFIG.DDL_LOCATION, active['filename']) - #logger.fdebug('checking file existance: %s' % filelocation) + #logger.fdebug('b4-checking file existance: %s' % filelocation) + if all(['External' in active['site'], active['link_type'] == 'DDL-Ext']): + filelocation = active['tmp_filename'] + elif 'DDL' in active['site'] and any([active['link_type'] == 'GC-Mega', active['link_type'] == 'GC-Pixel']): + filelocation = active['tmp_filename'] + else: + filelocation = os.path.join(mylar.CONFIG.DDL_LOCATION, active['filename']) + #logger.fdebug('after-checking file existance: %s' % filelocation) if os.path.exists(filelocation) is True: filesize = os.stat(filelocation).st_size #logger.fdebug('filesize: %s / remote: %s' % (filesize, active['remote_filesize'])) @@ -9011,7 +9059,7 @@ def editDetails(self, comicid, location=None, booktype=None, imageloaded=False, else: comicImage = None - if all([comlocation is None, booktype is None]): + if any([comlocation is None, booktype is None]): #this recreates the diretory structure if results['ComicVersion'] is not None: if results['ComicVersion'].isdigit(): diff --git a/requirements.txt b/requirements.txt index b4eec277..353ff993 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,13 +6,13 @@ #### ESSENTIAL LIBRARIES FOR MAIN FUNCTIONALITY #### APScheduler>=3.6.3 beautifulsoup4>=4.8.2 -cfscrape>=2.0.8 cheroot>=8.2.1 CherryPy>=18.5.0 configparser>=4.0.2 feedparser>=5.2.1 Mako>=1.1.0 natsort>=3.5.2 +packaging>=22.0 Pillow>=10.1 portend>=2.6 pystun>=0.1.0 From 85199b066abb560ac76062d9ad1b32d28530efb0 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Tue, 16 Jan 2024 00:15:43 -0500 Subject: [PATCH 02/32] restore cfscrape for the time being (will be removed shortly) --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 353ff993..599047f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ #### ESSENTIAL LIBRARIES FOR MAIN FUNCTIONALITY #### APScheduler>=3.6.3 beautifulsoup4>=4.8.2 +cfscrape>=2.0.8 cheroot>=8.2.1 CherryPy>=18.5.0 configparser>=4.0.2 From ffded2cfab779ee11dda10bd84f188b83f6a403b Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Tue, 16 Jan 2024 00:27:38 -0500 Subject: [PATCH 03/32] add pycryptodome to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 599047f3..8fc2c630 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ natsort>=3.5.2 packaging>=22.0 Pillow>=10.1 portend>=2.6 +pycryptodome>=3.19.0 pystun>=0.1.0 pytz>=2019.3 rarfile>=4.0 From b4a725d1212a16d8c2e2789c831f0870328b53c7 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Tue, 16 Jan 2024 00:33:47 -0500 Subject: [PATCH 04/32] FIX: tenacity add to requirements too --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8fc2c630..e971b149 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,7 @@ rarfile>=4.0 requests[socks]>=2.22.0 simplejson>=3.17.0 six>=1.13.0 +tenacity>=8.2.3 tzlocal>=2.0.0 urllib3<2 user_agent2>=2021.12.11 From c3809df800937ddaf4e9f82532ef9aa6382812b6 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Tue, 16 Jan 2024 07:56:08 -0500 Subject: [PATCH 05/32] FIX: invalid reference to dict item --- mylar/search_filer.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mylar/search_filer.py b/mylar/search_filer.py index 25e873b3..8819f7da 100644 --- a/mylar/search_filer.py +++ b/mylar/search_filer.py @@ -60,9 +60,13 @@ def _process_entry(self, entry, is_info): chktpb = is_info['chktpb'] provider_stat = is_info['provider_stat'] + try: + pack = entry['pack'] + except Exception: + pack = False + alt_match = False #logger.fdebug('entry: %s' % (entry,)) - # brief match here against 32p since it returns the direct issue number logger.fdebug("checking search result: %s" % entry['title']) # some nzbsites feel that comics don't deserve a nice regex to strip @@ -420,7 +424,7 @@ def _process_entry(self, entry, is_info): ' is not necessary: %s' % cleantitle ) # only send it to parser if it's not a DDL + pack (already parsed) - if entry['pack'] is True and 'DDL' in entry['site']: + if pack is True and 'DDL' in entry['site']: logger.fdebug('parsing pack...') ffc = filechecker.FileChecker() dnr = ffc.dynamic_replace(entry['series']) @@ -670,7 +674,7 @@ def _process_entry(self, entry, is_info): elif UseFuzzy == "1": yearmatch = True - if yearmatch is False and entry['pack'] is False: + if yearmatch is False and pack is False: return None annualize = False @@ -795,12 +799,7 @@ def _process_entry(self, entry, is_info): downloadit = False - try: - pack_test = entry['pack'] - except Exception: - pack_test = False - - if all(['DDL' in nzbprov, pack_test is True]): + if all(['DDL' in nzbprov, pack is True]): logger.fdebug( '[PACK-QUEUE] %s Pack detected for %s.' % (nzbprov, entry['filename']) From 37aba5baba6afeaf28219dc0363dcfff0c45c113 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Wed, 17 Jan 2024 00:22:59 -0500 Subject: [PATCH 06/32] FIX: DDL attempting items in a forever loop --- data/interfaces/default/queue_management.html | 9 ++++-- mylar/getcomics.py | 3 +- mylar/helpers.py | 29 ++++++++++++++----- mylar/search.py | 20 +++++++++++++ mylar/webserve.py | 11 +++---- 5 files changed, 54 insertions(+), 18 deletions(-) diff --git a/data/interfaces/default/queue_management.html b/data/interfaces/default/queue_management.html index 2aab49cc..83b140d5 100755 --- a/data/interfaces/default/queue_management.html +++ b/data/interfaces/default/queue_management.html @@ -267,13 +267,16 @@

HISTORY

var restartline = "('ddl_requeue?mode=restart&id="+String(full[5])+"',$(this));" var resumeline = "('ddl_requeue?mode=resume&id="+String(full[5])+"',$(this));" var removeline = "('ddl_requeue?mode=remove&id="+String(full[5])+"',$(this));" + tbl_options = ''; if (val == 'Completed' || val == 'Failed' || val == 'Downloading'){ - return 'RestartRemove'; + tbl_options += 'Restart'; } else if (val == 'Incomplete') { - return 'RestartResume'; + tbl_options += 'RestartResume'; } else if (val == 'Queued') { - return 'Start'; + tbl_options += 'Start'; } + tbl_options += 'Remove'; + return tbl_options; } }, ], diff --git a/mylar/getcomics.py b/mylar/getcomics.py index de1deca4..d0b437d9 100644 --- a/mylar/getcomics.py +++ b/mylar/getcomics.py @@ -627,8 +627,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin t_site = re.sub('link', '', lk['title'].lower()).strip() ltf = False if link_type_failure is not None: - logger.fdebug('link_type_failure: %s' % link_type_failure) - if [True for tst in link_type_failure if t_site.lower() in tst.lower()]: + if [True for tst in link_type_failure if t_site[:4].lower() in tst.lower()]: logger.fdebug('[REDO-FAILURE-DETECTION] detected previous invalid link for %s - ignoring this result' ' and seeing if anything else can be downloaded.' % t_site) ltf = True diff --git a/mylar/helpers.py b/mylar/helpers.py index 4182e060..bce6827a 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -3275,6 +3275,7 @@ def latestissue_update(): def ddl_downloader(queue): myDB = db.DBConnection() + link_type_failure = {} while True: if mylar.DDL_LOCK is True: time.sleep(5) @@ -3290,9 +3291,11 @@ def ddl_downloader(queue): mylar.DDL_QUEUED.append(item['id']) try: - link_type_failure = item['link_type_failure'] - except Exception as e: - link_type_failure = [] + link_type_failure[item['id']].append(item['link_type_failure']) + except Exception: + pass + + #logger.info('[%s] link_type_failure: %s' % (item['id'], link_type_failure)) logger.info('Now loading request from DDL queue: %s' % item['series']) @@ -3361,6 +3364,7 @@ def ddl_downloader(queue): #logger.fdebug('mylar.ddl_queued: %s' % mylar.DDL_QUEUED) mylar.DDL_QUEUED.remove(item['id']) + link_type_failure.pop(item['id']) #logger.fdebug('before-pack_issueids: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) pck_cnt = 0 if item['comicinfo'][0]['pack'] is True: @@ -3381,18 +3385,23 @@ def ddl_downloader(queue): else: try: ltf = ddzstat['links_exhausted'] - except Exception as e: + except KeyError: logger.info('[Status: %s] Failed to download item from %s : %s ' % (ddzstat['success'], item['link_type'], ddzstat)) - link_type_failure.append(item['link_type']) + try: + link_type_failure[item['id']].append(item['link_type']) + except KeyError: + link_type_failure[item['id']] = [item['link_type']] + logger.fdebug('[%s] link_type_failure: %s' % (item['id'], link_type_failure)) ggc = getcomics.GC() - ggc.parse_downloadresults(item['id'], item['mainlink'], item['comicinfo'], item['packinfo'], link_type_failure) + ggc.parse_downloadresults(item['id'], item['mainlink'], item['comicinfo'], item['packinfo'], link_type_failure[item['id']]) else: - logger.info('[REDO] Exhausted all available links [%s] for issueid %s and was not able to download anything' % (link_type_failure, item['issueid'])) + logger.info('[REDO] Exhausted all available links [%s] for issueid %s and was not able to download anything' % (link_type_failure[item['id']], item['issueid'])) nval = {'status': 'Failed', 'updated_date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M')} myDB.upsert('ddl_info', nval, ctrlval) #undo all snatched items, to previous status via item['id'] - this will be set to Skipped currently regardless of previous status reverse_the_pack_snatch(item['id'], item['comicid']) + link_type_failure.pop(item['id']) else: time.sleep(5) @@ -3471,7 +3480,11 @@ def search_queue(queue): 'ddl': False, 'download_info': None}) else: - ss_queue = mylar.search.searchforissue(item['issueid']) + try: + manual = item['manual'] + except Exception as e: + manual = False + ss_queue = mylar.search.searchforissue(item['issueid'], manual=manual) time.sleep(5) #arbitrary sleep to let the process attempt to finish pp'ing if mylar.SEARCHLOCK is True: diff --git a/mylar/search.py b/mylar/search.py index 7537993e..cc0f0bfd 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -2337,6 +2337,26 @@ def searchforissue(issueid=None, new=False, rsschecker=None, manual=False): mylar.SEARCHLOCK = False return + #if it's not manually initiated, make sure it's not already downloaded/snatched. + if not manual: + checkit = searchforissue_checker( + result['IssueID'], + result['ReleaseDate'], + result['IssueDate'], + result['DigitalDate'], + {'ComicName': result['ComicName'], + 'Issue_Number': result['Issue_Number'], + 'ComicID': result['ComicID'] + } + ) + if checkit['status'] is False: + logger.fdebug( + 'Issue is already in a Downloaded / Snatched status. If this is' + ' still wanted, perform a Manual search or mark issue as Skipped' + ' or Wanted.' + ) + return + allow_packs = False ComicID = result['ComicID'] if smode == 'story_arc': diff --git a/mylar/webserve.py b/mylar/webserve.py index 4d06f9fd..627cfc30 100644 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -2580,12 +2580,13 @@ def queueissue(self, mode, ComicName=None, ComicID=None, ComicYear=None, ComicIs logger.info('Marking %s : %s as wanted...' % (ComicName, ComicIssue)) myDB.upsert("annuals", newStatus, controlValueDict) moduletype = '[WANTED-SEARCH]' - passinfo = {'issueid': IssueID, - 'comicname': ComicName, - 'seriesyear': SeriesYear, - 'comicid': ComicID, + passinfo = {'issueid': IssueID, + 'comicname': ComicName, + 'seriesyear': SeriesYear, + 'comicid': ComicID, 'issuenumber': ComicIssue, - 'booktype': BookType} + 'booktype': BookType, + 'manual': manualsearch} if mode == 'want': From 0ab86a7c76950529bd38f286d5165ba8aa127941 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Wed, 17 Jan 2024 02:42:10 -0500 Subject: [PATCH 07/32] various fixes (see notes) --- mylar/cv.py | 5 ++++- mylar/getimage.py | 24 +++++++++++++++++++++++- mylar/helpers.py | 33 +++++++++++++++++++++++---------- mylar/search.py | 28 ++++++++++++++++++---------- mylar/updater.py | 5 ++++- mylar/webserve.py | 39 +++++++++++++++++++++++++++++++++++---- 6 files changed, 107 insertions(+), 27 deletions(-) diff --git a/mylar/cv.py b/mylar/cv.py index 5ff3d09c..2afd4356 100755 --- a/mylar/cv.py +++ b/mylar/cv.py @@ -49,7 +49,10 @@ def pulldetails(comicid, rtype, issueid=None, offset=1, arclist=None, comicidlis PULLURL = mylar.CVURL + str(cv_rtype) + '/?api_key=' + str(comicapi) + '&format=xml&' + str(searchset) + '&offset=' + str(offset) elif any([rtype == 'image', rtype == 'firstissue', rtype == 'imprints_first']): #this is used ONLY for CV_ONLY - PULLURL = mylar.CVURL + 'issues/?api_key=' + str(comicapi) + '&format=xml&filter=id:' + str(issueid) + '&field_list=cover_date,store_date,image' + if issueid: + PULLURL = mylar.CVURL + 'issues/?api_key=' + str(comicapi) + '&format=xml&filter=id:' + str(issueid) + '&field_list=cover_date,store_date,image' + else: + PULLURL = mylar.CVURL + 'volume/' + str(comicid) + '/?api_key=' + str(comicapi) + '&format=xml' + '&field_list=image' elif rtype == 'storyarc': PULLURL = mylar.CVURL + 'story_arcs/?api_key=' + str(comicapi) + '&format=xml&filter=name:' + str(issueid) + '&field_list=cover_date' elif rtype == 'comicyears': diff --git a/mylar/getimage.py b/mylar/getimage.py index 254b8427..234eebcc 100644 --- a/mylar/getimage.py +++ b/mylar/getimage.py @@ -18,6 +18,7 @@ import requests import zipfile from io import BytesIO +from pathlib import Path try: from PIL import Image @@ -214,7 +215,28 @@ def load_image(filename, resize=600): # used to load an image from file for display using the getimage method (w/out extracting) ie. series detail cover page with open(filename, 'rb') as i: imagefile = i.read() - img = Image.open( BytesIO( imagefile) ) + try: + img = Image.open( BytesIO( imagefile) ) + except Exception as e: + if filename.startswith(mylar.CONFIG.CACHE_DIR): + logger.warn('possible corrupt image - cannot open/retrieve properly. Deleteing existing entry and redownloading.') + comicid = Path(filename).stem + fullcomicid = '4050-' + str(comicid) + imageurl = mylar.cv.getComic(fullcomicid, 'image') + coverchk = helpers.getImage(comicid, imageurl['image'], overwrite=True) + if coverchk['status'] == 'retry': + coverchk = helpers.getImage(comicid, imageurl['image_alt'], overwrite=True) + if coverchk['status'] == 'success': + logger.info('successfully retrieved new image...') + with open(filename, 'rb') as i: + imagefile = i.read() + try: + img = Image.open( BytesIO( imagefile) ) + except Exception as e: + #maybe force-load here a random image of donkeys in a field or somethin + return + else: + return imdata = scale_image(img, "JPEG", resize) try: ComicImage = str(base64.b64encode(imdata), 'utf-8') diff --git a/mylar/helpers.py b/mylar/helpers.py index bce6827a..24161cda 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -3364,15 +3364,22 @@ def ddl_downloader(queue): #logger.fdebug('mylar.ddl_queued: %s' % mylar.DDL_QUEUED) mylar.DDL_QUEUED.remove(item['id']) - link_type_failure.pop(item['id']) - #logger.fdebug('before-pack_issueids: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) - pck_cnt = 0 - if item['comicinfo'][0]['pack'] is True: - logger.fdebug('[PACK DETECTION] Attempting to remove issueids from the pack dont-queue list') - for x,y in dict(mylar.PACK_ISSUEIDS_DONT_QUEUE).items(): - if y == item['id']: - pck_cnt +=1 - del mylar.PACK_ISSUEIDS_DONT_QUEUE[x] + try: + link_type_failure.pop(item['id']) + except KeyError: + pass + + try: + #logger.fdebug('before-pack_issueids: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) + pck_cnt = 0 + if item['comicinfo'][0]['pack'] is True: + logger.fdebug('[PACK DETECTION] Attempting to remove issueids from the pack dont-queue list') + for x,y in dict(mylar.PACK_ISSUEIDS_DONT_QUEUE).items(): + if y == item['id']: + pck_cnt +=1 + del mylar.PACK_ISSUEIDS_DONT_QUEUE[x] + except Exception: + pass #logger.fdebug('after-pack_issueids: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) logger.fdebug('Successfully removed %s issueids from pack queue list as download is completed.' % pck_cnt) @@ -4245,7 +4252,7 @@ def sizeof_fmt(num, suffix='B'): num /= 1024.0 return "%.1f%s%s" % (num, 'Yi', suffix) -def getImage(comicid, url, issueid=None, thumbnail_path=None, apicall=False): +def getImage(comicid, url, issueid=None, thumbnail_path=None, apicall=False, overwrite=False): if thumbnail_path is None: if os.path.exists(mylar.CONFIG.CACHE_DIR): @@ -4292,6 +4299,12 @@ def getImage(comicid, url, issueid=None, thumbnail_path=None, apicall=False): logger.warn('Unable to download image from CV URL link: %s [Status Code returned: %s]' % (url, statuscode)) coversize = 0 else: + if os.path.exists(coverfile) and overwrite: + try: + os.remove(coverfile) + except Exception: + pass + with open(coverfile, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks diff --git a/mylar/search.py b/mylar/search.py index cc0f0bfd..c2a73c6f 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -3187,7 +3187,15 @@ def searcher( ggc = getcomics.GC(issueid=tmp_issueid, comicid=ComicID) ggc.loadsite(nzbid, link) ddl_it = ggc.parse_downloadresults(nzbid, link, comicinfo, pack_info) - tnzbprov = ddl_it['site'] + tnzbprov = nzbprov + if ddl_it['success'] is True: + logger.info( + '[%s] Successfully snatched %s from DDL site. It is currently being queued' + ' to download in position %s' % (tnzbprov, nzbname, mylar.DDL_QUEUE.qsize()) + ) + else: + logger.info('[%s] Failed to retrieve %s from the DDL site.' % (tnzbprov, nzbname)) + return "ddl-fail" else: cinfo = {'id': nzbid, 'series': comicinfo[0]['ComicName'], @@ -3203,16 +3211,16 @@ def searcher( meganz = exs.MegaNZ(provider_stat=provider_stat) ddl_it = meganz.queue_the_download(cinfo, comicinfo, pack_info) - tnzbprov = 'Mega' + tnzbprov = 'DDL(External)' - if ddl_it['success'] is True: - logger.info( - '[%s] Successfully snatched %s from DDL site. It is currently being queued' - ' to download in position %s' % (tnzbprov, nzbname, mylar.DDL_QUEUE.qsize()) - ) - else: - logger.info('[%s] Failed to retrieve %s from the DDL site.' % (tnzbprov, nzbname)) - return "ddl-fail" + if ddl_it['success'] is True: + logger.info( + '[%s] Successfully snatched %s from DDL site. It is currently being queued' + ' to download in position %s' % (tnzbprov, nzbname, mylar.DDL_QUEUE.qsize()) + ) + else: + logger.info('[%s] Failed to retrieve %s from the DDL site.' % (tnzbprov, nzbname)) + return "ddl-fail" sent_to = "is downloading it directly via %s" % tnzbprov diff --git a/mylar/updater.py b/mylar/updater.py index 355603bd..18f0038d 100755 --- a/mylar/updater.py +++ b/mylar/updater.py @@ -2224,7 +2224,10 @@ def watchlist_updater(calledfrom=None, sched=False): '[BACKFILL-UPDATE] [%s] series need to be updated due to previous' ' failures: %s' % (len(prev_failed_updates), prev_failed_updates) ) - to_check = dict(to_check, **prev_failed_updates) + try: + to_check = dict(to_check, **prev_failed_updates) + except Exception as e: + to_check = prev_failed_updates #to_check.extend(prev_failed_updates) else: logger.info( diff --git a/mylar/webserve.py b/mylar/webserve.py index 627cfc30..a84d0d4f 100644 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -3414,7 +3414,11 @@ def loadupcoming(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSortD else: reverse_order = False - final_filtered = sorted(filtered, key=itemgetter(sortcolumn), reverse=reverse_order) + try: + final_filtered = sorted(filtered, key=itemgetter(sortcolumn), reverse=reverse_order) + except Exception as e: + final_filtered = sorted(filtered, key=itemgetter(0), reverse=reverse_order) + #filtered = sorted(issues_tmp, key=lambda x: x if isinstance(x[0], str) else "", reverse=True) if iDisplayLength == -1: @@ -3530,6 +3534,19 @@ def ddl_requeue(self, mode, id=None, issueid=None): itemlist = [] for x in items: + OneOff = False + comic = myDB.selectone( + "SELECT * from comics WHERE ComicID=? AND ComicName != 'None'", + [x['comicid']], + ).fetchone() + if comic is None: + comic = myDB.selectone( + "SELECT * from storyarcs WHERE IssueID=?", + [x['issueid']], + ).fetchone() + if comic is None: + OneOff = True + itemlist.append({'link': x['link'], 'mainlink': x['mainlink'], 'series': x['series'], @@ -3538,7 +3555,7 @@ def ddl_requeue(self, mode, id=None, issueid=None): 'size': x['size'], 'link_type': x['link_type'], 'pack': x['pack'], - 'oneoff': x['oneoff'], + 'oneoff': OneOff, 'filename': x['filename'], 'remote_filesize': x['remote_filesize'], 'comicid': x['comicid'], @@ -3550,11 +3567,25 @@ def ddl_requeue(self, mode, id=None, issueid=None): for item in itemlist: seriesname = item['series'] seriessize = item['size'] - if all([mylar.CONFIG.DDL_AUTORESUME is True, mode == 'resume', item['status'] != 'Completed']): + if any( + [ + item['link_type'] is None, + item['link_type'] == 'GC-Main', + item['link_type'] == 'GC-Mirror' + ] + ) and all( + [ + mylar.CONFIG.DDL_AUTORESUME is True, + mode == 'resume', + item['status'] != 'Completed' + ] + ): logger.fdebug('Attempting to resume....') try: filesize = os.stat(os.path.join(mylar.CONFIG.DDL_LOCATION, item['filename'])).st_size - except: + except Exception as e: + logger.warn('[DDL-REQUEUE] Unable to retrieve previous filesize (file was deleted/moved maybe).' + ' Resume unavailable - will restart download.') filesize = 0 logger.fdebug('resume set to resume at: %s bytes' % filesize) resume = filesize From 59ec8d392420268498629016d1527213f4449c68 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Wed, 17 Jan 2024 23:23:05 -0500 Subject: [PATCH 08/32] FIX: endless DDL looping when paywall site encountered, FIX: html_cache in cache_dir for GC html files --- mylar/config.py | 16 ++++++++++------ mylar/getcomics.py | 41 ++++++++++++++++++++++++++--------------- mylar/helpers.py | 16 ++++++++++++++++ 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/mylar/config.py b/mylar/config.py index d686a853..8328502e 100644 --- a/mylar/config.py +++ b/mylar/config.py @@ -170,7 +170,7 @@ 'CHECK_GITHUB' : (bool, 'Git', False), 'CHECK_GITHUB_ON_STARTUP' : (bool, 'Git', False), - 'ENFORCE_PERMS': (bool, 'Perms', True), + 'ENFORCE_PERMS': (bool, 'Perms', False), 'CHMOD_DIR': (str, 'Perms', '0777'), 'CHMOD_FILE': (str, 'Perms', '0660'), 'CHOWNER': (str, 'Perms', None), @@ -267,7 +267,7 @@ 'FOLDER_CACHE_LOCATION': (str, 'PostProcess', None), 'PROVIDER_ORDER': (str, 'Providers', None), - 'USENET_RETENTION': (int, 'Providers', 1500), + 'USENET_RETENTION': (int, 'Providers', 3500), 'NZB_DOWNLOADER': (int, 'Client', 0), #0': sabnzbd, #1': nzbget, #2': blackhole 'TORRENT_DOWNLOADER': (int, 'Client', 0), #0': watchfolder, #1': uTorrent, #2': rTorrent, #3': transmission, #4': deluge, #5': qbittorrent @@ -347,7 +347,7 @@ 'CT_NOTES_FORMAT': (str, 'Metatagging', 'Issue ID'), 'CT_SETTINGSPATH': (str, 'Metatagging', None), 'CMTAG_VOLUME': (bool, 'Metatagging', True), - 'CMTAG_START_YEAR_AS_VOLUME': (bool, 'Metatagging', False), + 'CMTAG_START_YEAR_AS_VOLUME': (bool, 'Metatagging', True), 'SETDEFAULTVOLUME': (bool, 'Metatagging', False), 'ENABLE_TORRENTS': (bool, 'Torrents', False), @@ -1104,7 +1104,7 @@ def configure(self, update=False, startup=False): if self.CLEANUP_CACHE is True: logger.fdebug('[Cache Cleanup] Cache Cleanup initiated. Will delete items from cache that are no longer needed.') - cache_types = ['*.nzb', '*.torrent', '*.zip', '*.html', 'mylar_*'] + cache_types = ['*.nzb', '*.torrent', '*.zip', '*.html', 'mylar_*', 'html_cache'] cntr = 0 for x in cache_types: for f in glob.glob(os.path.join(self.CACHE_DIR,x)): @@ -1343,13 +1343,17 @@ def configure(self, update=False, startup=False): if self.ENABLE_DDL: #make sure directory for mega downloads is created... mega_ddl_path = os.path.join(self.DDL_LOCATION, 'mega') + html_cache_path = os.path.join(self.CACHE_DIR, 'html_cache') if not os.path.isdir(mega_ddl_path): try: os.makedirs(mega_ddl_path) - #dcreate = filechecker.validateAndCreateDirectory(mega_ddl_path, create=True) - #if dcreate is False: except Exception as e: logger.error('Unable to create temp download directory [%s] for DDL-External. You will not be able to view the progress of the download.' % mega_ddl_path) + if not os.path.isdir(html_cache_path): + try: + os.makedirs(html_cache_path) + except Exception as e: + logger.error('Unable to create html_cache folder within the cache folder location [%s]. DDL will not work until this is corrected.' % html_cache_path) if len(self.DDL_PRIORITY_ORDER) > 0 and self.DDL_PRIORITY_ORDER != '[]': if type(self.DDL_PRIORITY_ORDER) != list: diff --git a/mylar/getcomics.py b/mylar/getcomics.py index d0b437d9..e14fe1a6 100644 --- a/mylar/getcomics.py +++ b/mylar/getcomics.py @@ -304,18 +304,18 @@ def search(self,is_info=None): return sorted(verified_matches, key=itemgetter('pack'), reverse=False) def loadsite(self, id, link): - self.cookie_receipt() - title = os.path.join(mylar.CONFIG.CACHE_DIR, 'getcomics-' + id) + title = os.path.join(mylar.CONFIG.CACHE_DIR, 'html_cache', 'getcomics-' + id) logger.fdebug('now loading info from local html to resolve via url: %s' % link) + self.cookie_receipt() #logger.fdebug('session cookies: %s' % (self.session.cookies,)) t = self.session.get( link, verify=True, headers=self.headers, stream=True, - timeout=(30,30) + timeout=(30,30) ) with open(title + '.html', 'wb') as f: @@ -524,10 +524,12 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin series = None year = None size = None - title = os.path.join(mylar.CONFIG.CACHE_DIR, 'getcomics-' + id) + title = os.path.join(mylar.CONFIG.CACHE_DIR, 'html_cache', 'getcomics-' + id) + if not os.path.exists(title): logger.fdebug('Unable to locate local cached html file - attempting to retrieve page results again..') self.loadsite(id, mainlink) + soup = BeautifulSoup(open(title + '.html', encoding='utf-8'), 'html.parser') i = 0 @@ -627,21 +629,29 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin t_site = re.sub('link', '', lk['title'].lower()).strip() ltf = False if link_type_failure is not None: - if [True for tst in link_type_failure if t_site[:4].lower() in tst.lower()]: + if [ + True + for tst in link_type_failure + if t_site[:4].lower() in tst.lower() + or all(["main" in tst.lower(), "download" in t_site.lower()]) + or all(["mirror" in tst.lower(), "mirror" in t_site.lower()]) + ]: logger.fdebug('[REDO-FAILURE-DETECTION] detected previous invalid link for %s - ignoring this result' ' and seeing if anything else can be downloaded.' % t_site) ltf = True + if not ltf: - gather_links.append({ - "series": series, - "site": t_site, - "year": year, - "issues": None, - "size": size, - "links": lk['href'], - "pack": comicinfo[0]['pack'] - }) - #logger.fdebug('gather_links so far: %s' % gather_links) + if 'sh.st' not in lk['href']: + gather_links.append({ + "series": series, + "site": t_site, + "year": year, + "issues": None, + "size": size, + "links": lk['href'], + "pack": comicinfo[0]['pack'] + }) + #logger.fdebug('gather_links so far: %s' % gather_links) count_bees +=1 #logger.fdebug('final valid_links: %s' % (valid_links)) @@ -843,6 +853,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin force_title = True if 'sh.st' in link: logger.fdebug('[Paywall-link detected] this is not a valid link') + link_matched = False else: if force_title: series = link['series'] diff --git a/mylar/helpers.py b/mylar/helpers.py index 24161cda..e351124a 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -3381,6 +3381,9 @@ def ddl_downloader(queue): except Exception: pass + # remove html file from cache if it's successful + ddl_cleanup(item['id']) + #logger.fdebug('after-pack_issueids: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) logger.fdebug('Successfully removed %s issueids from pack queue list as download is completed.' % pck_cnt) @@ -3389,6 +3392,7 @@ def ddl_downloader(queue): if ddzstat['filename'] is not None: path = os.path.join(path, ddzstat['filename']) logger.info('File successfully downloaded. Post Processing is not enabled - item retained here: %s' % (path,)) + ddl_cleanup(item['id']) else: try: ltf = ddzstat['links_exhausted'] @@ -3409,9 +3413,21 @@ def ddl_downloader(queue): #undo all snatched items, to previous status via item['id'] - this will be set to Skipped currently regardless of previous status reverse_the_pack_snatch(item['id'], item['comicid']) link_type_failure.pop(item['id']) + ddl_cleanup(item['id']) else: time.sleep(5) +def ddl_cleanup(id): + # remove html file from cache if it's successful + tlnk = 'getcomics-%s.html' % id + try: + os.remove(os.path.join(mylar.CONFIG.CACHE_DIR, 'html_cache', tlnk)) + except Exception as e: + logger.fdebug('[HTML-cleanup] Unable to remove html used for item from html_cache folder.' + ' Manual removal required or set `cleanup_cache=True` in the config.ini to' + ' clean cache items on every startup. If this was a Retry - ignore this.') + + def postprocess_main(queue): while True: if mylar.APILOCK is True: From 39793ee46127cb5aec674eb21c38bbc984516bb6 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Fri, 19 Jan 2024 02:43:59 -0500 Subject: [PATCH 09/32] FIX: DDL would not gather links due to digital typo and incorrect reference to variable --- mylar/getcomics.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mylar/getcomics.py b/mylar/getcomics.py index e14fe1a6..9b8f9248 100644 --- a/mylar/getcomics.py +++ b/mylar/getcomics.py @@ -784,21 +784,21 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin if not link_matched and site_lp == 'mega': sub_site_chk = [y for y in tmp_sites if 'mega' in y] if sub_site_chk: - kk = tmp_links[site_position['SD-Digtal:mega']] + kk = tmp_links[site_position['SD-Digital:mega']] logger.info('[MEGA] SD-Digital preference detected...attempting %s' % kk['series']) link_matched = True elif not link_matched and site_lp == 'pixeldrain': sub_site_chk = [y for y in tmp_sites if 'pixel' in y] if sub_site_chk: - kk = tmp_links[site_position['SD-Digtal:pixeldrain']] + kk = tmp_links[site_position['SD-Digital:pixeldrain']] logger.info('[PixelDrain] SD-Digital preference detected...attempting %s' % kk['series']) link_matched = True elif not link_matched and site_lp == 'mediafire': sub_site_chk = [y for y in tmp_sites if 'mediafire' in y] if sub_site_chk: - kk = tmp_links[site_position['SD-Digtal:mediafire']] + kk = tmp_links[site_position['SD-Digital:mediafire']] logger.info('[mediafire] SD-Digital preference detected...attempting %s' % kk['series']) link_matched = True @@ -812,9 +812,10 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin logger.info('[MIRROR-SERVER] SD-Digital preference detected...attempting %s' % kk['series']) link_matched = True - link = kk - series = link['series'] - #logger.fdebug('link: %s' % link) + if link_matched: + link = kk + series = link['series'] + #logger.fdebug('link: %s' % link) else: if not link_matched and site_lp == 'mega': sub_site_chk = [y for y in tmp_sites if 'mega' in y] From ac22bc4eef463aecc77bce42ecba5318b20fc564 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Sat, 20 Jan 2024 10:15:53 -0500 Subject: [PATCH 10/32] FIX: folder monitor would die on lock, FIX: spamming pack exclusions --- mylar/PostProcessor.py | 4 ++-- mylar/helpers.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index 3eca29f6..18eaa0be 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -3548,9 +3548,9 @@ def run(self): mylar.MONITOR_STATUS = 'Paused' helpers.job_management(write=True) else: - if mylar.API_LOCK is True: + if mylar.APILOCK is True: logger.info('%s Unable to initiate folder monitor as another process is currently using it or using post-processing.' % self.module) - return + return {'status': 'IN PROGRESS'} helpers.job_management(write=True, job='Folder Monitor', current_run=helpers.utctimestamp(), status='Running') mylar.MONITOR_STATUS = 'Running' logger.info('%s Checking folder %s for newly snatched downloads' % (self.module, mylar.CONFIG.CHECK_FOLDER)) diff --git a/mylar/helpers.py b/mylar/helpers.py index e351124a..cf716aa9 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -2605,6 +2605,7 @@ def issue_find_ids(ComicName, ComicID, pack, IssueNumber, pack_id): Int_IssueNumber = issuedigits(IssueNumber) valid = False + ignores = [] for iss in pack_issues: int_iss = issuedigits(str(iss)) for xb in issuelist: @@ -2620,8 +2621,12 @@ def issue_find_ids(ComicName, ComicID, pack, IssueNumber, pack_id): 'pack_id': pack_id}) break else: - logger.info('issue #%s exists in the pack and is already in a Downloaded state. Mark the issue as anything' - 'other than Wanted if you want the pack to be downloaded.' % iss) + ignores.append(iss) + + if ignores: + logger.info('[%s] These issues already exist in the pack and is already in a Downloaded state. Mark the issue as anything' + 'other than Wanted if you want the pack to be downloaded.' % ignores) + if valid: for wv in write_valids: mylar.PACK_ISSUEIDS_DONT_QUEUE[wv['issueid']] = wv['pack_id'] From 80309de30a6e1a3145cfdfdf8272a3d77567bb2e Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Sun, 21 Jan 2024 03:41:14 -0500 Subject: [PATCH 11/32] FIX: subset DDL site(s) to better handle failures/failed marking --- mylar/helpers.py | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/mylar/helpers.py b/mylar/helpers.py index cf716aa9..489c3a8e 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -3375,7 +3375,6 @@ def ddl_downloader(queue): pass try: - #logger.fdebug('before-pack_issueids: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) pck_cnt = 0 if item['comicinfo'][0]['pack'] is True: logger.fdebug('[PACK DETECTION] Attempting to remove issueids from the pack dont-queue list') @@ -3383,15 +3382,13 @@ def ddl_downloader(queue): if y == item['id']: pck_cnt +=1 del mylar.PACK_ISSUEIDS_DONT_QUEUE[x] + logger.fdebug('Successfully removed %s issueids from pack queue list as download is completed.' % pck_cnt) except Exception: pass # remove html file from cache if it's successful ddl_cleanup(item['id']) - #logger.fdebug('after-pack_issueids: %s' % mylar.PACK_ISSUEIDS_DONT_QUEUE) - logger.fdebug('Successfully removed %s issueids from pack queue list as download is completed.' % pck_cnt) - elif all([ddzstat['success'] is True, mylar.CONFIG.POST_PROCESSING is False]): path = ddzstat['path'] if ddzstat['filename'] is not None: @@ -3399,26 +3396,31 @@ def ddl_downloader(queue): logger.info('File successfully downloaded. Post Processing is not enabled - item retained here: %s' % (path,)) ddl_cleanup(item['id']) else: - try: - ltf = ddzstat['links_exhausted'] - except KeyError: - logger.info('[Status: %s] Failed to download item from %s : %s ' % (ddzstat['success'], item['link_type'], ddzstat)) + if item['site'] == 'DDL(GetComics)': try: - link_type_failure[item['id']].append(item['link_type']) + ltf = ddzstat['links_exhausted'] except KeyError: - link_type_failure[item['id']] = [item['link_type']] - logger.fdebug('[%s] link_type_failure: %s' % (item['id'], link_type_failure)) - ggc = getcomics.GC() - ggc.parse_downloadresults(item['id'], item['mainlink'], item['comicinfo'], item['packinfo'], link_type_failure[item['id']]) + logger.info('[Status: %s] Failed to download item from %s : %s ' % (ddzstat['success'], item['link_type'], ddzstat)) + try: + link_type_failure[item['id']].append(item['link_type']) + except KeyError: + link_type_failure[item['id']] = [item['link_type']] + logger.fdebug('[%s] link_type_failure: %s' % (item['id'], link_type_failure)) + ggc = getcomics.GC() + ggc.parse_downloadresults(item['id'], item['mainlink'], item['comicinfo'], item['packinfo'], link_type_failure[item['id']]) + else: + logger.info('[REDO] Exhausted all available links [%s] for issueid %s and was not able to download anything' % (link_type_failure[item['id']], item['issueid'])) + nval = {'status': 'Failed', + 'updated_date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M')} + myDB.upsert('ddl_info', nval, ctrlval) + #undo all snatched items, to previous status via item['id'] - this will be set to Skipped currently regardless of previous status + reverse_the_pack_snatch(item['id'], item['comicid']) + link_type_failure.pop(item['id']) + ddl_cleanup(item['id']) else: - logger.info('[REDO] Exhausted all available links [%s] for issueid %s and was not able to download anything' % (link_type_failure[item['id']], item['issueid'])) - nval = {'status': 'Failed', - 'updated_date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M')} - myDB.upsert('ddl_info', nval, ctrlval) - #undo all snatched items, to previous status via item['id'] - this will be set to Skipped currently regardless of previous status - reverse_the_pack_snatch(item['id'], item['comicid']) - link_type_failure.pop(item['id']) - ddl_cleanup(item['id']) + logger.info('[Status: %s] Failed to download item from %s : %s ' % (ddzstat['success'], item['site'], ddzstat)) + myDB.action('DELETE FROM ddl_info where id=?', [item['id']]) + mylar.search.FailedMark(item['issueid'], item['comicid'], item['id'], ddzstat['filename'], item['site']) else: time.sleep(5) From ecf48e3ec6c75e1b6b922f50389207236b16b61d Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Tue, 23 Jan 2024 00:52:57 -0500 Subject: [PATCH 12/32] FIX: DDL would error when determining available links in some cases --- mylar/getcomics.py | 62 +++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/mylar/getcomics.py b/mylar/getcomics.py index 9b8f9248..a84b2561 100644 --- a/mylar/getcomics.py +++ b/mylar/getcomics.py @@ -734,7 +734,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin logger.fdebug('priority ddl enabled - checking %s' % site_lp) if site_check: #any([('HD-Upscaled', 'SD-Digital', 'HD-Digital') in tmp_sites]): if mylar.CONFIG.DDL_PREFER_UPSCALED: - if site_lp == 'mega': + if not link_matched and site_lp == 'mega': sub_site_chk = [y for y in tmp_sites if 'mega' in y] if sub_site_chk: if any('HD-Upscaled' in ssc for ssc in sub_site_chk): @@ -780,37 +780,37 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin kk = tmp_links[site_position['HD-Digital:download now']] logger.info('[MAIN-SERVER] HD-Digital preference detected...attempting %s' % kk['series']) link_matched = True - else: - if not link_matched and site_lp == 'mega': - sub_site_chk = [y for y in tmp_sites if 'mega' in y] - if sub_site_chk: - kk = tmp_links[site_position['SD-Digital:mega']] - logger.info('[MEGA] SD-Digital preference detected...attempting %s' % kk['series']) - link_matched = True - elif not link_matched and site_lp == 'pixeldrain': - sub_site_chk = [y for y in tmp_sites if 'pixel' in y] - if sub_site_chk: - kk = tmp_links[site_position['SD-Digital:pixeldrain']] - logger.info('[PixelDrain] SD-Digital preference detected...attempting %s' % kk['series']) - link_matched = True - - elif not link_matched and site_lp == 'mediafire': - sub_site_chk = [y for y in tmp_sites if 'mediafire' in y] - if sub_site_chk: - kk = tmp_links[site_position['SD-Digital:mediafire']] - logger.info('[mediafire] SD-Digital preference detected...attempting %s' % kk['series']) - link_matched = True - - elif not link_matched and site_lp == 'main': - try: - kk = tmp_links[site_position['SD-Digital:download now']] - logger.info('[MAIN-SERVER] SD-Digital preference detected...attempting %s' % kk['series']) - link_matched = True - except Exception as e: - kk = tmp_links[site_position['SD-Digital:mirror download']] - logger.info('[MIRROR-SERVER] SD-Digital preference detected...attempting %s' % kk['series']) - link_matched = True + if not link_matched and site_lp == 'mega': + sub_site_chk = [y for y in tmp_sites if 'mega' in y] + if sub_site_chk: + kk = tmp_links[site_position['SD-Digital:mega']] + logger.info('[MEGA] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + + elif not link_matched and site_lp == 'pixeldrain': + sub_site_chk = [y for y in tmp_sites if 'pixel' in y] + if sub_site_chk: + kk = tmp_links[site_position['SD-Digital:pixeldrain']] + logger.info('[PixelDrain] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + + elif not link_matched and site_lp == 'mediafire': + sub_site_chk = [y for y in tmp_sites if 'mediafire' in y] + if sub_site_chk: + kk = tmp_links[site_position['SD-Digital:mediafire']] + logger.info('[mediafire] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + + elif not link_matched and site_lp == 'main': + try: + kk = tmp_links[site_position['SD-Digital:download now']] + logger.info('[MAIN-SERVER] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + except Exception as e: + kk = tmp_links[site_position['SD-Digital:mirror download']] + logger.info('[MIRROR-SERVER] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True if link_matched: link = kk From f860f86287b8fac5ede0e9620ed9c7ccdba7fc02 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Tue, 23 Jan 2024 02:11:10 -0500 Subject: [PATCH 13/32] FIX: series.json schema update to v1.0.2, FIX: proper reload of issues table on regeneration of json --- data/interfaces/default/comicdetails_update.html | 2 +- mylar/series_metadata.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/interfaces/default/comicdetails_update.html b/data/interfaces/default/comicdetails_update.html index 3764bde4..7a31c2d7 100755 --- a/data/interfaces/default/comicdetails_update.html +++ b/data/interfaces/default/comicdetails_update.html @@ -117,7 +117,7 @@

- + series.json series.json diff --git a/mylar/series_metadata.py b/mylar/series_metadata.py index 8e94bb43..270225e4 100644 --- a/mylar/series_metadata.py +++ b/mylar/series_metadata.py @@ -161,7 +161,7 @@ def update_metadata(self): c_image = comic metadata = {} - metadata['version'] = '1.0.1' + metadata['version'] = '1.0.2' metadata['metadata'] = ( {'type': 'comicSeries', 'publisher': comic['ComicPublisher'], @@ -175,7 +175,7 @@ def update_metadata(self): 'booktype': booktype, 'age_rating': comic['AgeRating'], 'collects': clean_issue_list, - 'ComicImage': comic['ComicImageURL'], + 'comic_image': comic['ComicImageURL'], 'total_issues': comic['Total'], 'publication_run': comic['ComicPublished'], 'status': seriesStatus} From f5f08499bb718a3459ec8dd16d8afb26b3bed705 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Thu, 25 Jan 2024 22:28:21 -0500 Subject: [PATCH 14/32] FIX: feedparser would blow up when parsing non-digital dates in some cases --- mylar/search.py | 8 ++++++-- mylar/search_filer.py | 10 ++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/mylar/search.py b/mylar/search.py index c2a73c6f..899310e5 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -2339,13 +2339,17 @@ def searchforissue(issueid=None, new=False, rsschecker=None, manual=False): #if it's not manually initiated, make sure it's not already downloaded/snatched. if not manual: + if smode == 'story_arc': + issnumb = result['IssueNumber'] + else: + issnumb = result['Issue_Number'] checkit = searchforissue_checker( result['IssueID'], result['ReleaseDate'], result['IssueDate'], result['DigitalDate'], {'ComicName': result['ComicName'], - 'Issue_Number': result['Issue_Number'], + 'Issue_Number': issnumb, 'ComicID': result['ComicID'] } ) @@ -2504,7 +2508,7 @@ def searchforissue(issueid=None, new=False, rsschecker=None, manual=False): 'err': str(err), 'err_text': err_text, 'traceback': tracebackline, - 'comicname': comic['ComicName'], + 'comicname': result['ComicName'], 'issuenumber': result['Issue_Number'], #'seriesyear': SeriesYear, 'issueid': result['IssueID'], diff --git a/mylar/search_filer.py b/mylar/search_filer.py index 8819f7da..80c823db 100644 --- a/mylar/search_filer.py +++ b/mylar/search_filer.py @@ -377,12 +377,14 @@ def _process_entry(self, entry, is_info): '[CONV] %s is after store date of %s' % (pubdate, stdate) ) - except Exception: + except Exception as e: # if the above fails, drop down to the integer compare method # as a failsafe. - if ( - digitaldate != '0000-00-00' - and postdate_int >= digitaldate_int + if digitaldate is not None and all( + [ + digitaldate != '0000-00-00', + postdate_int >= digitaldate_int + ] ): logger.fdebug( '%s is after DIGITAL store date of %s' From 70b9bc35443fbc06e26bdb43a659870f7eb09149 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Fri, 26 Jan 2024 01:13:58 -0500 Subject: [PATCH 15/32] more fixes and improvements - see notes --- Mylar.py | 99 ++++++++ ascii_logo.nfo | 6 + data/interfaces/default/base.html | 15 ++ data/interfaces/default/storyarc_detail.html | 2 +- .../default/storyarc_detail.poster.html | 2 +- mylar/PostProcessor.py | 2 +- mylar/__init__.py | 3 +- mylar/api.py | 12 +- mylar/config.py | 27 ++- mylar/filechecker.py | 12 +- mylar/helpers.py | 15 ++ mylar/locg.py | 5 + mylar/mb.py | 12 +- mylar/webserve.py | 213 ++++++++++++++---- 14 files changed, 358 insertions(+), 67 deletions(-) create mode 100644 ascii_logo.nfo diff --git a/Mylar.py b/Mylar.py index 7a116cf8..a8811f4a 100755 --- a/Mylar.py +++ b/Mylar.py @@ -21,9 +21,108 @@ import re import threading import signal +import importlib sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'lib')) +class test_the_requires(object): + + def __init__(self): + if hasattr(sys, 'frozen'): + full_path = os.path.abspath(sys.executable) + else: + full_path = os.path.abspath(__file__) + + prog_dir = os.path.dirname(full_path) + data_dir = prog_dir + if len(sys.argv) > 0: + ddir = [x for x in sys.argv if 'datadir' in sys.argv] + if ddir: + ddir = re.sub('--datadir=', ''.join(ddir)).strip() + data_dir = re.sub('--datadir ', ddir).strip() + + docker = False + d_path = '/proc/self/cgroup' + if os.path.exists('/.dockerenv') or 'KUBERNETES_SERVICE_HOST' in os.environ or os.path.isfile(d_path) and any('docker' in line for line in open(d_path)): + print('[DOCKER-AWARE] Docker installation detected.') + docker = True + + self.req_file_present = True + self.reqfile = os.path.join(data_dir, 'requirements.txt') + + if any([docker, data_dir != prog_dir]) and not os.path.isfile(self.reqfile): + self.reqfile = os.path.join(prog_dir, 'requirements.txt') + + if not os.path.isfile(self.reqfile): + self.req_file_present = False + + self.nfo_file = os.path.join(data_dir, 'ascii_logo.nfo') + if not self.nfo_file: + self.nfo_file = os.path.join(prog_dir, 'ascii_logo.nfo') + + if not self.nfo_file: + print('[WARNING] Unable to load ascii_logo. You\'re missing something cool...') + else: + with open(self.nfo_file, 'r') as f: + for line in f: + print(line.rstrip()) + print(f'{"-":-<60}') + + self.ops = ['==', '>=', '<='] + self.mod_list = {} + self.mappings = {'APScheduler': 'apscheduler', + 'beautifulsoup4': 'bs4', + 'Pillow': 'PIL', + 'pycryptodome': 'Crypto', + 'pystun': 'stun', + 'PySocks': 'socks'} + + def check_it(self): + if not self.req_file_present: + print('[REQUIREMENTS MISSING] Unable to locate requirements.txt in %s. Make sure it exists, or use --data-dir to specify location' % self.reqfile) + sys.exit() + + with open(self.reqfile, 'r') as file: + for line in file.readlines(): + operator = [x for x in self.ops if x in line] + if operator: + operator = ''.join(operator) + lf = line.find(operator) + module_name = line[:lf].strip() + module_version = line[lf+len(operator):].strip() + if module_name == 'requests[socks]': + self.mod_list['requests'] = module_version + module_name = 'PySocks' + self.mod_list[module_name] = {'version': module_version, 'operator': operator} + + failures = {} + for key,value in self.mod_list.items(): + try: + module = key + if key in self.mappings: + module = self.mappings[key] + try: + importlib.import_module(module, package=None) + except Exception: + importlib.import_module(module.lower(), package=None) + except (ModuleNotFoundError) as e: + failures[key] = value + + if failures: + if 'PySocks' in failures: + print('[MODULES UNAVAILABLE] Some modules are missing and may need to be installed via pip before proceeding. PySocks is required only if using proxies.') + else: + print('[MODULES UNAVAILABLE] Required modules are missing and need to be installed via pip before proceeding.') + print('[MODULES UNAVAILABLE] Reinstall each of the listed module(s) below or reinstall the included requirements.txt file') + print('[MODULES UNAVAILABLE] The following modules are missing:') + for modname, modreq in failures.items(): + print('[MODULES UNAVAILABLE] %s %s%s' % (modname, modreq['operator'], modreq['version'])) + if all(['PySocks' in failures, len(failures)>1]) or 'PySocks' not in failures: + sys.exit() + +t = test_the_requires() +t.check_it() + import mylar from mylar import ( diff --git a/ascii_logo.nfo b/ascii_logo.nfo new file mode 100644 index 00000000..c3b5b033 --- /dev/null +++ b/ascii_logo.nfo @@ -0,0 +1,6 @@ + __ _ _____ __ + _| _| _ __ ___ _ _| | __ _ _ _|___ / |_ |_ + (_) | | '_ ` _ \| | | | |/ _` | '__||_ \ | (_) + _ _ _| | | | | | | | |_| | | (_| | | ___) | | |_ _ _ +(_|_|_) | |_| |_| |_|\__, |_|\__,_|_| |____/ | (_|_|_) + |__| |___/ |__| diff --git a/data/interfaces/default/base.html b/data/interfaces/default/base.html index ac230e2c..04381f14 100755 --- a/data/interfaces/default/base.html +++ b/data/interfaces/default/base.html @@ -141,6 +141,9 @@ ${prov}:${st_image}   %endfor +

+ @@ -109,6 +110,7 @@ %endif + %endfor @@ -135,12 +137,14 @@ "destroy": true, "sDom": '<"clear"Af><"clear"lp><"clear">rt<"clear"ip>', "columnDefs": [ - { "orderable": false, "targets": [5, 7, 10] }, - { "visible": false, "targets": [5, 7, 10] }, + { "orderable": false, "targets": [5, 7, 10, 11] }, + { "visible": false, "targets": [5, 7, 10, 11] }, { "type": 'num', "targets": 5 }, { "type": 'num', "targets": 7 }, + { "type": 'num', "targets": 11 }, { "orderData": [ 5, 7 ], "targets": 6 }, { "orderData": 10, "targets": 9 }, + { "orderData": 11, "targets": 3 }, { "order": [[7, 'asc'],[1, 'asc']] } ], "lengthMenu": [[10, 15, 25, 50, 100, 200, -1], [10, 15, 25, 50, 100, 200, 'All' ]], @@ -163,12 +167,14 @@ "destroy": true, "sDom": '<"clear"f><"clear"lp><"clear">rt<"clear"ip>', "columnDefs": [ - { "orderable": false, "targets": [5, 7, 10] }, - { "visible": false, "targets": [5, 7, 10] }, + { "orderable": false, "targets": [5, 7, 10, 11] }, + { "visible": false, "targets": [5, 7, 10, 11] }, { "type": 'num', "targets": 5 }, { "type": 'num', "targets": 7 }, + { "type": 'num', "targets": 11 }, { "orderData": [ 5, 7 ], "targets": 6 }, { "orderData": 10, "targets": 9 }, + { "orderData": 11, "targets": 3 }, { "order": [[7, 'asc'],[1, 'asc']] } ], "lengthMenu": [[10, 15, 25, 50, 100, 200, -1], [10, 15, 25, 50, 100, 200, 'All' ]], diff --git a/mylar/helpers.py b/mylar/helpers.py index 54e57057..564403df 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -1561,6 +1561,7 @@ def havetotals(refreshit=None): "ComicYear": comic['ComicYear'], "ComicImage": comicImage, "LatestIssue": comic['LatestIssue'], + "IntLatestIssue": comic['IntLatestIssue'], "LatestDate": comic['LatestDate'], "ComicVolume": cversion, "ComicPublished": cpub, diff --git a/mylar/webserve.py b/mylar/webserve.py index 16b1b98f..7398e8fb 100644 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -358,6 +358,8 @@ def loadhome(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSortDir_0 filtered.sort(key=lambda x: (x['percent'] is None, x['percent'] == '', x['percent']), reverse=sSortDir_0 == "desc") filtered.sort(key=lambda x: (x['haveissues'] is None, x['haveissues'] == '', x['haveissues']), reverse=sSortDir_0 == "desc") filtered.sort(key=lambda x: (x['percent'] is None, x['percent'] == '', x['percent']), reverse=sSortDir_0 == "desc") + elif sortcolumn == 'LatestIssue': + filtered.sort(key=lambda x: (x['IntLatestIssue'] is None, x['IntLatestIssue'] == '', x['IntLatestIssue']), reverse=sSortDir_0 == "asc") else: filtered.sort(key=lambda x: (x[sortcolumn] is None, x[sortcolumn] == '', x[sortcolumn]), reverse=sSortDir_0 == "desc") if iDisplayLength != -1: @@ -3370,25 +3372,29 @@ def loadupcoming(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSortD if mylar.CONFIG.FAILED_DOWNLOAD_HANDLING is True: statline += " OR Status='Failed'" - iss_query = "SELECT ComicName, Issue_Number, ReleaseDate, Status, ComicID, IssueID, DateAdded from issues WHERE %s" % statline + iss_query = "SELECT ComicName, Issue_Number, Int_IssueNumber, ReleaseDate, Status, ComicID, IssueID, DateAdded from issues WHERE %s" % statline issues = myDB.select(iss_query) arcs = {} if mylar.CONFIG.UPCOMING_STORYARCS is True: - arcs_query = "SELECT Storyarc, StoryArcID, IssueArcID, ComicName, IssueNumber, ReleaseDate, Status, ComicID, IssueID, DateAdded from storyarcs WHERE %s" % statline + arcs_query = "SELECT Storyarc, StoryArcID, IssueArcID, ComicName, IssueNumber, Int_IssueNumber, ReleaseDate, Status, ComicID, IssueID, DateAdded from storyarcs WHERE %s" % statline arclist = myDB.select(arcs_query) for arc in arclist: + arc_int_number = arc['Int_IssueNumber'] + if arc_int_number is None: + arc_int_number = 0 arcs[arc['IssueID']] = {'storyarc': arc['Storyarc'], 'storyarcid': arc['StoryArcID'], 'issuearcid': arc['IssueArcID'], 'comicid': arc['ComicID'], 'comicname': arc['ComicName'], 'issuenumber': arc['IssueNumber'], + 'int_issuenumber': arc_int_number, 'releasedate': arc['ReleaseDate'], 'status': arc['Status'], 'dateadded': arc['DateAdded']} if mylar.CONFIG.ANNUALS_ON: - annuals_query = "SELECT ReleaseComicName as ComicName, Issue_Number as Issue_Number, ReleaseDate, Status, ComicID, IssueID, DateAdded FROM annuals WHERE NOT Deleted AND %s" % statline + annuals_query = "SELECT ReleaseComicName as ComicName, Issue_Number as Issue_Number, Int_IssueNumber, ReleaseDate, Status, ComicID, IssueID, DateAdded FROM annuals WHERE NOT Deleted AND %s" % statline annuals_list = myDB.select(annuals_query) issues += annuals_list @@ -3462,10 +3468,10 @@ def loadupcoming(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSortD watcharc = None try: - filtered.append([row['ComicName'], row['Issue_Number'], row['ReleaseDate'], row['IssueID'], tier, row['ComicID'], row['Status'], storyarc, storyarcid, issuearcid, watcharc]) + filtered.append([row['ComicName'], row['Issue_Number'], row['ReleaseDate'], row['IssueID'], tier, row['ComicID'], row['Status'], storyarc, storyarcid, issuearcid, watcharc, row['Int_IssueNumber']]) except Exception as e: #logger.warn('danger Wil Robinson: %s' % (e,)) - filtered.append([row['ComicName'], row['Issue_Number'], row['ReleaseDate'], row['IssueID'], tier, row['ComicID'], row['Status'], None, None, None, watcharc]) + filtered.append([row['ComicName'], row['Issue_Number'], row['ReleaseDate'], row['IssueID'], tier, row['ComicID'], row['Status'], None, None, None, watcharc, row['Int_IssueNumber']]) if mylar.CONFIG.UPCOMING_STORYARCS is True: for key, ark in arcs.items(): @@ -3514,7 +3520,7 @@ def loadupcoming(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSortD matched = True if matched is True: - filtered.append([ark['comicname'], ark['issuenumber'], ark['releasedate'], key, tier, ark['comicid'], ark['status'], ark['storyarc'], ark['storyarcid'], ark['issuearcid'], "oneoff"]) + filtered.append([ark['comicname'], ark['issuenumber'], ark['releasedate'], key, tier, ark['comicid'], ark['status'], ark['storyarc'], ark['storyarcid'], ark['issuearcid'], "oneoff", ark['int_issuenumber']]) #logger.fdebug('[%s] one-off arcs: %s' % (len(arcs), arcs,)) @@ -3523,7 +3529,7 @@ def loadupcoming(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSortD if iSortCol_0 == '1': sortcolumn = 0 #'comicame' elif iSortCol_0 == '2': - sortcolumn = 1 #'issuenumber' + sortcolumn = 11 #'issuenumber' elif iSortCol_0 == '3': sortcolumn = 2 #'releasedate' elif iSortCol_0 == '4': @@ -3541,8 +3547,6 @@ def loadupcoming(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSortD except Exception as e: final_filtered = sorted(filtered, key=itemgetter(0), reverse=reverse_order) - #filtered = sorted(issues_tmp, key=lambda x: x if isinstance(x[0], str) else "", reverse=True) - if iDisplayLength == -1: rows = final_filtered else: From 8beb2fc7a8ef3f65f7cbcfda58ebe0a1071e95b4 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Tue, 6 Feb 2024 23:53:35 -0500 Subject: [PATCH 19/32] FIX:(#1502) sabnzbd version detection problem on startup re: versioning format --- mylar/config.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mylar/config.py b/mylar/config.py index 3d29e6b3..db2f797a 100644 --- a/mylar/config.py +++ b/mylar/config.py @@ -1429,12 +1429,6 @@ def configure(self, update=False, startup=False): elif self.SAB_PRIORITY == "4": self.SAB_PRIORITY = "Paused" else: self.SAB_PRIORITY = "Default" - if self.SAB_VERSION is not None: - config.set('SABnzbd', 'sab_version', self.SAB_VERSION) - if int(re.sub("[^0-9]", '', self.SAB_VERSION).strip()) < int(re.sub("[^0-9]", '', '0.8.0').strip()) and self.SAB_CLIENT_POST_PROCESSING is True: - logger.warn('Your SABnzbd client is less than 0.8.0, and does not support Completed Download Handling which is enabled. Disabling CDH.') - self.SAB_CLIENT_POST_PROCESSING = False - mylar.USE_WATCHDIR = False mylar.USE_UTORRENT = False mylar.USE_RTORRENT = False From 88090b365c014b5111f70a711ddd753a082cc67a Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Wed, 7 Feb 2024 00:05:45 -0500 Subject: [PATCH 20/32] FIX: Remove some additional information from cleaned ini due to previous config additions --- mylar/carepackage.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mylar/carepackage.py b/mylar/carepackage.py index bc45d0ce..14faedbb 100644 --- a/mylar/carepackage.py +++ b/mylar/carepackage.py @@ -52,6 +52,7 @@ def __init__(self, maintenance=False): ('PUSHBULLET', 'pushbullet_apikey'), ('NMA', 'nma_apikey'), ('TELEGRAM', 'telegram_token'), + ('GOTIFY', 'gotify_token'), ('CV', 'comicvine_api'), ('Seedbox', 'seedbox_user'), ('Seedbox', 'seedbox_pass'), @@ -63,7 +64,9 @@ def __init__(self, maintenance=False): ('AutoSnatch', 'pp_sshport'), ('Email', 'email_password'), ('Email', 'email_user'), - ('DISCORD', 'discord_webhook_url') + ('DISCORD', 'discord_webhook_url'), + ('DDL', 'external_username'), + ('DDL', 'external_apikey'), } self.hostname_list = { ('SABnzbd', 'sab_host'), @@ -78,7 +81,12 @@ def __init__(self, maintenance=False): ('AutoSnatch', 'pp_sshhost'), ('Tablet', 'tab_host'), ('Seedbox', 'seedbox_host'), - ('Email', 'email_server') + ('GOTIFY', 'gotify_server_url'), + ('Email', 'email_server'), + ('DDL', 'external_server'), + ('DDL', 'flaresolverr_url'), + ('DDL', 'http_proxy'), + ('DDL', 'https_proxy'), } def loaders(self): From d3d80597608a0bf5d87b758f30e2e252db6b8a6d Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Wed, 7 Feb 2024 03:50:29 -0500 Subject: [PATCH 21/32] FIX: would fail to set selection when previous failures resulted in main link being used --- mylar/getcomics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mylar/getcomics.py b/mylar/getcomics.py index a84b2561..b3a5c4bd 100644 --- a/mylar/getcomics.py +++ b/mylar/getcomics.py @@ -860,11 +860,12 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin series = link['series'] link_matched = True - dl_selection = link['site'] else: logger.info('No valid items available that I am able to download from. Not downloading...') return {'success': False, 'links_exhausted': link_type_failure} + dl_selection = link['site'] + logger.fdebug( '[%s] Now downloading: %s [%s] / %s ... this can take a while' ' (go get some take-out)...' % (dl_selection, series, year, size) From 7fc9493e580ff4980112da85c4975630137f4403 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Thu, 8 Feb 2024 01:39:09 -0500 Subject: [PATCH 22/32] IMP: Detect and handle CV reusing existing ComicIDs --- data/interfaces/carbon/css/style.css | 17 ++-- .../default/comicdetails_update.html | 53 ++++++++++- data/interfaces/default/css/style.css | 9 +- data/interfaces/default/index-alphaindex.html | 31 +++++-- data/interfaces/default/index.html | 35 +++++--- mylar/__init__.py | 7 +- mylar/cv.py | 27 +++++- mylar/helpers.py | 9 +- mylar/importer.py | 17 ++++ mylar/webserve.py | 87 ++++++++++++++++++- 10 files changed, 256 insertions(+), 36 deletions(-) diff --git a/data/interfaces/carbon/css/style.css b/data/interfaces/carbon/css/style.css index 3dca4abc..eabb6dd6 100644 --- a/data/interfaces/carbon/css/style.css +++ b/data/interfaces/carbon/css/style.css @@ -1376,7 +1376,6 @@ div#artistheader h2 a { vertical-align: middle; } #series_table { - background-color: #FFF; padding: 20px; width: 960px !important; } @@ -1430,19 +1429,16 @@ div#artistheader h2 a { text-align: center; vertical-align: middle; font-size: 12px; - background-color: #353a41; } #series_table td#name { min-width: 290px; text-align: center; vertical-align: middle; - background-color: #353a41; } #series_table td#year { max-width: 25px; text-align: center; vertical-align: middle; - background-color: #353a41; } #series_table td#havepercent, #series_table td#totalcount { @@ -1461,31 +1457,26 @@ div#artistheader h2 a { max-width: 35px; text-align: center; vertical-align: middle; - background-color: #353a41; } #series_table td#issue { max-width: 30px; text-align: center; vertical-align: middle; - background-color: #353a41; } #series_table td#status { max-width: 50px; text-align: center; vertical-align: middle; - background-color: #353a41; } #series_table td#published { max-width: 55px; text-align: center; vertical-align: middle; - background-color: #353a41; } #series_table td#have { max-width: 80px; text-align: center; vertical-align: middle; - background-color: #353a41; } #manageheader { margin-top: 45px; @@ -2601,3 +2592,11 @@ img.mylarload.lastbad { .py_vers { color: #EE3232; } +.cv_row { + display: flex; + text-align: center; +} +.cv_column { + flex: 50%; + text-align: center; +} diff --git a/data/interfaces/default/comicdetails_update.html b/data/interfaces/default/comicdetails_update.html index 7a31c2d7..3ea23230 100755 --- a/data/interfaces/default/comicdetails_update.html +++ b/data/interfaces/default/comicdetails_update.html @@ -6,6 +6,9 @@ %> <%def name="headerIncludes()"> +
Refresh Comic @@ -51,6 +54,7 @@ <%def name="body()"> +


@@ -1397,8 +1401,41 @@

"+obj['message']+"

"); + if (obj['status'] == 'success'){ + $('#ajaxMsg').addClass('success').fadeIn().delay(3000).fadeOut(); + } else { + $('#ajaxMsg').addClass('error').fadeIn().delay(3000).fadeOut(); + } + }); + } - function count_checks(tabletype){ + function count_checks(tabletype){ if (tabletype == 'issues'){ var checks = document.getElementsByName("issueid[]"); cc = document.getElementById("markissue"); @@ -1532,6 +1569,20 @@

ComicVine has decided to remove this volume and has replaced it with an entirely different volume. You have 2 options:
\ +

Remove this volume from your watchlist
(all issues/annuals)
\ +
\ +

\ +
\ +
Retain existing information but no new
information will be available (ie. Paused)
\ +
\ +

'); + $("#cv_removed_dialog").dialog({ + modal: true, + width: "50%", + }); + } $( "#tabs" ).tabs({ cache: false, }); diff --git a/data/interfaces/default/css/style.css b/data/interfaces/default/css/style.css index 9efc2c56..5ed0125c 100755 --- a/data/interfaces/default/css/style.css +++ b/data/interfaces/default/css/style.css @@ -1337,7 +1337,6 @@ div#artistheader h2 a { vertical-align: middle; } #series_table { - background-color: #FFF; padding: 20px; width: 960px !important; } @@ -2481,3 +2480,11 @@ img.mylarload.lastbad { .py_vers { color: #EE3232; } +.cv_row { + display: flex; + text-align: center; +} +.cv_column { + flex: 50%; + text-align: center; +} diff --git a/data/interfaces/default/index-alphaindex.html b/data/interfaces/default/index-alphaindex.html index 2e8c434b..d8333c9d 100755 --- a/data/interfaces/default/index-alphaindex.html +++ b/data/interfaces/default/index-alphaindex.html @@ -47,14 +47,16 @@ comic_percent = comic['percent'] - if comic['Status'] == 'Paused': + if comic['cv_removed'] == 1: + grade = 'U' + elif comic['Status'] == 'Paused': grade = 'T' elif comic['Status'] == 'Loading': grade = 'L' elif comic['Status'] == 'Error': grade = 'X' else: - grade = 'A' + grade = 'Z' comicpub = comic['ComicPublisher'] try: @@ -64,12 +66,14 @@ pass comicname = comic['ComicSortName'] - try: - if len(comic['ComicSortName']) > 55: - comicname = comic['ComicSortName'][:55] + '...' - except: - pass - + if comicname is not None: + try: + if len(comic['ComicSortName']) > 55: + comicname = comic['ComicSortName'][:55] + '...' + except: + pass + else: + comicname = comic['ComicName'] comicline = comicname comictype = comic['Type'] @@ -116,6 +120,17 @@

ComicID Series SizeLink % Status UpdatedStatus Active
+
+ + + + + + + +
CV removed Paused series Failed/Error Loading
+
+ <%def name="headIncludes()"> diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index b8d643bf..4eb69173 100755 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -7,7 +7,7 @@ <%def name="body()"> -
+
@@ -25,6 +25,16 @@
+
+ + + + + + + +
CV removed Paused series Failed/Error Loading
+
<%def name="headIncludes()"> @@ -37,7 +47,6 @@ diff --git a/mylar/__init__.py b/mylar/__init__.py index 3b440e29..e3a3f45d 100644 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -795,7 +795,7 @@ def dbcheck(): except sqlite3.OperationalError: logger.warn('Unable to update readinglist table to new storyarc table format.') - c.execute('CREATE TABLE IF NOT EXISTS comics (ComicID TEXT UNIQUE, ComicName TEXT, ComicSortName TEXT, ComicYear TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, Have INTEGER, Total INTEGER, ComicImage TEXT, FirstImageSize INTEGER, ComicPublisher TEXT, PublisherImprint TEXT, ComicLocation TEXT, ComicPublished TEXT, NewPublish TEXT, LatestIssue TEXT, intLatestIssue INT, LatestDate TEXT, Description TEXT, DescriptionEdit TEXT, QUALalt_vers TEXT, QUALtype TEXT, QUALscanner TEXT, QUALquality TEXT, LastUpdated TEXT, AlternateSearch TEXT, UseFuzzy TEXT, ComicVersion TEXT, SortOrder INTEGER, DetailURL TEXT, ForceContinuing INTEGER, ComicName_Filesafe TEXT, AlternateFileName TEXT, ComicImageURL TEXT, ComicImageALTURL TEXT, DynamicComicName TEXT, AllowPacks TEXT, Type TEXT, Corrected_SeriesYear TEXT, Corrected_Type TEXT, TorrentID_32P TEXT, LatestIssueID TEXT, Collects CLOB, IgnoreType INTEGER, AgeRating TEXT, FilesUpdated TEXT, seriesjsonPresent INT, dirlocked INTEGER)') + c.execute('CREATE TABLE IF NOT EXISTS comics (ComicID TEXT UNIQUE, ComicName TEXT, ComicSortName TEXT, ComicYear TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, Have INTEGER, Total INTEGER, ComicImage TEXT, FirstImageSize INTEGER, ComicPublisher TEXT, PublisherImprint TEXT, ComicLocation TEXT, ComicPublished TEXT, NewPublish TEXT, LatestIssue TEXT, intLatestIssue INT, LatestDate TEXT, Description TEXT, DescriptionEdit TEXT, QUALalt_vers TEXT, QUALtype TEXT, QUALscanner TEXT, QUALquality TEXT, LastUpdated TEXT, AlternateSearch TEXT, UseFuzzy TEXT, ComicVersion TEXT, SortOrder INTEGER, DetailURL TEXT, ForceContinuing INTEGER, ComicName_Filesafe TEXT, AlternateFileName TEXT, ComicImageURL TEXT, ComicImageALTURL TEXT, DynamicComicName TEXT, AllowPacks TEXT, Type TEXT, Corrected_SeriesYear TEXT, Corrected_Type TEXT, TorrentID_32P TEXT, LatestIssueID TEXT, Collects CLOB, IgnoreType INTEGER, AgeRating TEXT, FilesUpdated TEXT, seriesjsonPresent INT, dirlocked INTEGER, cv_removed INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS issues (IssueID TEXT, ComicName TEXT, IssueName TEXT, Issue_Number TEXT, DateAdded TEXT, Status TEXT, Type TEXT, ComicID TEXT, ArtworkURL Text, ReleaseDate TEXT, Location TEXT, IssueDate TEXT, DigitalDate TEXT, Int_IssueNumber INT, ComicSize TEXT, AltIssueNumber TEXT, IssueDate_Edit TEXT, ImageURL TEXT, ImageURL_ALT TEXT, forced_file INT)') c.execute('CREATE TABLE IF NOT EXISTS snatched (IssueID TEXT, ComicName TEXT, Issue_Number TEXT, Size INTEGER, DateAdded TEXT, Status TEXT, FolderName TEXT, ComicID TEXT, Provider TEXT, Hash TEXT, crc TEXT)') c.execute('CREATE TABLE IF NOT EXISTS upcoming (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, IssueDate TEXT, Status TEXT, DisplayComicName TEXT)') @@ -1006,6 +1006,11 @@ def dbcheck(): except sqlite3.OperationalError: c.execute('ALTER TABLE comics ADD COLUMN seriesjsonPresent INT') + try: + c.execute('SELECT cv_removed from comics') + except sqlite3.OperationalError: + c.execute('ALTER TABLE comics ADD COLUMN cv_removed INT') + try: c.execute('SELECT DynamicComicName from comics') if CONFIG.DYNAMIC_UPDATE < 3: diff --git a/mylar/cv.py b/mylar/cv.py index 93866342..5bc565b9 100755 --- a/mylar/cv.py +++ b/mylar/cv.py @@ -15,7 +15,7 @@ import re import time import pytz -from mylar import logger, helpers +from mylar import db, logger, helpers import mylar from bs4 import BeautifulSoup as Soup from xml.parsers.expat import ExpatError @@ -1479,3 +1479,28 @@ def basenum_mapping(ordinal=False): 'xi': '9'} return basenums + +def check_that_biatch(comicid, oldinfo, newinfo): + failures = 0 + if newinfo['ComicName'] is not None: + if newinfo['ComicName'].lower() != oldinfo['comicname'].lower(): + failures +=1 + if newinfo['ComicYear'] is not None: + if newinfo['ComicYear'] != oldinfo['comicyear']: + failures +=1 + if newinfo['ComicPublisher'] is not None: + if newinfo['ComicPublisher'] != oldinfo['publisher']: + failures +=1 + if newinfo['ComicURL'] is not None: + if newinfo['ComicURL'] != oldinfo['detailurl']: + failures +=1 + + if failures > 2: + # if > 50% failure (> 2/4 mismatches) assume removed... + logger.warn('[%s] Detected CV removing existing data for series [%s (%s)] and replacing it with [%s (%s)].' + 'This is a failure for this series and will be paused until fixed manually' % + (failures, oldinfo['comicname'], oldinfo['comicyear'], newinfo['ComicName'], newinfo['ComicYear']) + ) + return True + + return False diff --git a/mylar/helpers.py b/mylar/helpers.py index 564403df..bf0cff15 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -1528,7 +1528,8 @@ def havetotals(refreshit=None): try: cpub = re.sub('(N)', '', comic['ComicPublished']).strip() except Exception as e: - logger.warn('[Error: %s] No Publisher found for %s - you probably want to Refresh the series when you get a chance.' % (e, comic['ComicName'])) + if comic['cv_removed'] == 0: + logger.warn('[Error: %s] No Publisher found for %s - you probably want to Refresh the series when you get a chance.' % (e, comic['ComicName'])) cpub = None comictype = comic['Type'] @@ -1553,6 +1554,9 @@ def havetotals(refreshit=None): else: comicImage = comic['ComicImage'] + #cv_removed: 0 = series is present on CV + # 1 = series has been removed from CV + # 2 = series has been removed from CV but retaining what mylar has in it's db comics.append({"ComicID": comic['ComicID'], "ComicName": comic['ComicName'], @@ -1574,7 +1578,8 @@ def havetotals(refreshit=None): "DateAdded": comic['LastUpdated'], "Type": comic['Type'], "Corrected_Type": comic['Corrected_Type'], - "displaytype": comictype}) + "displaytype": comictype, + "cv_removed": comic['cv_removed']}) return comics def filesafe(comic): diff --git a/mylar/importer.py b/mylar/importer.py index e3350cc1..aa19e1f9 100644 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -113,6 +113,11 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No logger.warn('Error trying to validate/create directory. Aborting this process at this time.') return {'status': 'incomplete'} oldcomversion = dbcomic['ComicVersion'] #store the comicversion and chk if it exists before hammering. + db_check_values = {'comicname': dbcomic['ComicName'], + 'comicyear': dbcomic['ComicYear'], + 'publisher': dbcomic['ComicPublisher'], + 'detailurl': dbcomic['DetailURL'], + 'total_count': dbcomic['Total']} if dbcomic is None or bypass is False: newValueDict = {"ComicName": "Comic ID: %s" % (comicid), @@ -131,6 +136,7 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No aliases = None FirstImageSize = 0 old_description = None + db_check_values = None myDB.upsert("comics", newValueDict, controlValueDict) @@ -159,6 +165,17 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No else: sortname = comic['ComicName'] + if db_check_values is not None: + if comic['ComicURL'] != db_check_values['detailurl']: + logger.warn('[CORRUPT-COMICID-DETECTION-ENABLED] ComicID may have been removed from CV' + ' and replaced with an entirely different series/volume. Checking some values' + ' to make sure before proceeding...' + ) + i_choose_violence = cv.check_that_biatch(comicid, db_check_values, comic) + if i_choose_violence: + myDB.upsert("comics", {'Status': 'Paused', 'cv_removed': 1}, {'ComicID': comicid}) + return {'status': 'incomplete'} + comic['Corrected_Type'] = fixed_type if fixed_type is not None and fixed_type != comic['Type']: logger.info('Forced Comic Type to : %s' % comic['Corrected_Type']) diff --git a/mylar/webserve.py b/mylar/webserve.py index 7398e8fb..116e357d 100644 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -274,11 +274,23 @@ def config_check(self): def home(self, **kwargs): if mylar.START_UP is True: self.config_check() + # pass the proper table colors here + if mylar.CONFIG.INTERFACE == 'default': + legend_colors = {'paused': '#f9cbe6', + 'removed': '#ddd', + 'loading': '#ebf5ff', + 'failed': '#ffdddd'} + else: + legend_colors = {'paused': '#bd915a', + 'removed': '#382f64', + 'loading': '#1c5188', + 'failed': '#641716'} + if mylar.CONFIG.ALPHAINDEX == True: comics = helpers.havetotals() - return serve_template(templatename="index-alphaindex.html", title="Home", comics=comics, alphaindex=mylar.CONFIG.ALPHAINDEX) + return serve_template(templatename="index-alphaindex.html", title="Home", comics=comics, alphaindex=mylar.CONFIG.ALPHAINDEX, legend_colors=legend_colors) else: - return serve_template(templatename="index.html", title="Home") + return serve_template(templatename="index.html", title="Home", legend_colors=legend_colors) home.exposed = True def loadhome(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSortDir_0="desc", sSearch="", **kwargs): @@ -366,7 +378,7 @@ def loadhome(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=5, sSortDir_0 rows = filtered[iDisplayStart:(iDisplayStart + iDisplayLength)] else: rows = filtered - rows = [[row['ComicPublisher'], row['ComicName'], row['ComicYear'], row['LatestIssue'], row['LatestDate'], row['recentstatus'], row['Status'], row['percent'], row['haveissues'], row['totalissues'], row['ComicID'], row['displaytype'], row['ComicVolume']] for row in rows] + rows = [[row['ComicPublisher'], row['ComicName'], row['ComicYear'], row['LatestIssue'], row['LatestDate'], row['recentstatus'], row['Status'], row['percent'], row['haveissues'], row['totalissues'], row['ComicID'], row['displaytype'], row['ComicVolume'], row['cv_removed']] for row in rows] return json.dumps({ 'iTotalDisplayRecords': len(filtered), 'iTotalRecords': len(resultlist), @@ -9315,3 +9327,72 @@ def return_checks(self): return json.dumps(mylar.REQS) return_checks.exposed = True + + def fix_cv_removed(self, comicid, opts, delete_dir=False): + if delete_dir == 1: + delete_dir = True + else: + delete_dir = False + myDB = db.DBConnection() + cc = myDB.selectone('SELECT comicname, comicyear, comiclocation from comics where ComicID=?', [comicid]).fetchone() + comicname = cc['comicname'] + comicyear = cc['comicyear'] + seriesdir = cc['comiclocation'] + + if opts == 'delete': + if not cc: + logger.warn('[CV-REMOVAL-DETECTION] Unable to locate comicid in db - this series does not exist currently..') + return json.dumps({'status': 'failure', 'message': 'Series has already been removed from the watchlist'}) + + myDB.action("DELETE FROM comics WHERE ComicID=?", [comicid]) + myDB.action("DELETE FROM issues WHERE ComicID=?", [comicid]) + if mylar.CONFIG.ANNUALS_ON: + myDB.action("DELETE FROM annuals WHERE ComicID=?", [comicid]) + myDB.action('DELETE from upcoming WHERE ComicID=?', [comicid]) + myDB.action('DELETE from readlist WHERE ComicID=?', [comicid]) + myDB.action('UPDATE weekly SET Status="Skipped" WHERE ComicID=? AND Status="Wanted"', [comicid]) + warnings = 0 + if delete_dir: + logger.fdebug('Remove directory on series removal enabled.') + if seriesdir is not None: + if os.path.exists(seriesdir): + logger.fdebug('Attempting to remove the directory and contents of : %s' % seriesdir) + try: + shutil.rmtree(seriesdir) + except: + logger.warn('Unable to remove directory after removing series from Mylar.') + warnings += 1 + else: + logger.info('Successfully removed directory: %s' % (seriesdir)) + else: + logger.warn('Unable to remove directory as it does not exist in : %s' % seriesdir) + warnings += 1 + else: + logger.warn('Unable to remove directory as it does not exist.') + warnings += 1 + + helpers.ComicSort(sequence='update') + + c_image = os.path.join(mylar.CONFIG.CACHE_DIR, comicid + '.jpg') + if os.path.exists(c_image): + try: + os.remove(c_image) + except Exception as e: + warnings += 1 + logger.warn('[CV-REMOVAL-DETECTION] Unable to remove the image file from the cache (%s).' + ' You may have to delete the file manually' % c_image) + else: + logger.fdebug('image file already removed from cache for %s (%s) - [%s]' % (comicname, comicyear, comicid)) + + linemsg = 'Successfully removed %s (%s) from the watchlist' % (comicname, comicyear) + if warnings > 0: + linemsg += '[%s warnings]' % warnings + + return json.dumps({'status': 'success', 'message': linemsg}) + + else: + myDB.upsert("comics", {'Status': 'Paused', 'cv_removed': 2}, {'ComicID': comicid}) + logger.info('[CV-REMOVAL-DETECTION] Successfully retained %s (%s) and is now in a Paused status' % (comicname, comicyear)) + linemsg = 'Successfully Paused %s (%s) due to CV removal' % (comicname, comicyear) + return json.dumps({'status': 'success', 'message': linemsg}) + fix_cv_removed.exposed = True From 4bbf9ab02fb8b35427e61592f9ee59a61bdd2920 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Fri, 16 Feb 2024 00:45:52 -0500 Subject: [PATCH 23/32] bunch of late night fixes...(#1510)(#1507) and some more --- data/interfaces/default/base.html | 6 +++--- mylar/__init__.py | 2 ++ mylar/filechecker.py | 21 ++++++++++++--------- mylar/helpers.py | 16 ++++++++++++++-- mylar/importer.py | 19 ++++++++++++++++--- mylar/updater.py | 2 +- mylar/webserve.py | 1 + 7 files changed, 49 insertions(+), 18 deletions(-) diff --git a/data/interfaces/default/base.html b/data/interfaces/default/base.html index 04381f14..00762de7 100755 --- a/data/interfaces/default/base.html +++ b/data/interfaces/default/base.html @@ -354,7 +354,7 @@ if (data.status == 'success'){ $('#ajaxMsg').addClass('success').fadeIn().delay(3000).fadeOut(); console.log('data.comicid:'+data.comicid) - if ( (data.tables == 'both' || data.tables == 'tables') && ( document.body.innerHTML.search(data.comicid) || tt.value == 'history' || tt.value == 'search_results') ){ + if ( ( tt.value != "config" ) && (data.tables == 'both' || data.tables == 'tables') && ( document.body.innerHTML.search(data.comicid) || tt.value == "history" || tt.value == "search_results") ){ console.log('reloading table1...'); reload_table(); } @@ -362,7 +362,7 @@ console.log('reloading table2...'); reload_table(); } - if (data.tables == 'both' || data.tables == 'tabs'){ + if ( (data.tables == 'both' || data.tables == 'tabs') && ( tt.value != "config") ) { reload_tabs(); } if( data.comicid == cid && document.getElementById("page_name").value == 'series_detail'){ @@ -424,7 +424,7 @@ var tables = $('table.display').DataTable(); var tt = document.getElementById("page_name"); if(typeof(tt) != 'undefined' && tt != null){ - if (tt.value != "weekly" && tt.value != "import_results" && tt.value != "manage_comics" && tt.value != "manage_issues" && tt.value != "manage_failed" && tt.value != "reading_list" && tt.value != "storyarcs_index" && tt.value != "storyarc_detail") { + if (tt.value != "weekly" && tt.value != "import_results" && tt.value != "manage_comics" && tt.value != "manage_issues" && tt.value != "manage_failed" && tt.value != "reading_list" && tt.value != "storyarcs_index" && tt.value != "storyarc_detail" && tt.value != "config") { // this is required so it doesn't error if on the weekly page // once weekly & other pages are converted to dynamic loading, this if can be removed tables.ajax.reload(null,false); diff --git a/mylar/__init__.py b/mylar/__init__.py index e3a3f45d..234a2d14 100644 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -179,6 +179,8 @@ 'ALPHA', 'OMEGA', 'BLACK', + 'DARK', + 'LIGHT', 'AU', 'AI', 'INH', diff --git a/mylar/filechecker.py b/mylar/filechecker.py index f275b9e8..62c56693 100755 --- a/mylar/filechecker.py +++ b/mylar/filechecker.py @@ -786,15 +786,18 @@ def parseit(self, path, filename, subpath=None): lastissue_label = sf lastissue_mod_position = file_length elif x > 0: - logger.fdebug('I have encountered a decimal issue #: %s' % sf) - possible_issuenumbers.append({'number': sf, - 'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)}) - 'mod_position': self.char_file_position(modfilename, sf, lastmod_position), - 'validcountchk': validcountchk}) + if x == float('inf') and split_file.index(sf, lastissue_position) <= 2: + logger.fdebug('infinity wording detected - position places it within series title boundaries..') + else: + logger.fdebug('I have encountered a decimal issue #: %s' % sf) + possible_issuenumbers.append({'number': sf, + 'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)}) + 'mod_position': self.char_file_position(modfilename, sf, lastmod_position), + 'validcountchk': validcountchk}) - lastissue_position = split_file.index(sf, lastissue_position) - lastissue_label = sf - lastissue_mod_position = file_length + lastissue_position = split_file.index(sf, lastissue_position) + lastissue_label = sf + lastissue_mod_position = file_length else: raise ValueError except ValueError as e: @@ -1501,7 +1504,7 @@ def matchIT(self, series_info): if qmatch_chk is None: qmatch_chk = 'match' if qmatch_chk is not None: - #logger.fdebug('[MATCH: ' + series_info['series_name'] + '] ' + filename) + #logger.fdebug('[%s][MATCH: %s][seriesALT: %s] %s' % (qmatch_chk, seriesalt, series_info['series_name'], filename)) enable_annual = False annual_comicid = None if any(re.sub('[\|\s]','', x.lower()).strip() == re.sub('[\|\s]','', nspace_seriesname.lower()).strip() for x in self.AS_Alt): diff --git a/mylar/helpers.py b/mylar/helpers.py index bf0cff15..2f2d58e4 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -1116,8 +1116,20 @@ def issuedigits(issnum): try: int_issnum = (int(issb4dec) * 1000) + (int(issaftdec) * 10) except ValueError: - #logger.fdebug('This has no issue # for me to get - Either a Graphic Novel or one-shot.') - int_issnum = 999999999999999 + try: + ordtot = 0 + if any(ext == issaftdec.upper() for ext in mylar.ISSUE_EXCEPTIONS): + inu = 0 + while (inu < len(issaftdec)): + ordtot += ord(issaftdec[inu].lower()) #lower-case the letters for simplicty + inu+=1 + int_issnum = (int(issb4dec) * 1000) + ordtot + except Exception as e: + logger.warn('error: %s' % e) + ordtot = 0 + if ordtot == 0: + #logger.error('This has no issue # for me to get - Either a Graphic Novel or one-shot.') + int_issnum = 999999999999999 elif all([ '[' in issnum, ']' in issnum ]): issnum_tmp = issnum.find('[') int_issnum = int(issnum[:issnum_tmp].strip()) * 1000 diff --git a/mylar/importer.py b/mylar/importer.py index aa19e1f9..33de4227 100644 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -1384,9 +1384,22 @@ def updateissuedata(comicid, comicname=None, issued=None, comicIssues=None, call #int_issnum = str(issnum) int_issnum = (int(issb4dec) * 1000) + (int(issaftdec) * 10) except ValueError: - logger.error('This has no issue # for me to get - Either a Graphic Novel or one-shot.') - updater.no_searchresults(comicid) - return {'status': 'failure'} + try: + ordtot = 0 + if any(ext == issaftdec.upper() for ext in mylar.ISSUE_EXCEPTIONS): + logger.fdebug('issue_exception detected..') + inu = 0 + while (inu < len(issaftdec)): + ordtot += ord(issaftdec[inu].lower()) #lower-case the letters for simplicty + inu+=1 + int_issnum = (int(issb4dec) * 1000) + ordtot + except Exception as e: + logger.warn('error: %s' % e) + ordtot = 0 + if ordtot == 0: + logger.error('This has no issue # for me to get - Either a Graphic Novel or one-shot.') + updater.no_searchresults(comicid) + return {'status': 'failure'} elif all([ '[' in issnum, ']' in issnum ]): issnum_tmp = issnum.find('[') int_issnum = int(issnum[:issnum_tmp].strip()) * 1000 diff --git a/mylar/updater.py b/mylar/updater.py index 4852d30e..1f8e9b5a 100755 --- a/mylar/updater.py +++ b/mylar/updater.py @@ -1787,7 +1787,7 @@ def forceRescan(ComicID, archive=None, module=None, recheck=False): if pause_status: issStatus = old_status - logger.fdefbug('[PAUSE_ANNUAL_CHECK_STATUS_CHECK] series is paused, keeping status of %s for issue #%s' % (issStatus, chk['Issue_Number'])) + logger.fdebug('[PAUSE_ANNUAL_CHECK_STATUS_CHECK] series is paused, keeping status of %s for issue #%s' % (issStatus, chk['Issue_Number'])) else: #if old_status == "Skipped": # if mylar.CONFIG.AUTOWANT_ALL: diff --git a/mylar/webserve.py b/mylar/webserve.py index 116e357d..ca296857 100644 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -431,6 +431,7 @@ def comicDetails(self, ComicID, addbyid=0, **kwargs): 'ComicLocation': None, 'AlternateSearch': None, 'AlternateFileName': None, + 'cv_removed': 0, 'DetailURL': 'https://comicvine.com/volume/4050-%s' % ComicID} else: secondary_folders = None From e63b4b496b0b71767ccdee7f4beb1322651a70b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20S=C3=A1nchez?= Date: Mon, 19 Feb 2024 12:49:48 +0100 Subject: [PATCH 24/32] Fixed typo --- data/interfaces/default/comicdetails_update.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/interfaces/default/comicdetails_update.html b/data/interfaces/default/comicdetails_update.html index 3ea23230..88794a1d 100755 --- a/data/interfaces/default/comicdetails_update.html +++ b/data/interfaces/default/comicdetails_update.html @@ -417,7 +417,7 @@

- Use this insetad of CV name during post-processing / renaming + Use this instead of CV name during post-processing / renaming

From 438dbadf303f86767fbfa0d506389d9f2b9312a6 Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 2 Mar 2024 19:59:34 +0000 Subject: [PATCH 25/32] fix casing when invoking updater.dbUpdate --- mylar/series_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mylar/series_metadata.py b/mylar/series_metadata.py index 270225e4..8b234eaa 100644 --- a/mylar/series_metadata.py +++ b/mylar/series_metadata.py @@ -40,7 +40,7 @@ def update_metadata(self): for cid in self.comiclist: if self.refreshSeries is True: - updater.dbupdate(cid, calledfrom='json_api') + updater.dbUpdate(cid, calledfrom='json_api') myDB = db.DBConnection() comic = myDB.selectone('SELECT * FROM comics WHERE ComicID=?', [cid]).fetchone() From 86751d62df54635f4310421e6c44dc5f3e48cf3f Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Sun, 3 Mar 2024 03:25:17 -0500 Subject: [PATCH 26/32] FIXES...FIXES...FIXES..(and a few improvements) --- data/interfaces/default/base.html | 38 +-- .../default/comicdetails_update.html | 6 +- data/interfaces/default/config.html | 49 +--- data/interfaces/default/index.html | 10 +- data/interfaces/default/searchresults.html | 19 +- lib/comictaggerlib/comicvinetalker.py | 4 +- lib/comictaggerlib/main.py | 1 + lib/comictaggerlib/settings.py | 4 + mylar/PostProcessor.py | 13 +- mylar/config.py | 243 +++++++++++++++--- mylar/downloaders/mediafire.py | 7 +- mylar/getcomics.py | 67 +++-- mylar/importer.py | 51 +++- mylar/notifiers.py | 6 +- mylar/rsscheck.py | 54 ++-- mylar/search.py | 72 +----- mylar/search_filer.py | 4 - mylar/webserve.py | 65 +++-- mylar/weeklypull.py | 17 +- 19 files changed, 451 insertions(+), 279 deletions(-) diff --git a/data/interfaces/default/base.html b/data/interfaces/default/base.html index 00762de7..43c43dfc 100755 --- a/data/interfaces/default/base.html +++ b/data/interfaces/default/base.html @@ -123,23 +123,31 @@
<% mylar.PROVIDER_STATUS = {} - for ko, vo in sorted(mylar.CONFIG.PROVIDER_ORDER.items()): - mylar.PROVIDER_STATUS.update({vo : 'success'}) - for kb in mylar.PROVIDER_BLOCKLIST: - if vo == kb['site']: - mylar.PROVIDER_STATUS.update({vo : 'fail'}) - break + prov_fail = False + try: + for ko, vo in sorted(mylar.CONFIG.PROVIDER_ORDER.items()): + mylar.PROVIDER_STATUS.update({vo : 'success'}) + for kb in mylar.PROVIDER_BLOCKLIST: + if vo == kb['site']: + mylar.PROVIDER_STATUS.update({vo : 'fail'}) + break + except Exception: + prov_fail = True %> Providers: - %for prov, stats in sorted(mylar.PROVIDER_STATUS.items()): - <% - if stats == 'success': - st_image = '' - else: - st_image = '' - %> - ${prov}:${st_image}   - %endfor + %if prov_fail: + Unable to determine Provider Status' at this time... + %else: + %for prov, stats in sorted(mylar.PROVIDER_STATUS.items()): + <% + if stats == 'success': + st_image = '' + else: + st_image = '' + %> + ${prov}:${st_image}   + %endfor + %endif
-
-
- NZB.SU -
-
-
- -
-
- - - ( only needed for RSS feed ) -
-
- - -
-
- - -
-
-
- -
-
- DOGNZB -
-
-
- -
-
- - -
-
- - -
-
-
-
Use Experimental Search @@ -3380,8 +3337,8 @@

Search results${ Digital/TPB/GN/HC + +

-
+

@@ -143,6 +155,9 @@

Search results${ } function addseries(comicid, query_id, com_location, booktype){ + //var markupcoming = document.getElementById("markupcoming").checked; + //var markall = document.getElementById("markall").checked; + if(typeof(com_location) === "undefined" || com_location === null){ com_location = null; } @@ -153,7 +168,7 @@

Search results${ $.when($.ajax({ type: "GET", url: "addbyid", - data: { comicid: comicid, query_id: query_id, com_location: com_location, booktype: booktype}, + data: { comicid: comicid, query_id: query_id, com_location: com_location, booktype: booktype}, //, markupcoming: markupcoming, markall: markall}, success: function(response) { //obj = JSON.parse(response); }, diff --git a/lib/comictaggerlib/comicvinetalker.py b/lib/comictaggerlib/comicvinetalker.py index 1e40d4c7..d1d7f8d2 100644 --- a/lib/comictaggerlib/comicvinetalker.py +++ b/lib/comictaggerlib/comicvinetalker.py @@ -21,7 +21,6 @@ import time import datetime import sys -from user_agent2 import generate_user_agent from bs4 import BeautifulSoup @@ -102,8 +101,7 @@ def __init__(self): self.log_func = None - #self.cv_headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246'} - self.cv_headers = {'User-Agent': generate_user_agent(os='win', device_type='desktop')} + self.cv_headers = {'User-Agent': ComicVineTalker.cv_user_agent} def setLogFunc(self, log_func): self.log_func = log_func diff --git a/lib/comictaggerlib/main.py b/lib/comictaggerlib/main.py index 039be407..643e3043 100644 --- a/lib/comictaggerlib/main.py +++ b/lib/comictaggerlib/main.py @@ -54,6 +54,7 @@ def ctmain(): SETTINGS.save() ComicVineTalker.api_key = SETTINGS.cv_api_key + ComicVineTalker.cv_user_agent = SETTINGS.cv_user_agent signal.signal(signal.SIGINT, signal.SIG_DFL) diff --git a/lib/comictaggerlib/settings.py b/lib/comictaggerlib/settings.py index 75aa6436..2e08dd31 100644 --- a/lib/comictaggerlib/settings.py +++ b/lib/comictaggerlib/settings.py @@ -98,6 +98,7 @@ def setDefaultValues(self): self.remove_html_tables = False self.cv_api_key = "" self.notes_format = 'Issue ID' + self.cv_user_agent = None # CBL Tranform settings @@ -270,6 +271,8 @@ def readline_generator(f): self.cv_api_key = self.config.get('comicvine', 'cv_api_key') if self.config.has_option('comicvine', 'notes_format'): self.notes_format = self.config.get('comicvine', 'notes_format') + if self.config.has_option('comicvine', 'cv_user_agent'): + self.cv_user_agent = self.config.get('comicvine', 'cv_user_agent') if self.config.has_option( 'cbl_transform', 'assume_lone_credit_is_primary'): @@ -424,6 +427,7 @@ def save(self): 'comicvine', 'remove_html_tables', self.remove_html_tables) self.config.set('comicvine', 'cv_api_key', self.cv_api_key) self.config.set('comicvine', 'notes_format', self.notes_format) + self.config.set('comicvine', 'cv_user_agent', self.cv_user_agent) if not self.config.has_section('cbl_transform'): self.config.add_section('cbl_transform') diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index 133fa798..66330411 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -767,7 +767,18 @@ def Process(self): if wv['ComicPublished'] is None: logger.fdebug('Publication Run cannot be generated - probably due to an incomplete Refresh. Manually refresh the following series and try again: %s (%s)' % (wv['ComicName'], wv['ComicYear'])) continue - if any([wv['Status'] == 'Paused', bool(wv['ForceContinuing']) is True]) or (wv['Have'] == wv['Total'] and not any(['Present' in wv['ComicPublished'], helpers.now()[:4] in wv['ComicPublished']])): + if (wv['Status'] == 'Paused' and any( + [ + wv['cv_removed'] == 2, + bool(wv['ForceContinuing']) is True + ] + )) or (wv['Have'] == wv['Total'] and not any( + [ + 'Present' in wv['ComicPublished'], + helpers.now()[:4] in wv['ComicPublished'] + ] + ) + ): dbcheck = myDB.selectone('SELECT Status FROM issues WHERE ComicID=? and Int_IssueNumber=?', [wv['ComicID'], tmp_iss]).fetchone() if not dbcheck and mylar.CONFIG.ANNUALS_ON: dbcheck = myDB.selectone('SELECT Status FROM annuals WHERE ComicID=? and Int_IssueNumber=?', [wv['ComicID'], tmp_iss]).fetchone() diff --git a/mylar/config.py b/mylar/config.py index db2f797a..7e472f00 100644 --- a/mylar/config.py +++ b/mylar/config.py @@ -155,7 +155,7 @@ 'CV_ONLY': (bool, 'CV', True), 'CV_ONETIMER': (bool, 'CV', True), 'CVINFO': (bool, 'CV', False), - 'CV_USER_AGENT': (str, 'CV', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246'), + 'CV_USER_AGENT': (str, 'CV', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'), 'IMPRINT_MAPPING_TYPE': (str, 'CV', 'CV'), # either 'CV' for ComicVine or 'JSON' for imprints.json to choose which naming to use for imprints 'LOG_DIR' : (str, 'Logs', None), @@ -298,15 +298,6 @@ 'BLACKHOLE_DIR': (str, 'Blackhole', None), - 'NZBSU': (bool, 'NZBsu', False), - 'NZBSU_UID': (str, 'NZBsu', None), - 'NZBSU_APIKEY': (str, 'NZBsu', None), - 'NZBSU_VERIFY': (bool, 'NZBsu', True), - - 'DOGNZB': (bool, 'DOGnzb', False), - 'DOGNZB_APIKEY': (str, 'DOGnzb', None), - 'DOGNZB_VERIFY': (bool, 'DOGnzb', True), - 'NEWZNAB': (bool, 'Newznab', False), 'EXTRA_NEWZNABS': (str, 'Newznab', ""), @@ -461,6 +452,13 @@ 'ENABLE_PUBLIC': ('Torrents', 'ENABLE_TPSE'), 'PUBLIC_VERIFY': ('Torrents', 'TPSE_VERIFY'), 'IGNORED_PUBLISHERS': ('CV', 'BLACKLISTED_PUBLISHERS'), + 'NZBSU': ('NZBsu', 'nzbsu', bool, None), + 'NZBSU_UID': ('NZBsu', 'nzbsu_uid', str, None), + 'NZBSU_APIKEY': ('NZBsu', 'nzbsu_apikey', str, None), + 'NZBSU_VERIFY': ('NZBsu', 'nzbsu_verify', bool, None), + 'DOGNZB': ('DOGnzb', 'dognzb', bool, None), + 'DOGNZB_APIKEY': ('DOGnzb', 'dognzb_apikey', str, None), + 'DOGNZB_VERIFY': ('DOGnzb', 'dognzb_verify', bool, None), }) class Config(object): @@ -480,7 +478,7 @@ def config_vals(self, update=False): count = 0 #this is the current version at this particular point in time. - self.newconfig = 12 + self.newconfig = 14 OLDCONFIG_VERSION = 0 if count == 0: @@ -504,6 +502,7 @@ def config_vals(self, update=False): setattr(self, 'MINIMAL_INI', MINIMALINI) config_values = [] + for k,v in _CONFIG_DEFINITIONS.items(): xv = [] xv.append(k) @@ -599,6 +598,22 @@ def config_vals(self, update=False): elif k == 'MINIMAL_INI': config.set(v[1], k.lower(), str(self.MINIMAL_INI)) + #this section retains values of variables that are no longer being saved to the ini + #in case they are needed prior to wiping out things + self.OLD_VALUES = {} + for b, bv in _BAD_DEFINITIONS.items(): + if len(bv) == 4: #removal of option... + if bv[1] not in self.OLD_VALUES: + try: + if bv[2] == bool: + self.OLD_VALUES[bv[1]] = config.getboolean(bv[0], bv[1]) + elif bv[2] == str: + self.OLD_VALUES[bv[1]] = config.get(bv[0], bv[1]) + elif bv[2] == int: + self.OLD_VALUES[bv[1]] = config.getint(bv[0], bv[1]) + except (configparser.NoSectionError, configparser.NoOptionError): + pass + def read(self, startup=False): self.config_vals() @@ -641,13 +656,14 @@ def read(self, startup=False): cc = maintenance.Maintenance('backup') bcheck = cc.backup_files(cfg=True, dbs=False, backupinfo=backupinfo) - if self.CONFIG_VERSION < 12: + if self.CONFIG_VERSION < 14: print('Attempting to update configuration..') #8-torznab multiple entries merged into extra_torznabs value #9-remote rtorrent ssl option #10-encryption of all keys/passwords. #11-provider ids #12-ddl seperation into multiple providers, new keys, update tables + #13-remove dognzb and nzbsu as independent options (throw them under newznabs if present) self.config_update() setattr(self, 'OLDCONFIG_VERSION', str(self.CONFIG_VERSION)) setattr(self, 'CONFIG_VERSION', self.newconfig) @@ -738,6 +754,118 @@ def config_update(self): config.set('DDL', 'enable_getcomics', self.ENABLE_GETCOMICS) #tables will be updated by checking the OLDCONFIG_VERSION in __init__ logger.info('Successfully updated config to version 12 ( multiple DDL provider option )') + if self.newconfig < 15: + # remove nzbsu and dognzb as individual options + # if data exists already, add them as newznab options (if not already there or via Prowlarr) + try: + for chk_e in [self.OLD_VALUES['nzbsu_apikey'], self.OLD_VALUES['dognzb_apikey']]: + if chk_e is not None: + if chk_e[:5] == '^~$z$': + nz = encrypted.Encryptor(chk_e) + nz_stat = nz.decrypt_it() + if nz_stat['status'] is True: + if chk_e == self.OLD_VALUES['nzbsu_apikey']: + self.OLD_VALUES['nzbsu_apikey'] = nz_stat['password'] + else: + self.OLD_VALUES['dognzb_apikey'] = nz_stat['password'] + except Exception as e: + logger.error('error: %s' % e) + + extra_newznabs, extra_torznabs = self.get_extras() + enz = [] + dogs = [] + nzbsus = [] + try: + ncnt = 0 + for en in extra_newznabs: + dognzb_found = nzbsu_found = False + ben = list(en) + if ben[1] is not None: + n_name = ben[0].lower() + if n_name is None: + n_name = '' + if ben[3][:5] == '^~$z$': + nz = encrypted.Encryptor(ben[3]) + nz_stat = nz.decrypt_it() + if nz_stat['status'] is True: + ben[3] = nz_stat['password'] + + # prowlarr's url does not contain the actual url, hope the name contains it... + if 'nzb.su' in ben[1].lower() or (any(['nzb.su' in n_name.lower(), 'nzbsu' in re.sub(r'\s', '', n_name).lower()]) and 'prowlarr' in n_name.lower()): + nzbsus.append(tuple(ben)) + nzbsu_found = True + elif 'dognzb' in ben[1].lower() or all(['dognzb' in re.sub(r'\s', '', n_name).lower(), 'prowlarr' in n_name.lower()]): + dogs.append(tuple(ben)) + dognzb_found = True + if not any([dognzb_found, nzbsu_found]): + enz.append(tuple(ben)) + ncnt +=1 + except Exception as e: + logger.warn('error: %s' % e) + + if self.OLD_VALUES['nzbsu']: + mylar.PROVIDER_START_ID+=1 + tsnzbsu = '' if self.OLD_VALUES['nzbsu_uid'] is None else self.OLD_VALUES['nzbsu_uid'] + nzbsus.append(('nzb.su', 'https://api.nzb.su', '1', self.OLD_VALUES['nzbsu_apikey'], tsnzbsu, str(int(self.OLD_VALUES['nzbsu'])), mylar.PROVIDER_START_ID)) + if self.OLD_VALUES['dognzb']: + mylar.PROVIDER_START_ID+=1 + dogs.append(('DOGnzb', 'https://api.dognzb.cr', '1', self.OLD_VALUES['dognzb_apikey'], '', str(int(self.OLD_VALUES['dognzb'])), mylar.PROVIDER_START_ID)) + + #loop thru nzbsus and dogs entries and only keep one (in order of priority): Enabled, Prowlarr, newznab + keep_nzbsu = None + keep_dognzb = None + keep_it = None + kcnt = 0 + for ggg in [nzbsus, dogs]: + for gg in sorted(ggg, key=itemgetter(5), reverse=True): + try: + if gg[5] == '1': + if gg[0] is not None: + if 'Prowlarr' in gg[0]: + keep_it = gg + if keep_it is None and gg[0] is not None: + if 'Prowlarr' in gg[0]: + keep_it = gg + if keep_it is None: + keep_it = gg + except Exception as e: + logger.error('error: %s' % e) + + if kcnt == 0 and keep_it is not None: + enz.append(keep_it) + keep_nzbsu = keep_it + elif kcnt == 1 and keep_it is not None: + enz.append(keep_it) + keep_dognzb = keep_it + keep_it = None + kcnt+=1 + + logger.fdebug('keep_nzbsu: %s' % (keep_nzbsu,)) + logger.fdebug('keep_dognzb: %s' % (keep_dognzb,)) + + try: + config.remove_option('NZBsu', 'nzbsu') + config.remove_option('NZBsu', 'nzbsu_uid') + config.remove_option('NZBsu', 'nzbsu_apikey') + config.remove_option('NZBsu', 'nzbsu_verify') + except configparser.NoSectionError: + pass + else: + config.remove_section('NZBsu') + try: + config.remove_option('DOGnzb', 'dognzb') + config.remove_option('DOGnzb', 'dognzb_verify') + config.remove_option('DOGnzb', 'dognzb_apikey') + except configparser.NoSectionError: + pass + else: + config.remove_section('DOGnzb') + + setattr(self, 'EXTRA_NEWZNABS', enz) + setattr(self, 'EXTRA_TORZNABS', extra_torznabs) + myDB = db.DBConnection() + chk_tbl = myDB.action("DELETE FROM provider_searches where id=102 or id=103") + self.writeconfig(startup=False) logger.info('Configuration upgraded to version %s' % self.newconfig) @@ -938,8 +1066,6 @@ def encrypt_items(self, mode='encrypt', updateconfig=False): 'SAB_PASSWORD': ('SABnzbd', 'sab_password', self.SAB_PASSWORD), 'SAB_APIKEY': ('SABnzbd', 'sab_apikey', self.SAB_APIKEY), 'NZBGET_PASSWORD': ('NZBGet', 'nzbget_password', self.NZBGET_PASSWORD), - 'NZBSU_APIKEY': ('NZBsu', 'nzbsu_apikey', self.NZBSU_APIKEY), - 'DOGNZB_APIKEY': ('DOGnzb', 'dognzb_apikey', self.DOGNZB_APIKEY), 'UTORRENT_PASSWORD': ('uTorrent', 'utorrent_password', self.UTORRENT_PASSWORD), 'TRANSMISSION_PASSWORD': ('Transmission', 'transmission_password', self.TRANSMISSION_PASSWORD), 'DELUGE_PASSWORD': ('Deluge', 'deluge_password', self.DELUGE_PASSWORD), @@ -1217,12 +1343,16 @@ def configure(self, update=False, startup=False): self.IGNORE_HAVETOTAL = False logger.warn('You cannot have both ignore_total and ignore_havetotal enabled in the config.ini at the same time. Set only ONE to true - disabling both until this is resolved.') - if len(self.MASS_PUBLISHERS) > 0: + if len(self.MASS_PUBLISHERS) > 0 and self.MASS_PUBLISHERS != '[]': if type(self.MASS_PUBLISHERS) != list: try: self.MASS_PUBLISHERS = json.loads(self.MASS_PUBLISHERS) except Exception as e: - logger.warn('[MASS_PUBLISHERS] Unable to convert publishers [%s]. Error returned: %s' % (self.MASS_PUBLISHERS, e)) + try: + tmp_publishers = json.dumps(self.MASS_PUBLISHERS) + self.MASS_PUBLISHERS = json.loads(tmp_publishers) + except Exception as e: + logger.warn('[MASS_PUBLISHERS] Unable to convert publishers [%s]. Error returned: %s' % (self.MASS_PUBLISHERS, e)) logger.info('[MASS_PUBLISHERS] Auto-add for weekly publishers set to: %s' % (self.MASS_PUBLISHERS,)) if len(self.IGNORE_SEARCH_WORDS) > 0 and self.IGNORE_SEARCH_WORDS != '[]': @@ -1311,6 +1441,54 @@ def configure(self, update=False, startup=False): else: logger.fdebug('Successfully created ComicTagger Settings location.') + #make sure the user_agent is running a current version and write it to the .ComicTagger file for use with CT + if '42.0.2311.135' in self.CV_USER_AGENT: + self.CV_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36' + + ct_settingsfile = os.path.join(self.CT_SETTINGSPATH, 'settings') + if os.path.exists(ct_settingsfile): + ct_config = configparser.ConfigParser() + def readline_generator(f): + line = f.readline() + while line: + yield line + line = f.readline() + + ct_config.read_file( + readline_generator(codecs.open(ct_settingsfile, "r", "utf8"))) + + tmp_agent = None + if ct_config.has_option('comicvine', 'cv_user_agent'): + tmp_agent = ct_config.get('comicvine', 'cv_user_agent') + + if tmp_agent != self.CV_USER_AGENT: + #update + try: + with codecs.open(ct_settingsfile, 'r', 'utf8') as ct_read: + ct_lines = ct_read.readlines() + + process_next = False + cv_line = 'cv_user_agent = %s' % self.CV_USER_AGENT + with codecs.open(ct_settingsfile, encoding='utf8', mode='w+') as ct_file: + for line in ct_lines: + if 'cv_user_agent' in line: + line = cv_line + + elif '[comicvine]' not in line and process_next: + ct_file.write(cv_line+'\n') + process_next = False + + if tmp_agent is None and '[comicvine]' in line: + process_next = True + + ct_file.write(line) + + logger.fdebug('Updated CT Settings with new CV user agent string.') + except IOError as e: + logger.warn("Error writing configuration file: %s" % e) + else: + logger.info('[CV_USER_AGENT] Agent already identical in comictagger session.') + #make sure queues are running here... if startup is False: if self.POST_PROCESSING is True and ( all([self.NZB_DOWNLOADER == 0, self.SAB_CLIENT_POST_PROCESSING is True]) or all([self.NZB_DOWNLOADER == 1, self.NZBGET_CLIENT_POST_PROCESSING is True]) ): @@ -1519,11 +1697,12 @@ def get_extras(self): for x in ex: x_cat = x[4] - if '#' in x_cat: - x_t = x[4].split('#') - x_cat = ','.join(x_t) - if x_cat[0] == ',': - x_cat = re.sub(',', '#', x_cat, 1) + if x_cat: + if '#' in x_cat: + x_t = x[4].split('#') + x_cat = ','.join(x_t) + if x_cat[0] == ',': + x_cat = re.sub(',', '#', x_cat, 1) try: if cnt == 0: x_newzcat.append((x[0],x[1],x[2],x[3],x_cat,x[5],int(x[6]))) @@ -1617,12 +1796,6 @@ def provider_sequence(self): #if self.ENABLE_PUBLIC: # PR.append('public torrents') # PR_NUM +=1 - if self.NZBSU: - PR.append('nzb.su') - PR_NUM +=1 - if self.DOGNZB: - PR.append('dognzb') - PR_NUM +=1 if self.EXPERIMENTAL: PR.append('Experimental') PR_NUM +=1 @@ -1635,7 +1808,7 @@ def provider_sequence(self): PR.append('DDL(External)') PR_NUM +=1 - PPR = ['32p', 'nzb.su', 'dognzb', 'Experimental', 'DDL(GetComics)', 'DDL(External)'] + PPR = ['Experimental', 'DDL(GetComics)', 'DDL(External)'] if self.NEWZNAB: for ens in self.EXTRA_NEWZNABS: if str(ens[5]) == '1': # if newznabs are enabled @@ -1805,10 +1978,6 @@ def write_out_provider_searches(self): t_id = 201 elif any(['experimental' in prov_t, 'Experimental' in prov_t]): t_id = 101 - elif 'dog' in prov_t: - t_id = 102 - elif any(['nzb.su' in prov_t, 'nzbsu' in prov_t]): - t_id = 103 else: nnf = False if self.EXTRA_NEWZNABS: @@ -1845,12 +2014,6 @@ def write_out_provider_searches(self): tmp_prov = 'experimental' t_type = 'experimental' t_id = 101 - elif 'dog' in tmp_prov: - t_type = 'dognzb' - t_id = 102 - elif any(['nzb.su' in tmp_prov, 'nzbsu' in tmp_prov]): - t_type = 'nzb.su' - t_id = 103 else: nnf = False if self.EXTRA_NEWZNABS: @@ -1876,14 +2039,12 @@ def write_out_provider_searches(self): tprov = None if tprov: - if (any(['nzb.su' in tmp_prov, 'nzbsu' in tmp_prov]) and tprov['type'] != 'nzb.su') or (tmp_prov == 'Experimental'): + if tmp_prov == 'Experimental': # needed to ensure the type is set properly for this provider ptype = tprov['type'] if tmp_prov == 'Experimental': myDB.action("DELETE FROM provider_searches where id=101") tmp_prov = 'experimental' - else: - ptype = 'nzb.su' ctrls = {'id': tprov['id'], 'provider': tmp_prov} vals = {'active': tprov['active'], 'lastrun': tprov['lastrun'], 'type': ptype, 'hits': tprov['hits']} write = True diff --git a/mylar/downloaders/mediafire.py b/mylar/downloaders/mediafire.py index da4ad61c..61c83586 100644 --- a/mylar/downloaders/mediafire.py +++ b/mylar/downloaders/mediafire.py @@ -127,7 +127,12 @@ def mediafire_dl(self, url, id, fileinfo, issueid): logger.fdebug('[MediaFire] Content has been removed - we should move on to the next one at this point.') return {"success": False, "filename": None, "path": None, "link_type_failure": 'GC-Media'} - logger.fdebug('[MediaFire] download completed - donwloaded %s / %s' % (os.stat(filepath).st_size, fileinfo['filesize'])) + try: + filesize = os.stat(filepath).st_size + except FileNotFoundError: + return {"success": false, "filenme": None, "path": None} + else: + logger.fdebug('[MediaFire] download completed - downloaded %s / %s' % (filesize, fileinfo['filesize'])) logger.fdebug('[MediaFire] ddl_linked - filename: %s' % fileinfo['filename']) diff --git a/mylar/getcomics.py b/mylar/getcomics.py index b3a5c4bd..30411cc8 100644 --- a/mylar/getcomics.py +++ b/mylar/getcomics.py @@ -544,9 +544,15 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin logger.info('[DDL-GATHERER-OF-LINKAGE] Now compiling release information & available links...') while True: #logger.fdebug('count_bees: %s' % count_bees) - f = beeswax[count_bees] - option_find = beeswax[count_bees] - linkage = f.find("div", {"class": "aio-pulse"}) + try: + f = beeswax[count_bees] + option_find = beeswax[count_bees] + linkage = f.find("div", {"class": "aio-pulse"}) + except Exception as e: + if looped_thru_once is False: + valid_links[multiple_links].update({'links': gather_links}) + break + #logger.fdebug('linkage: %s' % linkage) if not linkage: linkage_test = f.text.strip() @@ -649,7 +655,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin "issues": None, "size": size, "links": lk['href'], - "pack": comicinfo[0]['pack'] + "pack": pack }) #logger.fdebug('gather_links so far: %s' % gather_links) count_bees +=1 @@ -784,23 +790,38 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin if not link_matched and site_lp == 'mega': sub_site_chk = [y for y in tmp_sites if 'mega' in y] if sub_site_chk: - kk = tmp_links[site_position['SD-Digital:mega']] - logger.info('[MEGA] SD-Digital preference detected...attempting %s' % kk['series']) - link_matched = True + try: + kk = tmp_links[site_position['SD-Digital:mega']] + logger.info('[MEGA] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + except KeyError: + kk = tmp_links[site_position['normal:mega']] + logger.info('[MEGA] mega preference detected...attempting %s' % kk['series']) + link_matched = True elif not link_matched and site_lp == 'pixeldrain': sub_site_chk = [y for y in tmp_sites if 'pixel' in y] if sub_site_chk: - kk = tmp_links[site_position['SD-Digital:pixeldrain']] - logger.info('[PixelDrain] SD-Digital preference detected...attempting %s' % kk['series']) - link_matched = True + try: + kk = tmp_links[site_position['SD-Digital:pixeldrain']] + logger.info('[PixelDrain] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + except KeyError: + kk = tmp_links[site_position['normal:pixeldrain']] + logger.info('[PixelDrain] PixelDrain preference detected...attempting %s' % kk['series']) + link_matched = True elif not link_matched and site_lp == 'mediafire': sub_site_chk = [y for y in tmp_sites if 'mediafire' in y] if sub_site_chk: - kk = tmp_links[site_position['SD-Digital:mediafire']] - logger.info('[mediafire] SD-Digital preference detected...attempting %s' % kk['series']) - link_matched = True + try: + kk = tmp_links[site_position['SD-Digital:mediafire']] + logger.info('[mediafire] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + except KeyError: + kk = tmp_links[site_position['normal:mediafire']] + logger.info('[mediafire] mediafire preference detected...attempting %s' % kk['series']) + link_matched = True elif not link_matched and site_lp == 'main': try: @@ -808,9 +829,19 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin logger.info('[MAIN-SERVER] SD-Digital preference detected...attempting %s' % kk['series']) link_matched = True except Exception as e: - kk = tmp_links[site_position['SD-Digital:mirror download']] - logger.info('[MIRROR-SERVER] SD-Digital preference detected...attempting %s' % kk['series']) - link_matched = True + try: + kk = tmp_links[site_position['SD-Digital:mirror download']] + logger.info('[MIRROR-SERVER] SD-Digital preference detected...attempting %s' % kk['series']) + link_matched = True + except KeyError: + try: + kk = tmp_links[site_position['normal:download now']] + logger.info('[MAIN-SERVER] main preference detected...attempting %s' % kk['series']) + link_matched = True + except KeyError: + kk = tmp_links[site_position['normal:mirror download']] + logger.info('[MIRROR-SERVER] main-mirror preference detected...attempting %s' % kk['series']) + link_matched = True if link_matched: link = kk @@ -938,7 +969,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin "issues": issues, "size": size, "links": linked, - "pack": comicinfo[0]['pack'] + "pack": pack } ) else: @@ -980,7 +1011,7 @@ def parse_downloadresults(self, id, mainlink, comicinfo=None, packinfo=None, lin "issues": issues, "size": size, "links": linked, - "pack": comicinfo[0]['pack'] + "pack": pack } ) diff --git a/mylar/importer.py b/mylar/importer.py index 33de4227..d30c3184 100644 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -214,13 +214,20 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No CV_NoYearGiven = "no" #if the SeriesYear returned by CV is blank or none (0000), let's use the gcd one. - if any([comic['ComicYear'] is None, comic['ComicYear'] == '0000', comic['ComicYear'][-1:] == '-']): + if any([comic['ComicYear'] is None, comic['ComicYear'] == '0000', comic['ComicYear'][-1:] == '-', comic['ComicYear'] == '2099']): if mylar.CONFIG.CV_ONLY: #we'll defer this until later when we grab all the issues and then figure it out logger.info('Uh-oh. I cannot find a Series Year for this series. I am going to try analyzing deeper.') SeriesYear = cv.getComic(comicid, 'firstissue', comic['FirstIssueID']) - if not SeriesYear: - return + if not SeriesYear or SeriesYear == '2099': + try: + if int(comic['ComicYear']) == 2099: + logger.fdebug('Incorrect Series year detected (%s) ...' + ' Correcting to current year as this is probably a new series' % (comic['ComicYear']) + ) + SeriesYear = str(datetime.datetime.now().year) + except Exception as e: + return if SeriesYear == '0000': logger.info('Ok - I could not find a Series Year at all. Loading in the issue data now and will figure out the Series Year.') CV_NoYearGiven = "yes" @@ -286,6 +293,8 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No # let's remove the non-standard characters here that will break filenaming / searching. comicname_filesafe = helpers.filesafe(u_comicnm) + dir_rename = False + if comlocation is None: comic_values = {'ComicName': comic['ComicName'], @@ -308,9 +317,43 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No else: comsubpath = comlocation.replace(mylar.CONFIG.DESTINATION_DIR, '').strip() + #check for year change and rename the folder to the corrected year... + if comic['ComicYear'] == '2099' and SeriesYear: + badyears = [i.start() for i in re.finditer('2099', comlocation)] + num_bad = len(badyears) + if num_bad == 1: + new_location = re.sub('2099', SeriesYear, comlocation) + dir_rename = True + elif num_bad > 1: + #assume right-most is the year cause anything else isn't very smart anyways... + new_location = comlocation[:badyears[num_bad-1]] + SeriesYear + comlocation[badyears[num_bad-1]+1:] + dir_rename = True + + if dir_rename and all([new_location != comlocation, os.path.isdir(comlocation)]): + logger.fdebug('Attempting to rename existing location [%s]' % (comlocation)) + try: + # make sure 2 levels up in strucure exist + if not os.path.exists(os.path.split ( os.path.split(new_location)[0] ) [0] ): + logger.fdebug('making directory: %s' % os.path.split(os.path.split(new_location)[0])[0]) + os.mkdir(os.path.split(os.path.split(new_location)[0])[0]) + # make sure parent directory exists + if not os.path.exists(os.path.split(new_location)[0]): + logger.fdebug('making directory: %s' % os.path.split(new_location)[0]) + os.mkdir(os.path.split(new_location)[0]) + logger.info('Renaming directory: %s --> %s' % (comlocation,new_location)) + shutil.move(comlocation, new_location) + except Exception as e: + if 'No such file or directory' in e: + if mylar.CONFIG.CREATE_FOLDERS: + checkdirectory = filechecker.validateAndCreateDirectory(new_location, True) + if not checkdirectory: + logger.warn('Error trying to validate/create directory. Aborting this process at this time.') + else: + logger.warn('Unable to rename existing directory: %s' % e) + #moved this out of the above loop so it will chk for existance of comlocation in case moved #if it doesn't exist - create it (otherwise will bugger up later on) - if comlocation is not None: + if not dir_rename and comlocation is not None: if os.path.isdir(comlocation): logger.info('Directory (' + comlocation + ') already exists! Continuing...') else: diff --git a/mylar/notifiers.py b/mylar/notifiers.py index 83e8eecf..de24f982 100644 --- a/mylar/notifiers.py +++ b/mylar/notifiers.py @@ -475,7 +475,7 @@ def notify(self, text, attachment_text, snatched_nzb=None, prov=None, sent_to=No def test_notify(self): return self.notify('Test Message', 'Release the Ninjas!') - + class MATTERMOST: def __init__(self, test_webhook_url=None): self.webhook_url = mylar.CONFIG.MATTERMOST_WEBHOOK_URL if test_webhook_url is None else test_webhook_url @@ -731,7 +731,7 @@ def notify(self, text, attachment_text, snatched_nzb=None, prov=None, sent_to=No # Error logging sent_successfuly = True - if not response.status_code == 204 or response.status_code == 200: + if not all([response.status_code == 204, response.status_code == 200]): logger.info(module + 'Could not send notification to Discord (webhook_url=%s). Response: [%s]' % (self.webhook_url, response.text)) sent_successfuly = False @@ -779,7 +779,7 @@ def notify(self, text, attachment_text, snatched_nzb=None, prov=None, sent_to=No } } } - + try: response = requests.post(self.webhook_url, json=payload, verify=True) except Exception as e: diff --git a/mylar/rsscheck.py b/mylar/rsscheck.py index bf6a48dc..9acb9aa4 100755 --- a/mylar/rsscheck.py +++ b/mylar/rsscheck.py @@ -503,7 +503,7 @@ def _parse_feed(site, url, verify, payload=None): if str(newznab_host[5]) == '1': newznab_hosts.append(newznab_host) - providercount = len(newznab_hosts) + int(mylar.CONFIG.EXPERIMENTAL is True) + int(mylar.CONFIG.NZBSU is True) + int(mylar.CONFIG.DOGNZB is True) + providercount = len(newznab_hosts) + int(mylar.CONFIG.EXPERIMENTAL is True) logger.fdebug('[RSS] You have enabled ' + str(providercount) + ' NZB RSS search providers.') if providercount > 0: @@ -528,29 +528,6 @@ def _parse_feed(site, url, verify, payload=None): if check == 'disable': helpers.disable_provider('experimental') - if mylar.CONFIG.NZBSU is True: - num_items = "&num=100" if forcerss else "" # default is 25 - params = {'t': '7030', - 'dl': '1', - 'i': mylar.CONFIG.NZBSU_UID, - 'r': mylar.CONFIG.NZBSU_APIKEY, - 'num_items': num_items} - check = _parse_feed('nzb.su', 'https://api.nzb.su/rss', bool(int(mylar.CONFIG.NZBSU_VERIFY)), params) - if check == 'disable': - helpers.disable_provider(site) - - if mylar.CONFIG.DOGNZB is True: - #default is 100 - params = {'cat': '7030', - 'o': 'xml', - 'apikey': mylar.CONFIG.DOGNZB_APIKEY, - 't': 'search', - 'dl': '1'} - - check = _parse_feed('dognzb', 'https://api.dognzb.cr/api', bool(int(mylar.CONFIG.DOGNZB_VERIFY)), params) - if check == 'disable': - helpers.disable_provider(site) - for newznab_host in newznab_hosts: site = newznab_host[0].rstrip() (newznabuid, _, newznabcat) = (newznab_host[4] or '').partition('#') @@ -602,24 +579,19 @@ def _parse_feed(site, url, verify, payload=None): continue else: titlename = entry.title - if site == 'dognzb': - #because the rss of dog doesn't carry the enclosure item, we'll use the newznab size value - size = 0 + size = 0 + try: + # experimental, newznab + size = entry.enclosures[0]['length'] + except Exception as e: if 'newznab' in entry and 'size' in entry['newznab']: size = entry['newznab']['size'] - else: - # experimental, nzb.su, newznab - size = entry.enclosures[0]['length'] # Link - # dognzb, nzb.su, newznab link = entry.link #Remove the API keys from the url to allow for possible api key changes - if site == 'dognzb': - link = re.sub(mylar.CONFIG.DOGNZB_APIKEY, '', link).strip() - else: - link = link[:link.find('&i=')].strip() + link = link[:link.find('&i=')].strip() feeddata.append({'Site': site, 'Title': titlename, @@ -1551,11 +1523,15 @@ def ddlrss_pack_detect(title, link): # if it's a pack - remove the issue-range and the possible issue years # (cause it most likely will span) and pass thru as separate items if pack is True: - title = re.sub(issues, '', title).strip() + try: + title = re.sub(issues, '', title).strip() # kill any brackets in the issue line here. - issues = re.sub(r'[\(\)\[\]]', '', issues).strip() - if title.endswith('#'): - title = title[:-1].strip() + issues = re.sub(r'[\(\)\[\]]', '', issues).strip() + except Exception as e: + return + else: + if title.endswith('#'): + title = title[:-1].strip() return {'title': title, 'issues': issues, 'pack': pack, 'link': link} else: diff --git a/mylar/search.py b/mylar/search.py index 899310e5..00438107 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -368,14 +368,7 @@ def search_init( current_prov = get_current_prov(searchprov) logger.info('current_prov: %s' % (current_prov)) - if current_prov.get('dognzb') and all( - [mylar.CONFIG.DOGNZB == 0, provider_blocked] - ): - # since dognzb could hit the 100 daily api limit during the middle - # of a search run, check here on each pass to make sure it's not - # disabled (it gets auto-disabled on maxing out the API hits) - break - elif all( + if all( [ not provider_blocked, ''.join(current_prov.keys()) in checked_once, @@ -587,16 +580,8 @@ def provider_order(initial_run=False): torznabs += 1 # nzb provider selection## - # 'dognzb' or 'nzb.su' or 'experimental' nzbprovider = [] nzbp = 0 - if mylar.CONFIG.NZBSU is True and not helpers.block_provider_check('nzb.su'): - nzbprovider.append('nzb.su') - nzbp += 1 - if mylar.CONFIG.DOGNZB is True and not helpers.block_provider_check('dognzb'): - nzbprovider.append('dognzb') - nzbp += 1 - # -------- # Xperimental if mylar.CONFIG.EXPERIMENTAL is True and not helpers.block_provider_check( @@ -722,14 +707,7 @@ def NZB_SEARCH( provider_stat = provider_stat.get(list(provider_stat.keys())[0]) #logger.info('nzbprov: %s' % (nzbprov)) #logger.fdebug('provider_stat_after: %s' % (provider_stat)) - - if nzbprov == 'nzb.su': - apikey = mylar.CONFIG.NZBSU_APIKEY - verify = bool(int(mylar.CONFIG.NZBSU_VERIFY)) - elif nzbprov == 'dognzb': - apikey = mylar.CONFIG.DOGNZB_APIKEY - verify = bool(int(mylar.CONFIG.DOGNZB_VERIFY)) - elif nzbprov == 'experimental': + if nzbprov == 'experimental': apikey = 'none' verify = False elif provider_stat['type'] == 'torznab': @@ -890,13 +868,8 @@ def NZB_SEARCH( while findloop < findcount: logger.fdebug('findloop: %s / findcount: %s' % (findloop, findcount)) comsrc = comsearch - if nzbprov == 'dognzb' and not mylar.CONFIG.DOGNZB: - is_info['foundc']['status'] = False - done = True - break - if any([nzbprov == '32P', nzbprov == 'Public Torrents', 'DDL' in nzbprov, nzbprov == 'experimental']): - # 32p directly stores the exact issue, no need to iterate over variations - # of the issue number. DDL iteration is handled in it's own module as is experimental. + if any([nzbprov == 'Public Torrents', 'DDL' in nzbprov, nzbprov == 'experimental']): + # DDL iteration is handled in it's own module as is experimental. findloop = 99 if done is True: # and seperatealpha == "no": @@ -1055,19 +1028,7 @@ def NZB_SEARCH( if nzbprov == '': verified_matches = "no results" elif nzbprov != 'experimental': - if nzbprov == 'dognzb': - findurl = ( - "https://api.dognzb.cr/api?t=search&q=" - + str(comsearch) - + "&o=xml&cat=7030" - ) - elif nzbprov == 'nzb.su': - findurl = ( - "https://api.nzb.su/api?t=search&q=" - + str(comsearch) - + "&o=xml&cat=7030" - ) - elif provider_stat['type'] == 'newznab': + if provider_stat['type'] == 'newznab': # let's make sure the host has a '/' at the end, if not add it. host_newznab_fix = host_newznab if not host_newznab_fix.endswith('api'): @@ -1587,8 +1548,6 @@ def searchforissue(issueid=None, new=False, rsschecker=None, manual=False): )) or any( [ - mylar.CONFIG.NZBSU is True, - mylar.CONFIG.DOGNZB is True, mylar.CONFIG.EXPERIMENTAL is True, ] ) @@ -2050,8 +2009,6 @@ def searchforissue(issueid=None, new=False, rsschecker=None, manual=False): ) or ( any( [ - mylar.CONFIG.NZBSU is True, - mylar.CONFIG.DOGNZB is True, mylar.CONFIG.EXPERIMENTAL is True, mylar.CONFIG.ENABLE_GETCOMICS is True, mylar.CONFIG.ENABLE_EXTERNAL_SERVER is True, @@ -2552,8 +2509,6 @@ def searchIssueIDList(issuelist): ) ) or any( [ - mylar.CONFIG.NZBSU is True, - mylar.CONFIG.DOGNZB is True, mylar.CONFIG.EXPERIMENTAL is True, ] ) @@ -2960,7 +2915,7 @@ def searcher( payload = None headers = {'User-Agent': str(mylar.USER_AGENT)} # link doesn't have the apikey - add it and use ?t=get for newznab based. - if provider_stat['type'] == 'newznab' or nzbprov == 'nzb.su': + if provider_stat['type'] == 'newznab': # need to basename the link so it just has the id/hash. # rss doesn't store apikey, have to put it back. if provider_stat['type'] == 'newznab': @@ -2982,10 +2937,6 @@ def searcher( if uid is not None: payload['i'] = uid verify = bool(newznab[2]) - else: - down_url = 'https://api.nzb.su/api' - apikey = mylar.CONFIG.NZBSU_APIKEY - verify = bool(mylar.CONFIG.NZBSU_VERIFY) if nzbhydra is True: down_url = link @@ -2999,11 +2950,6 @@ def searcher( else: down_url = link - elif nzbprov == 'dognzb': - # dognzb - need to add back in the dog apikey - down_url = urljoin(link, str(mylar.CONFIG.DOGNZB_APIKEY)) - verify = bool(mylar.CONFIG.DOGNZB_VERIFY) - else: # experimental - direct link. down_url = link @@ -4079,12 +4025,6 @@ def generate_id(nzbprov, link, comicname): path_parts = url_parts[2].rpartition('/') nzbtempid = path_parts[2] nzbid = re.sub('.torrent', '', nzbtempid).rstrip() - elif nzbprov == 'nzb.su': - nzbid = os.path.splitext(link)[0].rsplit('/', 1)[1] - elif nzbprov == 'dognzb': - url_parts = urlparse(link) - path_parts = url_parts[2].rpartition('/') - nzbid = path_parts[0].rsplit('/', 1)[1] elif 'newznab' in nzbprov: # if in format of http://newznab/getnzb/.nzb&i=1&r=apikey tmpid = urlparse(link)[ diff --git a/mylar/search_filer.py b/mylar/search_filer.py index 5b5ff5c9..48b5f63e 100644 --- a/mylar/search_filer.py +++ b/mylar/search_filer.py @@ -869,8 +869,6 @@ def _process_entry(self, entry, is_info): if nowrite is False: if any( [ - nzbprov == 'dognzb', - nzbprov == 'nzb.su', nzbprov == 'experimental', 'newznab' in nzbprov, ] @@ -1072,8 +1070,6 @@ def _process_entry(self, entry, is_info): if nowrite is False: if any( [ - nzbprov == 'dognzb', - nzbprov == 'nzb.su', nzbprov == 'experimental', 'newznab' in nzbprov, provider_stat['type'] == 'newznab', diff --git a/mylar/webserve.py b/mylar/webserve.py index ca296857..c24d081a 100644 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -1380,13 +1380,32 @@ def addComic(self, comicid, comicname=None, comicyear=None, comicimage=None, com raise cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % comicid) addComic.exposed = True - def addbyid(self, comicid, calledby=None, imported=None, ogcname=None, nothread=False, seriesyear=None, query_id=None, com_location=None, booktype=None): + def addbyid(self, comicid, calledby=None, imported=None, ogcname=None, nothread=False, seriesyear=None, query_id=None, com_location=None, booktype=None, markupcoming=None, markall=None): mismatch = "no" if com_location == 'null': com_location = None if booktype == 'null': booktype = None logger.info('com_location: %s' % com_location) + + writeit = False + if markupcoming: + autoup = False + if markupcoming == 'true': + autoup = True + if autoup != mylar.CONFIG.AUTOWANT_UPCOMING: + mylar.CONFIG.AUTOWANT_UPCOMING = autoup + writeit = True + if markall: + autoall = False + if markall == 'true': + autoall = True + if autoall != mylar.CONFIG.AUTOWANT_ALL: + mylar.CONFIG.AUTOWANT_ALL = autoall + writeit = True + if writeit: + mylar.CONFIG.writeconfig(values={'autowant_all': mylar.CONFIG.AUTOWANT_ALL, 'autowant_upcoming': mylar.CONFIG.AUTOWANT_UPCOMING}) + if query_id is not None: myDB = db.DBConnection() query_chk = myDB.selectone("SELECT * FROM tmp_searches where query_id=? AND comicid=?", [query_id, comicid]).fetchone() @@ -2520,23 +2539,23 @@ def retryissue(self, IssueID, ComicID, ComicName=None, IssueNumber=None, Release newznabinfo = None link = None - if fullprov == 'nzb.su': - if not mylar.CONFIG.NZBSU: - logger.error('nzb.su is not enabled - unable to process retry request until provider is re-enabled.') - continue - # http://nzb.su/getnzb/ea1befdeee0affd663735b2b09010140.nzb&i=&r= - link = 'http://nzb.su/getnzb/' + str(id) + '.nzb&i=' + str(mylar.CONFIG.NZBSU_UID) + '&r=' + str(mylar.CONFIG.NZBSU_APIKEY) - logger.info('fetched via nzb.su. Retrying the send : ' + str(link)) - retried = True - elif fullprov == 'dognzb': - if not mylar.CONFIG.DOGNZB: - logger.error('Dognzb is not enabled - unable to process retry request until provider is re-enabled.') - continue - # https://dognzb.cr/fetch/5931874bf7381b274f647712b796f0ac/ - link = 'https://dognzb.cr/fetch/' + str(id) + '/' + str(mylar.CONFIG.DOGNZB_APIKEY) - logger.info('fetched via dognzb. Retrying the send : ' + str(link)) - retried = True - elif fullprov == 'experimental': + #if fullprov == 'nzb.su': + # if not mylar.CONFIG.NZBSU: + # logger.error('nzb.su is not enabled - unable to process retry request until provider is re-enabled.') + # continue + # # http://nzb.su/getnzb/ea1befdeee0affd663735b2b09010140.nzb&i=&r= + # link = 'http://nzb.su/getnzb/' + str(id) + '.nzb&i=' + str(mylar.CONFIG.NZBSU_UID) + '&r=' + str(mylar.CONFIG.NZBSU_APIKEY) + # logger.info('fetched via nzb.su. Retrying the send : ' + str(link)) + # retried = True + #elif fullprov == 'dognzb': + # if not mylar.CONFIG.DOGNZB: + # logger.error('Dognzb is not enabled - unable to process retry request until provider is re-enabled.') + # continue + # # https://dognzb.cr/fetch/5931874bf7381b274f647712b796f0ac/ + # link = 'https://dognzb.cr/fetch/' + str(id) + '/' + str(mylar.CONFIG.DOGNZB_APIKEY) + # logger.info('fetched via dognzb. Retrying the send : ' + str(link)) + # retried = True + if fullprov == 'experimental': if not mylar.CONFIG.EXPERIMENTAL: logger.error('Experimental is not enabled - unable to process retry request until provider is re-enabled.') continue @@ -6782,13 +6801,6 @@ def config(self): "qbittorrent_loadaction": mylar.CONFIG.QBITTORRENT_LOADACTION, "blackhole_dir": mylar.CONFIG.BLACKHOLE_DIR, "usenet_retention": mylar.CONFIG.USENET_RETENTION, - "nzbsu": helpers.checked(mylar.CONFIG.NZBSU), - "nzbsu_uid": mylar.CONFIG.NZBSU_UID, - "nzbsu_api": mylar.CONFIG.NZBSU_APIKEY, - "nzbsu_verify": helpers.checked(mylar.CONFIG.NZBSU_VERIFY), - "dognzb": helpers.checked(mylar.CONFIG.DOGNZB), - "dognzb_api": mylar.CONFIG.DOGNZB_APIKEY, - "dognzb_verify": helpers.checked(mylar.CONFIG.DOGNZB_VERIFY), "experimental": helpers.checked(mylar.CONFIG.EXPERIMENTAL), "enable_torznab": helpers.checked(mylar.CONFIG.ENABLE_TORZNAB), "extra_torznabs": sorted(mylar.CONFIG.EXTRA_TORZNABS, key=itemgetter(5), reverse=True), @@ -7278,8 +7290,7 @@ def arcOptions(self, StoryArcID=None, StoryArcName=None, read2filename=0, storya def configUpdate(self, **kwargs): checked_configs = ['enable_https', 'launch_browser', 'backup_on_start', 'syno_fix', 'auto_update', 'annuals_on', 'api_enabled', 'nzb_startup_search', 'enforce_perms', 'sab_to_mylar', 'torrent_local', 'torrent_seedbox', 'rtorrent_ssl', 'rtorrent_verify', 'rtorrent_startonload', - 'enable_torrents', 'enable_rss', 'nzbsu', 'nzbsu_verify', - 'dognzb', 'dognzb_verify', 'experimental', 'enable_torrent_search', 'enable_32p', 'enable_torznab', + 'enable_torrents', 'enable_rss', 'experimental', 'enable_torrent_search', 'enable_32p', 'enable_torznab', 'newznab', 'use_minsize', 'use_maxsize', 'ddump', 'failed_download_handling', 'sab_client_post_processing', 'nzbget_client_post_processing', 'failed_auto', 'post_processing', 'enable_check_folder', 'enable_pre_scripts', 'enable_snatch_script', 'enable_extra_scripts', 'enable_meta', 'cbr2cbz_only', 'ct_tag_cr', 'ct_tag_cbl', 'ct_cbz_overwrite', 'cmtag_start_year_as_volume', 'cmtag_volume', 'setdefaultvolume', diff --git a/mylar/weeklypull.py b/mylar/weeklypull.py index d47573c3..6a5d1a8f 100755 --- a/mylar/weeklypull.py +++ b/mylar/weeklypull.py @@ -1383,10 +1383,19 @@ def mass_publishers(publishers, weeknumber, year): if type(publishers) == list and len(publishers) == 0: publishers = None - if type(publishers) == str: - publishers = json.loads(publishers) - if len(publishers) == 0: - publishers = None + if type(publishers) != list: + try: + publishers = json.loads(publishers) + except Exception as e: + try: + tmp_publishers = json.dumps(publishers) + publishers = json.loads(tmp_publishers) + except Exception as e: + logger.warn('[MASS PUBLISHERS] Unable to convert mass publishers value in current state. Error: %s' % e) + publishers = None + + if len(publishers) == 0: + publishers = None if publishers is None: watchlist = myDB.select('SELECT * FROM weekly WHERE weeknumber=? and year=?', [weeknumber, year]) From 0299c268dbaf93af19c8cf3effe9c6af9b7f171a Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Sun, 3 Mar 2024 14:03:06 -0500 Subject: [PATCH 27/32] FIX: variable error on startup due to removal of su/dog --- mylar/config.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/mylar/config.py b/mylar/config.py index 7e472f00..201f9d2c 100644 --- a/mylar/config.py +++ b/mylar/config.py @@ -769,7 +769,7 @@ def config_update(self): else: self.OLD_VALUES['dognzb_apikey'] = nz_stat['password'] except Exception as e: - logger.error('error: %s' % e) + pass extra_newznabs, extra_torznabs = self.get_extras() enz = [] @@ -803,13 +803,20 @@ def config_update(self): except Exception as e: logger.warn('error: %s' % e) - if self.OLD_VALUES['nzbsu']: - mylar.PROVIDER_START_ID+=1 - tsnzbsu = '' if self.OLD_VALUES['nzbsu_uid'] is None else self.OLD_VALUES['nzbsu_uid'] - nzbsus.append(('nzb.su', 'https://api.nzb.su', '1', self.OLD_VALUES['nzbsu_apikey'], tsnzbsu, str(int(self.OLD_VALUES['nzbsu'])), mylar.PROVIDER_START_ID)) - if self.OLD_VALUES['dognzb']: - mylar.PROVIDER_START_ID+=1 - dogs.append(('DOGnzb', 'https://api.dognzb.cr', '1', self.OLD_VALUES['dognzb_apikey'], '', str(int(self.OLD_VALUES['dognzb'])), mylar.PROVIDER_START_ID)) + try: + if self.OLD_VALUES['nzbsu']: + mylar.PROVIDER_START_ID+=1 + tsnzbsu = '' if self.OLD_VALUES['nzbsu_uid'] is None else self.OLD_VALUES['nzbsu_uid'] + nzbsus.append(('nzb.su', 'https://api.nzb.su', '1', self.OLD_VALUES['nzbsu_apikey'], tsnzbsu, str(int(self.OLD_VALUES['nzbsu'])), mylar.PROVIDER_START_ID)) + except Exception as e: + pass + + try: + if self.OLD_VALUES['dognzb']: + mylar.PROVIDER_START_ID+=1 + dogs.append(('DOGnzb', 'https://api.dognzb.cr', '1', self.OLD_VALUES['dognzb_apikey'], '', str(int(self.OLD_VALUES['dognzb'])), mylar.PROVIDER_START_ID)) + except Exception as e: + pass #loop thru nzbsus and dogs entries and only keep one (in order of priority): Enabled, Prowlarr, newznab keep_nzbsu = None From 11f7af0b5fe15f5e234407453e75d8be57b6a209 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Sun, 3 Mar 2024 20:31:53 -0500 Subject: [PATCH 28/32] FIX: db table error on new installations when bumping config version --- mylar/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mylar/config.py b/mylar/config.py index 201f9d2c..c8c5e024 100644 --- a/mylar/config.py +++ b/mylar/config.py @@ -871,7 +871,12 @@ def config_update(self): setattr(self, 'EXTRA_NEWZNABS', enz) setattr(self, 'EXTRA_TORZNABS', extra_torznabs) myDB = db.DBConnection() - chk_tbl = myDB.action("DELETE FROM provider_searches where id=102 or id=103") + try: + chk_tbl = myDB.action("DELETE FROM provider_searches where id=102 or id=103") + except Exception as e: + #if the table doesn't exist yet, it'll get created after the config loads on new installs. + pass + self.writeconfig(startup=False) logger.info('Configuration upgraded to version %s' % self.newconfig) From bbefdf9d8496e0ae8f395a771ac815fed16804b9 Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Mon, 4 Mar 2024 12:53:08 -0500 Subject: [PATCH 29/32] FIX: Startup error on new installs due to db check at incorrect time --- mylar/config.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mylar/config.py b/mylar/config.py index c8c5e024..e379f0ca 100644 --- a/mylar/config.py +++ b/mylar/config.py @@ -847,9 +847,6 @@ def config_update(self): keep_it = None kcnt+=1 - logger.fdebug('keep_nzbsu: %s' % (keep_nzbsu,)) - logger.fdebug('keep_dognzb: %s' % (keep_dognzb,)) - try: config.remove_option('NZBsu', 'nzbsu') config.remove_option('NZBsu', 'nzbsu_uid') @@ -872,13 +869,13 @@ def config_update(self): setattr(self, 'EXTRA_TORZNABS', extra_torznabs) myDB = db.DBConnection() try: - chk_tbl = myDB.action("DELETE FROM provider_searches where id=102 or id=103") + ccd = myDB.select("PRAGMA table_info(provider_searches)") + if ccd: + chk_tbl = myDB.action("DELETE FROM provider_searches where id=102 or id=103") except Exception as e: #if the table doesn't exist yet, it'll get created after the config loads on new installs. pass - self.writeconfig(startup=False) - logger.info('Configuration upgraded to version %s' % self.newconfig) def check_section(self, section, key): From 16f22b77378c0547bfc779b7ece7141305044bab Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Thu, 14 Mar 2024 01:27:47 -0400 Subject: [PATCH 30/32] FIX: searchresults filter would filter everything if used --- mylar/webserve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mylar/webserve.py b/mylar/webserve.py index c24d081a..319f68a3 100644 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -1031,6 +1031,7 @@ def loadSearchResults(self, query=None, iDisplayStart=0, iDisplayLength=25, iSor 'publisher': rt['publisher'], 'comicid': rt['comicid'], 'comicyear': rt['comicyear'], + 'volume': rt['volume'], 'issues': int(rt['issues']), 'deck': rt['deck'], 'url': rt['url'], From b1006f9543bcbb2797ae6b3bd1e85831f95e601d Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Thu, 14 Mar 2024 01:54:55 -0400 Subject: [PATCH 31/32] FIX: booktype would fail to detect non-print volumes properly due to a previous PR change --- mylar/cv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mylar/cv.py b/mylar/cv.py index 5bc565b9..ee8101b9 100755 --- a/mylar/cv.py +++ b/mylar/cv.py @@ -1294,7 +1294,7 @@ def get_imprint_volume_and_booktype(series, comicyear, publisher, firstissueid, else: comic['Type'] = 'Print' - if comic_desc != 'None' and comic['Type'] == 'None': + if comic_desc != 'None' and comic['Type'] == 'Print': if 'print' in comic_desc[:60].lower() and all(['also available as a print' not in comic_desc.lower(), 'for the printed edition' not in comic_desc.lower(), 'print edition can be found' not in comic_desc.lower(), 'reprints' not in comic_desc.lower()]): comic['Type'] = 'Print' elif all(['digital' in comic_desc[:60].lower(), 'graphic novel' not in comic_desc[:60].lower(), 'digital edition can be found' not in comic_desc.lower()]): From c06efde70be2519f3879de97996896af1f3ffd6f Mon Sep 17 00:00:00 2001 From: evilhero <909424+evilhero@users.noreply.github.com> Date: Thu, 14 Mar 2024 03:00:03 -0400 Subject: [PATCH 32/32] Removal of () in folder structure, link to CV for items within collected editions --- data/interfaces/default/comicdetails_update.html | 2 +- mylar/filers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/interfaces/default/comicdetails_update.html b/data/interfaces/default/comicdetails_update.html index fa041510..108f264a 100755 --- a/data/interfaces/default/comicdetails_update.html +++ b/data/interfaces/default/comicdetails_update.html @@ -181,7 +181,7 @@

%s" % (cid[re.sub('4050-', '', x['comicid']).strip()]['comicid'], x['series']) except: - watch = "%s" % (x['series']) + watch = "%s" % (x['comicid'],x['series']) else: watch = x['series'] if cnt == 0: diff --git a/mylar/filers.py b/mylar/filers.py index b82f2c2b..c5a810e5 100644 --- a/mylar/filers.py +++ b/mylar/filers.py @@ -160,7 +160,7 @@ def folder_create(self, booktype=None, update_loc=None, secondary=None, imprint= chunk_f = re.compile(r'\s+') chunk_folder_format = chunk_f.sub(' ', chunk_f_f) - chunk_folder_format = re.sub("[()|[]]", '', chunk_folder_format).strip() + chunk_folder_format = re.sub(r'\(\)|\[\]', '', chunk_folder_format).strip() ccf = chunk_folder_format.find('/ ') if ccf != -1: chunk_folder_format = chunk_folder_format[:ccf+1] + chunk_folder_format[ccf+2:]