Skip to content

Commit

Permalink
Merge branch 'python3-dev'
Browse files Browse the repository at this point in the history 
FIX: urllib3 breaking changes
FIX:(mylar3#1344) Attempted fix for CF errors being returned when using GC
FIX: When Alternate Filename is blank, it writes a blank space
FIX: Imprint mappings (thanks @qubidt)
FIX: When determining match to download, TPB series spanning multiple years would fail to match
FIX: During import, if title contained One-Shot - would not find the match on CV
FIX: Storyarcs failing to properly generate file path locations in OPDS
  • Loading branch information
@evilhero
evilhero committed May 9, 2023
2 parents 4db40cd + 8fbd89d commit eca80ef
Show file tree
Hide file tree
Showing 8 changed files with 75 additions and 30 deletions.
12 changes: 6 additions & 6 deletions data/interfaces/default/comicdetails_update.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,7 @@ <h3><i><span id="series_status" style="margin-bottom:0px;margin-top:10px;positio
%endif
<div class="row">
<label>Volume Number</label>
<input type="text" name="comic_version" value="${comic['ComicVersion']}" size="20">
<input type="text" name="comic_version" placeholder="${comic['ComicVersion']}" onfocus="if(this.value==this.defaultValue) this.value='';" value="${comic['ComicVersion']}" size="20">
<small>Format: vN where N is a number 0-infinity</small>
</div>

Expand All @@ -404,21 +404,21 @@ <h3><i><span id="series_status" style="margin-bottom:0px;margin-top:10px;positio

<div class="row">
<label>Alternate Search Names</label>
<input type="text" name="alt_search" value="${comic['AlternateSearch']}" size="90" />
<a href="#" title="Seperate multiple entries with ##"><img src="images/info32.png" height="16" alt="" /></a>
<input type="text" name="alt_search" placeholder="${comic['AlternateSearch']}" onfocus="if(this.value==this.defaultValue) this.value='';" value="${comic['AlternateSearch']}" size="90" />
<a href="#" title="Seperate multiple entries with ##"><img src="images/info32.png" height="16" alt=""></a>
<small>Alternate comic names to be searched in case naming is different<small>
</div>

<div class="row">
<label>Alternate File-Naming</label>
<input type="text" name="alt_filename" value="${comic['AlternateFileName']}" size="90">
<a href="#" title="Alternate File Naming"><img src="images/info32.png" height="16" alt="" /></a>
<input type="text" name="alt_filename" placeholder="${comic['AlternateFileName']}" onfocus="if(this.value==this.defaultValue) this.value='';" value="${comic['AlternateFileName']}" size="90">
<a href="#" title="Alternate File Naming"><img src="images/info32.png" height="16" alt=""></a>
<small>Use this insetad of CV name during post-processing / renaming</small>
</div>

<div class="row">
<label>Publication Imprint</label>
<input type="text" name="publisher_imprint" value="${comic['PublisherImprint']}" size="90">
<input type="text" name="publisher_imprint" placeholder="${comic['PublisherImprint']}" onfocus="if(this.value==this.defaultValue) this.value='';" value="${comic['PublisherImprint']}" size="90">
</div>

<div class="row">
Expand Down
6 changes: 4 additions & 2 deletions mylar/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,9 +208,11 @@
SESSION_ID = None
UPDATE_VALUE = {}
REQS = {}
GC_URL = 'https://getcomics.org'
IMPRINT_MAPPING = {
#ComicVine: imprint.json
'Homage': 'Homage Comics',
'Homage Comics': 'Homage',
'Max Comics': 'MAX',
'Mailbu': 'Malibu Comics',
'Milestone': 'Milestone Comics',
'Skybound': 'Skybound Entertainment',
Expand Down Expand Up @@ -241,7 +243,7 @@ def initialize(config_file):
MONITOR_SCHEDULER, SEARCH_SCHEDULER, RSS_SCHEDULER, WEEKLY_SCHEDULER, VERSION_SCHEDULER, UPDATER_SCHEDULER, \
SCHED_RSS_LAST, SCHED_WEEKLY_LAST, SCHED_MONITOR_LAST, SCHED_SEARCH_LAST, SCHED_VERSION_LAST, SCHED_DBUPDATE_LAST, COMICINFO, SEARCH_TIER_DATE, \
BACKENDSTATUS_CV, BACKENDSTATUS_WS, PROVIDER_STATUS, EXT_IP, ISSUE_EXCEPTIONS, PROVIDER_START_ID, GLOBAL_MESSAGES, CHECK_FOLDER_CACHE, FOLDER_CACHE, SESSION_ID, \
MAINTENANCE_UPDATE, MAINTENANCE_DB_COUNT, MAINTENANCE_DB_TOTAL, UPDATE_VALUE, REQS
MAINTENANCE_UPDATE, MAINTENANCE_DB_COUNT, MAINTENANCE_DB_TOTAL, UPDATE_VALUE, REQS, IMPRINT_MAPPING, GC_URL

cc = mylar.config.Config(config_file)
CONFIG = cc.read(startup=True)
Expand Down
26 changes: 16 additions & 10 deletions mylar/getcomics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def cookie_receipt(self, main_url=None):
# (ie. http://192.168.2.2:8191/v1 )
main_url = mylar.CONFIG.FLARESOLVERR_URL
else:
main_url = 'https://getcomics.info'
main_url = mylar.GC_URL
else:
flare_test = True

Expand All @@ -56,7 +56,7 @@ def cookie_receipt(self, main_url=None):
#get the coookies here for use down-below
get_cookies = self.session.post(
main_url,
json={'url': 'https://getcomics.info', 'cmd': 'request.get'},
json={'url': mylar.GC_URL, 'cmd': 'request.get'},
verify=False,
headers=self.flare_headers,
timeout=30,
Expand Down Expand Up @@ -131,14 +131,14 @@ def __init__(self, query=None, issueid=None, comicid=None, oneoff=False, session
self.headers = {
'Accept-encoding': 'gzip',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1',
'Referer': 'https://getcomics.info/',
'Referer': mylar.GC_URL,
}

self.session = requests.Session()

self.session_path = session_path if session_path is not None else os.path.join(mylar.CONFIG.SECURE_DIR, ".gc_cookies.dat")

self.url = 'https://getcomics.info'
self.url = mylar.GC_URL

self.query = query #{'comicname', 'issue', year'}

Expand All @@ -155,7 +155,7 @@ def __init__(self, query=None, issueid=None, comicid=None, oneoff=False, session
self.headers = {
'Accept-encoding': 'gzip',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1',
'Referer': 'https://getcomics.info/',
'Referer': mylar.GC_URL,
}

self.provider_stat = provider_stat
Expand Down Expand Up @@ -244,7 +244,7 @@ def search(self,is_info=None):
verify=True,
headers=self.headers,
stream=True,
timeout=30,
timeout=(30,10)
)

write_time = time.time()
Expand Down Expand Up @@ -363,7 +363,7 @@ def loadsite(self, id, link):
link,
verify=True,
stream=True,
timeout=30,
timeout=(30,10)
)

with open(title + '.html', 'wb') as f:
Expand Down Expand Up @@ -793,7 +793,7 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files
verify=True,
headers=self.headers,
stream=True,
timeout=30,
timeout=(30,10)
)

filename = os.path.basename(
Expand All @@ -805,7 +805,8 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files
if filename is not None:
file, ext = os.path.splitext(filename)
filename = '%s[__%s__]%s' % (file, issueid, ext)
logger.info('filename: %s' % filename)

logger.fdebug('filename: %s' % filename)

if remote_filesize == 0:
try:
Expand All @@ -821,7 +822,7 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files
verify=True,
headers=self.headers,
stream=True,
timeout=30,
timeout=(30,10)
)
filename = os.path.basename(
urllib.parse.unquote(t.url)
Expand Down Expand Up @@ -918,6 +919,11 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files
f.write(chunk)
f.flush()

except requests.exceptions.Timeout as e:
logger.error('[ERROR] download has timed out due to inactivity...' % e)
mylar.DDL_LOCK = False
return {"success": False, "filename": filename, "path": None}

except Exception as e:
logger.error('[ERROR] %s' % e)
mylar.DDL_LOCK = False
Expand Down
23 changes: 22 additions & 1 deletion mylar/helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,17 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N
comicnzb= myDB.selectone("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
series = comicnzb['ComicName']
if comicnzb['AlternateFileName'] is None or comicnzb['AlternateFileName'] == 'None':
if any(
[
comicnzb['AlternateFileName'] is None,
comicnzb['AlternateFileName'] == 'None'
]
) or all(
[
comicnzb['AlternateFileName'] is not None,
comicnzb['AlternateFileName'].strip() == ''
]
):
seriesfilename = series
else:
seriesfilename = comicnzb['AlternateFileName']
Expand Down Expand Up @@ -3590,6 +3600,17 @@ def script_env(mode, vars):
except OSError as e:
logger.warn("Unable to run extra_script: " + str(script_cmd))
return False
except TypeError as e:
bad_environment = False
for key, value in mylar_env.items():
if not isinstance(key, str) or not isinstance(value, str):
bad_environment = True
if key in os.environ:
logger.error('Invalid global environment variable: {k!r} = {v!r}'.format(k=key, v=value))
else:
logger.error('Invalid Mylar environment variable: {k!r} = {v!r}'.format(k=key, v=value))
if not bad_environment:
raise e
else:
return True

Expand Down
2 changes: 1 addition & 1 deletion mylar/opds.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -919,7 +919,7 @@ def _StoryArc(self, **kwargs):
metainfo = issuedetails.get('metadata', None)
if not metainfo:
metainfo = [{'writer': None,'summary': ''}]
fileloc = os.path.join(comic['ComicLocation'],issue['Location'])
fileloc = issue['fileloc']
cb, _ = open_archive(fileloc)
if cb is None:
self.data = self._error_with_message('Can\'t open archive')
Expand Down
17 changes: 13 additions & 4 deletions mylar/search_filer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -729,10 +729,19 @@ def checker(self, entries, is_info=None):
booktype == 'HC',
booktype == 'GN',
]
) and (
int(F_ComicVersion) == int(findcomiciss)
and filecomic['justthedigits'] is None
):
) and any([
all(
[
int(F_ComicVersion) == int(findcomiciss)
and filecomic['justthedigits'] is None
]
), all(
[
int(F_ComicVersion) == int(findcomiciss)
and ComicYear == parsed_comic['issue_year']
]
)
]):
logger.fdebug(
'%s detected - reassigning volume %s to match as the'
' issue number based on Volume'
Expand Down
17 changes: 12 additions & 5 deletions mylar/webserve.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6090,7 +6090,8 @@ def preSearchit(self, ComicName, comiclist=None, mimp=0, volume=None, displaycom

mode='series'
displaycomic = helpers.filesafe(ComicName)
displaycomic = re.sub('[\-]','', displaycomic).strip()
if 'one-shot' not in displaycomic.lower():
displaycomic = re.sub('[\-]','', displaycomic).strip()
displaycomic = re.sub('\s+', ' ', displaycomic).strip()
logger.fdebug('[IMPORT] displaycomic : %s' % displaycomic)
logger.fdebug('[IMPORT] comicname : %s' % ComicName)
Expand Down Expand Up @@ -6867,7 +6868,10 @@ def comic_config(self, com_location, ComicID, alt_search=None, fuzzy_year=None,
newValues['AgeRating'] = age_rating

if all([publisher_imprint is not None, publisher_imprint != 'None', publisher_imprint != '']):
newValues['PublisherImprint'] = publisher_imprint
if re.sub(r'\s', '', publisher_imprint) == '':
newValues['PublisherImprint'] = 'None'
else:
newValues['PublisherImprint'] = publisher_imprint
else:
newValues['PublisherImprint'] = 'None'

Expand Down Expand Up @@ -6914,10 +6918,13 @@ def comic_config(self, com_location, ComicID, alt_search=None, fuzzy_year=None,

newValues['TorrentID_32P'] = torrentid_32p

if alt_filename is None or alt_filename == 'None':
newValues['AlternateFileName'] = "None"
if alt_filename is not None:
if re.sub(r'\s', '', alt_filename) == '':
newValues['AlternateFileName'] = "None"
else:
newValues['AlternateFileName'] = str(alt_filename)
else:
newValues['AlternateFileName'] = str(alt_filename)
newValues['AlternateFileName'] = "None"

#force the check/creation of directory com_location here
updatedir = True
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,5 @@ requests>=2.22.0
simplejson>=3.17.0
six>=1.13.0
tzlocal>=2.0.0
urllib3>=1.25.7
urllib3<2
user_agent2>=2021.12.11

0 comments on commit eca80ef

Please sign in to comment.