Merge branch 'python3-dev'

IMP:(mylar3#1421) (mass-add): Do not add series if in ignored publisher list (@morsok)
IMP: webserve: introduce /prometheus_metrics with queue metrics (@OddBloke)
IMP: helpers: convert queue_info to a generator (@OddBloke)
IMP: helpers: convert queue_info to return a namedtuple (@OddBloke)
IMP: webserve: refactor queue information generation to helpers (@OddBloke)
FIX:(mylar3#1432) story arc banner upload field could accept null values in error
FIX: Working around accidentally invalid API key (starting with None) (@Nebelherr23)
FIX: weeklypull: handle invalid annuallink values gracefully Emit a WARNING log message instead of a traceback. (@OddBloke)
FIX: Correct wording about worker repair (@OddBloke - mylar3#1417)
FIX: Downloading annuals would work intermittently (usually not working thru GC)
FIX: Post-processing would not match annuals properly and ignore them during a run
FIX: DDL failing to search/download due to CF/FS problems
FIX: getcomics: release the DDL_LOCK at the end of downloadit (@OddBloke)
FIX: getcomics: add fallthrough return value to downloadit (@OddBloke)

data/interfaces/default/manage.html

@@ -286,27 +286,27 @@ <h1 class="clearfix">Manage</h1>
                            </tr>
                         </thead>
                         <tbody>
-                           %for q_name, (is_up, length) in queues.items():
+                           %for q in queues:
                               <%
-                                    if is_up:
+                                    if q.is_alive:
                                         text = "Up"
                                         grade = '#7AEE52'
-                                    elif is_up is not None:
+                                    elif q.is_alive is not None:
                                         text = "Down"
                                         grade = '#EC7564'
                                     else:
                                         text = "Never Started"
                                         grade = '#52A9EE'
                                %>
                               <tr>
-                                <td style="width: 50px;text-align: center;">${q_name}</td>
-                                <td style="width: 50px;text-align: center;">${length}</td>
+                                <td style="width: 50px;text-align: center;">${q.name}</td>
+                                <td style="width: 50px;text-align: center;">${q.size}</td>
                                 <td style="width: 50px;text-align: center;color:${grade}">${text}</td>
                               </tr>
                            %endfor
                         </tbody>
            </table>
-           </br><small><center>Workers are only started if your config requires them. The DDL worker can be restarted <a href="/queueManage">here</a>: other workers require a restart of Mylar to repair.</center></small>
+           </br><small><center>Workers are only started if your config requires them. Down workers require a restart of Mylar to repair.</center></small>
         </div>
 
 

data/interfaces/default/storyarc_detail.html

@@ -100,10 +100,10 @@ <h1><p style="display:inline;float:right;">${storyarcname}</h1>
                                  optpos = '</br><div style=\"display:block;float:right;position:absolute;right:20px;top:120px;\">'
                          %>
                      ${optpos}
-                         <form action="downloadBanner" method="GET">
+                         <form name="banner_stuff" onsubmit="return checkValids(event)" action="downloadBanner" method="POST">
                          <label style="display:inline;float:right;">Banner Image Retrieval</label></br>
-                               <input style="display:inline;float:right;" type="text" title="enter full url where image resides" placeholder="full URL where image resides" onfocus="if(this.value==this.defaultValue) this.value='';" name="url" size="25">
-                               <input style="display:inline;float:right;" type="image" src="images/download_icon.png" height="20" width="20" alt="submit"></br>
+                               <input style="display:inline;float:right;" id="banner_url" type="text" title="enter full url where image resides" placeholder="full URL where image resides" name="url" size="25">
+                               <input style="display:inline;float:right;" id="bannner_click" type="image" src="images/download_icon.png" height="20" width="20" alt="submit"></br>
                                <input type="hidden" name="storyarcid" value="${storyarcid}">
                                <input type="hidden" name="storyarcname" value="${storyarcname| u}">
                           %if storyarcbanner is not None:
@@ -310,6 +310,31 @@ <h1><p style="display:inline;float:right;">${storyarcname}</h1>
                },
             });
         }            
+        function checkValids(e) {
+            var urlchk = document.getElementById("banner_url");
+            if (urlchk.value == ""){
+                e.preventDefault();
+                //alert("1 - no url");
+                urlchk.placeholder = "Invalid URL";
+                timeout = setTimeout(function() {
+                  urlchk.placeholder = "full URL where image resides";
+                }, 5000);
+                return false;
+            } else {
+                if (urlchk.value.startsWith("http")){
+                    //alert("url provided");
+                    return true;
+                }
+            }
+            //alert("2 - no url");
+            e.preventDefault();
+            urlchk.placeholder = "Invalid URL";
+            timeout = setTimeout(function() {
+              urlchk.placeholder = "full URL where image resides";
+            }, 5000);
+            return false;
+        }
+
 	function initThisPage() {
                 $(function() {
                         $( "#tabs" ).tabs();

data/interfaces/default/storyarc_detail.poster.html

@@ -101,10 +101,11 @@ <h1 class="clearfix"><a href="storyarc_main"><img src="${icons['ReadingList-icon
                              optpos = '<div style=\"display:block;float:right;position:relative;right:20px;top:50px;\">'
                          %>
                      ${optpos}
-                         <form action="downloadBanner" method="GET">
+
+                         <form name="banner_stuff" onsubmit="return checkValids(event)" action="downloadBanner" method="POST">
                          <label style="display:inline;float:right;">Banner Image Retrieval</label></br>
-                               <input style="display:inline;float:right;" type="text" title="enter full url where image resides" placeholder="full URL where image resides" onfocus="if(this.value==this.defaultValue) this.value='';" name="url" size="25">
-                               <input style="display:inline;float:right;" type="image" src="images/download_icon.png" height="20" width="20" alt="submit"></br>
+                               <input style="display:inline;float:right;" id="banner_url" type="text" title="enter full url where image resides" placeholder="full URL where image resides" name="url" size="25">
+                               <input style="display:inline;float:right;" id="bannner_click" type="image" src="images/download_icon.png" height="20" width="20" alt="submit"></br>
                                <input type="hidden" name="storyarcid" value="${storyarcid}">
                                <input type="hidden" name="storyarcname" value="${storyarcname| u}">
                           %if storyarcbanner is not None:
@@ -311,7 +312,32 @@ <h1 class="clearfix"><a href="storyarc_main"><img src="${icons['ReadingList-icon
                  alert('ERROR'+data.responseText);
                },
             });
-        }            
+        }
+        function checkValids(e) {
+            var urlchk = document.getElementById("banner_url");
+            if (urlchk.value == ""){
+                e.preventDefault();
+                //alert("1 - no url");
+                urlchk.placeholder = "Invalid URL";
+                timeout = setTimeout(function() {
+                  urlchk.placeholder = "full URL where image resides";
+                }, 5000);
+                return false;
+            } else {
+                if (urlchk.value.startsWith("http")){
+                    //alert("url provided");
+                    return true;
+                }
+            }
+            //alert("2 - no url");
+            e.preventDefault();
+            urlchk.placeholder = "Invalid URL";
+            timeout = setTimeout(function() {
+              urlchk.placeholder = "full URL where image resides";
+            }, 5000);
+            return false;
+        }
+
 	function initThisPage() {
                 $(function() {
                         $( "#tabs" ).tabs();

mylar/config.py

@@ -1139,6 +1139,12 @@ def configure(self, update=False, startup=False):
         if any([self.COMICVINE_API is None, self.COMICVINE_API == 'None', self.COMICVINE_API == '']):
             logger.error('No User Comicvine API key specified. I will not work very well due to api limits - http://api.comicvine.com/ and get your own free key.')
             self.COMICVINE_API = None
+        # Check if Comicvine API key starts with None, thus making it invalid
+        elif self.COMICVINE_API[:4] == 'None':
+            # Notify user of what's going on
+            logger.warn('Comicvine API key starts with a None, working around for now, please fix')
+            # Set the actual API key, so mylar does not appear broken from the start
+            self.COMICVINE_API = self.COMICVINE_API[4:]
 
         if self.SEARCH_INTERVAL < 360:
             logger.fdebug('Search interval too low. Resetting to 6 hour minimum')

mylar/filechecker.py

@@ -1388,7 +1388,17 @@ def matchIT(self, series_info):
         n_name = 'annual'
         if mylar.CONFIG.ANNUALS_ON:
             ann_year_check = re.findall(r'(\d{4})(?=[\s]|annual\b|$)', self.watchcomic, flags=re.I)
-            if all(['annual' in nspace_watchcomic.lower(), 'annual' not in series_name.lower()]):
+            if all(
+                      [
+                          'annual' in nspace_watchcomic.lower(),
+                          'annual' not in series_name.lower()
+                      ]
+                ) or all(
+                      [
+                          'annual' not in nspace_watchcomic.lower(),
+                          'annual' in series_name.lower()
+                      ]
+                ):
                 annualisation = True
                 justthedigits = 'Annual'
                 if series_info['issue_number'] is not None:

mylar/getcomics.py

@@ -129,7 +129,6 @@ def __init__(self, query=None, issueid=None, comicid=None, oneoff=False, session
         }
 
         self.headers = {
-            'Accept-encoding': 'gzip',
             'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1',
             'Referer': mylar.GC_URL,
         }
@@ -156,12 +155,6 @@ def __init__(self, query=None, issueid=None, comicid=None, oneoff=False, session
 
         self.search_format = ['"%s #%s (%s)"', '%s #%s (%s)', '%s #%s', '%s %s']
 
-        self.headers = {
-            'Accept-encoding': 'gzip',
-            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1',
-            'Referer': mylar.GC_URL,
-        }
-
         self.provider_stat = provider_stat
 
     def search(self,is_info=None):
@@ -318,6 +311,7 @@ def loadsite(self, id, link):
         t = self.session.get(
             link,
             verify=True,
+            headers=self.headers,
             stream=True,
             timeout=(30,10)
         )
@@ -773,6 +767,7 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files
         myDB = db.DBConnection()
         filename = None
         self.cookie_receipt()
+        self.headers['Accept-encoding'] = 'gzip'
         try:
             with requests.Session() as s:
                 if resume is not None:
@@ -787,7 +782,7 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files
                     verify=True,
                     headers=self.headers,
                     stream=True,
-                    timeout=(30,10)
+                    timeout=(30,30)
                 )
 
                 filename = os.path.basename(
@@ -956,6 +951,9 @@ def downloadit(self, id, link, mainlink, resume=None, issueid=None, remote_files
                     new_path = dst_path
                 return {"success": True, "filename": filename, "path": new_path}
 
+        mylar.DDL_LOCK = False
+        return {"success": False, "filename": filename, "path": None}
+
     def issue_list(self, pack):
         # packlist = [x.strip() for x in pack.split(',)]
         packlist = pack.replace('+', ' ').replace(',', ' ').split()

mylar/helpers.py

@@ -34,6 +34,7 @@
 import gzip
 import os, errno
 import urllib
+from collections import namedtuple
 from urllib.parse import urljoin
 from io import StringIO
 from apscheduler.triggers.interval import IntervalTrigger
@@ -3485,6 +3486,23 @@ def cdh_monitor(queue, item, nzstat, readd=False):
             logger.error('process error: %s' % e)
     return
 
+
+QueueInfo = namedtuple("QueueInfo", ("name", "is_alive", "size"))
+
+
+def queue_info():
+    yield from (
+        QueueInfo(queue_name, thread_obj.is_alive() if thread_obj is not None else None, queue.qsize())
+        for (queue_name, thread_obj, queue) in [
+            ("AUTO-COMPLETE-NZB", mylar.NZBPOOL, mylar.NZB_QUEUE),
+            ("AUTO-SNATCHER", mylar.SNPOOL, mylar.SNATCHED_QUEUE),
+            ("DDL-QUEUE", mylar.DDLPOOL, mylar.DDL_QUEUE),
+            ("POST-PROCESS-QUEUE", mylar.PPPOOL, mylar.PP_QUEUE),
+            ("SEARCH-QUEUE", mylar.SEARCHPOOL, mylar.SEARCH_QUEUE),
+        ]
+    )
+
+
 def script_env(mode, vars):
     #mode = on-snatch, pre-postprocess, post-postprocess
     #var = dictionary containing variables to pass

mylar/search_filer.py

@@ -903,8 +903,14 @@ def _process_entry(self, entry, is_info):
                     ):
                         intIss = 1000
                     else:
-                        if annualize is True and len(re.sub('[^0-9]', '', parsed_comic['issue_number']).strip()) == 4:
-                            intIss = 1000
+                        if annualize is True:
+                            if len(re.sub('[^0-9]', '', parsed_comic['issue_number']).strip()) == 4:
+                                intIss = 1000
+                            elif parsed_comic['issue_number'] is not None:
+                                intIss = helpers.issuedigits(parsed_comic['issue_number'])
+                            else:
+                                # if issue_number is None, assume it's #1 of the annual
+                                intIss = 1000
                         else:
                             intIss = 9999999999
                 if filecomic['justthedigits'] is not None:

mylar/webserve.py

@@ -3962,16 +3962,7 @@ def manage(self):
                             'jobname': jb['JobName'],
                             'status': status})
             jobresults = tmp
-        queues = {
-            queue_name: (thread_obj.is_alive() if thread_obj is not None else None, queue.qsize())
-            for (queue_name, thread_obj, queue) in [
-                ("AUTO-COMPLETE-NZB", mylar.NZBPOOL, mylar.NZB_QUEUE),
-                ("AUTO-SNATCHER", mylar.SNPOOL, mylar.SNATCHED_QUEUE),
-                ("DDL-QUEUE", mylar.DDLPOOL, mylar.DDL_QUEUE),
-                ("POST-PROCESS-QUEUE", mylar.PPPOOL, mylar.PP_QUEUE),
-                ("SEARCH-QUEUE", mylar.SEARCHPOOL, mylar.SEARCH_QUEUE),
-            ]
-        }
+        queues = helpers.queue_info()
         return serve_template(templatename="manage.html", title="Manage", mylarRoot=mylarRoot, jobs=jobresults, queues=queues, scan_info=scan_info)
     manage.exposed = True
 
@@ -7376,6 +7367,27 @@ def api(self, *args, **kwargs):
 
     api.exposed = True
 
+    def prometheus_metrics(self):
+        logger.debug("Responding to Prometheus metrics request")
+        cherrypy.response.headers['Content-Type'] = "text/plain"
+        q_metrics = {name: {} for name in ["alive", "size", "started"]}
+        for q in helpers.queue_info():
+            normalised_name = q.name.replace("-", "_")
+
+            q_metrics["size"][normalised_name] = q.size
+            q_metrics["started"][normalised_name] = "1" if q.is_alive is not None else "0"
+            q_metrics["alive"][normalised_name] = "1" if q.is_alive else "0"
+
+        response = ""
+        for metric, items in q_metrics.items():
+            full_metric = f"mylar_queue_{metric}"
+            response += f"# TYPE {full_metric} gauge\n"
+            for name, value in items.items():
+                response += '%s{queue="%s"} %s\n' % (full_metric, name, value)
+            response += "\n"
+        return response
+
+    prometheus_metrics.exposed = True
 
     def opds(self, *args, **kwargs):
         from mylar.opds import OPDS

mylar/weeklypull.py

@@ -1033,7 +1033,14 @@ def new_pullcheck(weeknumber, pullyear, comic1off_name=None, comic1off_id=None,
                 if mylar.CONFIG.ANNUALS_ON is True:
                     annualidmatch = [x for x in weeklylist if week['comicid'] is not None and ([xa for xa in x['AnnualIDs'] if int(xa['ComicID']) == int(week['comicid'])])]
                     if not annualidmatch:
-                        annualidmatch = [x for x in weeklylist if week['annuallink'] is not None and (int(x['ComicID']) == int(week['annuallink']))]
+                        annual_link = week['annuallink']
+                        if annual_link is not None:
+                            try:
+                                annual_link = int(annual_link)
+                            except ValueError:
+                                logger.warn("[WEEKLY-PULL] %s #%s has an invalid annuallink value (%s): walksoftly data may be invalid; skipping", week['ComicName'], week['ISSUE'], week['annuallink'])
+                                continue
+                        annualidmatch = [x for x in weeklylist if annual_link is not None and (int(x['ComicID']) == annual_link)]
 
                 #The above will auto-match against ComicID if it's populated on the pullsite, otherwise do name-matching.
                 namematch = [ab for ab in weeklylist if ab['DynamicName'] == week['dynamicname']]
@@ -1403,6 +1410,9 @@ def mass_publishers(publishers, weeknumber, year):
         for wt in watchlist:
             if wt['ComicID'] not in watchlibrary and wt['ComicID'] is not None:
                 if not {"comicid": wt['ComicID'], "comicname": wt['COMIC']} in mylar.ADD_LIST.queue:
+                    if wt['Publisher'] in mylar.CONFIG.IGNORED_PUBLISHERS:
+                        logger.info("[SHIZZLE-WHIZZLE] %s is in your ignored_publishers list skipping %s either it's a configuration issue or a mismatch in the weekly pull-list" % (wt['Publisher'], wt['COMIC']))
+                        continue
                     watch.append({"comicid": wt['ComicID'], "comicname": wt['COMIC'], "seriesyear": wt['seriesyear']})
 
     if len(watch) > 0: