Merge pull request #583 from WebarchivCZ/updates

Session age, pip bumps, TC seeds, #579, #576, #569, #582

Seeder/blacklists/views.py

@@ -24,10 +24,10 @@ class ListView(BlacklistView, TemplateView):
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
-        last_change = timezone.now().replace(microsecond=0) -\
-            models.Blacklist.last_change().replace(microsecond=0)
-
-        context['last_change'] = last_change
+        last_change = models.Blacklist.last_change()
+        if last_change:
+            context['last_change'] = (timezone.now().replace(microsecond=0) -
+                                      last_change.replace(microsecond=0))
         context['blacklists'] = models.Blacklist.objects.all()
         return context
 

Seeder/core/dashboard_data.py

@@ -236,6 +236,9 @@ def get_queryset(self):
         random.seed(str(self.user) + str(timezone.now().date()))
         # Include all sources, both owned by user and not
         qa_sources = source_models.Source.objects.needs_qa()
+        # Don't continue if there are no QA sources
+        if qa_sources.count() == 0:
+            return qa_sources
         # Randomly select up to N sources for QA with at most M tries
         random_qa = set()  # collect pks
         tries = 0

Seeder/harvests/models.py

@@ -2,6 +2,7 @@
 
 from itertools import chain
 from datetime import date
+from django.contrib import messages
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
@@ -96,7 +97,7 @@ def get_custom_seeds(self):
         return set(self.custom_seeds.splitlines())
 
     def get_custom_sources_seeds(self):
-        seeds = Seed.objects.archiving().filter(
+        seeds = Seed.objects.filter(
             source__in=self.custom_sources.all())
         return set(seeds.values_list('url', flat=True)) - self.get_blacklisted()
 
@@ -341,8 +342,12 @@ def freeze_seeds(self):
         """
         Freezes the seeds to preserve them for later use
         """
-        self.seeds_frozen = '\n'.join(self.get_seeds())
-        self.save()
+        seeds = self.get_seeds()
+        if len(seeds) > 0:
+            self.seeds_frozen = '\n'.join(seeds)
+            self.save()
+            return True     # frozen correctly
+        return False        # not frozen
 
     @property
     def is_oneshot(self):
@@ -496,4 +501,6 @@ def freeze_urls(sender, instance, **kwargs):
     :type instance: Harvest
     """
     if instance.status == Harvest.STATE_RUNNING and not instance.seeds_frozen:
-        instance.freeze_seeds()
+        # If cannot freeze correctly, reset Harvest status
+        if not instance.freeze_seeds():
+            instance.status = Harvest.STATE_PLANNED

Seeder/harvests/templates/topic_collection.html

@@ -117,6 +117,7 @@ <h4>{% trans 'Seeds' %}</h4>
         <h4>{% trans 'Links' %}:</h4>
             <a href="{% url 'harvests:topic_collection_edit' pk=object.pk %}" class="btn btn-primary btn-warning btn-block">{% trans 'Edit' %}</a>
             <a href="{% url 'harvests:topic_collection_history' pk=object.pk %}" class="btn btn-primary btn-info btn-block">{% trans 'History' %}</a>
+            <a href="{% url 'harvests:topic_collection_urls' pk=object.pk %}" class="btn btn-primary btn-info btn-block" target="_blank">{% trans 'Seed URLs' %}</a>
             <br />
             <form action='{% url 'harvests:topic_collection_update_slug' pk=object.pk %}' method='post'>
                 {% csrf_token %}

Seeder/harvests/urls.py

@@ -35,6 +35,8 @@ def to_url(self, value):
          name='topic_collection_edit'),
     path('<int:pk>/collection_detail', CollectionDetail.as_view(),
          name='topic_collection_detail'),
+    path('<int:pk>/collection_urls', CollectionListUrls.as_view(),
+         name='topic_collection_urls'),
     path('<int:pk>/collection_history', CollectionHistory.as_view(),
          name='topic_collection_history'),
     path('<int:pk>/toggle_publish', CollectionTogglePublish.as_view(),

Seeder/harvests/views.py

@@ -407,7 +407,7 @@ class ReorderCollections(View):
     def get(self, request, *args, **kwargs):
         collections = models.TopicCollection.objects.order_by('order')
         for i, tc in enumerate(collections):
-            tc.order = i+1
+            tc.order = i + 1
             tc.save()
         return HttpResponseRedirect(reverse('harvests:topic_collection_list'))
 
@@ -434,6 +434,20 @@ class CollectionDetail(TCView, DetailView, CommentViewGeneric):
     template_name = 'topic_collection.html'
 
 
+class CollectionListUrls(TCView, DetailView, TemplateView):
+    """
+    List all seeds for a specific topic collection.
+    """
+    template_name = 'urls.html'
+
+    def get_context_data(self, **kwargs):
+        self.object = self.get_object()
+        context = super().get_context_data(**kwargs)
+        context['head_lines'] = [f"# {self.object.title}"]
+        context['urls'] = self.object.get_seeds()
+        return context
+
+
 class CollectionHistory(TCView, generic_views.HistoryView):
     """
         History of changes to TopicCollections