HQ: mark the item as done when host is excluded

internetarchive · Apr 26, 2024 · 52b2551 · 52b2551
1 parent 60e711f
commit 52b2551
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/internal/pkg/crawl/worker.go b/internal/pkg/crawl/worker.go
@@ -34,6 +34,11 @@ func (c *Crawl) Worker() {
 
 		// If the host of the item is in the host exclusion list, we skip it
 		if utils.StringInSlice(item.Host, c.ExcludedHosts) || !c.checkIncludedHosts(item.Host) {
+			if c.UseHQ {
+				// If we are using the HQ, we want to mark the item as done
+				c.HQFinishedChannel <- item
+			}
+
 			continue
 		}