feat(dl): Add *new* --workers to set download threads/workers

The previously named --workers which is now --downloads specified how many tracks to download, not how many threads/workers are used per-download.

It defaults to nothing, which each downloader then has their own defaults. All current downloaders though currently default to `min(32, (os.cpu_count() or 1) + 4)`, which is also the default for `ThreadPoolExecutor` in general.

This also brings a side effect of changing DASH and HLS's forced max_workers of 16 to now a more appropriate default but more importantly actually configurable. You can set a default in your config under `dl.workers`.

CONFIG.md

@@ -153,6 +153,13 @@ For example to set the default primary language to download to German,
 lang: de
 ```
 
+to set how many tracks to download concurrently to 4 and download threads to 16,
+
+```yaml
+downloads: 4
+workers: 16
+```
+
 to set `--bitrate=CVBR` for the AMZN service,
 
 ```yaml

devine/commands/dl.py

@@ -131,6 +131,8 @@ class dl:
                   help="Disable folder creation for TV Shows.")
     @click.option("--no-source", is_flag=True, default=False,
                   help="Disable the source tag from the output file name and path.")
+    @click.option("--workers", type=int, default=None,
+                  help="Max workers/threads to download with per-track. Default depends on the downloader.")
     @click.option("--downloads", type=int, default=1,
                   help="Amount of tracks to download concurrently.")
     @click.pass_context
@@ -275,6 +277,7 @@ def result(
         no_proxy: bool,
         no_folder: bool,
         no_source: bool,
+        workers: Optional[int],
         downloads: int,
         *_: Any,
         **__: Any
@@ -528,6 +531,7 @@ def result(
                                     vaults_only=vaults_only,
                                     export=export
                                 ),
+                                max_workers=workers,
                                 progress=tracks_progress_callables[i]
                             )
                             for i, track in enumerate(title.tracks)

devine/core/manifests/dash.py

@@ -236,6 +236,7 @@ def download_track(
         progress: partial,
         session: Optional[Session] = None,
         proxy: Optional[str] = None,
+        max_workers: Optional[int] = None,
         license_widevine: Optional[Callable] = None
     ):
         if not session:
@@ -472,7 +473,7 @@ def download_track(
             headers=session.headers,
             cookies=session.cookies,
             proxy=proxy,
-            max_workers=16
+            max_workers=max_workers
         ):
             file_downloaded = status_update.get("file_downloaded")
             if file_downloaded:

devine/core/manifests/hls.py

@@ -197,6 +197,7 @@ def download_track(
         progress: partial,
         session: Optional[Session] = None,
         proxy: Optional[str] = None,
+        max_workers: Optional[int] = None,
         license_widevine: Optional[Callable] = None
     ) -> None:
         if not session:
@@ -280,7 +281,7 @@ def download_track(
             headers=session.headers,
             cookies=session.cookies,
             proxy=proxy,
-            max_workers=16
+            max_workers=max_workers
         ):
             file_downloaded = status_update.get("file_downloaded")
             if file_downloaded:

devine/core/tracks/subtitle.py

@@ -153,9 +153,10 @@ def download(
         self,
         session: requests.Session,
         prepare_drm: partial,
+        max_workers: Optional[int] = None,
         progress: Optional[partial] = None
     ):
-        super().download(session, prepare_drm, progress)
+        super().download(session, prepare_drm, max_workers, progress)
         if not self.path:
             return
 

devine/core/tracks/track.py

@@ -135,6 +135,7 @@ def download(
         self,
         session: Session,
         prepare_drm: partial,
+        max_workers: Optional[int] = None,
         progress: Optional[partial] = None
     ):
         """Download and optionally Decrypt this Track."""
@@ -191,6 +192,7 @@ def cleanup():
                     progress=progress,
                     session=session,
                     proxy=proxy,
+                    max_workers=max_workers,
                     license_widevine=prepare_drm
                 )
             elif self.descriptor == self.Descriptor.DASH:
@@ -201,6 +203,7 @@ def cleanup():
                     progress=progress,
                     session=session,
                     proxy=proxy,
+                    max_workers=max_workers,
                     license_widevine=prepare_drm
                 )
             elif self.descriptor == self.Descriptor.URL:
@@ -236,7 +239,8 @@ def cleanup():
                             filename=save_path.name,
                             headers=session.headers,
                             cookies=session.cookies,
-                            proxy=proxy
+                            proxy=proxy,
+                            max_workers=max_workers
                         ):
                             file_downloaded = status_update.get("file_downloaded")
                             if not file_downloaded: