fetch-assets: test all downloaded files with zstd

Test them before moving them to the final location.
This makes the download fial of there is some file corruption etc.

This adds a dependency on the zstd exectuable for the fetch-assets
command.

Motivated by msys2/msys2-main-server#42

msys2_autobuild/cmd_fetch_assets.py

@@ -3,6 +3,7 @@
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Any, Dict, List, Tuple
+import subprocess
 
 from github.GitReleaseAsset import GitReleaseAsset
 
@@ -140,10 +141,16 @@ def file_is_uptodate(path: str, asset: GitReleaseAsset) -> bool:
     print("Pass --fetch-all to fetch all packages.")
     print("Pass --delete to clear the target directory")
 
+    def verify_file(path: str, target: str) -> None:
+        try:
+            subprocess.run(["zstd", "--quiet", "--test", path], capture_output=True, check=True, text=True)
+        except subprocess.CalledProcessError as e:
+            raise Exception(f"zstd test failed for {target!r}: {e.stderr}") from e
+
     def fetch_item(item: Tuple[str, GitReleaseAsset]) -> Tuple[str, GitReleaseAsset]:
         asset_path, asset = item
         if not args.pretend:
-            download_asset(asset, asset_path)
+            download_asset(asset, asset_path, verify_file)
         return item
 
     with ThreadPoolExecutor(8) as executor:

msys2_autobuild/gh.py

@@ -9,7 +9,7 @@
 from functools import lru_cache
 from hashlib import sha256
 from pathlib import Path
-from typing import Any, Dict, Generator, List, Optional
+from typing import Any, Dict, Generator, List, Optional, Callable
 
 import requests
 from github import Github
@@ -141,7 +141,8 @@ def get_asset_mtime_ns(asset: GitReleaseAsset) -> int:
     return int(asset.updated_at.timestamp() * (1000 ** 3))
 
 
-def download_asset(asset: GitReleaseAsset, target_path: str) -> None:
+def download_asset(asset: GitReleaseAsset, target_path: str,
+                   onverify: Callable[[str, str], None] | None = None) -> None:
     assert asset_is_complete(asset)
     session = get_requests_session(nocache=True)
     with session.get(asset.browser_download_url, stream=True, timeout=REQUESTS_TIMEOUT) as r:
@@ -154,6 +155,8 @@ def download_asset(asset: GitReleaseAsset, target_path: str) -> None:
                     h.write(chunk)
             mtime_ns = get_asset_mtime_ns(asset)
             os.utime(temppath, ns=(mtime_ns, mtime_ns))
+            if onverify is not None:
+                onverify(temppath, target_path)
             shutil.move(temppath, target_path)
         finally:
             try: