diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index e2048caf6b..bd33afaf05 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -2,6 +2,10 @@ name: Pull Request Check on: pull_request: + paths-ignore: + - '.github/**' + - 'README.md' + - 'tools/**' branches: - main @@ -19,8 +23,21 @@ jobs: - uses: hmarr/debug-action@v2 name: "debug: ${{github.event_name}}" + - name: Clean out files part 1 + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - uses: actions/checkout@v4 + - name: Clean out files part 2 + run: | + rm -rf .git + echo "Free Space" + df -h + - name: env - better defaulting of env vars; id: env run: | @@ -64,3 +81,7 @@ jobs: touch .github_check python3 tools/build_data.py python3 tools/build_release.py --do-check + + - name: Check Free Space + run: | + df -h diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fa0529a7b6..1a953f37db 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,15 +15,20 @@ jobs: - uses: hmarr/debug-action@v2 name: "debug: ${{github.event_name}}" - - uses: actions/checkout@v4 - - - name: Clean out files for more space + - name: Clean out files part 1 run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + - uses: actions/checkout@v4 + + - name: Clean out files part 2 + run: | rm -rf .git + echo "Free Space" + df -h - name: env - better defaulting of env vars; id: env @@ -64,19 +69,6 @@ jobs: ###################################################################### ## Only run these if it is the PortMaster-New repo - - uses: robinraju/release-downloader@v1.8 - if: ${{ steps.env.outputs.RELEASE_REPO }} == "PortMaster-New" - with: - out-file-path: "releases" - repository: "PortsMaster/PortMaster-Runtime" - tag: "runtimes" - fileName: "*.squashfs" - - - name: Remove Zulu JDK - if: ${{ steps.env.outputs.RELEASE_REPO }} == "PortMaster-New" - run: | - rm -f releases/zulu* - - uses: robinraju/release-downloader@v1.8 if: ${{ steps.env.outputs.RELEASE_REPO }} == "PortMaster-New" with: @@ -99,7 +91,6 @@ jobs: python3 tools/build_data.py python3 tools/build_release.py "${{steps.date.outputs.date}}" rm -fv releases/.gitignore - rm -f releases/*.squashfs - name: "Prepare Release" uses: ncipollo/release-action@v1 @@ -128,6 +119,10 @@ jobs: repo: ${{ steps.env.outputs.RELEASE_REPO }} owner: ${{ steps.env.outputs.RELEASE_ORG }} + - name: Check Free Space + run: | + df -h + - name: Release Info id: info run: | diff --git a/.gitignore b/.gitignore index 7ed157ced1..24d0327aa5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ # Special file, ignore it. :) .github_check +.hash_cache diff --git a/runtimes/.gitignore b/runtimes/.gitignore new file mode 100644 index 0000000000..4d4868450e --- /dev/null +++ b/runtimes/.gitignore @@ -0,0 +1,2 @@ +# Autogenerated by tools/build_data.py +mono-6.12.0.122-aarch64.squashfs diff --git a/runtimes/frt_2.1.6.squashfs b/runtimes/frt_2.1.6.squashfs new file mode 100644 index 0000000000..10517a1f29 Binary files /dev/null and b/runtimes/frt_2.1.6.squashfs differ diff --git a/runtimes/frt_3.0.6_v1.squashfs b/runtimes/frt_3.0.6_v1.squashfs new file mode 100644 index 0000000000..3f420be320 Binary files /dev/null and b/runtimes/frt_3.0.6_v1.squashfs differ diff --git a/runtimes/frt_3.1.2.squashfs b/runtimes/frt_3.1.2.squashfs new file mode 100644 index 0000000000..ec47814d70 Binary files /dev/null and b/runtimes/frt_3.1.2.squashfs differ diff --git a/runtimes/frt_3.2.3.squashfs b/runtimes/frt_3.2.3.squashfs new file mode 100644 index 0000000000..24941d06d7 Binary files /dev/null and b/runtimes/frt_3.2.3.squashfs differ diff --git a/runtimes/frt_3.3.4.squashfs b/runtimes/frt_3.3.4.squashfs new file mode 100644 index 0000000000..0f5d3d8be7 Binary files /dev/null and b/runtimes/frt_3.3.4.squashfs differ diff --git a/runtimes/frt_3.4.5.squashfs b/runtimes/frt_3.4.5.squashfs new file mode 100644 index 0000000000..46eb7142b6 Binary files /dev/null and b/runtimes/frt_3.4.5.squashfs differ diff --git a/runtimes/frt_3.5.2.squashfs b/runtimes/frt_3.5.2.squashfs new file mode 100644 index 0000000000..8896ba8454 Binary files /dev/null and b/runtimes/frt_3.5.2.squashfs differ diff --git a/runtimes/frt_4.0.4.squashfs b/runtimes/frt_4.0.4.squashfs new file mode 100644 index 0000000000..dc223fa54a Binary files /dev/null and b/runtimes/frt_4.0.4.squashfs differ diff --git a/runtimes/frt_4.1.3.squashfs b/runtimes/frt_4.1.3.squashfs new file mode 100644 index 0000000000..73cf2bba60 Binary files /dev/null and b/runtimes/frt_4.1.3.squashfs differ diff --git a/runtimes/mono-6.12.0.122-aarch64.squashfs.part.001 b/runtimes/mono-6.12.0.122-aarch64.squashfs.part.001 new file mode 100644 index 0000000000..f1ec7c0d2b Binary files /dev/null and b/runtimes/mono-6.12.0.122-aarch64.squashfs.part.001 differ diff --git a/runtimes/mono-6.12.0.122-aarch64.squashfs.part.002 b/runtimes/mono-6.12.0.122-aarch64.squashfs.part.002 new file mode 100644 index 0000000000..07860ba4b5 Binary files /dev/null and b/runtimes/mono-6.12.0.122-aarch64.squashfs.part.002 differ diff --git a/runtimes/mono-6.12.0.122-aarch64.squashfs.part.003 b/runtimes/mono-6.12.0.122-aarch64.squashfs.part.003 new file mode 100644 index 0000000000..4269ccc1a6 Binary files /dev/null and b/runtimes/mono-6.12.0.122-aarch64.squashfs.part.003 differ diff --git a/runtimes/mono-6.12.0.122-aarch64.squashfs.part.004 b/runtimes/mono-6.12.0.122-aarch64.squashfs.part.004 new file mode 100644 index 0000000000..7b69fdb6c6 Binary files /dev/null and b/runtimes/mono-6.12.0.122-aarch64.squashfs.part.004 differ diff --git a/runtimes/mono-6.12.0.122-aarch64.squashfs.part.005 b/runtimes/mono-6.12.0.122-aarch64.squashfs.part.005 new file mode 100644 index 0000000000..fcc1d18a8e Binary files /dev/null and b/runtimes/mono-6.12.0.122-aarch64.squashfs.part.005 differ diff --git a/runtimes/runtimes.json b/runtimes/runtimes.json new file mode 100644 index 0000000000..1bf00e2f80 --- /dev/null +++ b/runtimes/runtimes.json @@ -0,0 +1,13 @@ +{ + "frt_2.1.6.squashfs": "Godot/FRT 2.1.6", + "frt_3.0.6_v1.squashfs": "Godot/FRT 3.0.5_v1", + "frt_3.1.2.squashfs": "Godot/FRT 3.1.2", + "frt_3.2.3.squashfs": "Godot/FRT 3.2.3", + "frt_3.3.4.squashfs": "Godot/FRT 3.3.4", + "frt_3.4.5.squashfs": "Godot/FRT 3.4.5", + "frt_3.5.2.squashfs": "Godot/FRT 3.5.2", + "frt_4.0.4.squashfs": "Godot/FRT 4.0.4", + "frt_4.1.3.squashfs": "Godot/FRT 4.1.3", + "mono-6.12.0.122-aarch64.squashfs": "Mono 6.12.0.122", + "solarus-1.6.5.squashfs": "Solarus 1.6.5" +} \ No newline at end of file diff --git a/runtimes/solarus-1.6.5.squashfs b/runtimes/solarus-1.6.5.squashfs new file mode 100644 index 0000000000..6bc2cd3f5a Binary files /dev/null and b/runtimes/solarus-1.6.5.squashfs differ diff --git a/tools/build_data.py b/tools/build_data.py index 989a5d6d15..8e59247555 100644 --- a/tools/build_data.py +++ b/tools/build_data.py @@ -24,9 +24,13 @@ ############################################################################# ROOT_DIR = Path('.') +CACHE_FILE = ROOT_DIR / '.hash_cache' MANIFEST_FILE = ROOT_DIR / 'manifest.json' STATUS_FILE = ROOT_DIR / 'ports_status.json' PORTS_DIR = ROOT_DIR / 'ports' +RUNTIMES_DIR = ROOT_DIR / 'runtimes' + +GITHUB_RUN = (ROOT_DIR / '.github_check').is_file() LARGEST_FILE = (1024 * 1024 * 90) CHUNK_SIZE = (1024 * 1024 * 50) @@ -147,8 +151,19 @@ def combine_large_files(port_dir, large_file_name, large_file_parts): out_fh.write(data) + if GITHUB_RUN: + # Delete the part files to reduce file system usage. + Path(large_file_part).unlink() + + +def check_large_files(port_dir, large_files, hash_cache=None): + if hash_cache is not None: + files_hash = hash_cache.get_files_hash + file_hash = hash_cache.get_file_hash + else: + files_hash = hash_files + file_hash = hash_file -def check_large_files(port_dir, large_files): for large_file_name, large_file_parts in large_files.items(): large_file_name = Path(large_file_name) @@ -156,10 +171,10 @@ def check_large_files(port_dir, large_files): file_md5 = None if len(large_file_parts) > 0: - parts_md5 = hash_files(large_file_parts) + parts_md5 = files_hash(large_file_parts) if large_file_name.is_file(): - file_md5 = hash_file(large_file_name) + file_md5 = file_hash(large_file_name) if file_md5 == None and parts_md5 == None: error(port_dir.name, "Wut?") @@ -174,12 +189,21 @@ def check_large_files(port_dir, large_files): def main(argv): + hash_cache = None + + if not GITHUB_RUN: + hash_cache = HashCache(CACHE_FILE) + for port_dir in sorted(PORTS_DIR.iterdir(), key=lambda x: str(x).casefold()): if not port_dir.is_dir(): continue large_files = load_port(port_dir) - check_large_files(port_dir, large_files) + check_large_files(port_dir, large_files, hash_cache) + + # Build any large runtimes. + large_files = load_port(RUNTIMES_DIR) + check_large_files(RUNTIMES_DIR, large_files, hash_cache) errors = 0 warnings = 0 @@ -198,6 +222,9 @@ def main(argv): print(" " + "\n ".join(messages['errors']) + "\n") errors += 1 + if hash_cache is not None: + hash_cache.save_cache() + if '--do-check' in argv: if errors > 0: return 255 diff --git a/tools/build_release.py b/tools/build_release.py index fc5e39236a..ceb8d84e3e 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -57,13 +57,18 @@ ROOT_DIR = Path('.') +CACHE_FILE = ROOT_DIR / '.hash_cache' RELEASE_DIR = ROOT_DIR / 'releases' +RUNTIMES_DIR = ROOT_DIR / 'runtimes' MANIFEST_FILE = RELEASE_DIR / 'manifest.json' STATUS_FILE = RELEASE_DIR / 'ports_status.json' PORTS_DIR = ROOT_DIR / 'ports' +GITHUB_RUN = (ROOT_DIR / '.github_check').is_file() + LARGEST_FILE = (1024 * 1024 * 90) + ############################################################################# ## Read CONFIG file. REPO_CONFIG = { @@ -111,19 +116,6 @@ def current_release_url(release_id): return f"https://github.com/{REPO_CONFIG['RELEASE_ORG']}/{REPO_CONFIG['RELEASE_REPO']}/releases/download/{release_id}/" -def runtime_nicename(runtime): - if runtime.startswith("frt"): - return ("Godot/FRT {version}").format(version=runtime.split('_', 1)[1].rsplit('.', 1)[0]) - - if runtime.startswith("mono"): - return ("Mono {version}").format(version=runtime.split('-', 1)[1].rsplit('-', 1)[0]) - - if "jdk" in runtime and runtime.startswith("zulu11"): - return ("JDK {version}").format(version=runtime.split('-')[2][3:]) - - return runtime - - def file_type(port_file): if port_file.is_dir(): return PORT_DIR @@ -138,7 +130,13 @@ def file_type(port_file): return UNKNOWN_FILE -def load_port(port_dir, manifest, registered, port_status, quick_build=False): +def load_port(port_dir, manifest, registered, port_status, quick_build=False, hash_cache=None): + if hash_cache is not None: + hash_func = hash_cache.get_file_hash + + else: + hash_func = hash_file + port_data = { 'name': None, 'port_json': None, @@ -323,9 +321,10 @@ def load_port(port_dir, manifest, registered, port_status, quick_build=False): large_files[str(file_name)] = True if not quick_build: - temp = hash_file(file_name) - manifest[port_file_name] = temp - port_manifest.append((port_file_name, temp)) + file_hash = hash_func(file_name) + + manifest[port_file_name] = file_hash + port_manifest.append((port_file_name, file_hash)) for large_file, large_file_status in large_files.items(): if not large_file_status: @@ -582,14 +581,41 @@ def port_info(file_name, ports_json, ports_status): ports_json[clean_name]['source']['url'] = current_release_url(ports_status[clean_name]['release_id']) + (file_name.name.replace(" ", ".").replace("..", ".")) -def util_info(file_name, util_json): +def util_info(file_name, util_json, ports_status, runtimes_json): clean_name = name_cleaner(file_name.name) file_md5 = hash_file(file_name) + file_size = file_name.stat().st_size if file_name.name.lower().endswith('.squashfs'): - name = runtime_nicename(file_name.name) - url = "https://github.com/PortsMaster/PortMaster-Runtime/releases/download/runtimes/" + (file_name.name.replace(" ", ".").replace("..", ".")) + + default_status = { + 'date_added': TODAY, + 'date_updated': TODAY, + 'md5': file_md5, + 'size': file_size, + 'release_id': CURRENT_RELEASE_ID, + } + + if clean_name not in ports_status: + ports_status[clean_name] = default_status + + if GITHUB_RUN: + file_name.rename(RELEASE_DIR / file_name.name) + file_name = RELEASE_DIR / file_name.name + + elif ports_status[clean_name]['md5'] != file_md5: + ports_status[clean_name]['md5'] = file_md5 + ports_status[clean_name]['size'] = file_size + ports_status[clean_name]['release_id'] = CURRENT_RELEASE_ID + ports_status[clean_name]['date_updated'] = TODAY + + if GITHUB_RUN: + file_name.rename(RELEASE_DIR / file_name.name) + file_name = RELEASE_DIR / file_name.name + + name = runtimes_json.get(clean_name, clean_name) + url = current_release_url(ports_status[clean_name]['release_id']) + (file_name.name.replace(" ", ".").replace("..", ".")) else: name = file_name.name @@ -598,7 +624,7 @@ def util_info(file_name, util_json): util_json[clean_name] = { "name": name, 'md5': file_md5, - 'size': file_name.stat().st_size, + 'size': file_size, 'url': url, } @@ -671,12 +697,25 @@ def generate_ports_json(all_ports, port_status): utils.append(RELEASE_DIR / 'PortMaster.zip') utils.append(RELEASE_DIR / 'images.zip') - utils.extend(RELEASE_DIR.glob('*.squashfs')) + runtimes_json = {} + + if RUNTIMES_DIR.is_dir(): + if (RUNTIMES_DIR / 'runtimes.json').is_file(): + # print(f"Loading runtimes.json") + + with open((RUNTIMES_DIR / 'runtimes.json'), 'r') as fh: + runtimes_json = json.load(fh) + + # print(json.dumps(runtimes_json, indent=4)) + + utils.extend(RUNTIMES_DIR.glob('*.squashfs')) for file_name in sorted(utils, key=lambda x: str(x).casefold()): util_info( file_name, - ports_json_output['utils'] + ports_json_output['utils'], + port_status, + runtimes_json ) with open(RELEASE_DIR / 'ports.json', 'w') as fh: @@ -727,6 +766,10 @@ def main(argv): old_manifest = {} port_status = {} + file_cache = None + + if not GITHUB_RUN: + file_cache = HashCache(CACHE_FILE) registered = { 'dirs': {}, @@ -790,12 +833,11 @@ def main(argv): return 0 - for port_dir in sorted(PORTS_DIR.iterdir(), key=lambda x: str(x).casefold()): if not port_dir.is_dir(): continue - port_data = load_port(port_dir, new_manifest, registered, port_status) + port_data = load_port(port_dir, new_manifest, registered, port_status, hash_cache=file_cache) if port_data is None: status['broken'] += 1 @@ -844,7 +886,7 @@ def main(argv): (PORTS_DIR / port_name) not in bad_ports): continue - if Path('.github_check').is_file(): + if GITHUB_RUN: for warning in messages['warnings']: print(f"::warning file=ports/{port_name}::{warning}") warnings += 1 @@ -874,6 +916,9 @@ def main(argv): print("") print(f"Total Ports: {status['total']}") + if file_cache is not None: + file_cache.save_cache() + if '--do-check' in argv: if errors > 0: return 255 diff --git a/tools/libs/util.py b/tools/libs/util.py index 1796763ffc..627e722886 100644 --- a/tools/libs/util.py +++ b/tools/libs/util.py @@ -283,6 +283,138 @@ def port_info_load(raw_info, source_name=None, do_default=False): return port_info +class HashCache(): + CACHE_ATTRS = ('st_size', 'st_mtime') + DEBUG_CACHE = False + + def __init__(self, file_name): + self.file_name = file_name + self._cache = dict() + self._stats = { + 'hits': 0, + 'misses': 0, + 'loaded': 0, + 'new': 0, + } + + if file_name.is_file(): + self.load_cache() + + def _stat_file(self, file_name): + file_name = Path(file_name) + if not file_name.is_file(): + return None + + stat = Path(file_name).stat() + return ':'.join([ + str(getattr(stat, attr, None)) + for attr in self.CACHE_ATTRS]) + + def load_cache(self): + self._cache.clear() + self._stats = { + 'hits': 0, + 'misses': 0, + 'loaded': 0, + 'new': 0, + } + + with open(self.file_name, 'r') as fh: + data = json.load(fh) + + invalidated = 0 + cache_items = 0 + total_items = 0 + + # This takes a few extra seconds, but can be worth it. + if self.DEBUG_CACHE: + print("Loading PM Cache:") + + for file_name, file_data in data.items(): + total_items += 1 + + stat = self._stat_file(file_name) + if stat is None: + invalidated += 1 + continue + + if stat != file_data[0]: + invalidated += 1 + continue + + cache_items += 1 + self._cache[file_name] = file_data + + if self.DEBUG_CACHE: + print(f"- invalidated: {invalidated}") + print(f"- loaded items: {cache_items}") + print(f"- total items: {total_items}") + print("") + + def save_cache(self): + if self.DEBUG_CACHE: + print("Saving PM Cache:") + print(f"- cache hits: {self._stats['hits']}") + print(f"- cache misses: {self._stats['misses']}") + print(f"- new items: {self._stats['new']}") + print(f"- total cache size: {len(self._cache)}") + + with open(self.file_name, 'w') as fh: + json.dump(self._cache, fh, indent=2) + + def get_file_hash(self, file_name): + file_name = str(file_name) + stat = self._stat_file(file_name) + + if file_name in self._cache: + if self._cache[file_name][0] == stat: + self._stats['hits'] += 1 + return self._cache[file_name][1] + + else: + self._stats['misses'] += 1 + + else: + self._stats['new'] += 1 + + file_hash = hash_file(file_name) + self._cache[file_name] = [stat, file_hash, None] + + return file_hash + + def get_files_hash(self, file_names): + all_result = None + + for file_name in file_names: + stat = self._stat_file(file_name) + + if file_name in self._cache: + if self._cache[file_name][0] == stat: + if all_result is None: + all_result = self._cache[file_name][2] + + else: + self._stats['misses'] += 1 + break + + else: + self._stats['new'] += 1 + break + + else: + if all_result is not None: + self._stats['hits'] += 1 + return all_result + + + all_md5, file_data = hash_files_2(file_names) + for file_name, file_md5 in file_data: + stat = self._stat_file(file_name) + self._cache[file_name] = [stat, file_md5, all_md5] + + return all_md5 + + def fetch_bytes(url): try: # Open the URL @@ -395,6 +527,26 @@ def hash_files(file_list): return md5.hexdigest() +def hash_files_2(file_list): + all_md5 = hashlib.md5() + results = [] + + for file_name in file_list: + file_md5 = hashlib.md5() + + with open(file_name, 'rb') as fh: + while True: + data = fh.read(4096 * 10) + if len(data) == 0: + break + + file_md5.update(data) + all_md5.update(data) + + results.append((file_name, file_md5.hexdigest())) + + return all_md5.hexdigest(), results + def hash_file_handle(fh): md5 = hashlib.md5() @@ -425,6 +577,7 @@ def datetime_compare(time_a, time_b=None): 'PORT_INFO_ATTR_ATTRS', 'PORT_INFO_GENRES', 'MESSAGES', + 'HashCache', 'datetime_compare', 'error', 'fetch_bytes',