WIP: Switch to fetchFromGithub

.gitignore

@@ -12,3 +12,4 @@ ros-indigo
 ros-kinetic
 ros-lunar
 .coverage
+*.ignore

superflore/generate_installers.py

@@ -19,6 +19,7 @@
 from superflore.utils import info
 from superflore.utils import ok
 from superflore.utils import warn
+import tarfile
 
 
 def generate_installers(
@@ -84,6 +85,16 @@ def generate_installers(
                 )
             )
             failed = failed + 1
+        except tarfile.ReadError:
+            failed_msg = 'Failed to generate %s tarfile.ReadError' % what_generating
+            err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))
+            bad_installers.append(pkg)
+            failed = failed + 1
+        except FileNotFoundError:
+            failed_msg = 'Failed to generate %s FileNotFoundError' % what_generating
+            err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))
+            bad_installers.append(pkg)
+            failed = failed + 1
         except KeyError:
             failed_msg = 'Failed to generate %s' % what_generating
             err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))

superflore/generators/nix/nix_expression.py

@@ -117,6 +117,49 @@ def _to_nix_list(it: Iterable[str]) -> str:
     def _to_nix_parameter(dep: str) -> str:
         return dep.split('.')[0]
 
+    @property
+    def url_host(self):
+        # https://github.com/owner/repo/whatever => "github.com"
+        if self.src_url and "//" in self.src_url:
+            after_the_https_part = self.src_url.split("//")[1]
+            first_slash_index = after_the_https_part.index("/")
+            if first_slash_index == -1:
+                first_slash_index = len(after_the_https_part)
+            return after_the_https_part[0:first_slash_index]
+
+    def attempt_extract_github_tar_data(self):
+        # returns [owner, repo, revision]
+        required_ending = ".tar.gz"
+        if self.url_host != "github.com" or not self.src_url.endswith(required_ending):
+            return None
+        else:
+            no_https = "/".join(self.src_url.split("//")[1:])
+            folders = no_https.split("/")
+            # what is expected:
+                # folders[0] == "github.com"
+                # folders[1] == the owner
+                # folders[2] == the repo name
+                # folders[3] == "archive"
+                # folders[4:] == the tag + ".tar.gz"
+
+            # if unexpected format
+            if len(folders) < 5:
+                return None
+
+            owner = folders[1]
+            repo = folders[2]
+
+            # the stuff after "archive" is a tag
+            # so if there is no "archive" thats a problem
+            if folders[3] != "archive":
+                return None
+
+            tag_name_pieces = folders[4:]
+            # chop off the .tar.gz part
+            tag_name_pieces[-1] = tag_name_pieces[-1][:-len(required_ending)]
+            tag = "/".join(tag_name_pieces)
+            return [ owner, repo, tag ]
+
     def get_text(self, distributor: str, license_name: str) -> str:
         """
         Generate the Nix expression, given the distributor line
@@ -140,27 +183,69 @@ def get_text(self, distributor: str, license_name: str) -> str:
                                         self.native_build_inputs |
                                         self.propagated_native_build_inputs)))
                          ) + ' }:'
-
-        ret += dedent('''
-        buildRosPackage {{
-          pname = "ros-{distro_name}-{name}";
-          version = "{version}";
-
-          src = fetchurl {{
-            url = "{src_url}";
-            name = "{src_name}";
-            sha256 = "{src_sha256}";
-          }};
-
-          buildType = "{build_type}";
-        ''').format(
-            distro_name=self.distro_name,
-            name=self.name,
-            version=self.version,
-            src_url=self.src_url,
-            src_name=self.src_name,
-            src_sha256=self.src_sha256,
-            build_type=self.build_type)
+
+        # if possible (for checksum reasons) switch to github fetcher
+        maybe_github_data = self.attempt_extract_github_tar_data()
+        if maybe_github_data:
+            owner, repo, rev = maybe_github_data
+            # super slow, but is the only reliable way AFAIK (would be a lot better to do this asyncly)
+            import subprocess
+            try:
+                # this sha256 can be different from the tarball sha256 (e.g. self.src_256)
+                sha256 = subprocess.check_output(['nix-prefetch', 'fetchFromGitHub', '--quiet', "--repo", repo, "--owner", owner, "--rev", rev ]).decode('utf-8')[0:-1]
+            except Exception as error:
+                print(f'''if you're seeing this warning a lot, please install nix-prefetch then re-run this script''')
+                sha256 = self.src_sha256
+
+            ret += dedent('''
+            buildRosPackage {{
+              pname = "ros-{distro_name}-{name}";
+              version = "{version}";
+
+              src = let
+                  fetchFromGithub = (builtins.import (builtins.fetchTarball ({{ url = "https://github.com/NixOS/nixpkgs/archive/aa0e8072a57e879073cee969a780e586dbe57997.tar.gz"; }})) ({{}})).fetchFromGitHub;
+                in
+                  fetchFromGithub {{
+                    owner = "{owner}";
+                    repo = "{repo}";
+                    rev = "{rev}";
+                    sha256 = "{sha256}";
+                  }};
+
+              buildType = "{build_type}";
+            ''').format(
+                distro_name=self.distro_name,
+                name=self.name,
+                owner=owner,
+                repo=repo,
+                rev=rev,
+                version=self.version,
+                src_url=self.src_url,
+                src_name=self.src_name,
+                sha256=sha256,
+                build_type=self.build_type)
+        # otherwise fallback on more generic fetchurl
+        else:
+            ret += dedent('''
+            buildRosPackage {{
+              pname = "ros-{distro_name}-{name}";
+              version = "{version}";
+
+              src = fetchurl {{
+                url = "{src_url}";
+                name = "{src_name}";
+                sha256 = "{src_sha256}";
+              }};
+
+              buildType = "{build_type}";
+            ''').format(
+                distro_name=self.distro_name,
+                name=self.name,
+                version=self.version,
+                src_url=self.src_url,
+                src_name=self.src_name,
+                src_sha256=self.src_sha256,
+                build_type=self.build_type)
 
         if self.build_inputs:
             ret += "  buildInputs = {};\n" \

superflore/generators/nix/nix_package.py

@@ -48,12 +48,15 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str,
         else:
             info("downloading archive version for package '{}'..."
                  .format(name))
-            retry_on_exception(download_file, src_uri, archive_path,
-                               retry_msg="network error downloading '{}'"
-                               .format(src_uri),
-                               error_msg="failed to download archive for '{}'"
-                               .format(name))
-            downloaded_archive = True
+            try:
+                retry_on_exception(download_file, src_uri, archive_path,
+                                retry_msg="network error downloading '{}'"
+                                .format(src_uri),
+                                error_msg="failed to download archive for '{}'"
+                                .format(name))
+                downloaded_archive = True
+            except Exception as error:
+                pass
 
         if downloaded_archive or archive_path not in sha256_cache:
             sha256_cache[archive_path] = hashlib.sha256(

superflore/generators/nix/run.py

@@ -77,6 +77,8 @@ def main():
         preserve_existing = False
     elif args.only:
         parser.error('Invalid args! --only requires specifying --ros-distro')
+    else:
+        raise Exception(f'''Please add one of the following:\n    --all\n    --ros_distro DISTRO_NAME_HERE''')
     if not selected_targets:
         selected_targets = get_distros_by_status('active') + \
             get_distros_by_status('rolling')

tests/bootstrap_nix_dryrun.sh

@@ -0,0 +1,61 @@
+# if git doesnt exist
+if [ -z "$(command -v "git")" ]
+then
+    echo "not even sure how you got this code without git, please install git"
+    exit
+fi
+
+# if python3 doesnt exist
+if [ -z "$(command -v "python3")" ]
+then
+    echo "please install python3"
+    exit
+fi
+
+# if nix doesnt exist
+if [ -z "$(command -v "nix")" ]
+then
+    echo "please install nix"
+    exit
+fi
+
+# if nix-prefetch doesnt exist
+if [ -z "$(command -v "nix-prefetch")" ]
+then
+    nix-env -iA nixpkgs.nix-prefetch -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/aa0e8072a57e879073cee969a780e586dbe57997.tar.gz
+fi
+
+
+# 
+# actual setup
+# 
+# if superflore-gen-nix doesnt exist
+if [ -z "$(command -v "superflore-gen-nix")" ]
+then
+    sudo python3 ./setup.py install
+    sudo rosdep init
+    rosdep update
+fi
+
+# 
+# actual run
+# 
+sudo python3 ./setup.py install
+
+tar_cache_folder=".temp.ignore/tar_cache/"
+mkdir -p "$tar_cache_folder"
+
+output_folder=".temp.ignore/nix-ros-overlay/"
+
+if ! [ -d "$output_folder" ]
+then
+    cd "$(dirname "$output_folder")"
+    git clone [email protected]:jeff-hykin/nix-ros-overlay.git
+    cd -
+fi
+
+superflore-gen-nix \
+    --dry-run \
+    --all \
+    --output-repository-path "$output_folder" \
+    --tar-archive-dir "$tar_cache_folder"