0.1.2

pygbag/__init__.py

@@ -3,4 +3,33 @@
 try:
     sched_yield
 except:
-    __import__("builtins").sched_yield = lambda : None
+    __import__("builtins").sched_yield = lambda: None
+
+import sys, traceback
+
+
+def print_exception(e, out=sys.stderr, **kw):
+    kw["file"] = out
+    traceback.print_exc(**kw)
+
+
+sys.print_exception = print_exception
+
+import builtins
+
+
+def ESC(*argv):
+    for arg in argv:
+        sys.__stdout__.write(chr(0x1B))
+        sys.__stdout__.write(arg)
+
+
+def CSI(*argv):
+    for arg in argv:
+        sys.__stdout__.write(chr(0x1B))
+        sys.__stdout__.write("[")
+        sys.__stdout__.write(arg)
+
+
+builtins.ESC = ESC
+builtins.CSI = CSI

pygbag/__main__.py

@@ -1,10 +1,11 @@
+import asyncio
 import sys
 from .__init__ import __version__
 
 print(f" *pygbag {__version__}*")
 
 from pathlib import Path
-from .app import main, main_run
+from .app import main_run
 
 if __name__ == "__main__":
-    main_run(Path(sys.argv[-1]).resolve())
+    asyncio.run(main_run(Path(sys.argv[-1]).resolve()))

pygbag/app.py

@@ -1,7 +1,9 @@
+import asyncio
 import sys
 import os
 import argparse
 
+
 from pathlib import Path
 import hashlib
 import urllib
@@ -42,10 +44,10 @@
 
 
 def main():
-    main_run(Path(sys.argv[-1]).resolve())
+    asyncio.run(main_run(Path(sys.argv[-1]).resolve()))
 
 
-def main_run(patharg, cdn=DEFAULT_CDN):
+async def main_run(patharg, cdn=DEFAULT_CDN):
     global DEFAULT_PORT, DEFAULT_SCRIPT
 
     if patharg.is_file():
@@ -77,6 +79,14 @@ def main_run(patharg, cdn=DEFAULT_CDN):
     build_dir.mkdir(exist_ok=True)
 
     cache_dir = app_folder.joinpath("build/web-cache")
+
+    if devmode and cache_dir.is_dir():
+        print("DEVMODE: clearing cache")
+        if shutil.rmtree.avoids_symlink_attacks:
+            shutil.rmtree(cache_dir.as_posix())
+        else:
+            print("can't clear cache : rmtree is not safe")
+
     cache_dir.mkdir(exist_ok=True)
 
     version = "0.0.0"
@@ -199,9 +209,7 @@ def main_run(patharg, cdn=DEFAULT_CDN):
 """
     )
 
-    pack.archive(f"{app_name}.apk", app_folder, build_dir)
-
-    from . import testserver
+    await pack.archive(f"{app_name}.apk", app_folder, build_dir)
 
     CC = {
         "cdn": args.cdn,
@@ -258,7 +266,8 @@ def cache_file(remote_url, suffix):
 
             try:
                 template_file, headers = web.get(tmpl_url, tmpl)
-            except:
+            except Exception as e:
+                print(e)
                 print(f"CDN {args.cdn} is not responding : not running test server")
                 args.build = True
 
@@ -294,8 +303,9 @@ def cache_file(remote_url, suffix):
         cached locallly at {icon_file}
         """
                 )
-            except:
-                print(f"{icon_url} not found")
+            except Exception as e:
+                sys.print_exception(e)
+                print(f"{icon_url} caching error :", e)
 
     if icon_file.is_file():
         shutil.copyfile(icon_file, build_dir.joinpath("favicon.png"))
@@ -328,11 +338,26 @@ def cache_file(remote_url, suffix):
 """
             )
 
-            pack.web_archive(f"{app_name}.apk", build_dir)
-            raise SystemExit
+            await pack.web_archive(f"{app_name}.apk", build_dir)
+            return
 
         elif not args.build:
-            testserver.run_code_server(args, CC)
+            if 1:
+                ESC("(0")
+                CSI("104;93m")
+                print("l--------------k")
+                print("x      علي     x")
+                print("m--------------j")
+                CSI("0m")
+                ESC("(B")
+
+                CSI("2;10r")
+
+                from . import testserver
+
+                testserver.run_code_server(args, CC)
+            else:
+                import uasyncio
         else:
             print(
                 f"""

pygbag/filtering.py

@@ -1,6 +1,41 @@
+from pathlib import Path
 
+dbg = True
 
 
-def filter(*argv, **kw):
-    pass
+def filter(walked):
+    global dbg
+    for folder, filenames in walked:
+        blocking = False
 
+        for block in ["/.git", "/.github", "/build", "/venv"]:
+            if folder.match(block):
+                if dbg:
+                    print("REJ 1", folder)
+                blocking = True
+                break
+
+            fx = folder.as_posix()
+
+            if fx.startswith(f"{block}/"):
+                if dbg:
+                    print("REJ 2", folder)
+                blocking = True
+                break
+
+        if blocking:
+            continue
+
+        for filename in filenames:
+            if filename in [".gitignore"]:
+                if dbg:
+                    print("REJ 3", folder, filename)
+                continue
+
+            ext = filename.rsplit(".", 1)[-1].lower()
+            if ext in ["pyc", "pyx", "pyd", "pyi", "exe", "log"]:
+                if dbg:
+                    print("REJ 4", folder, filename)
+                continue
+
+            yield Path(folder), Path(folder).joinpath(filename)

pygbag/gathering.py

@@ -1,16 +1,18 @@
 from os import walk
 from pathlib import Path
 
+
 class Error(Exception):
     pass
 
 
-def gather(root:Path,*kw):
+def gather(root: Path, *kw):
     if root.is_file():
         if root.name == "main.py":
             raise Error("project must be a folder or an archive")
 
-
     for current, dirnames, filenames in walk(root):
-        print(current, len(dirnames), len(filenames))
-        yield
+        rel = Path("/").joinpath(Path(current).relative_to(root))
+
+        # print(rel, len(dirnames), len(filenames))
+        yield rel, filenames

pygbag/optimizing.py

@@ -1,2 +1,3 @@
-def optimize(*argv, **kw):
-    pass
+def optimize(filenames, **kw):
+    for filename in filenames:
+        yield filename

pygbag/pack.py

@@ -137,18 +137,31 @@ def pack_files(zf, parent, zfolders, newpath):
 
                 # folders to skip __pycache__
                 # extensions to skip : pyc pyx pyd pyi
-
-                ext = f.rsplit(".", 1)[-1].lower()
-
-                if ext in ["pyc", "pyx", "pyd", "pyi", "exe"]:
-                    continue
+                if not f.count("."):
+                    print(
+                        f"""
+    ==================================================================
+    {f} has no extension
+    ==================================================================
+                    """
+                    )
+                    zpath = list(zfolders)
+                    zpath.append(f)
+                    src = "/".join(zpath)
+                    name = f
+                    ext = ""
+                else:
+
+                    name, ext = f.rsplit(".", 1)
+                    ext = ext.lower()
+
+                    if ext in ["pyc", "pyx", "pyd", "pyi", "exe", "log"]:
+                        continue
 
                 zpath = list(zfolders)
                 zpath.append(f)
                 src = "/".join(zpath)
 
-                name, ext = f.rsplit(".", 1)
-
                 if ext == "png":
                     maybe = f"{name}-pygbag.png"
                     if Path(maybe).is_file():
@@ -169,19 +182,32 @@ def pack_files(zf, parent, zfolders, newpath):
         LEVEL -= 1
 
 
-def archive(apkname, target_folder, build_dir=None):
+async def archive(apkname, target_folder, build_dir=None):
     global COUNTER, TRUNCATE, ASSETS, HAS_MAIN, HAS_STATIC
     TRUNCATE = len(target_folder.as_posix())
     if build_dir:
         apkname = build_dir.joinpath(apkname).as_posix()
 
-    files = []
-    for f in gather(target_folder):
-        files.append(f)
+    walked = []
+    for folder, filenames in gather(target_folder):
+        walked.append([folder, filenames])
         sched_yield()
 
+    filtered = []
+    last = ""
+    for infolder, fullpath in filter(walked):
+        if last != infolder:
+            print(f"Now in {infolder}")
+            last = infolder
 
+        # print(" " * 4, fullpath)
+        filtered.append(fullpath)
+        sched_yield()
 
+    packlist = []
+    for filename in optimize(filtered):
+        packlist.append(filename)
+        sched_yield()
 
     try:
         with zipfile.ZipFile(
@@ -207,7 +233,7 @@ def archive(apkname, target_folder, build_dir=None):
         print(f"INFO: {len(MP3OPT)} mp3 format files were swapped for packing")
 
 
-def web_archive(apkname, build_dir):
+async def web_archive(apkname, build_dir):
     archfile = build_dir.with_name("web.zip")
     if archfile.is_file():
         archfile.unlink()

pygbag/web.py

@@ -2,6 +2,8 @@
 import os.path
 
 import urllib
+import urllib.request
+
 
 from pathlib import Path
 
@@ -17,6 +19,7 @@ class ssl:
 
 # https://stackoverflow.com/questions/42098126/mac-osx-python-ssl-sslerror-ssl-certificate-verify-failed-certificate-verify
 def fixcert():
+    wd = os.getcwd()
     import stat
     import subprocess
 
@@ -67,35 +70,46 @@ def fixcert():
     os.symlink(relpath_to_certifi_cafile, openssl_cafile)
     print(" -- setting permissions")
     os.chmod(openssl_cafile, STAT_0o775)
+    os.chdir(wd)
     print(" -- update complete")
 
 
 def get(url, path):
     error = None
     data_file = None
-    while True:
-        try:
-            data_file, header = urllib.request.urlretrieve(url, path)
-            break
-
-        except urllib.error.HTTPError as e:
-            error = e
-            break
-        except urllib.error.URLError as e:
-            error = e
-            break
-        except ssl.SSLCertVerificationError:
-            print("Trying to fix certificate error")
-            fixcert()
-
-        finally:
+    try:
+        print(f'urllib.request.urlretrieve("{url}", "{path}")')
+        while data_file is None:
+            try:
+                data_file, header = urllib.request.urlretrieve(url, path)
+
+            except urllib.error.HTTPError as e:
+                error = e
+
+            except urllib.error.URLError as e:
+                error = e
+
+            except ssl.SSLCertVerificationError:
+                print("Trying to fix certificate error")
+                fixcert()
+                continue
+
+            if data_file is not None:
+                break
+
             if error:
                 print("WARNING: web.get", e)
-                raise error
 
-            if data_fileNone:
-                return Path(data_file), header
-            raise Exception(f"cannot cache {url} to {path}")
+            time.sleep(5)
+            print("retrying in 5 seconds")
+
+    finally:
+        if data_file is not None:
+            return Path(data_file), header
+        # this is normal in dev mode for favicon and templates because
+        # proxy is not yet started.
+        print(f"NO DATA RECEIVED FOR {url}")
+        raise Exception(f"cannot cache {url} to {path}")
 
 
 if __name__ == "__main__":