refecator

commune/cli/__init__.py

@@ -1,4 +1,4 @@
 from .cli import Cli  
 def main():
-    Cli().forward()
+    Cli()
 

commune/cli/cli.py

@@ -5,6 +5,8 @@
 print = c.print
 class Cli:
     desc = 'commune cli for running functions'
+    def __init__(self):
+        self.forward()
     def forward(self):
         t0 = time.time()
         argv = sys.argv[1:]
@@ -85,7 +87,7 @@ def get_params(self, argv):
         return {'args': args, 'kwargs': kwargs}
 
 
-    def get_fn(self, argv:list, init_kwargs:dict={}, default_fn:str='forward', default_module:str='module'):
+    def get_fn(self, argv:list, init_kwargs:dict={}, default_fn:str='forward', default_module:str='module', helper_fns=['code']):
         if len(argv) == 0:
             fn = default_fn
         else:

commune/module.py

@@ -32,7 +32,7 @@ class c:
     repo_path  = os.path.dirname(root_path) # the path to the repo
     docs_path = repo_path + '/docs'
     storage_path = os.path.expanduser(f'~/.{repo_name}')
-    modules_path = os.path.dirname(__file__) + '/modules'
+    modules_path = libpath + '/modules'
 
     cache = {} # cache for module objects
     shortcuts =  {
@@ -119,6 +119,21 @@ def filepath(cls, obj=None) -> str:
     def objectpath(cls, obj=None) -> str:
         return c.classes(cls.filepath(obj))[-1]
 
+    @classmethod 
+    def obj2code(self, path='./', search=None):
+        obj2code = {}
+        for obj in c.objs(path):
+            if search != None and str(search) not in obj:
+                continue
+
+            try:
+                obj2code[obj] = c.code(obj)
+            except:
+                pass
+        return obj2code.keys()
+
+
+
     def file2size(self, path:str='./', reverse=True) -> int:
         file2size =  {k:len(str(v)) for k,v in c.file2text(path).items()}
         file2size = dict(sorted(file2size.items(), key=lambda x: x[1], reverse=reverse))
@@ -617,7 +632,7 @@ def rm(cls, path:str,
     def glob(cls,  path =None, files_only:bool = True, recursive:bool=True):
         import glob
         path = cls.resolve_path(path)
-        if os.path.isdir(path):
+        if os.path.isdir(path) and not path.endswith('**'):
             path = os.path.join(path, '**')
         paths = glob.glob(path, recursive=recursive)
         if files_only:
@@ -1864,6 +1879,26 @@ def docs(self, path='./', search=None):
         files =  c.files(path)
         readmes = [f for f in files if f.endswith('.md')]
         return {k.replace(c.abspath('~') +'/', '~/'):c.get_text(k) for k in readmes}
+
+    home_modules_path = os.path.expanduser('~/modules')
+
+    def export_modules(self, path=home_modules_path):
+        fromto_path = []
+        avoid_terms =['__pycache__']
+        for f in c.glob(c.modules_path):
+            print(f)
+            if any([term in f for term in avoid_terms]):
+                continue
+            to_f = c.home_modules_path + '/' + f[len(c.modules_path) + 1:]
+            fromto_path += [[f, to_f]]
+
+        return fromto_path
+
+
+    def fixit(self):
+        avoid_terms = ['routes.json']
+        return [c.rm(f) for f in c.files() if f.endswith('.json')  if not any([at in f for at in avoid_terms])]
+
 c.add_routes()
 Module = c # Module is alias of c
 if __name__ == "__main__":

commune/server/client.py

@@ -14,8 +14,7 @@ def __init__( self,
                  network: Optional[bool] = 'local', 
                  **kwargs):
         self.key  = c.get_key(key, create_if_not_exists=True)
-        self.namespace = c.namespace(network=network)
-        self.url = module if c.is_url(module) else self.namespace.get(module)
+        self.url = module if c.is_url(module) else c.namespace().get(module,module)
         self.session = requests.Session()
 
     @classmethod
@@ -24,7 +23,7 @@ def call(cls,
                 *args,
                 kwargs = None,
                 params = None,
-                module : str = 'module',
+                module : str = None,
                 network:str = 'local',
                 key: Optional[str] = None, # defaults to module key (c.default_key)
                 timeout=40,
@@ -54,10 +53,11 @@ def client(cls, module:str = 'module', network : str = 'local', virtual:bool = T
     def get_url(self, fn, mode='http'):
         if '/' in str(fn):  
             url, fn = '/'.join(fn.split('/')[:-1]), fn.split('/')[-1]
-            if url in self.namespace:
-                url = self.namespace[url]
+            if not c.is_url(url):
+                url = c.namespace().get(url, url)
         else:
             url = self.url
+        assert c.is_url(url), f'{url}'
         url = url if url.startswith(mode) else f'{mode}://{url}'
         return f"{url}/{fn}/"
 
@@ -72,8 +72,7 @@ def get_params(self,params: Union[list, dict] = None, args = None, kwargs = None
                 args = params
             else:
                 raise Exception(f'Invalid params {params}')
-        params =  {"args": args, "kwargs": kwargs}
-        return params
+        return {"args": args, "kwargs": kwargs}
 
     def forward(self, 
                 fn  = 'info', 

commune/server/server.py

@@ -108,15 +108,24 @@ def get_info(self, module):
             "time": c.time(),
             "schema": {fn: c.fn_schema(getattr(module, fn )) for fn in module.fns if hasattr(module, fn)},
         }
+
     def set_port(self, port:Optional[int]=None, port_attributes = ['port', 'server_port']):
         name = self.module.name
         if port == None:
-            for k in port_attributes:
-                if hasattr(self.module, k):
-                    port = getattr(self.module, k)
-                    c.kill_port(port)
-                    break
-        port = port or c.free_port()
+            in_port_attribute = any([k for k in port_attributes])
+            if in_port_attribute:
+                for k in port_attributes:
+                    if hasattr(self.module, k):
+                        port = getattr(self.module, k)
+                        c.kill_port(port)
+                        break
+            else:
+                port = port or c.free_port()
+                namespace = self.namespace()
+                if name in namespace:
+                    port = int(namespace.get(name).split(':')[-1])
+        if str(port) == 'None':
+            port = c.free_port()
         while c.port_used(port):
             c.kill_port(port)
             c.sleep(1)
@@ -197,7 +206,7 @@ def  resolve_path(self, path):
     def processes(cls):
         return self.processes()
 
-    def wait_for_server(cls,
+    def wait_for_server(self,
                           name: str ,
                           network: str = 'local',
                           timeout:int = 600,
@@ -208,7 +217,7 @@ def wait_for_server(cls,
         # rotating status thing
         c.print(f'waiting for {name} to start...', color='cyan')
         while time_waiting < timeout:
-            namespace = cls.namespace(network=network, max_age=max_age)
+            namespace = self.namespace(network=network, max_age=max_age)
             if name in namespace:
                 try:
                     result = c.call(namespace[name]+'/info')
@@ -217,7 +226,7 @@ def wait_for_server(cls,
                     result.pop('schema', None)
                     return result
                 except Exception as e:
-                    c.print(f'Error getting info for {name} --> {e}', color='red')
+                    c.print(f'Error getting info for {name} --> {c.detailed_error(e)}', color='red')
             c.sleep(sleep_interval)
             # c.print(c.logs(name, tail=10, mode='local'))
 
@@ -256,7 +265,7 @@ def add_endpoint(self, name, fn):
     def test(cls, **kwargs):
         from .test import Test
         return Test().test()
-    
+
     def kill(self, name:str, verbose:bool = True, **kwargs):
         process_name = self.resolve_process_name(name)
         try:
@@ -361,10 +370,12 @@ def serve_background(self,
         env = env or {}
         if '/' in fn:
             module, fn = fn.split('/')
-        if self.server_exists(module):
-            self.kill(name)
-
+        name = name or module
         process_name = self.resolve_process_name(name)
+
+        if self.process_exists(process_name):
+            self.kill(process_name)
+
         cmd = f"pm2 start {c.filepath()} --name {process_name} --interpreter {interpreter} -f"
         cmd = cmd  if autorestart else ' --no-autorestart' 
         params_str = json.dumps({'module': module ,  'fn': fn, 'params': params or {}}).replace('"', "'")

commune/utils/misc.py

@@ -11,9 +11,10 @@ def tqdm(*args, **kwargs):
     return tqdm(*args, **kwargs)
 
 def find_word( word:str, path='./')-> str:
+    import os
     import commune as c
     path = os.path.abspath(path)
-    files = get_files(path)
+    files = c.files(path)
     progress = c.tqdm(len(files))
     found_files = {}
     for f in files:
@@ -24,7 +25,6 @@ def find_word( word:str, path='./')-> str:
             lines = text.split('\n')
         except Exception as e:
             continue
-
         line2text = {i:line for i, line in enumerate(lines) if word in line}
         found_files[f[len(path)+1:]]  = line2text
         progress.update(1)

commune/utils/network.py

@@ -1,6 +1,12 @@
 
 from typing import *
 import os
+import sys
+import subprocess
+import psutil
+import socket
+import time
+
 def port_free( *args, **kwargs) -> bool:
     return not port_used(*args, **kwargs)
 
@@ -231,45 +237,46 @@ def unreserve_ports(*ports, var_path='reserved_ports' ):
     c.put(var_path, reserved_ports)
     return c.reserved_ports()
 
-def kill_port(port):
-    import socket
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    try:
-        sock.bind(('0.0.0.0', port))
-    except socket.error:
-        print(f"Port {port} is in use")
-    finally:
-        sock.close()
-
-    if port_used(port):
 
-        import os
-        # kill it without lsof
+def kill_port(port, timeout=10):
+    try:
+        # Check operating system
+        operating_system = sys.platform
 
-
-
-
-
-
+        if operating_system == "windows":
+            # Windows command
+            command = f"for /f \"tokens=5\" %a in ('netstat -aon ^| find \":{port}\"') do taskkill /F /PID %a"
+            subprocess.run(command, shell=True)
 
-
-
+        elif operating_system in ["linux", "darwin"]:  # Linux or MacOS
+            # Unix command
+            command = f"lsof -i tcp:{port} | grep LISTEN | awk '{{print $2}}' | xargs kill -9"
+            subprocess.run(command, shell=True)
+        t0 = time.time()
+        while port_used(port):
+            if time.time() - t0 > timeout:
+                raise Exception(f'Timeout for killing port {port}')
+
+        print(f"Process on port {port} has been killed")
+        return True
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return False
 
-    assert port_used(port) == False, f'Port {port} is still in use'
-
-    return port
 
 
 def kill_ports(ports = None, *more_ports):
+    import commune as c
     ports = ports or used_ports()
     if isinstance(ports, int):
         ports = [ports]
     if '-' in ports:
         ports = list(range([int(p) for p in ports.split('-')]))
     ports = list(ports) + list(more_ports)
-    for port in ports:
-        kill_port(port)
-    return check_used_ports()
+    for p in ports:
+        kill_port(p)
+    return used_ports()
 
 def is_port_public(port:int, ip:str=None, timeout=0.5):
     import socket

commune/utils/storage.py

@@ -377,26 +377,6 @@ def type2files( path:str='./', **kwargs):
 def type2filecount( path:str='./', **kwargs):
     return {k: len(v) for k,v in type2files(path, **kwargs).items()}
 
-def get_files( path ='./', 
-              search=None,
-              avoid_terms = None,
-              include_terms = None,
-               recursive:bool = True, files_only:bool = True):
-    import glob
-    path = os.path.abspath(path)
-    if os.path.isdir(path):
-        path = os.path.join(path, '**')
-    paths = glob.glob(path, recursive=recursive)
-    if files_only:
-        paths =  list(filter(lambda f:os.path.isfile(f), paths))
-    if avoid_terms != None:
-        paths = [p for p in paths if not any([term in p for term in avoid_terms])]
-    if include_terms != None:
-        paths = [p for p in paths if any([term in p for term in include_terms])]
-    if search != None:
-        paths = [p for p in paths if search in p]
-    return paths
-
 def abspath(path:str):
     return os.path.abspath(os.path.expanduser(path))