Update s3-sync and rclone to sync

src/forge/main.py

@@ -13,7 +13,7 @@
 from .run import cli_run, run
 from .ssh import cli_ssh, ssh
 from .configure import cli_configure, configure, check_env_yaml
-from .s3_sync import cli_s3_sync, s3_sync
+from .sync import cli_sync, s3_sync
 from .stop import cli_stop, stop
 from .start import cli_start, start
 from .yaml_loader import load_config
@@ -23,7 +23,7 @@
     cli_create,
     cli_destroy,
     cli_rsync,
-    cli_s3_sync,
+    cli_sync,
     cli_run,
     cli_engine,
     cli_configure,
@@ -86,8 +86,6 @@ def execute(config):
         destroy(config)
     elif job == 'rsync':
         status = rsync(config)
-    elif job == 's3-sync':
-        status = s3_sync(config)
     elif job == 'run':
         status = run(config)
     elif job == 'engine':
@@ -100,6 +98,8 @@ def execute(config):
         stop(config)
     elif job == 'start':
         start(config)
+    elif job == 'sync':
+        status = s3_sync(config)
     elif job == 'cleanup':
         cleanup(config)
 

src/forge/parser.py

@@ -142,7 +142,7 @@ def add_action_args(parser):
     """
     action_grp = parser.add_argument_group('Action Arguments')
     action_grp.add_argument('--rsync_path', '--rsync-path')
-    action_grp.add_argument('--rclone_path', '--rclone-path')
+    action_grp.add_argument('--sync_path', '--rclone-path')
     action_grp.add_argument('--run_cmd', '--run-cmd')
     action_grp.add_argument('--all', action='store_true', dest='rr_all')
 

src/forge/s3_sync.py → src/forge/sync.py

@@ -13,15 +13,15 @@
 logger = logging.getLogger(__name__)
 
 
-def cli_s3_sync(subparsers):
+def cli_sync(subparsers):
     """adds s3-sync parser to subparser
 
     Parameters
     ----------
     subparsers : argparse.ArgumentParser
         Argument parser for Forge.main
     """
-    parser = subparsers.add_parser('s3-sync', description='Rclone user content to EC2 instance')
+    parser = subparsers.add_parser('sync', description='Rclone user content to EC2 instance')
     add_basic_args(parser)
 
     add_general_args(parser)
@@ -62,7 +62,7 @@ def _rclone(config, ip):
         pem_secret = config['forge_pem_secret']
         region = config['region']
         profile = config.get('aws_profile')
-        rsync_loc = config.get('rclone_path', config.get('app_dir'))
+        rsync_loc = config.get('sync_path', config.get('app_dir'))
 
         with key_file(pem_secret, region, profile) as pem_path:
             logger.info('Copying source %s to EC2.', rsync_loc)
@@ -74,29 +74,29 @@ def _rclone(config, ip):
                 output = subprocess.check_output(
                     cmd, stderr=subprocess.STDOUT, shell=True, universal_newlines=True
                 )
-                logger.info('Rclone successful:\n%s', output)
+                logger.info('Sync successful:\n%s', output)
                 return 0
             except subprocess.CalledProcessError as exc:
-                logger.error('Rclone failed:\n%s', exc.output)
+                logger.error('Sync failed:\n%s', exc.output)
                 return exc.returncode
 
     n_list = get_nlist(config)
 
     for n in n_list:
         try:
-            logger.info('Trying to rclone to %s...', n)
+            logger.info('Trying to sync to %s...', n)
             details = ec2_ip(n, config)
             targets = get_ip(details, ('running',))
             logger.debug('Instance target details are %s', targets)
             if not targets or len(targets[0]) != 2:
-                logger.error('Could not find any valid instances to rsync to')
+                logger.error('Could not find any valid instances to sync to')
                 continue
 
             for ip, _ in targets:
-                logger.info('Rclone destination is %s', ip)
+                logger.info('Sync destination is %s', ip)
                 rval = _rclone(config, ip)
                 if rval:
-                    raise ValueError('Rsync command unsuccessful, ending attempts.')
+                    raise ValueError('Sync command unsuccessful, ending attempts.')
         except ValueError as e:
             logger.error('Got error %s when trying to rclone.', e)
             try:

src/forge/yaml_loader.py

@@ -145,11 +145,11 @@ def _get_type(x):
         Optional('on_demand_failover'): And(bool),
         Optional('ratio'): And(list),
         Optional('ram'): And(list, error='Invalid RAM'),
-        Optional('rclone_path'): And(str),
         Optional('rsync_path'): And(str),
         Optional('run_cmd'): And(str, len, error='Invalid run_cmd'),
         Optional('service'): And(str, len, Or('single', 'cluster'), error='Invalid Service'),
         Optional('spot_retries'): And(Use(int), positive_int),
+        Optional('sync_path'): And(str),
         Optional('user_data'): And(list),
         Optional('valid_time'): And(Use(int), positive_int),
         Optional('workers'): And(Use(int), positive_int),