From 9bfd8e58ef49e15e2dbc7ee417b8d36676454865 Mon Sep 17 00:00:00 2001 From: opencm Date: Tue, 4 Apr 2017 17:04:01 +0300 Subject: [PATCH 1/4] Update create.py --- components/stage/scripts/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/stage/scripts/create.py b/components/stage/scripts/create.py index b7a6976df..5c8c968ee 100644 --- a/components/stage/scripts/create.py +++ b/components/stage/scripts/create.py @@ -22,7 +22,7 @@ def install_stage(): nodejs_source_url = ctx_properties['nodejs_tar_source_url'] stage_source_url = ctx_properties['stage_tar_source_url'] - + print "test" # injected as an input to the script ctx.instance.runtime_properties['influxdb_endpoint_ip'] = \ os.environ.get('INFLUXDB_ENDPOINT_IP') From 1e54bf86b17dd07f09d0378a985956e7ab1a2ff3 Mon Sep 17 00:00:00 2001 From: limor-gs Date: Wed, 5 Apr 2017 15:31:13 +0300 Subject: [PATCH 2/4] Ignore ui on community build --- components/stage/scripts/create.py | 49 +++++++++++++++++------------- components/stage/scripts/start.py | 12 +++++--- components/stage/scripts/stop.py | 12 +++++--- components/utils.py | 35 ++++++++++++--------- 4 files changed, 63 insertions(+), 45 deletions(-) diff --git a/components/stage/scripts/create.py b/components/stage/scripts/create.py index 5c8c968ee..50c4c861e 100644 --- a/components/stage/scripts/create.py +++ b/components/stage/scripts/create.py @@ -6,8 +6,8 @@ from cloudify import ctx ctx.download_resource( - join('components', 'utils.py'), - join(dirname(__file__), 'utils.py')) + join('components', 'utils.py'), + join(dirname(__file__), 'utils.py')) import utils # NOQA STAGE_SERVICE_NAME = 'stage' @@ -22,7 +22,9 @@ def install_stage(): nodejs_source_url = ctx_properties['nodejs_tar_source_url'] stage_source_url = ctx_properties['stage_tar_source_url'] - print "test" + print "stage_source_url={0}".format(stage_source_url) + ctx.instance.runtime_properties['ignore_ui'] = 'False' + # injected as an input to the script ctx.instance.runtime_properties['influxdb_endpoint_ip'] = \ os.environ.get('INFLUXDB_ENDPOINT_IP') @@ -42,25 +44,30 @@ def install_stage(): utils.mkdir(stage_home) utils.mkdir(stage_log_path) - utils.create_service_user(stage_user, stage_home) - - ctx.logger.info('Installing NodeJS...') - nodejs = utils.download_cloudify_resource(nodejs_source_url, - STAGE_SERVICE_NAME) - utils.untar(nodejs, nodejs_home) - ctx.logger.info('Installing Cloudify Stage (UI)...') - stage = utils.download_cloudify_resource(stage_source_url, - STAGE_SERVICE_NAME) - utils.untar(stage, stage_home) - - ctx.logger.info('Fixing permissions...') - utils.chown(stage_user, stage_group, stage_home) - utils.chown(stage_user, stage_group, nodejs_home) - utils.chown(stage_user, stage_group, stage_log_path) - - utils.logrotate(STAGE_SERVICE_NAME) - utils.systemd.configure(STAGE_SERVICE_NAME) + stage = utils.download_cloudify_resource( + stage_source_url, STAGE_SERVICE_NAME, avoid_failure=True) + if not stage: + ctx.instance.runtime_properties['ignore_ui'] = 'True' + print "***ignore ui***" + ignore_ui = ctx.instance.runtime_properties['ignore_ui'] + print ctx.instance.runtime_properties['ignore_ui'].__class__ + print "ignore_ui={0}".format(ignore_ui) + if ignore_ui != 'True': + print "**inside if**" + utils.create_service_user(stage_user, stage_home) + ctx.logger.info('Installing NodeJS...') + nodejs = utils.download_cloudify_resource(nodejs_source_url, + STAGE_SERVICE_NAME) + utils.untar(nodejs, nodejs_home) + utils.untar(stage, stage_home) + ctx.logger.info('Fixing permissions...') + utils.chown(stage_user, stage_group, stage_home) + utils.chown(stage_user, stage_group, nodejs_home) + utils.chown(stage_user, stage_group, stage_log_path) + + utils.logrotate(STAGE_SERVICE_NAME) + utils.systemd.configure(STAGE_SERVICE_NAME) def main(): diff --git a/components/stage/scripts/start.py b/components/stage/scripts/start.py index d1582e8ce..8a79c5de7 100644 --- a/components/stage/scripts/start.py +++ b/components/stage/scripts/start.py @@ -5,12 +5,14 @@ from cloudify import ctx ctx.download_resource( - join('components', 'utils.py'), - join(dirname(__file__), 'utils.py')) + join('components', 'utils.py'), + join(dirname(__file__), 'utils.py')) import utils # NOQA STAGE_SERVICE_NAME = 'stage' - -ctx.logger.info('Starting Stage (UI) Service...') -utils.start_service(STAGE_SERVICE_NAME) +ignore_ui = ctx.instance.runtime_properties['ignore_ui'] +print "ignore_ui={0}".format(ignore_ui) +if ignore_ui != 'True': + ctx.logger.info('Starting Stage (UI) Service...') + utils.start_service(STAGE_SERVICE_NAME) diff --git a/components/stage/scripts/stop.py b/components/stage/scripts/stop.py index c37bdcd08..a93f8a60b 100644 --- a/components/stage/scripts/stop.py +++ b/components/stage/scripts/stop.py @@ -5,12 +5,14 @@ from cloudify import ctx ctx.download_resource( - join('components', 'utils.py'), - join(dirname(__file__), 'utils.py')) + join('components', 'utils.py'), + join(dirname(__file__), 'utils.py')) import utils # NOQA STAGE_SERVICE_NAME = 'stage' - -ctx.logger.info('Stopping Stage (UI) Service...') -utils.systemd.stop(STAGE_SERVICE_NAME) +ignore_ui = ctx.instance.runtime_properties['ignore_ui'] +print "ignore_ui={0}".format(ignore_ui) +if ignore_ui != 'True': + ctx.logger.info('Stopping Stage (UI) Service...') + utils.systemd.stop(STAGE_SERVICE_NAME) diff --git a/components/utils.py b/components/utils.py index 92332bc98..1c835b5ff 100644 --- a/components/utils.py +++ b/components/utils.py @@ -405,26 +405,33 @@ def get_file_name_from_url(url): return os.path.basename(disassembled.path) -def download_cloudify_resource(url, service_name, destination=None): +def download_cloudify_resource( + url, service_name, destination=None, avoid_failure=False): """Downloads a resource and saves it as a cloudify resource. The resource will be saved under the appropriate service resource path and will be used in case of operation execution failure after the resource has already been downloaded. """ - if destination: - source_res_path, _ = resource_factory.create(url, - destination, - service_name, - source_resource=True, - render=False) - copy(source_res_path, destination) - else: - res_name = os.path.basename(url) - source_res_path, _ = resource_factory.create(url, res_name, - service_name, - source_resource=True, - render=False) + try: + if destination: + source_res_path, _ = resource_factory.create(url, + destination, + service_name, + source_resource=True, + render=False) + copy(source_res_path, destination) + else: + res_name = os.path.basename(url) + source_res_path, _ = resource_factory.create(url, res_name, + service_name, + source_resource=True, + render=False) + except Exception: + print "source_res_path={0}".format(source_res_path) + if avoid_failure: + return None + return source_res_path From 25c30a6e38c40df4e7fa72293510bb3235170904 Mon Sep 17 00:00:00 2001 From: limor-gs Date: Thu, 6 Apr 2017 08:52:25 +0300 Subject: [PATCH 3/4] Ignore ui on community build --- components/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/utils.py b/components/utils.py index 1c835b5ff..444dca590 100644 --- a/components/utils.py +++ b/components/utils.py @@ -428,7 +428,7 @@ def download_cloudify_resource( source_resource=True, render=False) except Exception: - print "source_res_path={0}".format(source_res_path) + # print "source_res_path={0}".format(source_res_path) if avoid_failure: return None From adbda638f3f3791c20148b3348194180d97370a2 Mon Sep 17 00:00:00 2001 From: limor-gs Date: Wed, 12 Apr 2017 13:30:21 +0300 Subject: [PATCH 4/4] Update community single tar url --- components/stage/scripts/create.py | 51 ++++++++++------------ components/stage/scripts/start.py | 6 +-- components/stage/scripts/stop.py | 6 +-- components/utils.py | 69 ++++++++++++++++++------------ 4 files changed, 70 insertions(+), 62 deletions(-) diff --git a/components/stage/scripts/create.py b/components/stage/scripts/create.py index 50c4c861e..595117b3e 100644 --- a/components/stage/scripts/create.py +++ b/components/stage/scripts/create.py @@ -22,8 +22,7 @@ def install_stage(): nodejs_source_url = ctx_properties['nodejs_tar_source_url'] stage_source_url = ctx_properties['stage_tar_source_url'] - print "stage_source_url={0}".format(stage_source_url) - ctx.instance.runtime_properties['ignore_ui'] = 'False' + ctx.instance.runtime_properties['skip_stage_installation'] = 'False' # injected as an input to the script ctx.instance.runtime_properties['influxdb_endpoint_ip'] = \ @@ -36,38 +35,32 @@ def install_stage(): stage_user = 'stage' stage_group = 'stage' - utils.set_selinux_permissive() + ctx.logger.info('Installing Cloudify Stage (UI)...') + stage = utils.download_cloudify_resource( + stage_source_url, STAGE_SERVICE_NAME, skip_stage_installation=True) + if stage is None: + ctx.instance.runtime_properties['skip_stage_installation'] = 'True' + ctx.logger.info("Skipping Stage installation") + return + utils.set_selinux_permissive() utils.copy_notice(STAGE_SERVICE_NAME) - utils.mkdir(nodejs_home) utils.mkdir(stage_home) utils.mkdir(stage_log_path) - - ctx.logger.info('Installing Cloudify Stage (UI)...') - stage = utils.download_cloudify_resource( - stage_source_url, STAGE_SERVICE_NAME, avoid_failure=True) - if not stage: - ctx.instance.runtime_properties['ignore_ui'] = 'True' - print "***ignore ui***" - ignore_ui = ctx.instance.runtime_properties['ignore_ui'] - print ctx.instance.runtime_properties['ignore_ui'].__class__ - print "ignore_ui={0}".format(ignore_ui) - if ignore_ui != 'True': - print "**inside if**" - utils.create_service_user(stage_user, stage_home) - ctx.logger.info('Installing NodeJS...') - nodejs = utils.download_cloudify_resource(nodejs_source_url, - STAGE_SERVICE_NAME) - utils.untar(nodejs, nodejs_home) - utils.untar(stage, stage_home) - ctx.logger.info('Fixing permissions...') - utils.chown(stage_user, stage_group, stage_home) - utils.chown(stage_user, stage_group, nodejs_home) - utils.chown(stage_user, stage_group, stage_log_path) - - utils.logrotate(STAGE_SERVICE_NAME) - utils.systemd.configure(STAGE_SERVICE_NAME) + utils.create_service_user(stage_user, stage_home) + ctx.logger.info('Installing NodeJS...') + nodejs = utils.download_cloudify_resource(nodejs_source_url, + STAGE_SERVICE_NAME) + utils.untar(nodejs, nodejs_home) + utils.untar(stage, stage_home) + ctx.logger.info('Fixing permissions...') + utils.chown(stage_user, stage_group, stage_home) + utils.chown(stage_user, stage_group, nodejs_home) + utils.chown(stage_user, stage_group, stage_log_path) + + utils.logrotate(STAGE_SERVICE_NAME) + utils.systemd.configure(STAGE_SERVICE_NAME) def main(): diff --git a/components/stage/scripts/start.py b/components/stage/scripts/start.py index 8a79c5de7..2db19a823 100644 --- a/components/stage/scripts/start.py +++ b/components/stage/scripts/start.py @@ -11,8 +11,8 @@ STAGE_SERVICE_NAME = 'stage' -ignore_ui = ctx.instance.runtime_properties['ignore_ui'] -print "ignore_ui={0}".format(ignore_ui) -if ignore_ui != 'True': +skip_stage_installation = ctx.instance.runtime_properties['skip_stage_installation'] +print "skip_stage_installation={0}".format(skip_stage_installation) +if skip_stage_installation != 'True': ctx.logger.info('Starting Stage (UI) Service...') utils.start_service(STAGE_SERVICE_NAME) diff --git a/components/stage/scripts/stop.py b/components/stage/scripts/stop.py index a93f8a60b..23bf0c115 100644 --- a/components/stage/scripts/stop.py +++ b/components/stage/scripts/stop.py @@ -11,8 +11,8 @@ STAGE_SERVICE_NAME = 'stage' -ignore_ui = ctx.instance.runtime_properties['ignore_ui'] -print "ignore_ui={0}".format(ignore_ui) -if ignore_ui != 'True': +skip_stage_installation = ctx.instance.runtime_properties['skip_stage_installation'] +print "skip_stage_installation={0}".format(skip_stage_installation) +if skip_stage_installation != 'True': ctx.logger.info('Stopping Stage (UI) Service...') utils.systemd.stop(STAGE_SERVICE_NAME) diff --git a/components/utils.py b/components/utils.py index 444dca590..07d7fa915 100644 --- a/components/utils.py +++ b/components/utils.py @@ -406,31 +406,31 @@ def get_file_name_from_url(url): def download_cloudify_resource( - url, service_name, destination=None, avoid_failure=False): + url, service_name, destination=None, skip_stage_installation=False): """Downloads a resource and saves it as a cloudify resource. The resource will be saved under the appropriate service resource path and will be used in case of operation execution failure after the resource has already been downloaded. """ - try: - if destination: - source_res_path, _ = resource_factory.create(url, - destination, - service_name, - source_resource=True, - render=False) - copy(source_res_path, destination) - else: - res_name = os.path.basename(url) - source_res_path, _ = resource_factory.create(url, res_name, - service_name, - source_resource=True, - render=False) - except Exception: - # print "source_res_path={0}".format(source_res_path) - if avoid_failure: + if destination: + + source_res_path, _ = resource_factory.create(url, + destination, + service_name, + skip_stage_installation, + source_resource=True, + render=False) + if source_res_path is None and skip_stage_installation: return None + copy(source_res_path, destination) + else: + + res_name = os.path.basename(url) + source_res_path, _ = resource_factory.create(url, res_name, + service_name, + source_resource=True, + render=False) return source_res_path @@ -570,13 +570,16 @@ def yum_install(source, service_name): sudo(['yum', 'install', '-y', source_path]) -def get_filepath_from_pkg_name(filename): +def get_filepath_from_pkg_name(filename, skip_stage_installation=False): local_filepath_list = \ [fn for fn in glob.glob(os.path.join(CLOUDIFY_SOURCES_PATH, filename)) if not os.path.basename(fn).startswith(SINGLE_TAR_PREFIX)] if not local_filepath_list: - ctx.abort_operation("File: {0} does not exist in sources path: {1}". - format(filename, CLOUDIFY_SOURCES_PATH)) + if skip_stage_installation: + return None + else: + ctx.abort_operation("File: {0} does not exist in sources path: {1}". + format(filename, CLOUDIFY_SOURCES_PATH)) if len(local_filepath_list) > 1: ctx.abort_operation("More than one file: {0} found in sources path:" " {1}".format(filename, CLOUDIFY_SOURCES_PATH)) @@ -1076,7 +1079,8 @@ class BlueprintResourceFactory(object): RESOURCES_JSON_FILE = '__resources.json' def create(self, source, destination, service_name, user_resource=False, - source_resource=False, render=True, load_ctx=True): + source_resource=False, skip_stage_installation=False, + render=True, load_ctx=True): """A Factory used to create a local copy of a resource upon deployment. This copy allows to later reuse the resource for upgrade/rollback purposes. @@ -1096,6 +1100,9 @@ def create(self, source, destination, service_name, user_resource=False, context of the script. :return: The local resource file path and destination. """ + print "***source={0}, destination={1}, service_name={2}, user_resource=={3}," \ + "source_resource={4}, render={5}, load_ctx={6}".format( + source, destination, service_name, user_resource, source_resource, render, load_ctx) resource_name = os.path.basename(destination) if is_upgrade: self._archive_resources(service_name) @@ -1110,7 +1117,7 @@ def create(self, source, destination, service_name, user_resource=False, # The local path is decided according to whether we are in upgrade local_resource_path = self._get_local_file_path(service_name, resource_name) - + print "***local_resource_path={0}".format(local_resource_path) if self._is_download_required(local_resource_path, render): mkdir(os.path.dirname(local_resource_path)) if user_resource: @@ -1121,8 +1128,12 @@ def create(self, source, destination, service_name, user_resource=False, render=render, load_ctx=load_ctx) elif source_resource: - self._download_source_resource(source, - local_resource_path) + res = True + res = self._download_source_resource(source, + local_resource_path, + skip_stage_installation) + if not res: + return None, None elif render: self._download_resource_and_render(source, local_resource_path, @@ -1196,7 +1207,8 @@ def _download_resource_and_render(self, source, dest, service_name, move(tmp_file, dest) @staticmethod - def _download_source_resource(source, local_resource_path): + def _download_source_resource(source, local_resource_path, + skip_stage_installation=False): is_url = source.startswith(('http://', 'https://', 'ftp://', 'file://')) filename = get_file_name_from_url(source) if is_url else source @@ -1204,7 +1216,10 @@ def _download_source_resource(source, local_resource_path): if is_manager_package: local_filepath = os.path.join(CLOUDIFY_SOURCES_PATH, filename) else: - local_filepath = get_filepath_from_pkg_name(filename) + local_filepath = get_filepath_from_pkg_name( + filename, skip_stage_installation) + if local_filepath is None and skip_stage_installation: + return None if is_url: if not os.path.isfile(local_filepath):