CFY-5334 add SSL support

components/elasticsearch/scripts/create.py

@@ -314,7 +314,7 @@ def main():
     es_endpoint_ip = ctx_properties['es_endpoint_ip']
     es_endpoint_port = ctx_properties['es_endpoint_port']
 
-    if utils.is_upgrade:
+    if utils.is_upgrade or utils.is_rollback:
         dump_upgrade_data()
 
     if not es_endpoint_ip:

components/manager/scripts/configure_manager.py

@@ -26,6 +26,10 @@
 )
 import utils  # NOQA
 
+# This MUST be invoked by the first node, before upgrade snapshot is created.
+rest_host = ctx.instance.runtime_properties['internal_rest_host']
+utils.clean_rollback_resources_if_necessary(rest_host)
+
 NODE_NAME = 'manager-config'
 
 
@@ -54,14 +58,15 @@ def _configure_security_properties():
 
     if security_enabled:
         # agent access-control settings
-        agents_rest_username = agent_config['rest_username']
-        agents_rest_password = agent_config['rest_password']
-        ctx.instance.runtime_properties['agents_rest_username'] = \
-            agents_rest_username
-        ctx.instance.runtime_properties['agents_rest_password'] = \
-            agents_rest_password
-        ctx.logger.info('agents_rest_username: {0}'.
-                        format(agents_rest_username))
+        agents_rest_username = agent_config.get('rest_username')
+        agents_rest_password = agent_config.get('rest_password')
+        if agents_rest_username and agents_rest_password:
+            ctx.instance.runtime_properties['agents_rest_username'] = \
+                agents_rest_username
+            ctx.instance.runtime_properties['agents_rest_password'] = \
+                agents_rest_password
+            ctx.logger.info('agents_rest_username: {0}'.
+                            format(agents_rest_username))
 
     if security_enabled and ssl_enabled:
         # manager SSL settings

components/manager/scripts/create.py

@@ -13,9 +13,6 @@
 
 NODE_NAME = 'manager-resources'
 
-# This MUST be invoked by the first node, before upgrade snapshot is created.
-utils.clean_rollback_resources_if_necessary()
-
 ctx_properties = utils.ctx_factory.create(NODE_NAME)
 
 

components/manager/scripts/creation_validation.py

@@ -10,7 +10,6 @@
 import utils  # NOQA
 
 IMMUTABLE_PROPERTIES = [
-    'security',
     'ssh_user'
 ]
 

components/manager/scripts/sanity/sanity.py

@@ -44,7 +44,7 @@ def _upload_app_blueprint(app_tar):
         app_data = f.read()
     length = os.path.getsize(app_tar)
 
-    headers = utils.create_maintenance_headers()
+    headers = {}
     headers['Content-Length'] = length
     headers['Content-Type'] = 'application/octet-stream'
     params = urllib.urlencode(
@@ -53,9 +53,10 @@ def _upload_app_blueprint(app_tar):
 
     endpoint = '{0}/blueprints/{1}'.format(_get_url_prefix(), BLUEPRINT_ID)
     url = endpoint + '?' + params
-    utils.http_request(url,
+    utils.rest_request(url,
                        data=app_data,
-                       headers=headers)
+                       headers=headers,
+                       method='PUT')
 
 
 def _deploy_app():
@@ -69,20 +70,20 @@ def _deploy_app():
         'blueprint_id': BLUEPRINT_ID,
         'inputs': dep_inputs
     }
-    headers = utils.create_maintenance_headers()
-    headers.update({'content-type': 'application/json'})
+    headers = {'content-type': 'application/json'}
 
-    utils.http_request(
+    utils.rest_request(
             '{0}/deployments/{1}'.format(_get_url_prefix(), DEPLOYMENT_ID),
             data=json.dumps(data),
-            headers=headers)
+            headers=headers,
+            method='PUT')
 
     # Waiting for create deployment env to end
     utils.repetitive(
         utils.wait_for_workflow,
         deployment_id=DEPLOYMENT_ID,
         workflow_id='create_deployment_environment',
-        url_prefix=_get_url_prefix(),
+        rest_host=manager_ip,
         timeout_msg='Timed out while waiting for '
                     'deployment {0} to be created'.format(DEPLOYMENT_ID))
 
@@ -92,10 +93,9 @@ def _install_sanity_app():
         'deployment_id': DEPLOYMENT_ID,
         'workflow_id': 'install'
     }
-    headers = utils.create_maintenance_headers()
-    headers.update({'content-type': 'application/json'})
+    headers = {'content-type': 'application/json'}
 
-    resp = utils.http_request(
+    resp = utils.rest_request(
             '{0}/executions'.format(_get_url_prefix()),
             method='POST',
             data=json.dumps(data),
@@ -108,31 +108,30 @@ def _install_sanity_app():
         interval=30,
         deployment_id=DEPLOYMENT_ID,
         workflow_id='install',
-        url_prefix=_get_url_prefix(),
+        rest_host=manager_ip,
         timeout_msg='Timed out while waiting for '
                     'deployment {0} to install'.format(DEPLOYMENT_ID))
 
-    resp_content = resp.readlines()
-    json_resp = json.loads(resp_content[0])
+    json_resp = json.loads(resp.content)
     return json_resp['id']
 
 
 def _assert_logs_and_events(execution_id):
-    headers = utils.create_maintenance_headers()
     params = urllib.urlencode(
             dict(execution_id=execution_id,
                  type='cloudify_log'))
 
     endpoint = '{0}/events'.format(_get_url_prefix())
     url = endpoint + '?' + params
-    resp = utils.http_request(url, method='GET', headers=headers, timeout=30)
+    resp = utils.rest_request(url,
+                              method='GET',
+                              timeout=30)
     if not resp:
         ctx.abort_operation("Can't connect to elasticsearch")
     if resp.code != 200:
         ctx.abort_operation('Failed to retrieve logs/events')
 
-    resp_content = resp.readlines()
-    json_resp = json.loads(resp_content[0])
+    json_resp = json.loads(resp.content)
 
     if 'items' not in json_resp or not json_resp['items']:
         ctx.abort_operation('No logs/events received')
@@ -165,10 +164,9 @@ def _uninstall_sanity_app():
         'deployment_id': DEPLOYMENT_ID,
         'workflow_id': 'uninstall'
     }
-    headers = utils.create_maintenance_headers()
-    headers.update({'content-type': 'application/json'})
+    headers = {'content-type': 'application/json'}
 
-    utils.http_request(
+    utils.rest_request(
         '{0}/executions'.format(_get_url_prefix()),
         method='POST',
         data=json.dumps(data),
@@ -181,20 +179,18 @@ def _uninstall_sanity_app():
         interval=30,
         deployment_id=DEPLOYMENT_ID,
         workflow_id='uninstall',
-        url_prefix=_get_url_prefix(),
+        rest_host=manager_ip,
         timeout_msg='Timed out while waiting for '
                     'deployment {0} to uninstall.'.format(DEPLOYMENT_ID))
 
 
 def _delete_sanity_deployment():
     if not _is_sanity_dep_exist():
         return
-    headers = utils.create_maintenance_headers()
 
-    resp = utils.http_request(
+    resp = utils.rest_request(
         '{0}/deployments/{1}'.format(_get_url_prefix(), DEPLOYMENT_ID),
-        method='DELETE',
-        headers=headers)
+        method='DELETE')
 
     if resp.code != 200:
         ctx.abort_operation('Failed deleting '
@@ -205,11 +201,9 @@ def _delete_sanity_deployment():
 def _delete_sanity_blueprint():
     if not _is_sanity_blueprint_exist():
         return
-    headers = utils.create_maintenance_headers()
-    resp = utils.http_request(
+    resp = utils.rest_request(
         '{0}/blueprints/{1}'.format(_get_url_prefix(), BLUEPRINT_ID),
-        method='DELETE',
-        headers=headers)
+        method='DELETE')
 
     if resp.code != 200:
         ctx.abort_operation('Failed deleting '
@@ -223,23 +217,19 @@ def _delete_key_file():
 
 
 def _is_sanity_dep_exist(should_fail=False):
-    headers = utils.create_maintenance_headers()
-    res = utils.http_request(
+    res = utils.rest_request(
         '{0}/deployments/{1}'.format(_get_url_prefix(), DEPLOYMENT_ID),
         method='GET',
-        headers=headers,
         should_fail=should_fail)
     if not res:
         return False
     return res.code == 200
 
 
 def _is_sanity_blueprint_exist(should_fail=False):
-    headers = utils.create_maintenance_headers()
-    res = utils.http_request(
+    res = utils.rest_request(
             '{0}/blueprints/{1}'.format(_get_url_prefix(), BLUEPRINT_ID),
             method='GET',
-            headers=headers,
             should_fail=should_fail)
     if not res:
         return False
@@ -276,7 +266,15 @@ def perform_sanity():
     perform_sanity()
 
 if utils.is_upgrade or utils.is_rollback:
-    utils.restore_upgrade_snapshot()
+    # Restore the snapshot at the end of the workflow.
+    utils.restore_upgrade_snapshot(manager_ip)
 
 if utils.is_upgrade:
+    # To keep the upgrade workflow idempotent, this flag is used to figure
+    # out if the next upgrade should dispose of old rollback data.
     utils.set_upgrade_success_in_upgrade_meta()
+
+if utils.is_rollback:
+    # remove data created by the upgrade process.
+    utils.remove(utils.UPGRADE_METADATA_FILE)
+    utils.remove(utils.ES_UPGRADE_DUMP_PATH)

components/nginx/scripts/preconfigure.py

@@ -13,20 +13,24 @@
 EXTERNAL_REST_CERT_PATH = '/root/cloudify/ssl/external_rest_host.crt'
 
 NGINX_SERVICE_NAME = 'nginx'
-ctx_properties = {'service_name': NGINX_SERVICE_NAME}
 
 
 def preconfigure_nginx():
 
     target_runtime_props = ctx.target.instance.runtime_properties
+    source_runtime_props = ctx.source.instance.runtime_properties
+
     # this is used by nginx's default.conf to select the relevant configuration
     rest_protocol = target_runtime_props['rest_protocol']
-
+    rest_host = target_runtime_props['internal_rest_host']
     # TODO: NEED TO IMPLEMENT THIS IN CTX UTILS
-    ctx.source.instance.runtime_properties['rest_protocol'] = rest_protocol
+    source_runtime_props['rest_protocol'] = rest_protocol
+    ctx.logger.info('setting rest host to {}'.format(rest_host))
+    source_runtime_props['rest_host'] = rest_host
     if rest_protocol == 'https':
+
         utils.deploy_rest_certificates(
-            internal_rest_host=target_runtime_props['internal_rest_host'],
+            internal_rest_host=rest_host,
             external_rest_host=target_runtime_props['external_rest_host'])
 
         # get rest public certificate for output later

components/nginx/scripts/start.py

@@ -25,12 +25,7 @@ def check_response(response):
 utils.start_service(NGINX_SERVICE_NAME, append_prefix=False)
 utils.systemd.verify_alive(NGINX_SERVICE_NAME, append_prefix=False)
 
-nginx_url = '{0}://127.0.0.1/api/v2.1/version'.format(
-    ctx.instance.runtime_properties['rest_protocol'])
+rest_host = ctx.instance.runtime_properties['rest_host']
+nginx_url = '{0}/api/v2.1/blueprints'.format(rest_host)
 
-headers = {}
-if utils.is_upgrade or utils.is_rollback:
-    headers = utils.create_maintenance_headers()
-
-utils.verify_service_http(NGINX_SERVICE_NAME, nginx_url, check_response,
-                          headers=headers)
+utils.verify_service_http(NGINX_SERVICE_NAME, nginx_url, check_response)

components/restservice/scripts/preconfigure.py

@@ -10,5 +10,7 @@ def preconfigure_restservice():
     ctx.logger.info('security_config is: {0}'.format(security_config))
     ctx.source.instance.runtime_properties['security_configuration'] = \
         security_config
+    ctx.source.instance.runtime_properties['rest_host'] = \
+        ctx.target.instance.runtime_properties['internal_rest_host']
 
 preconfigure_restservice()

components/restservice/scripts/start.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 
 import json
+import httplib
 import urllib2
-import urlparse
 from os.path import join, dirname
 
 from cloudify import ctx
@@ -23,22 +23,10 @@ def verify_restservice(url):
     that also requires the storage backend to be up, so if it works, there's
     a good chance everything is configured correctly.
     """
-    blueprints_url = urlparse.urljoin(url, 'api/v2.1/blueprints')
-
-    headers = utils.get_auth_headers(True)
-
-    if utils.is_upgrade or utils.is_rollback:
-        # if we're doing an upgrade, we're in maintenance mode - this request
-        # is safe to perform in maintenance mode, so let's bypass the check
-        headers = utils.create_maintenance_headers()
-    else:
-        headers = utils.get_auth_headers(True)
-
-    req = urllib2.Request(blueprints_url, headers=headers)
-
+    blueprints_url = '{0}/{1}'.format(url, 'api/v2.1/blueprints')
     try:
-        response = urllib2.urlopen(req)
-    except urllib2.URLError as e:
+        response = utils.rest_request(blueprints_url)
+    except (urllib2.URLError, httplib.HTTPException) as e:
         ctx.abort_operation('REST service returned an invalid response: {0}'
                             .format(e))
     if response.code == 401:
@@ -50,7 +38,7 @@ def verify_restservice(url):
                             .format(response.code))
 
     try:
-        json.load(response)
+        json.loads(response.content)
     except ValueError as e:
         ctx.abort_operation('REST service returned malformed JSON: {0}'
                             .format(e))
@@ -61,6 +49,6 @@ def verify_restservice(url):
 
 utils.systemd.verify_alive(REST_SERVICE_NAME)
 
-restservice_url = 'http://{0}:{1}'.format('127.0.0.1', 8100)
+restservice_url = ctx.instance.runtime_properties['rest_host']
 utils.verify_service_http(REST_SERVICE_NAME, restservice_url)
 verify_restservice(restservice_url)

components/riemann/scripts/preconfigure.py

@@ -1,14 +1,7 @@
 #!/usr/bin/env python
 
-from os.path import join, dirname
-
 from cloudify import ctx
 
-ctx.download_resource(
-    join('components', 'utils.py'),
-    join(dirname(__file__), 'utils.py'))
-import utils  # NOQA
-
 
 ctx.source.instance.runtime_properties['rest_host'] = \
     ctx.target.instance.runtime_properties['internal_rest_host']