Refactor ece object

galaxy.yml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ---
-version: 2.3.24
+version: 2.3.27
 namespace: expedient
 name: elastic
 readme: README.md
@@ -26,4 +26,4 @@ tags:
 dependencies: {}
 repository: https://github.com/Expedient/ansible-collection-elastic
 homepage: https://github.com/Expedient/ansible-collection-elastic
-issues: https://github.com/Expedient/ansible-collection-elastic/issues
+issues: https://github.com/Expedient/ansible-collection-elastic/issues

plugins/module_utils/ece.py

@@ -38,7 +38,7 @@ def __init__(self, module):
       }
       payload = dumps(data)
       headers = {'Content-Type': 'application/json'}
-      response = open_url(url, data=payload, headers=headers, method='POST', validate_certs=self.validate_certs, timeout=120)
+      response = open_url(url, data=payload, headers=headers, method='POST', validate_certs=self.validate_certs, timeout=300)
       content = loads(response.read())
       self.token = content['token']
 

plugins/module_utils/ece_apiproxy.py

@@ -47,15 +47,15 @@ def __init__(self, module):
     self.ece_auth = ECE(module)
 
 
-  def send_api_request(self, endpoint, method, data=None, headers={}, timeout=120, space_id='default', no_kbnver=False, version=None):
+  def send_api_request(self, endpoint, method, data=None, headers=[{}], timeout=600, space_id='default', no_kbnver=False, version=None):
 
     if endpoint.startswith('_'):
       url = f'https://{self.host}:{self.port}/api/v1/deployments/{self.deployment_id}/{self.resource_type}/{self.ref_id}/proxy/{endpoint}'
     else:
       url = f'https://{self.host}:{self.port}/api/v1/deployments/{self.deployment_id}/{self.resource_type}/{self.ref_id}/proxy/s/{space_id}/api/{endpoint}'
 
-    headers = {'Authorization': f'Bearer {self.ece_auth.token}'}
     payload = None
+    headers['Authorization'] = f'Bearer {self.ece_auth.token}'
     headers['Content-Type'] = 'application/json'
     headers['X-Management-Request'] = 'True'
 
@@ -70,20 +70,19 @@ def send_api_request(self, endpoint, method, data=None, headers={}, timeout=120,
       headers=headers, 
       method=method, 
       validate_certs=self.validate_certs,
-      timeout=120
+      timeout=timeout
       )
     if response.reason != 'No Content':
       content = loads(response.read())
     else:
       content = ''
     return content
 
-  def send_file_api_request(self, endpoint, method, data=None, headers={}, file=None, timeout=120, space_id = "default", no_kbnver=False, version=None, *args, **kwargs):
+  def send_file_api_request(self, endpoint, method, data=None, headers=[{}], file=None, timeout=600, space_id = "default", no_kbnver=False, version=None, *args, **kwargs):
 
     url = f'https://{self.host}:{self.port}/api/v1/deployments/{self.deployment_id}/{self.resource_type}/{self.ref_id}/proxy/s/{space_id}/api/{endpoint}'
 
     response = None
-    headers = {}
     headers['Authorization'] =  f'Bearer {self.ece_auth.token}'
     headers['X-Management-Request'] = 'True'
     #headers['Content-Type'] = 'application/json'

plugins/module_utils/kibana.py

@@ -55,15 +55,15 @@ def __init__(self, module):
       self.version = self.get_cluster_version()
       self.major_version,self.minor_version,self.patch_version = self.version.split(".")
 
-  def send_api_request(self, endpoint, method, data = None, headers = {}, timeout = 120, space_id = "default", no_kbnver = False,*args, **kwargs):
+  def send_api_request(self, endpoint, method, data = None, headers = {}, timeout = 600, space_id = "default", no_kbnver = False,*args, **kwargs):
 
     if self.deployment_info:
       result = self.ece_api_proxy.send_api_request(endpoint, method, data, headers, timeout, space_id, no_kbnver)
     else:
       result = self.send_kibana_api_request(endpoint, method, data, headers, timeout, space_id, no_kbnver)
     return result
 
-  def send_kibana_api_request(self, endpoint, method, data=None, headers={}, timeout=120, space_id = "default", no_kbnver = False, *args, **kwargs):
+  def send_kibana_api_request(self, endpoint, method, data=None, headers={}, timeout = 600, space_id = "default", no_kbnver = False, *args, **kwargs):
 
     if space_id != "default":
       url = f'https://{self.host}:{self.port}/s/{space_id}/api/{endpoint}'
@@ -77,8 +77,16 @@ def send_kibana_api_request(self, endpoint, method, data=None, headers={}, timeo
     if self.version and no_kbnver == False:
       headers['kbn-version'] = self.version
     try:
-      response = open_url(url, data=payload, method=method, validate_certs=self.validate_certs, headers=headers,
-                          force_basic_auth=True, url_username=self.username, url_password=self.password, timeout=timeout)
+      response = open_url(
+        url, 
+        data=payload, 
+        method=method, 
+        validate_certs=self.validate_certs, 
+        headers=headers,
+        force_basic_auth=True, 
+        url_username=self.username, 
+        url_password=self.password, 
+        timeout=timeout)
     except HTTPError as e:
       raise e ## This allows errors raised during the request to be inspected while debugging
     if response.msg == 'No Content' and str(response.status).startswith('2'):
@@ -96,7 +104,7 @@ def send_kibana_api_request(self, endpoint, method, data=None, headers={}, timeo
       else:
         return response_list[0]
 
-  def send_epr_api_request(self, endpoint, method, data=None, headers={}, timeout=120):
+  def send_epr_api_request(self, endpoint, method, data=None, headers={}, timeout=600):
     url = f'https://epr.elastic.co/{endpoint}'
     payload = None
     if data:
@@ -111,15 +119,15 @@ def send_epr_api_request(self, endpoint, method, data=None, headers={}, timeout=
       raise e ## This allows errors raised during the request to be inspected while debugging
     return loads(response.read())
 
-  def send_file_api_request(self, endpoint, method, data = None,  headers = {}, file = None, timeout = 120, space_id = "default", no_kbnver = False,*args, **kwargs):
+  def send_file_api_request(self, endpoint, method, data = None,  headers = {}, file = None, timeout = 600, space_id = "default", no_kbnver = False,*args, **kwargs):
 
     if self.deployment_info:
       result = self.ece_api_proxy.send_file_api_request(endpoint, method, data, headers, file, timeout, space_id, no_kbnver)
     else:
-      result = self.send_kibana_file_api_request(endpoint, method, data, headers, file, space_id )
+      result = self.send_kibana_file_api_request(endpoint, method, data, headers, file, space_id, timeout )
     return result
 
-  def send_kibana_file_api_request(self, endpoint, method, data=None, headers={}, file=None, space_id = "default", *args, **kwargs):
+  def send_kibana_file_api_request(self, endpoint, method, data=None, headers={}, file=None, space_id = "default", timeout = 600, *args, **kwargs):
 
     if space_id != "default":
       url = f'https://{self.host}:{self.port}/s/{space_id}/api/{endpoint}'
@@ -143,7 +151,7 @@ def send_kibana_file_api_request(self, endpoint, method, data=None, headers={},
           auth=(self.username, self.password),
           files={'file': open(file,'rb')}, 
           headers=headers,
-          timeout=60
+          timeout=timeout
         )
       except HTTPError as e:
         raise e ## This allows errors raised during the request to be inspected while debugging
@@ -355,19 +363,35 @@ def delete_action(self, action):
 
   # Elastic Security Rules functions
 
+  def get_security_rule_byid(self, rule_id):
+    endpoint = "detection_engine/rules?id=" + str(rule_id)
+    rule_object = self.send_api_request(endpoint, 'GET')
+    return rule_object
+
   def update_security_rule(self, body):
     endpoint = "detection_engine/rules"
-    update_rule = self.send_api_request(endpoint, 'PATCH', data=body)
+    rule_object = self.get_security_rule_byid(body['id'])
+    rule_object.pop('updated_at')
+    rule_object.pop('updated_by')
+    rule_object.pop('created_at')
+    rule_object.pop('created_by')
+    rule_object.pop('execution_summary')
+    rule_object.pop('rule_id')
+    rule_object.pop('related_integrations')
+    rule_object.pop('immutable')
+    rule_object.pop('required_fields')
+    rule_object.pop('setup')
+    rule_object.update(body)
+    update_rule = self.send_api_request(endpoint, 'PUT', data=rule_object)
     return update_rule
 
   def get_security_rules(self, page_size, page_no):
     endpoint = "detection_engine/rules/_find?page=" + str(page_no) + "&per_page=" + str(page_size)
     rules = self.send_api_request(endpoint, 'GET')
     return rules
 
-  def get_security_rules_byfilter(self, rule_name):
+  def get_security_rules_byfilter(self, rule_name, page_size = 500):
     page_no = 1
-    page_size = 100
     filter_scrubbed = urllib.parse.quote(str(rule_name))
     endpoint = "detection_engine/rules/_find?page=" + str(page_no) + "&per_page=" + str(page_size) + "&filter=alert.attributes.name:" + filter_scrubbed
     rules = self.send_api_request(endpoint, 'GET')
@@ -417,11 +441,10 @@ def enable_security_rule_action(
     update_rule = self.update_security_rule(body)
     return update_rule
 
-  def activate_security_rule(self, rule_name):
+  def activate_security_rule(self, rule_name, page_size = 500):
 
     #### Getting first page of rules
     page_number = 1
-    page_size = 100
     rules = self.get_security_rules_byfilter(rule_name)
     noOfRules = rules['total']
     allrules = rules['data']
@@ -441,7 +464,13 @@ def activate_security_rule(self, rule_name):
   # Elastic Integration functions
 
   def get_integrations(self):
-    if int(self.major_version) > 8 or (int(self.major_version) == 8 and int(self.minor_version) >= 6):
+    if 'self.major_version' in locals():
+      major_version = self.major_version
+      minor_version = self.minor_version
+    else:
+      [major_version,minor_version,patch_version] = self.deployment_info['version'].split('.')
+
+    if int(major_version) > 8 or (int(major_version) == 8 and int(minor_version) >= 6):
       all_integration_flag = "prerelease"
     else:
       all_integration_flag = "experimental"
@@ -528,7 +557,7 @@ def update_pkg_policy(self,pkgpolicy_id,body):
         input_no = input_no + 1
       if not self.module.check_mode:
         endpoint = "fleet/package_policies/" + pkgpolicy_id
-        pkg_policy_update = self.send_api_request(endpoint, 'PUT', data=body, timeout=300)
+        pkg_policy_update = self.send_api_request(endpoint, 'PUT', data=body)
       else:
         pkg_policy_update = "Cannot proceed with check_mode set to " + self.module.check_mode
       return pkg_policy_update
@@ -544,10 +573,10 @@ def get_pkg_policy(self,pkg_policy_name):
 
   def get_elatic_package_repository_package_info(self, package_name, package_version):
     endpoint = "package/" + package_name + "/" + package_version
-    epr_object = self.send_epr_api_request(endpoint, 'GET', timeout=300)
+    epr_object = self.send_epr_api_request(endpoint, 'GET')
     return epr_object
 
-  def create_pkg_policy(self,pkg_policy_name, pkg_policy_desc, agent_policy_id, integration_object, namespace="default", var_list=None):
+  def create_pkg_policy(self,pkg_policy_name, pkg_policy_desc, agent_policy_id, integration_object, space_id="default", var_list=None):
     pkg_policy_object = self.get_pkg_policy(pkg_policy_name)
     epr_object = self.get_elatic_package_repository_package_info(integration_object['name'], integration_object['version'])
 
@@ -633,7 +662,7 @@ def create_pkg_policy(self,pkg_policy_name, pkg_policy_desc, agent_policy_id, in
 
       body = {
         "name": pkg_policy_name,
-        "namespace": namespace.lower(),
+        "namespace": space_id.lower(),
         "description": pkg_policy_desc,
         "force": True,
         "enabled": True,
@@ -649,7 +678,7 @@ def create_pkg_policy(self,pkg_policy_name, pkg_policy_desc, agent_policy_id, in
       body_JSON = dumps(body)
       endpoint = 'fleet/package_policies'
       if not self.module.check_mode:
-        pkg_policy_object = self.send_api_request(endpoint, 'POST', data=body_JSON, timeout=300)
+        pkg_policy_object = self.send_api_request(endpoint, 'POST', data=body_JSON)
       else:
         pkg_policy_object = "Cannot proceed with check_mode set to " + self.module.check_mode
 
@@ -690,7 +719,7 @@ def get_all_agent_policys(self, perPage = 500):
     agent_policy_objects = self.send_api_request(endpoint, 'GET')
     return agent_policy_objects
 
-  def create_agent_policy(self, agent_policy_id, agent_policy_name, agent_policy_desc, namespace="default", monitoring=[]):
+  def create_agent_policy(self, agent_policy_id, agent_policy_name, agent_policy_desc, space_id="default", monitoring=[]):
     if agent_policy_id:
       agent_policy_object = self.get_agent_policy_byid(agent_policy_id)
     else:
@@ -699,7 +728,7 @@ def create_agent_policy(self, agent_policy_id, agent_policy_name, agent_policy_d
     if not agent_policy_object:
       body = {
           "name": agent_policy_name,
-          "namespace": namespace.lower(),
+          "namespace": space_id.lower(),
           "description": agent_policy_desc,
           "monitoring_enabled": monitoring
       }
@@ -810,7 +839,7 @@ def update_saved_object(self, saved_object, object_type, object_id, object_attri
   def export_saved_object(self,
       object_type, 
       object_id, 
-      space_id, 
+      space_id = "default", 
       includeReferencesDeep = True, 
       excludeExportDetails = True, 
       *args, 
@@ -835,35 +864,33 @@ def export_saved_object(self,
     export_object = self.send_api_request(endpoint, 'POST', data=body_JSON, headers = headers, space_id = space_id, no_kbnver = True)
     return export_object
 
-  def import_saved_object(self, object_attributes, space_id = "default", overwrite = False, createNewCopies = True):
-    importObjectJSON = tempfile.NamedTemporaryFile(delete=False,suffix='.ndjson', prefix='saved_object_')
-    #object_attributes_json = loads(object_attributes)
-    import_file = open(importObjectJSON.name, 'a')
-    #for i in object_attributes_json:
-    #  import_file.write(dumps(i) + '\n')
-    import_file.write(object_attributes)
-    import_file.close()
-    importObjectJSON.close()
-    endpoint = f'saved_objects/_import?createNewCopies={createNewCopies}&overwrite={overwrite}'
-    import_object = self.send_file_api_request(endpoint, 'POST', file=importObjectJSON.name, space_id = space_id)
-    os.remove(importObjectJSON.name)
+  def import_saved_object(self, object_attributes = [], space_id = "default", overwrite = False, createNewCopies = True):
+    for object_attribute in object_attributes:
+      importObjectJSON = tempfile.NamedTemporaryFile(delete=False,suffix='.ndjson', prefix='saved_object_')
+      import_file = open(importObjectJSON.name, 'a')
+      import_file.write(object_attribute)
+      import_file.close()
+      importObjectJSON.close()
+      endpoint = f'saved_objects/_import?createNewCopies={createNewCopies}&overwrite={overwrite}'
+      import_object = self.send_file_api_request(endpoint, 'POST', file=importObjectJSON.name, space_id = space_id)
+      os.remove(importObjectJSON.name)
     return import_object
 
   def get_fleet_server_hosts(self):
     endpoint = 'fleet/settings'
     result = self.send_api_request(endpoint, 'GET')
     return result['item']['fleet_server_hosts']
 
-  def set_fleet_server_hosts(self, hosts: list):
-    endpoint = 'fleet/settings'
+  def set_fleet_server_hosts(self, hosts: list, name='Default', default=True):
+    endpoint = 'fleet/fleet_server_hosts/fleet-default-fleet-server-host' 
     headers = {'kbn-xsrf': True}
     body = {
-        'fleet_server_hosts': hosts
+        'is_default': default,
+        'name': name,
+        'host_urls': hosts
       }
 
-    body_json = dumps(body)
-
-    result = self.send_api_request(endpoint, 'PUT', headers=headers, data=body_json)
+    result = self.send_api_request(endpoint, 'PUT', headers=headers, data=body)
     return result
 
   def get_fleet_elasticsearch_hosts(self):
@@ -926,12 +953,16 @@ def create_space(
     result = self.send_api_request(endpoint, 'POST', data = body_json)
     return result
 
-# Elastic User Role
+#  User Role
 
   def get_userrole(self, name):
-    endpoint  = f'security/role/{name}'
-    userrole_object = self.send_api_request(endpoint, 'GET')
-    return userrole_object
+    endpoint  = f'security/role'    
+    userrole_objects = self.send_api_request(endpoint, 'GET')
+    target_userrole = None
+    for userrole_object in userrole_objects:
+      if userrole_object['name'].lower() == name.lower():
+        target_userrole = userrole_object
+    return target_userrole
 
   def create_userrole(self, 
                    name,