Add retry for retrieving IP addresses from server

There is an intermittent issue with server objects being returned from
the CLC API where the IP address cannot be retrieved.  A method was
added that will attempt to retry the API call until the IP addresses are
present.

clc_ansible_module/clc_server.py

@@ -1136,12 +1136,8 @@ def _create_servers(self, module, clc, override_count=None):
             else:
                 # reload server details
                 server = clc.v2.Server(server.id)
-                server.data['ipaddress'] = server.details[
-                    'ipAddresses'][0]['internal']
+                server = ClcServer._retrieve_ip_addresses(module, server)
 
-                if add_public_ip and len(server.PublicIPs().public_ips) > 0:
-                    server.data['publicip'] = str(
-                        server.PublicIPs().public_ips[0])
                 created_server_ids.append(server.id)
             server_dict_array.append(server.data)
 
@@ -1430,13 +1426,7 @@ def _start_stop_servers(module, clc, server_ids):
         ClcServer._refresh_servers(module, changed_servers)
 
         for server in set(changed_servers + servers):
-            try:
-                server.data['ipaddress'] = server.details[
-                    'ipAddresses'][0]['internal']
-                server.data['publicip'] = str(
-                    server.PublicIPs().public_ips[0])
-            except (KeyError, IndexError):
-                pass
+            server = ClcServer._retrieve_ip_addresses(module, server)
 
             server_dict_array.append(server.data)
             result_server_ids.append(server.id)
@@ -1545,6 +1535,39 @@ def _find_group_recursive(module, group_list, lookup_group):
 
         return result
 
+    @staticmethod
+    def _retrieve_ip_addresses(module, server, poll_freq=2, retries=5):
+        """
+        Retrieve IP addresses for a CLC server, retrying if needed
+        :param module: the AnsibleModule instance to use
+        :param server: Server object for which to retrieve IP addresses
+        :param poll_freq: Poll frequency for retries
+        :param retries:  Number of retries
+        :return: Server object with IP addresses added to data dictionary
+        """
+        while 'ipAddresses' not in server.details:
+            if retries < 1:
+                module.fail_json(
+                    msg='Unable to retrieve IP addresses for server: '
+                        '{name}.'.format(
+                            name=server.name))
+            time.sleep(poll_freq)
+            retries -= 1
+            server.Refresh()
+
+        internal_ips = [ip['internal'] for ip
+                        in server.details['ipAddresses']
+                        if 'internal' in ip]
+        public_ips = [ip['public'] for ip
+                      in server.details['ipAddresses']
+                      if 'public' in ip]
+        if len(internal_ips) > 0:
+            server.data['ipaddress'] = internal_ips[0]
+        if len(public_ips) > 0:
+            server.data['publicip'] = public_ips[0]
+
+        return server
+
     @staticmethod
     def _create_clc_server(
             clc,

tests/test_clc_server.py

@@ -171,8 +171,8 @@ def test_process_request_exact_count_1_server_w_pubip(self,
         # Set Mock Server Return Values
         mock_server.id = 'TEST_SERVER'
         mock_server.data = {'name': 'TEST_SERVER'}
-        mock_server.details = {'ipAddresses': [{'internal': '1.2.3.4'}]}
-        mock_server.PublicIPs().public_ips = ['5.6.7.8']
+        mock_server.details = {'ipAddresses': [{'internal': '1.2.3.4'},
+                                               {'public': '5.6.7.8'}]}
 
         # Set Mock Request Return Values
         mock_single_request.Server.return_value = mock_server
@@ -255,8 +255,8 @@ def test_process_request_exact_count_1_server_w_no_alert_pol_name(self,
         # Set Mock Server Return Values
         mock_server.id = 'TEST_SERVER'
         mock_server.data = {'name': 'TEST_SERVER'}
-        mock_server.details = {'ipAddresses': [{'internal': '1.2.3.4'}]}
-        mock_server.PublicIPs().public_ips = ['5.6.7.8']
+        mock_server.details = {'ipAddresses': [{'internal': '1.2.3.4'},
+                                               {'public': '5.6.7.8'}]}
 
         # Set Mock Request Return Values
         mock_single_request.Server.return_value = mock_server
@@ -541,8 +541,8 @@ def test_process_request_start_server(self,
         mock_server.data = {'name': 'TEST_SERVER'}
         mock_server.powerState = 'stopped'
         mock_server.PowerOn.return_value = mock_request
-        mock_server.PublicIPs().public_ips.__getitem__.return_value = "5.6.7.8"
-        mock_server.details['ipAddresses'][0].__getitem__.return_value = "1.2.3.4"
+        mock_server.details = {'ipAddresses': [{'internal': '1.2.3.4'},
+                                               {'public': '5.6.7.8'}]}
 
         # Set Mock Group Values
         mock_group = mock.MagicMock()
@@ -1463,6 +1463,7 @@ def test_start_stop_servers(self, mock_clc_sdk):
         self.module.params = params
         mock_server = mock.MagicMock()
         mock_server.id = 'mockid1'
+        mock_server.details = {'ipAddresses': [{'internal': '1.2.3.4'}]}
         mock_servers = mock.MagicMock()
         mock_servers.id = 'temp1'
         mock_servers.Servers.return_value = [mock_server]