catch RequestLimitExceeded exceptions and do several retries with sle…

README.rst

@@ -1,10 +1,6 @@
 Backup Monkey
 =============
 
-.. image:: https://travis-ci.org/Answers4AWS/backup-monkey.png?branch=master
-   :target: https://travis-ci.org/Answers4AWS/backup-monkey
-   :alt: Build Status
-
 A monkey that makes sure you have a backup of your EBS volumes in case something goes wrong. 
 
 It is designed specifically for Amazon Web Services (AWS), and uses Python and Boto.
@@ -21,8 +17,10 @@ Usage
                          [--remove-only] [--verbose] [--version]
                          [--tags TAGS [TAGS ...]] [--reverse-tags]
                          [--label LABEL]
+                         [--snapshot-prefix SNAPSHOT_PREFIX]
                          [--cross-account-number CROSS_ACCOUNT_NUMBER]
                          [--cross-account-role CROSS_ACCOUNT_ROLE]
+                         [--path-to-graffiti-config PATH_TO_GRAFFITI_CONFIG]
 
     Loops through all EBS volumes, and snapshots them, then loops through all
     snapshots, and removes the oldest ones.
@@ -56,6 +54,11 @@ Usage
                               backup-monkey --max-snapshots-per-volume 6 --label daily
                               backup-monkey --max-snapshots-per-volume 4 --label weekly
                             You save 6 + 4 snapshots max. instead 4 or 6
+      --snapshot-prefix SNAPSHOT_PREFIX
+                           Created snapshots will contain this prefix. Only
+                           considers snapshots for removal that start with this
+                           prefix. Default: BACKUP_MONKEY
+
       --cross-account-number CROSS_ACCOUNT_NUMBER
                             Do a cross-account snapshot (this is the account
                             number to do snapshots on). NOTE: This requires that
@@ -66,6 +69,11 @@ Usage
                             The name of the role that backup-monkey will assume
                             when doing a cross-account snapshot. E.g. --cross-
                             account-role Snapshot
+      --path-to-graffiti-config PATH_TO_GRAFFITI_CONFIG
+                            backup-monkey can tag all created snapshots by using
+                            graffiti-monkey, if this is desired provide the
+                            absolute path to the graffiti config
+
 
 Examples
 --------
@@ -98,7 +106,7 @@ Alternatively, if you prefer to install from source:
 
 ::
 
-    git clone [email protected]:Answers4AWS/backup-monkey.git
+    git clone [email protected]:fsperling/backup-monkey.git
     cd backup-monkey
     python setup.py install
 

backup_monkey/cli.py

@@ -41,6 +41,8 @@ def run():
                         help='Only snapshot EBS volumes, do not remove old snapshots')
     parser.add_argument('--remove-only', action='store_true', default=False,
                         help='Only remove old snapshots, do not create new snapshots')
+    parser.add_argument('--snapshot-prefix', action='store', default="BACKUP_MONKEY",
+                        help='Created snapshots will contain this prefix. Only considers snapshots for removal that start with this prefix. Default: BACKUP_MONKEY')
     parser.add_argument('--verbose', '-v', action='count', 
                         help='enable verbose output (-vvv for more)')
     parser.add_argument('--version', action='version', version='%(prog)s ' + __version__,
@@ -55,6 +57,8 @@ def run():
                         help='Do a cross-account snapshot (this is the account number to do snapshots on). NOTE: This requires that you pass in the --cross-account-role parameter. E.g. --cross-account-number 111111111111 --cross-account-role Snapshot')
     parser.add_argument('--cross-account-role', action='store',
                         help='The name of the role that backup-monkey will assume when doing a cross-account snapshot. E.g. --cross-account-role Snapshot')
+    parser.add_argument('--path-to-graffiti-config', action='store', 
+                        help='backup-monkey can tag all created snapshots by using graffiti-monkey, if this is desired provide the absolute path to the graffiti config')
 
     args = parser.parse_args()
 
@@ -95,7 +99,9 @@ def run():
                               args.reverse_tags,
                               args.label,
                               args.cross_account_number,
-                              args.cross_account_role)
+                              args.cross_account_role,
+                              args.path_to_graffiti_config,
+                              args.snapshot_prefix)
 
         if not args.remove_only:
             monkey.snapshot_volumes()

backup_monkey/core.py

@@ -12,27 +12,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+import time
+import subprocess
 
-from boto.exception import NoAuthHandlerFound
+from exceptions import *
+
+import boto
 from boto import ec2
 
+
 from backup_monkey.exceptions import BackupMonkeyException
 
 __all__ = ('BackupMonkey', 'Logging')
 log = logging.getLogger(__name__)
 
 class BackupMonkey(object):
-    def __init__(self, region, max_snapshots_per_volume, tags, reverse_tags, label, cross_account_number, cross_account_role):
+    def __init__(self, region, max_snapshots_per_volume, tags, reverse_tags, label, cross_account_number, cross_account_role, graffiti_config, snapshot_prefix):
         self._region = region
-        self._prefix = 'BACKUP_MONKEY'
-        if label:
-            self._prefix += ' ' + label
+        self._prefix = snapshot_prefix
+        self._label = label
         self._snapshots_per_volume = max_snapshots_per_volume
         self._tags = tags
         self._reverse_tags = reverse_tags
         self._cross_account_number = cross_account_number
         self._cross_account_role = cross_account_role
         self._conn = self.get_connection()
+        self._tag_with_graffiti_config = graffiti_config
 
     def get_connection(self):
         ret = None
@@ -107,15 +112,34 @@ def snapshot_volumes(self):
         volumes = self.get_volumes_to_snapshot()
         log.info('Found %d volumes', len(volumes))
         for volume in volumes:            
-            description_parts = [self._prefix]
+            description_parts = [self._prefix + " " + self._label]
             description_parts.append(volume.id)
             if volume.attach_data.instance_id:
                 description_parts.append(volume.attach_data.instance_id)
             if volume.attach_data.device:
                 description_parts.append(volume.attach_data.device)
             description = ' '.join(description_parts)
             log.info('Creating snapshot of %s: %s', volume.id, description)
-            volume.create_snapshot(description)
+            for attempt in range(5):
+                try:
+                    snap = volume.create_snapshot(description)
+                    if self._tag_with_graffiti_config:
+                        cmd = ("graffiti-monkey --region " + self._region + " --config " + self._tag_with_graffiti_config + " --novolumes --snapshots").split()
+                        log.info("Tagging snapshot: %s", snap.id)
+                        subprocess.call(cmd + [str(snap.id)])
+                except boto.exception.EC2ResponseError, e:
+                    log.error("Encountered Error %s on volume %s", e.error_code, volume.id)
+                    break
+                except boto.exception.BotoServerError, e:
+                    log.error("Encountered Error %s on volume %s, waiting %d seconds then retrying", e.error_code, volume.id, attempt)
+                    time.sleep(attempt)
+                    break
+                else:
+                    break
+            else:
+                log.error("Encountered Error %s on volume %s, %d retries failed, continuing", e.error_code, volume.id, attempt)
+                continue
+
         return True
 
 
@@ -132,6 +156,9 @@ def remove_old_snapshots(self):
             if not snapshot.description.startswith(self._prefix):
                 log.debug('Skipping %s as prefix does not match', snapshot.id)
                 continue
+            if self._label and self._label not in snapshot.description:
+                log.debug('Skipping %s as label does not match', snapshot.id)
+                continue
             if not snapshot.status == 'completed':
                 log.debug('Skipping %s as it is not a complete snapshot', snapshot.id)
                 continue
@@ -147,7 +174,22 @@ def remove_old_snapshots(self):
             for i in range(self._snapshots_per_volume, num_snapshots):
                 snapshot = most_recent_snapshots[i]
                 log.info(' Deleting %s: %s', snapshot.id, snapshot.description)
-                snapshot.delete()
+                for attempt in range(5):
+                    try:
+                        snapshot.delete()
+                    except boto.exception.EC2ResponseError, e:
+                        log.error("Encountered Error %s on volume %s", e.error_code, volume_id)
+                        break
+                    except boto.exception.BotoServerError, e:
+                        log.error("Encountered Error %s on volume %s, waiting %d seconds then retrying", e.error_code, volume_id, attempt)
+                        time.sleep(attempt)
+                        break
+                    else:
+                        break
+                else:
+                    log.error("Encountered Error %s on volume %s, %d retries failed, continuing", e.error_code, volume_id, attempt)
+                    continue
+
         return True
 
 

requirements.txt

@@ -1 +1 @@
-boto==2.38.0
+boto>=2.38.0