From 404bbb6ed680547a8abba9bd209a6a01c0b26218 Mon Sep 17 00:00:00 2001 From: Felix Sperling Date: Fri, 13 May 2016 15:29:26 +0700 Subject: [PATCH 1/7] catch RequestLimitExceeded exceptions and do several retries with sleeps inbetween --- backup_monkey/core.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/backup_monkey/core.py b/backup_monkey/core.py index db7ed7d..974d505 100644 --- a/backup_monkey/core.py +++ b/backup_monkey/core.py @@ -12,10 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import time -from boto.exception import NoAuthHandlerFound +from exceptions import * + +import boto from boto import ec2 + from backup_monkey.exceptions import BackupMonkeyException __all__ = ('BackupMonkey', 'Logging') @@ -115,7 +119,22 @@ def snapshot_volumes(self): description_parts.append(volume.attach_data.device) description = ' '.join(description_parts) log.info('Creating snapshot of %s: %s', volume.id, description) - volume.create_snapshot(description) + for attempt in range(5): + try: + volume.create_snapshot(description) + except boto.exception.EC2ResponseError, e: + log.error("Encountered Error %s on volume %s", e.error_code, volume.id) + break + except boto.exception.BotoServerError, e: + log.error("Encountered Error %s on volume %s, waiting %d seconds then retrying", e.error_code, volume.id, attempt) + time.sleep(attempt) + break + else: + break + else: + log.error("Encountered Error %s on volume %s, %d retries failed, continuing", e.error_code, volume.id, attempt) + continue + return True @@ -147,7 +166,22 @@ def remove_old_snapshots(self): for i in range(self._snapshots_per_volume, num_snapshots): snapshot = most_recent_snapshots[i] log.info(' Deleting %s: %s', snapshot.id, snapshot.description) - snapshot.delete() + for attempt in range(5): + try: + snapshot.delete() + except boto.exception.EC2ResponseError, e: + log.error("Encountered Error %s on volume %s", e.error_code, volume.id) + break + except boto.exception.BotoServerError, e: + log.error("Encountered Error %s on volume %s, waiting %d seconds then retrying", e.error_code, volume.id, attempt) + time.sleep(attempt) + break + else: + break + else: + log.error("Encountered Error %s on volume %s, %d retries failed, continuing", e.error_code, volume.id, attempt) + continue + return True From 5e8d3d368ab589d4b2a003b17058ad18517906af Mon Sep 17 00:00:00 2001 From: Felix Sperling Date: Mon, 16 May 2016 18:53:39 +0700 Subject: [PATCH 2/7] use the right var name --- backup_monkey/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backup_monkey/core.py b/backup_monkey/core.py index 974d505..a2b9e3c 100644 --- a/backup_monkey/core.py +++ b/backup_monkey/core.py @@ -170,16 +170,16 @@ def remove_old_snapshots(self): try: snapshot.delete() except boto.exception.EC2ResponseError, e: - log.error("Encountered Error %s on volume %s", e.error_code, volume.id) + log.error("Encountered Error %s on volume %s", e.error_code, volume_id) break except boto.exception.BotoServerError, e: - log.error("Encountered Error %s on volume %s, waiting %d seconds then retrying", e.error_code, volume.id, attempt) + log.error("Encountered Error %s on volume %s, waiting %d seconds then retrying", e.error_code, volume_id, attempt) time.sleep(attempt) break else: break else: - log.error("Encountered Error %s on volume %s, %d retries failed, continuing", e.error_code, volume.id, attempt) + log.error("Encountered Error %s on volume %s, %d retries failed, continuing", e.error_code, volume_id, attempt) continue return True From 518f7823ad03127de4b779a9df5826b96ea11d12 Mon Sep 17 00:00:00 2001 From: Felix Sperling Date: Thu, 2 Jun 2016 15:15:06 +0800 Subject: [PATCH 3/7] add option to tag created snapshots with graffiti-monkey --- backup_monkey/cli.py | 5 ++++- backup_monkey/core.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/backup_monkey/cli.py b/backup_monkey/cli.py index 7786083..2e51861 100644 --- a/backup_monkey/cli.py +++ b/backup_monkey/cli.py @@ -55,6 +55,8 @@ def run(): help='Do a cross-account snapshot (this is the account number to do snapshots on). NOTE: This requires that you pass in the --cross-account-role parameter. E.g. --cross-account-number 111111111111 --cross-account-role Snapshot') parser.add_argument('--cross-account-role', action='store', help='The name of the role that backup-monkey will assume when doing a cross-account snapshot. E.g. --cross-account-role Snapshot') + parser.add_argument('--path-to-graffiti-config', action='store', + help='backup-monkey can tag all created snapshots by using graffiti-monkey, if this is desired provide the absolute path to the graffiti config') args = parser.parse_args() @@ -95,7 +97,8 @@ def run(): args.reverse_tags, args.label, args.cross_account_number, - args.cross_account_role) + args.cross_account_role, + args.path_to_graffiti_config) if not args.remove_only: monkey.snapshot_volumes() diff --git a/backup_monkey/core.py b/backup_monkey/core.py index a2b9e3c..124a9ad 100644 --- a/backup_monkey/core.py +++ b/backup_monkey/core.py @@ -13,6 +13,7 @@ # limitations under the License. import logging import time +import subprocess from exceptions import * @@ -26,7 +27,7 @@ log = logging.getLogger(__name__) class BackupMonkey(object): - def __init__(self, region, max_snapshots_per_volume, tags, reverse_tags, label, cross_account_number, cross_account_role): + def __init__(self, region, max_snapshots_per_volume, tags, reverse_tags, label, cross_account_number, cross_account_role, graffiti_config): self._region = region self._prefix = 'BACKUP_MONKEY' if label: @@ -37,6 +38,7 @@ def __init__(self, region, max_snapshots_per_volume, tags, reverse_tags, label, self._cross_account_number = cross_account_number self._cross_account_role = cross_account_role self._conn = self.get_connection() + self._tag_with_graffiti_config = graffiti_config def get_connection(self): ret = None @@ -121,7 +123,11 @@ def snapshot_volumes(self): log.info('Creating snapshot of %s: %s', volume.id, description) for attempt in range(5): try: - volume.create_snapshot(description) + snap = volume.create_snapshot(description) + if self._tag_with_graffiti_config: + cmd = ("graffiti-monkey --region " + self._region + " --config " + self._tag_with_graffiti_config + " --novolumes --snapshots").split() + log.info("Tagging snapshot: %s", snap.id) + subprocess.call(cmd + [str(snap.id)]) except boto.exception.EC2ResponseError, e: log.error("Encountered Error %s on volume %s", e.error_code, volume.id) break From bdb76cbe868e00072610ab46acbdd8044c872fcd Mon Sep 17 00:00:00 2001 From: Felix Sperling Date: Mon, 20 Jun 2016 13:10:42 +0700 Subject: [PATCH 4/7] make prefix configurable and also allow that the label does not have to be right after the prefix --- backup_monkey/cli.py | 5 ++++- backup_monkey/core.py | 12 +++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/backup_monkey/cli.py b/backup_monkey/cli.py index 2e51861..3b556e0 100644 --- a/backup_monkey/cli.py +++ b/backup_monkey/cli.py @@ -41,6 +41,8 @@ def run(): help='Only snapshot EBS volumes, do not remove old snapshots') parser.add_argument('--remove-only', action='store_true', default=False, help='Only remove old snapshots, do not create new snapshots') + parser.add_argument('--snapshot-prefix', action='store', default="BACKUP_MONKEY", + help='Only act on snapshots that start with this prefix') parser.add_argument('--verbose', '-v', action='count', help='enable verbose output (-vvv for more)') parser.add_argument('--version', action='version', version='%(prog)s ' + __version__, @@ -98,7 +100,8 @@ def run(): args.label, args.cross_account_number, args.cross_account_role, - args.path_to_graffiti_config) + args.path_to_graffiti_config, + args.snapshot_prefix) if not args.remove_only: monkey.snapshot_volumes() diff --git a/backup_monkey/core.py b/backup_monkey/core.py index 124a9ad..2e1c585 100644 --- a/backup_monkey/core.py +++ b/backup_monkey/core.py @@ -27,11 +27,10 @@ log = logging.getLogger(__name__) class BackupMonkey(object): - def __init__(self, region, max_snapshots_per_volume, tags, reverse_tags, label, cross_account_number, cross_account_role, graffiti_config): + def __init__(self, region, max_snapshots_per_volume, tags, reverse_tags, label, cross_account_number, cross_account_role, graffiti_config, snapshot_prefix): self._region = region - self._prefix = 'BACKUP_MONKEY' - if label: - self._prefix += ' ' + label + self._prefix = snapshot_prefix + self._label = label self._snapshots_per_volume = max_snapshots_per_volume self._tags = tags self._reverse_tags = reverse_tags @@ -113,7 +112,7 @@ def snapshot_volumes(self): volumes = self.get_volumes_to_snapshot() log.info('Found %d volumes', len(volumes)) for volume in volumes: - description_parts = [self._prefix] + description_parts = [self._prefix + " " + self._label] description_parts.append(volume.id) if volume.attach_data.instance_id: description_parts.append(volume.attach_data.instance_id) @@ -157,6 +156,9 @@ def remove_old_snapshots(self): if not snapshot.description.startswith(self._prefix): log.debug('Skipping %s as prefix does not match', snapshot.id) continue + if self._label and self._label not in snapshot.description: + log.debug('Skipping %s as label does not match', snapshot.id) + continue if not snapshot.status == 'completed': log.debug('Skipping %s as it is not a complete snapshot', snapshot.id) continue From f1504d84e286d426dfd0a60d38703a48672c93c3 Mon Sep 17 00:00:00 2001 From: Felix Sperling Date: Mon, 20 Jun 2016 13:17:53 +0700 Subject: [PATCH 5/7] update README --- README.rst | 18 +++++++++++++++--- backup_monkey/cli.py | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index be75f17..6eee599 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ Backup Monkey ============= -.. image:: https://travis-ci.org/Answers4AWS/backup-monkey.png?branch=master - :target: https://travis-ci.org/Answers4AWS/backup-monkey +.. image:: https://travis-ci.org/fsperling/backup-monkey.png?branch=master + :target: https://travis-ci.org/fsperling/backup-monkey :alt: Build Status A monkey that makes sure you have a backup of your EBS volumes in case something goes wrong. @@ -21,8 +21,10 @@ Usage [--remove-only] [--verbose] [--version] [--tags TAGS [TAGS ...]] [--reverse-tags] [--label LABEL] + [--snapshot-prefix SNAPSHOT_PREFIX] [--cross-account-number CROSS_ACCOUNT_NUMBER] [--cross-account-role CROSS_ACCOUNT_ROLE] + [--path-to-graffiti-config PATH_TO_GRAFFITI_CONFIG] Loops through all EBS volumes, and snapshots them, then loops through all snapshots, and removes the oldest ones. @@ -56,6 +58,11 @@ Usage backup-monkey --max-snapshots-per-volume 6 --label daily backup-monkey --max-snapshots-per-volume 4 --label weekly You save 6 + 4 snapshots max. instead 4 or 6 + --snapshot-prefix SNAPSHOT_PREFIX + Created snapshots will contain this prefix. Only + considers snapshots for removal that start with this + prefix. Default: BACKUP_MONKEY + --cross-account-number CROSS_ACCOUNT_NUMBER Do a cross-account snapshot (this is the account number to do snapshots on). NOTE: This requires that @@ -66,6 +73,11 @@ Usage The name of the role that backup-monkey will assume when doing a cross-account snapshot. E.g. --cross- account-role Snapshot + --path-to-graffiti-config PATH_TO_GRAFFITI_CONFIG + backup-monkey can tag all created snapshots by using + graffiti-monkey, if this is desired provide the + absolute path to the graffiti config + Examples -------- @@ -98,7 +110,7 @@ Alternatively, if you prefer to install from source: :: - git clone git@github.com:Answers4AWS/backup-monkey.git + git clone git@github.com:fsperling/backup-monkey.git cd backup-monkey python setup.py install diff --git a/backup_monkey/cli.py b/backup_monkey/cli.py index 3b556e0..c90abcd 100644 --- a/backup_monkey/cli.py +++ b/backup_monkey/cli.py @@ -42,7 +42,7 @@ def run(): parser.add_argument('--remove-only', action='store_true', default=False, help='Only remove old snapshots, do not create new snapshots') parser.add_argument('--snapshot-prefix', action='store', default="BACKUP_MONKEY", - help='Only act on snapshots that start with this prefix') + help='Created snapshots will contain this prefix. Only considers snapshots for removal that start with this prefix. Default: BACKUP_MONKEY') parser.add_argument('--verbose', '-v', action='count', help='enable verbose output (-vvv for more)') parser.add_argument('--version', action='version', version='%(prog)s ' + __version__, From 12d74d52105015ab83beb8f81f1699c2f53f966c Mon Sep 17 00:00:00 2001 From: Felix Sperling Date: Mon, 20 Jun 2016 13:18:45 +0700 Subject: [PATCH 6/7] remove travis ci pic --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index 6eee599..9eb714e 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,6 @@ Backup Monkey ============= -.. image:: https://travis-ci.org/fsperling/backup-monkey.png?branch=master - :target: https://travis-ci.org/fsperling/backup-monkey - :alt: Build Status - A monkey that makes sure you have a backup of your EBS volumes in case something goes wrong. It is designed specifically for Amazon Web Services (AWS), and uses Python and Boto. From 53887415d937a36619c78f1c99b51134c13ee795 Mon Sep 17 00:00:00 2001 From: Felix Sperling Date: Fri, 26 May 2017 17:30:15 +0800 Subject: [PATCH 7/7] dont lock to version, just min req --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8e6ed87..fcd217c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -boto==2.38.0 +boto>=2.38.0