From 81fda7d0227472db8eeef64d2fea2d9c1ca3e190 Mon Sep 17 00:00:00 2001 From: Vivian Date: Mon, 26 Mar 2018 16:56:57 -0700 Subject: [PATCH] NonNullUnicodeSetAttribute for blind cred and script upgrades (#166) --- CHANGELOG.md | 8 +- confidant/models/blind_credential.py | 6 +- .../models/non_null_unicode_set_attribute.py | 19 +++ confidant/models/service.py | 22 +--- confidant/routes/v1.py | 16 +++ confidant/scripts/manage.py | 10 +- confidant/scripts/migrate.py | 120 +++++++++++++++++- docs/source/basics/upgrade.html.markdown | 12 +- 8 files changed, 182 insertions(+), 31 deletions(-) create mode 100644 confidant/models/non_null_unicode_set_attribute.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 60782cf8..9fc3b388 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,5 @@ # Changelog -# 2 - ## 3.0.0 * This is a breaking release, if you're using blind credentials. This change @@ -11,6 +9,12 @@ This is due to a breaking change in pynamodb itself, which requires using specific versions of pynamodb to migrate the underlying data. +## 2.0.1 + +* Added additional logging in the v1 routes. +* Updated the migration script to include both Service and BlindCredential + migrations, as well as checks to ensure the migration was successful. + ## 2.0.0 WARNING: If you upgrade to this version, any new writes to blind credentials will be in a format that is only compatible in 1.11.0 forward. If you've diff --git a/confidant/models/blind_credential.py b/confidant/models/blind_credential.py index 2c2d588e..edc70c40 100644 --- a/confidant/models/blind_credential.py +++ b/confidant/models/blind_credential.py @@ -3,7 +3,6 @@ from pynamodb.models import Model from pynamodb.attributes import ( UnicodeAttribute, - UnicodeSetAttribute, NumberAttribute, LegacyBooleanAttribute, UTCDateTimeAttribute, @@ -14,6 +13,9 @@ from confidant.app import app from confidant.models.session_cls import DDBSession from confidant.models.connection_cls import DDBConnection +from confidant.models.non_null_unicode_set_attribute import ( + NonNullUnicodeSetAttribute +) class DataTypeDateIndex(GlobalSecondaryIndex): @@ -40,7 +42,7 @@ class Meta: data_type_date_index = DataTypeDateIndex() name = UnicodeAttribute() credential_pairs = JSONAttribute() - credential_keys = UnicodeSetAttribute(default=set([]), null=True) + credential_keys = NonNullUnicodeSetAttribute(default=set([]), null=True) enabled = LegacyBooleanAttribute(default=True) data_key = JSONAttribute() cipher_version = NumberAttribute() diff --git a/confidant/models/non_null_unicode_set_attribute.py b/confidant/models/non_null_unicode_set_attribute.py new file mode 100644 index 00000000..b9be33d5 --- /dev/null +++ b/confidant/models/non_null_unicode_set_attribute.py @@ -0,0 +1,19 @@ +from pynamodb.attributes import UnicodeSetAttribute + + +class NonNullUnicodeSetAttribute(UnicodeSetAttribute): + def __get__(self, instance, value): + ''' + Override UnicodeSetAttribute's __get__ method to return a set, rather + than None if the attribute isn't set. + ''' + if instance: + # Get the attribute. If the object doesn't have the attribute, + # ensure we return a set. + _value = instance.attribute_values.get(self.attr_name, set()) + # Attribute is assigned to None, return a set instead. + if _value is None: + _value = set() + return _value + else: + return self diff --git a/confidant/models/service.py b/confidant/models/service.py index ae231977..ba750f92 100644 --- a/confidant/models/service.py +++ b/confidant/models/service.py @@ -3,7 +3,6 @@ from pynamodb.models import Model from pynamodb.attributes import ( UnicodeAttribute, - UnicodeSetAttribute, NumberAttribute, UTCDateTimeAttribute, LegacyBooleanAttribute @@ -13,24 +12,9 @@ from confidant.app import app from confidant.models.session_cls import DDBSession from confidant.models.connection_cls import DDBConnection - - -class NonNullUnicodeSetAttribute(UnicodeSetAttribute): - def __get__(self, instance, value): - ''' - Override UnicodeSetAttribute's __get__ method to return a set, rather - than None if the attribute isn't set. - ''' - if instance: - # Get the attribute. If the object doesn't have the attribute, - # ensure we return a set. - _value = instance.attribute_values.get(self.attr_name, set()) - # Attribute is assigned to None, return a set instead. - if _value is None: - _value = set() - return _value - else: - return self +from confidant.models.non_null_unicode_set_attribute import ( + NonNullUnicodeSetAttribute +) class DataTypeDateIndex(GlobalSecondaryIndex): diff --git a/confidant/routes/v1.py b/confidant/routes/v1.py index d4ad8f2d..cb9fedc4 100644 --- a/confidant/routes/v1.py +++ b/confidant/routes/v1.py @@ -125,6 +125,7 @@ def get_service(id): try: credentials = _get_credentials(service.credentials) except KeyError: + logging.exception('KeyError occurred in getting credentials') return jsonify({'error': 'Decryption error.'}), 500 blind_credentials = _get_blind_credentials(service.blind_credentials) return jsonify({ @@ -145,6 +146,9 @@ def get_archive_service_revisions(id): try: service = Service.get(id) except DoesNotExist: + logging.warning( + 'Item with id {0} does not exist.'.format(id) + ) return jsonify({}), 404 if (service.data_type != 'service' and service.data_type != 'archive-service'): @@ -365,6 +369,9 @@ def get_credential(id): try: cred = Credential.get(id) except DoesNotExist: + logging.warning( + 'Item with id {0} does not exist.'.format(id) + ) return jsonify({}), 404 if (cred.data_type != 'credential' and cred.data_type != 'archive-credential'): @@ -404,6 +411,9 @@ def get_archive_credential_revisions(id): try: cred = Credential.get(id) except DoesNotExist: + logging.warning( + 'Item with id {0} does not exist.'.format(id) + ) return jsonify({}), 404 if (cred.data_type != 'credential' and cred.data_type != 'archive-credential'): @@ -852,6 +862,9 @@ def get_blind_credential(id): try: cred = BlindCredential.get(id) except DoesNotExist: + logging.warning( + 'Item with id {0} does not exist.'.format(id) + ) return jsonify({}), 404 if (cred.data_type != 'blind-credential' and cred.data_type != 'archive-blind-credential'): @@ -904,6 +917,9 @@ def get_archive_blind_credential_revisions(id): return jsonify({}), 404 if (cred.data_type != 'blind-credential' and cred.data_type != 'archive-blind-credential'): + logging.warning( + 'Item with id {0} does not exist.'.format(id) + ) return jsonify({}), 404 revisions = [] _range = range(1, cred.revision + 1) diff --git a/confidant/scripts/manage.py b/confidant/scripts/manage.py index 6f0cf52a..3f3e8f10 100644 --- a/confidant/scripts/manage.py +++ b/confidant/scripts/manage.py @@ -6,7 +6,10 @@ from confidant.scripts.utils import CreateDynamoTables from confidant.scripts.bootstrap import GenerateSecretsBootstrap from confidant.scripts.bootstrap import DecryptSecretsBootstrap -from confidant.scripts.migrate import MigrateSetAttribute +from confidant.scripts.migrate import ( + MigrateBlindCredentialSetAttribute, + MigrateServiceSetAttribute, +) manager = Manager(app.app) @@ -26,7 +29,10 @@ manager.add_command("create_dynamodb_tables", CreateDynamoTables) # Migration scripts -manager.add_command("migrate_set_attribute", MigrateSetAttribute) +manager.add_command("migrate_blind_cred_set_attribute", + MigrateBlindCredentialSetAttribute) +manager.add_command("migrate_service_set_attribute", + MigrateServiceSetAttribute) def main(): diff --git a/confidant/scripts/migrate.py b/confidant/scripts/migrate.py index 2413165c..fc18aedb 100644 --- a/confidant/scripts/migrate.py +++ b/confidant/scripts/migrate.py @@ -4,16 +4,134 @@ from confidant.app import app from confidant.models.blind_credential import BlindCredential +from confidant.models.service import Service + +import json +import six +from pynamodb.attributes import Attribute, UnicodeAttribute +from pynamodb.constants import STRING_SET +from pynamodb.models import Model app.logger.addHandler(logging.StreamHandler(sys.stdout)) app.logger.setLevel(logging.INFO) -class MigrateSetAttribute(Command): +def is_old_unicode_set(values): + if not values: + return False + return sum([x.startswith('"') for x in values]) > 0 + + +class SetMixin(object): + """ + Adds (de)serialization methods for sets + """ + def serialize(self, value): + """ + Serializes a set + + Because dynamodb doesn't store empty attributes, + empty sets return None + """ + if value is not None: + try: + iter(value) + except TypeError: + value = [value] + if len(value): + return [json.dumps(val) for val in sorted(value)] + return None + + def deserialize(self, value): + """ + Deserializes a set + """ + if value and len(value): + return set([json.loads(val) for val in value]) + + +class NewUnicodeSetAttribute(SetMixin, Attribute): + """ + A unicode set + """ + attr_type = STRING_SET + null = True + + def element_serialize(self, value): + """ + This serializes unicode / strings out as unicode strings. + It does not touch the value if it is already a unicode str + :param value: + :return: + """ + if isinstance(value, six.text_type): + return value + return six.u(str(value)) + + def element_deserialize(self, value): + return value + + def serialize(self, value): + if value is not None: + try: + iter(value) + except TypeError: + value = [value] + if len(value): + return [self.element_serialize(val) for val in sorted(value)] + return None + + def deserialize(self, value): + if value and len(value): + return set([self.element_deserialize(val) for val in value]) + + +class GeneralCredentialModel(Model): + class Meta(BlindCredential.Meta): + pass + + id = UnicodeAttribute(hash_key=True) + credential_keys = NewUnicodeSetAttribute(default=set([]), null=True) + + +class GeneralServiceModel(Model): + class Meta(Service.Meta): + pass + + id = UnicodeAttribute(hash_key=True) + credentials = NewUnicodeSetAttribute(default=set(), null=True) + blind_credentials = NewUnicodeSetAttribute(default=set(), null=True) + + +class MigrateBlindCredentialSetAttribute(Command): def run(self): + total = 0 + fail = 0 app.logger.info('Migrating UnicodeSetAttribute in BlindCredential') for cred in BlindCredential.data_type_date_index.query( 'blind-credential'): cred.save() + new_cred = GeneralCredentialModel.get(cred.id) + if is_old_unicode_set(new_cred.credential_keys): + fail += 1 + total += 1 + print("Fail: {}, Total: {}".format(fail, total)) + + +class MigrateServiceSetAttribute(Command): + + def run(self): + total = 0 + fail = 0 + app.logger.info('Migrating UnicodeSetAttribute in Service') + for service in Service.data_type_date_index.query( + 'service'): + service.save() + new_service = GeneralServiceModel.get(service.id) + if (is_old_unicode_set(new_service.credentials) or + is_old_unicode_set(new_service.blind_credentials)): + fail += 1 + total += 1 + print("Fail: {}, Total: {}".format(fail, total)) diff --git a/docs/source/basics/upgrade.html.markdown b/docs/source/basics/upgrade.html.markdown index acb9ed58..f372d935 100644 --- a/docs/source/basics/upgrade.html.markdown +++ b/docs/source/basics/upgrade.html.markdown @@ -12,9 +12,7 @@ document breaking changes and how to upgrade when they occur. ## Upgrading to 2.0.0 or 3.0.0 Due to breaking changes in PynamoDB, to upgrade to 2.0.0 or 3.0.0 may require -some data migration. It's only necessary to perform a data migration if you're -using blind credentials. If you're not using blind credentials, this change -isn't breaking and you can upgrade without migration. +some data migration. PynamoDB changed its data model over a series of releases, which requires the upgrade path for Confidant to follow the same model. To upgrade to 3.0.0, @@ -25,16 +23,20 @@ versions of Confidant. ### Performing the data migration -Confidant 2.0.0 ships with a maintenance script for the data migration: +Confidant 2.0.1 ships with two maintenance scripts for the data migration: ```bash cd /srv/confidant source venv/bin/activate # Encrypt the data -python manage.py migrate_set_attribute +python manage.py migrate_blind_cred_set_attribute +python manage.py migrate_service_set_attribute ``` +These scripts may fail intermittently. If any failures are occur, retry the +script until all objects are fully migrated. + 2.0.0 ships with the ability to enable a maintenance mode, which you may want to enable when upgrading to 2.0.0. Putting Confidant into maintenance mode will disallow any writes via the API, ensuring that blind credentials with the