From 520db11f4d012539cc5de2f63b9a1359b56ad991 Mon Sep 17 00:00:00 2001 From: Paragon Initiative Enterprises Date: Tue, 19 Jun 2018 19:15:57 -0400 Subject: [PATCH 1/6] Implement EncryptedRow. See #18 for motivation. --- src/CompoundIndex.php | 179 +++++++++++++ src/EncryptedField.php | 8 +- src/EncryptedRow.php | 488 ++++++++++++++++++++++++++++++++++++ tests/CompoundIndexTest.php | 43 ++++ tests/EncryptedRowTest.php | 249 ++++++++++++++++++ 5 files changed, 963 insertions(+), 4 deletions(-) create mode 100644 src/CompoundIndex.php create mode 100644 src/EncryptedRow.php create mode 100644 tests/CompoundIndexTest.php create mode 100644 tests/EncryptedRowTest.php diff --git a/src/CompoundIndex.php b/src/CompoundIndex.php new file mode 100644 index 0000000..b7d8233 --- /dev/null +++ b/src/CompoundIndex.php @@ -0,0 +1,179 @@ + $columns + */ + protected $columns; + + /** + * @var bool $fastHash + */ + protected $fastHash; + + /** + * @var array $hashConfig + */ + protected $hashConfig; + + /** + * @var string $name + */ + protected $name; + + /** + * @var int $outputLength + */ + protected $filterBits = 256; + + /** + * @var array> + */ + protected $columnTransforms = []; + + /** + * @var Compound + */ + private static $compounder; + + /** + * CompoundIndex constructor. + * + * @param string $name + * @param array $columns + * @param int $filterBits + * @param bool $fastHash + * @param array $hashConfig + */ + public function __construct( + $name, + array $columns = [], + $filterBits = 256, + $fastHash = false, + array $hashConfig = [] + ) { + $this->name = $name; + $this->columns = $columns; + $this->filterBits = $filterBits; + $this->fastHash = $fastHash; + $this->hashConfig = $hashConfig; + } + + /** + * @return Compound + */ + public static function getCompounder() + { + if (!self::$compounder) { + self::$compounder = new Compound(); + } + return self::$compounder; + } + + /** + * @param string $column + * @param TransformationInterface $tf + * @return self + */ + public function addTransform($column, TransformationInterface $tf) + { + $this->columnTransforms[$column][] = $tf; + return $this; + } + + /** + * @return array + */ + public function getColumns() + { + return $this->columns; + } + + /** + * @return string + */ + public function getName() + { + return $this->name; + } + + /** + * @return bool + */ + public function getFastHash() + { + return $this->fastHash; + } + + /** + * @return int + */ + public function getFilterBitLength() + { + return $this->filterBits; + } + + /** + * @return array + */ + public function getHashConfig() + { + return $this->hashConfig; + } + + /** + * @param string $column + * @return array + */ + public function getTransforms($column) + { + if (!\array_key_exists($column, $this->columns)) { + return []; + } + return $this->columnTransforms[$column]; + } + + /** + * Get a packed plaintext for use in creating a compound blind index + * This is a one-way transformation meant to be distinct from other inputs + * Not all elements of the row will be used. + * + * @param array $row + * + * @return string + * @throws \Exception + */ + public function getPacked(array $row) + { + /** @var array $pieces */ + $pieces = []; + /** @var string $col */ + foreach ($this->columns as $col) { + if (!\array_key_exists($col, $row)) { + continue; + } + /** @var string $piece */ + $piece = $row[$col]; + if (!empty($this->columnTransforms[$col])) { + foreach ($this->columnTransforms[$col] as $tf) { + if ($tf instanceof TransformationInterface) { + /** @var string $piece */ + $piece = $tf($piece); + } + } + } + $pieces[$col] = $piece; + } + $compounder = self::getCompounder(); + return (string) $compounder($pieces); + } +} diff --git a/src/EncryptedField.php b/src/EncryptedField.php index 8599925..72b514a 100644 --- a/src/EncryptedField.php +++ b/src/EncryptedField.php @@ -211,9 +211,9 @@ protected function getBlindIndexRaw( ); } - $Backend = $this->engine->getBackend(); + $backend = $this->engine->getBackend(); $subKey = new SymmetricKey( - $Backend, + $backend, \hash_hmac( 'sha256', Util::pack([$this->tableName, $this->fieldName, $name]), @@ -225,13 +225,13 @@ protected function getBlindIndexRaw( /** @var BlindIndex $index */ $index = $this->blindIndexes[$name]; if ($index->getFastHash()) { - return $Backend->blindIndexFast( + return $backend->blindIndexFast( $plaintext, $subKey, $index->getFilterBitLength() ); } - return $Backend->blindIndexSlow( + return $backend->blindIndexSlow( $plaintext, $subKey, $index->getFilterBitLength(), diff --git a/src/EncryptedRow.php b/src/EncryptedRow.php new file mode 100644 index 0000000..5691ea8 --- /dev/null +++ b/src/EncryptedRow.php @@ -0,0 +1,488 @@ + $fieldsToEncrypt + */ + protected $fieldsToEncrypt = []; + + /** + * @var array> $blindIndexes + */ + protected $blindIndexes = []; + + /** + * @var array $compoundIndexes + */ + protected $compoundIndexes = []; + + /** + * @var string $tableName + */ + protected $tableName; + + /** + * EncryptedFieldSet constructor. + * + * @param CipherSweet $engine + * @param string $tableName + */ + public function __construct(CipherSweet $engine, $tableName) + { + $this->engine = $engine; + $this->tableName = $tableName; + } + + /** + * @param string $fieldName + * @param string $type + * @return self + */ + public function addField($fieldName, $type = self::TYPE_TEXT) + { + $this->fieldsToEncrypt[$fieldName] = $type; + return $this; + } + + /** + * @param string $fieldName + * @return self + */ + public function addBooleanField($fieldName) + { + return $this->addField($fieldName, self::TYPE_BOOLEAN); + } + + /** + * @param string $fieldName + * @return self + */ + public function addFloatField($fieldName) + { + return $this->addField($fieldName, self::TYPE_FLOAT); + } + + /** + * @param string $fieldName + * @return self + */ + public function addIntegerField($fieldName) + { + return $this->addField($fieldName, self::TYPE_INT); + } + + /** + * @param string $fieldName + * @return self + */ + public function addTextField($fieldName) + { + return $this->addField($fieldName, self::TYPE_TEXT); + } + + /** + * @param string $column + * @param BlindIndex $index + * @return self + */ + public function addBlindIndex($column, BlindIndex $index) + { + $this->blindIndexes[$column][$index->getName()] = $index; + return $this; + } + + /** + * @param CompoundIndex $index + * @return self + */ + public function addCompoundIndex(CompoundIndex $index) + { + $this->compoundIndexes[$index->getName()] = $index; + return $this; + } + + /** + * @param string $name + * @param array $columns + * @param int $filterBits + * @param bool $fastHash + * @param array $hashConfig + * @return CompoundIndex + */ + public function createCompoundIndex( + $name, + array $columns = [], + $filterBits = 256, + $fastHash = false, + array $hashConfig = [] + ) { + $index = new CompoundIndex( + $name, + $columns, + $filterBits, + $fastHash, + $hashConfig + ); + $this->addCompoundIndex($index); + return $index; + } + + /** + * @param array $row + * @param string $column + * @param BlindIndex $index + * @return array + * + * @throws ArrayKeyException + * @throws Exception\CryptoOperationException + * @throws \SodiumException + */ + public function calcBlindIndex(array $row, $column, BlindIndex $index) + { + $name = $index->getName(); + $key = $this->engine->getBlindIndexRootKey( + $this->tableName, + $column + ); + + $k = $this->engine->getIndexTypeColumn( + $this->tableName, + $column, + $name + ); + return [ + 'type' => $k, + 'value' => + Hex::encode( + $this->calcBlindIndexRaw( + $row, + $column, + $index, + $key + ) + ) + ]; + } + + /** + * @param array $row + * @param CompoundIndex $index + * + * @return array + * @throws Exception\CryptoOperationException + */ + public function calcCompoundIndex(array $row, CompoundIndex $index) + { + $name = $index->getName(); + $key = $this->engine->getBlindIndexRootKey( + $this->tableName, + self::COMPOUND_SPECIAL + ); + + $k = $this->engine->getIndexTypeColumn( + $this->tableName, + self::COMPOUND_SPECIAL, + $name + ); + return [ + 'type' => $k, + 'value' => + Hex::encode( + $this->calcCompoundIndexRaw( + $row, + $index, + $key + ) + ) + ]; + } + + /** + * @param array $row + * @param string $column + * @param BlindIndex $index + * @param SymmetricKey|null $key + * + * @return string + * @throws Exception\CryptoOperationException + * @throws ArrayKeyException + * @throws \SodiumException + */ + public function calcBlindIndexRaw( + array $row, + $column, + BlindIndex $index, + SymmetricKey $key = null + ) { + if (!$key) { + $key = $this->engine->getBlindIndexRootKey( + $this->tableName, + $column + ); + } + + $backend = $this->engine->getBackend(); + /** @var string $name */ + $name = $index->getName(); + + /** @var SymmetricKey $subKey */ + $subKey = new SymmetricKey( + $backend, + \hash_hmac( + 'sha256', + Util::pack([$this->tableName, $column, $name]), + $key->getRawKey(), + true + ) + ); + if (!\array_key_exists($column, $this->fieldsToEncrypt)) { + throw new ArrayKeyException( + 'The field ' . $column . ' is not defined in this encrypted row.' + ); + } + /** @var string $fieldType */ + $fieldType = $this->fieldsToEncrypt[$column]; + + /** @var string|bool|int|float|null $unconverted */ + $unconverted = $row[$column]; + + /** @var string $plaintext */ + $plaintext = $this->convertToString($unconverted, $fieldType); + + /** @var BlindIndex $index */ + $index = $this->blindIndexes[$column][$name]; + if ($index->getFastHash()) { + return $backend->blindIndexFast( + $plaintext, + $subKey, + $index->getFilterBitLength() + ); + } + return $backend->blindIndexSlow( + $plaintext, + $subKey, + $index->getFilterBitLength(), + $index->getHashConfig() + ); + } + + /** + * @param array $row + * @param CompoundIndex $index + * @param SymmetricKey|null $key + * @return string + * + * @throws \Exception + * @throws Exception\CryptoOperationException + */ + public function calcCompoundIndexRaw( + array $row, + CompoundIndex $index, + SymmetricKey $key = null + ) { + if (!$key) { + $key = $this->engine->getBlindIndexRootKey( + $this->tableName, + self::COMPOUND_SPECIAL + ); + } + + $backend = $this->engine->getBackend(); + /** @var string $name */ + $name = $index->getName(); + + /** @var SymmetricKey $subKey */ + $subKey = new SymmetricKey( + $backend, + \hash_hmac( + 'sha256', + Util::pack([$this->tableName, self::COMPOUND_SPECIAL, $name]), + $key->getRawKey(), + true + ) + ); + + /** @var string $plaintext */ + $plaintext = $index->getPacked($row); + + /** @var CompoundIndex $index */ + $index = $this->compoundIndexes[$name]; + + if ($index->getFastHash()) { + return $backend->blindIndexFast( + $plaintext, + $subKey, + $index->getFilterBitLength() + ); + } + return $backend->blindIndexSlow( + $plaintext, + $subKey, + $index->getFilterBitLength(), + $index->getHashConfig() + ); + } + + /** + * @param array $row + * @return array> + * + * @throws ArrayKeyException + * @throws Exception\CryptoOperationException + * @throws \SodiumException + */ + public function getAllBlindIndexes(array $row) + { + $return = []; + foreach ($this->blindIndexes as $column => $blindIndexes) { + foreach ($blindIndexes as $blindIndex) { + $return[] = $this->calcBlindIndex($row, $column, $blindIndex); + } + } + foreach ($this->compoundIndexes as $name => $compoundIndex) { + $return[] = $this->calcCompoundIndex($row, $compoundIndex); + } + return $return; + } + + /** + * @param array $row + * @return array + * @throws Exception\CryptoOperationException + * @throws \SodiumException + */ + public function decryptRow(array $row) + { + $return = $row; + foreach ($this->fieldsToEncrypt as $field => $type) { + $key = $this->engine->getFieldSymmetricKey( + $this->tableName, + $field + ); + $plaintext = $this + ->engine + ->getBackend() + ->decrypt($row[$field], $key); + $return[$field] = $this->convertFromString($plaintext, $type); + } + return $return; + } + + /** + * @param array $row + * + * @return array + * @throws ArrayKeyException + * @throws Exception\CryptoOperationException + * @throws \SodiumException + */ + public function encryptRow(array $row) + { + $return = $row; + foreach ($this->fieldsToEncrypt as $field => $type) { + if (!\array_key_exists($field, $row)) { + throw new ArrayKeyException( + 'Expected value for column ' . $field. ' on array, nothing given.' + ); + } + /** @var string $plaintext */ + $plaintext = $this->convertToString($row[$field], $type); + $key = $this->engine->getFieldSymmetricKey( + $this->tableName, + $field + ); + $return[$field] = $this + ->engine + ->getBackend() + ->encrypt($plaintext, $key); + } + /** @var array $return */ + return $return; + } + + /** + * @param string $data + * @param string $type + * @return int|string|float|bool|null + * @throws \SodiumException + */ + protected function convertFromString($data, $type) + { + switch ($type) { + case self::TYPE_BOOLEAN: + return Util::chrToBool($data); + case self::TYPE_FLOAT: + return Util::stringToFloat($data); + case self::TYPE_INT: + return Util::stringToInt($data); + default: + return (string) $data; + } + } + /** + * @param int|string|float|bool|null $data + * @param string $type + * @return string + * @throws \SodiumException + */ + protected function convertToString($data, $type) + { + switch ($type) { + case self::TYPE_BOOLEAN: + if (!\is_null($data) && !\is_bool($data)) { + $data = !empty($data); + } + return Util::boolToChr($data); + case self::TYPE_FLOAT: + if (!\is_float($data)) { + throw new \TypeError('Expected a float'); + } + return Util::floatToString($data); + case self::TYPE_INT: + if (!\is_int($data)) { + throw new \TypeError('Expected an integer'); + } + return Util::intToString($data); + default: + return (string) $data; + } + } + + /** + * @param array $row + * @return array{0: array, 1: array>} + * + * @throws ArrayKeyException + * @throws Exception\CryptoOperationException + * @throws \SodiumException + */ + public function prepareRowForStorage(array $row) + { + return [ + $this->encryptRow($row), + $this->getAllBlindIndexes($row) + ]; + } +} diff --git a/tests/CompoundIndexTest.php b/tests/CompoundIndexTest.php new file mode 100644 index 0000000..cecb52e --- /dev/null +++ b/tests/CompoundIndexTest.php @@ -0,0 +1,43 @@ +addTransform('ssn', new LastFourDigits()); + $packed = $cIdx->getPacked([ + 'ssn' => '123-45-6789', + 'hivstatus' => true + ]); + $this->assertSame( + '{"ssn":"0400000000000000Njc4OQ==","hivstatus":true}', + $packed + ); + $packed = $cIdx->getPacked([ + 'ssn' => '123-45-6789', + 'hivstatus' => false + ]); + $this->assertSame( + '{"ssn":"0400000000000000Njc4OQ==","hivstatus":false}', + $packed + ); + } +} diff --git a/tests/EncryptedRowTest.php b/tests/EncryptedRowTest.php new file mode 100644 index 0000000..fc923a3 --- /dev/null +++ b/tests/EncryptedRowTest.php @@ -0,0 +1,249 @@ +fipsEngine = new CipherSweet( + new StringProvider( + $fips, + Hex::decode( + '4e1c44f87b4cdf21808762970b356891db180a9dd9850e7baf2a79ff3ab8a2fc' + ) + ) + ); + $this->naclEngine = new CipherSweet( + new StringProvider( + $nacl, + Hex::decode( + '4e1c44f87b4cdf21808762970b356891db180a9dd9850e7baf2a79ff3ab8a2fc' + ) + ) + ); + + $this->fipsRandom = new CipherSweet( + new StringProvider( + $fips, + \random_bytes(32) + ) + ); + $this->naclRandom = new CipherSweet( + new StringProvider( + $nacl, + \random_bytes(32) + ) + ); + } + + /** + * @throws ArrayKeyException + * @throws CryptoOperationException + * @throws \SodiumException + */ + public function testSimpleEncrypt() + { + $eF = (new EncryptedRow($this->fipsRandom, 'contacts')); + $eM = (new EncryptedRow($this->naclRandom, 'contacts')); + $eF->addTextField('message'); + $eM->addTextField('message'); + + $message = 'This is a test message: ' . \random_bytes(16); + $row = [ + 'message' => $message + ]; + + $fCipher = $eF->encryptRow($row); + $mCipher = $eM->encryptRow($row); + + $this->assertSame( + FIPSCrypto::MAGIC_HEADER, + Binary::safeSubstr($fCipher['message'], 0, 5) + ); + $this->assertSame( + ModernCrypto::MAGIC_HEADER, + Binary::safeSubstr($mCipher['message'], 0, 5) + ); + + $this->assertSame($row, $eF->decryptRow($fCipher)); + $this->assertSame($row, $eM->decryptRow($mCipher)); + } + + /** + * @throws CryptoOperationException + * @throws ArrayKeyException + * @throws \SodiumException + */ + public function testGetAllIndexes() + { + $row = [ + 'extraneous' => 'this is unecnrypted', + 'ssn' => '123-45-6789', + 'hivstatus' => true + ]; + $eF = $this->getExampleRow($this->fipsEngine, true); + $eM = $this->getExampleRow($this->naclEngine, true); + + $indexes = $eF->getAllBlindIndexes($row); + $this->assertEquals('abd9497d226601a2', $indexes[0]['value']); + $this->assertEquals('9c3d53214ab71d7f', $indexes[1]['value']); + + $indexes = $eM->getAllBlindIndexes($row); + $this->assertEquals('805815e4a43f6fd9', $indexes[0]['value']); + $this->assertEquals('1b8c1e1f8e122bd3', $indexes[1]['value']); + } + + /** + * @throws ArrayKeyException + * @throws CryptoOperationException + * @throws \SodiumException + */ + public function testEncrypt() + { + $row = [ + 'extraneous' => 'this is unecnrypted', + 'ssn' => '123-45-6789', + 'hivstatus' => true + ]; + $eF = $this->getExampleRow($this->fipsRandom, true); + $eM = $this->getExampleRow($this->naclRandom, true); + + /** @var EncryptedRow $engine */ + foreach ([$eM, $eF] as $engine) { + $store = $engine->encryptRow($row); + $this->assertSame($store['extraneous'], $row['extraneous']); + $this->assertNotSame($store['ssn'], $row['ssn']); + $this->assertNotSame($store['hivstatus'], $row['hivstatus']); + } + } + + /** + * @throws CryptoOperationException + * @throws \ParagonIE\CipherSweet\Exception\ArrayKeyException + * @throws \SodiumException + */ + public function testPrepareForStorage() + { + $eF = $this->getExampleRow($this->fipsRandom, true); + $eM = $this->getExampleRow($this->naclRandom, true); + + $rows = [ + [ + 'ssn' => '111-11-1111', + 'hivstatus' => false + ], + [ + 'ssn' => '123-45-6789', + 'hivstatus' => false + ], + [ + 'ssn' => '999-99-6789', + 'hivstatus' => false + ], + [ + 'ssn' => '123-45-1111', + 'hivstatus' => true + ], + [ + 'ssn' => '999-99-1111', + 'hivstatus' => true + ], + [ + 'ssn' => '123-45-6789', + 'hivstatus' => true + ] + ]; + foreach ([$eM, $eF] as $engine) { + foreach ($rows as $row) { + list($store, $indexes) = $engine->prepareRowForStorage($row); + $this->assertTrue(\is_array($store)); + $this->assertTrue(\is_string($store['ssn'])); + $this->assertTrue(\is_string($store['hivstatus'])); + $this->assertNotSame($row['ssn'], $store['ssn']); + $this->assertNotSame($row['hivstatus'], $store['hivstatus']); + $this->assertTrue(\is_array($indexes)); + } + } + } + + /** + * @param CipherSweet $backend + * @param bool $longer + * @param bool $fast + * + * @return EncryptedRow + * @throws BlindIndexNameCollisionException + */ + public function getExampleRow( + CipherSweet $backend, + $longer = false, + $fast = false + ) { + $row = (new EncryptedRow($backend, 'contacts')) + ->addTextField('ssn') + ->addBooleanField('hivstatus'); + + $row->addBlindIndex( + 'ssn', + new BlindIndex( + // Name (used in key splitting): + 'contact_ssn_last_four', + // List of Transforms: + [new LastFourDigits()], + // Output length (bytes) + $longer ? 64 : 16, + $fast + ) + ); + $row->createCompoundIndex( + 'contact_ssnlast4_hivstatus', + ['ssn', 'hivstatus'], + $longer ? 64 : 16, + $fast + ); + return $row; + } +} From eb2eb4308ae7f576c68bc03262452fcb5b1a75ae Mon Sep 17 00:00:00 2001 From: Paragon Initiative Enterprises Date: Tue, 19 Jun 2018 20:02:59 -0400 Subject: [PATCH 2/6] Avoid Argon2i puking --- tests/EncryptedRowTest.php | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/tests/EncryptedRowTest.php b/tests/EncryptedRowTest.php index fc923a3..6c13199 100644 --- a/tests/EncryptedRowTest.php +++ b/tests/EncryptedRowTest.php @@ -125,15 +125,10 @@ public function testGetAllIndexes() 'hivstatus' => true ]; $eF = $this->getExampleRow($this->fipsEngine, true); - $eM = $this->getExampleRow($this->naclEngine, true); $indexes = $eF->getAllBlindIndexes($row); $this->assertEquals('abd9497d226601a2', $indexes[0]['value']); $this->assertEquals('9c3d53214ab71d7f', $indexes[1]['value']); - - $indexes = $eM->getAllBlindIndexes($row); - $this->assertEquals('805815e4a43f6fd9', $indexes[0]['value']); - $this->assertEquals('1b8c1e1f8e122bd3', $indexes[1]['value']); } /** @@ -168,7 +163,6 @@ public function testEncrypt() public function testPrepareForStorage() { $eF = $this->getExampleRow($this->fipsRandom, true); - $eM = $this->getExampleRow($this->naclRandom, true); $rows = [ [ @@ -196,16 +190,14 @@ public function testPrepareForStorage() 'hivstatus' => true ] ]; - foreach ([$eM, $eF] as $engine) { - foreach ($rows as $row) { - list($store, $indexes) = $engine->prepareRowForStorage($row); - $this->assertTrue(\is_array($store)); - $this->assertTrue(\is_string($store['ssn'])); - $this->assertTrue(\is_string($store['hivstatus'])); - $this->assertNotSame($row['ssn'], $store['ssn']); - $this->assertNotSame($row['hivstatus'], $store['hivstatus']); - $this->assertTrue(\is_array($indexes)); - } + foreach ($rows as $row) { + list($store, $indexes) = $eF->prepareRowForStorage($row); + $this->assertTrue(\is_array($store)); + $this->assertTrue(\is_string($store['ssn'])); + $this->assertTrue(\is_string($store['hivstatus'])); + $this->assertNotSame($row['ssn'], $store['ssn']); + $this->assertNotSame($row['hivstatus'], $store['hivstatus']); + $this->assertTrue(\is_array($indexes)); } } From f8a9cfa8e71fc5d34c407686c1e6b1b502d29dae Mon Sep 17 00:00:00 2001 From: Paragon Initiative Enterprises Date: Thu, 21 Jun 2018 10:36:32 -0400 Subject: [PATCH 3/6] Boyscouting. --- src/EncryptedRow.php | 245 ++++++++++++++++++++++++++----------------- 1 file changed, 148 insertions(+), 97 deletions(-) diff --git a/src/EncryptedRow.php b/src/EncryptedRow.php index 5691ea8..15e8734 100644 --- a/src/EncryptedRow.php +++ b/src/EncryptedRow.php @@ -56,6 +56,8 @@ public function __construct(CipherSweet $engine, $tableName) } /** + * Define a field that will be encrypted. + * * @param string $fieldName * @param string $type * @return self @@ -67,6 +69,8 @@ public function addField($fieldName, $type = self::TYPE_TEXT) } /** + * Define a boolean field that will be encrypted. Nullable. + * * @param string $fieldName * @return self */ @@ -76,6 +80,8 @@ public function addBooleanField($fieldName) } /** + * Define a floating point number (decimal) field that will be encrypted. + * * @param string $fieldName * @return self */ @@ -85,6 +91,8 @@ public function addFloatField($fieldName) } /** + * Define an integer field that will be encrypted. + * * @param string $fieldName * @return self */ @@ -94,6 +102,8 @@ public function addIntegerField($fieldName) } /** + * Define a text field that will be encrypted. + * * @param string $fieldName * @return self */ @@ -103,6 +113,8 @@ public function addTextField($fieldName) } /** + * Add a normal blind index to this EncryptedRow object. + * * @param string $column * @param BlindIndex $index * @return self @@ -114,6 +126,8 @@ public function addBlindIndex($column, BlindIndex $index) } /** + * Add a compound blind index to this EncryptedRow object. + * * @param CompoundIndex $index * @return self */ @@ -124,6 +138,8 @@ public function addCompoundIndex(CompoundIndex $index) } /** + * Create a compound blind index then add it to this EncryptedRow object. + * * @param string $name * @param array $columns * @param int $filterBits @@ -149,6 +165,121 @@ public function createCompoundIndex( return $index; } + /** + * Get all of the blind indexes and compound indexes defined for this + * object, calculated from the input array. + * + * @param array $row + * @return array> + * + * @throws ArrayKeyException + * @throws Exception\CryptoOperationException + * @throws \SodiumException + */ + public function getAllBlindIndexes(array $row) + { + $return = []; + foreach ($this->blindIndexes as $column => $blindIndexes) { + foreach ($blindIndexes as $blindIndex) { + $return[] = $this->calcBlindIndex($row, $column, $blindIndex); + } + } + foreach ($this->compoundIndexes as $name => $compoundIndex) { + $return[] = $this->calcCompoundIndex($row, $compoundIndex); + } + return $return; + } + + /** + * Decrypt all of the appropriate fields in the given array. + * + * If any columns are defined in this object to be decrypted, the value + * will be decrypted in-place in the returned array. + * + * @param array $row + * @return array + * @throws Exception\CryptoOperationException + * @throws \SodiumException + */ + public function decryptRow(array $row) + { + $return = $row; + foreach ($this->fieldsToEncrypt as $field => $type) { + $key = $this->engine->getFieldSymmetricKey( + $this->tableName, + $field + ); + $plaintext = $this + ->engine + ->getBackend() + ->decrypt($row[$field], $key); + $return[$field] = $this->convertFromString($plaintext, $type); + } + return $return; + } + + /** + * Encrypt any of the appropriate fields in the given array. + * + * If any columns are defined in this object to be encrypted, the value + * will be encrypted in-place in the returned array. + * + * @param array $row + * + * @return array + * @throws ArrayKeyException + * @throws Exception\CryptoOperationException + * @throws \SodiumException + */ + public function encryptRow(array $row) + { + $return = $row; + foreach ($this->fieldsToEncrypt as $field => $type) { + if (!\array_key_exists($field, $row)) { + throw new ArrayKeyException( + 'Expected value for column ' . $field. ' on array, nothing given.' + ); + } + /** @var string $plaintext */ + $plaintext = $this->convertToString($row[$field], $type); + $key = $this->engine->getFieldSymmetricKey( + $this->tableName, + $field + ); + $return[$field] = $this + ->engine + ->getBackend() + ->encrypt($plaintext, $key); + } + /** @var array $return */ + return $return; + } + + /** + * Process an entire row, which means: + * + * 1. If any columns are defined in this object to be encrypted, the value + * will be encrypted in-place in the first array. + * 2. Blind indexes and compound indexes are calculated and stored in the + * second array. + * + * Calling encryptRow() and getAllBlindIndexes() is equivalent. + * + * @param array $row + * @return array{0: array, 1: array>} + * + * @throws ArrayKeyException + * @throws Exception\CryptoOperationException + * @throws \SodiumException + */ + public function prepareRowForStorage(array $row) + { + return [ + $this->encryptRow($row), + $this->getAllBlindIndexes($row) + ]; + } + /** * @param array $row * @param string $column @@ -159,7 +290,7 @@ public function createCompoundIndex( * @throws Exception\CryptoOperationException * @throws \SodiumException */ - public function calcBlindIndex(array $row, $column, BlindIndex $index) + protected function calcBlindIndex(array $row, $column, BlindIndex $index) { $name = $index->getName(); $key = $this->engine->getBlindIndexRootKey( @@ -193,7 +324,7 @@ public function calcBlindIndex(array $row, $column, BlindIndex $index) * @return array * @throws Exception\CryptoOperationException */ - public function calcCompoundIndex(array $row, CompoundIndex $index) + protected function calcCompoundIndex(array $row, CompoundIndex $index) { $name = $index->getName(); $key = $this->engine->getBlindIndexRootKey( @@ -230,7 +361,7 @@ public function calcCompoundIndex(array $row, CompoundIndex $index) * @throws ArrayKeyException * @throws \SodiumException */ - public function calcBlindIndexRaw( + protected function calcBlindIndexRaw( array $row, $column, BlindIndex $index, @@ -297,7 +428,7 @@ public function calcBlindIndexRaw( * @throws \Exception * @throws Exception\CryptoOperationException */ - public function calcCompoundIndexRaw( + protected function calcCompoundIndexRaw( array $row, CompoundIndex $index, SymmetricKey $key = null @@ -345,83 +476,6 @@ public function calcCompoundIndexRaw( ); } - /** - * @param array $row - * @return array> - * - * @throws ArrayKeyException - * @throws Exception\CryptoOperationException - * @throws \SodiumException - */ - public function getAllBlindIndexes(array $row) - { - $return = []; - foreach ($this->blindIndexes as $column => $blindIndexes) { - foreach ($blindIndexes as $blindIndex) { - $return[] = $this->calcBlindIndex($row, $column, $blindIndex); - } - } - foreach ($this->compoundIndexes as $name => $compoundIndex) { - $return[] = $this->calcCompoundIndex($row, $compoundIndex); - } - return $return; - } - - /** - * @param array $row - * @return array - * @throws Exception\CryptoOperationException - * @throws \SodiumException - */ - public function decryptRow(array $row) - { - $return = $row; - foreach ($this->fieldsToEncrypt as $field => $type) { - $key = $this->engine->getFieldSymmetricKey( - $this->tableName, - $field - ); - $plaintext = $this - ->engine - ->getBackend() - ->decrypt($row[$field], $key); - $return[$field] = $this->convertFromString($plaintext, $type); - } - return $return; - } - - /** - * @param array $row - * - * @return array - * @throws ArrayKeyException - * @throws Exception\CryptoOperationException - * @throws \SodiumException - */ - public function encryptRow(array $row) - { - $return = $row; - foreach ($this->fieldsToEncrypt as $field => $type) { - if (!\array_key_exists($field, $row)) { - throw new ArrayKeyException( - 'Expected value for column ' . $field. ' on array, nothing given.' - ); - } - /** @var string $plaintext */ - $plaintext = $this->convertToString($row[$field], $type); - $key = $this->engine->getFieldSymmetricKey( - $this->tableName, - $field - ); - $return[$field] = $this - ->engine - ->getBackend() - ->encrypt($plaintext, $key); - } - /** @var array $return */ - return $return; - } - /** * @param string $data * @param string $type @@ -441,7 +495,16 @@ protected function convertFromString($data, $type) return (string) $data; } } + /** + * Convert multiple data types to a string prior to encryption. + * + * The main goals here are: + * + * 1. Convert several data types to a string. + * 2. Leak no information about the original value in the + * output string length. + * * @param int|string|float|bool|null $data * @param string $type * @return string @@ -450,39 +513,27 @@ protected function convertFromString($data, $type) protected function convertToString($data, $type) { switch ($type) { + // Will return a 1-byte string: case self::TYPE_BOOLEAN: if (!\is_null($data) && !\is_bool($data)) { $data = !empty($data); } return Util::boolToChr($data); + // Will return a fixed-length string: case self::TYPE_FLOAT: if (!\is_float($data)) { throw new \TypeError('Expected a float'); } return Util::floatToString($data); + // Will return a fixed-length string: case self::TYPE_INT: if (!\is_int($data)) { throw new \TypeError('Expected an integer'); } return Util::intToString($data); + // Will return the original string, untouched: default: return (string) $data; } } - - /** - * @param array $row - * @return array{0: array, 1: array>} - * - * @throws ArrayKeyException - * @throws Exception\CryptoOperationException - * @throws \SodiumException - */ - public function prepareRowForStorage(array $row) - { - return [ - $this->encryptRow($row), - $this->getAllBlindIndexes($row) - ]; - } } From 8cf8b1becf3542650f7ab7c40b25a45c2500d518 Mon Sep 17 00:00:00 2001 From: Paragon Initiative Enterprises Date: Thu, 21 Jun 2018 10:38:33 -0400 Subject: [PATCH 4/6] Begin updating documentation. --- docs/solutions/01-boolean.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/solutions/01-boolean.md b/docs/solutions/01-boolean.md index 61841bf..c5ff51e 100644 --- a/docs/solutions/01-boolean.md +++ b/docs/solutions/01-boolean.md @@ -18,12 +18,21 @@ minimize data leaks. ## CipherSweet Features for Protecting Boolean Fields -### Util::boolToChr() and Util::chrToBool() +### EncryptedRow + + + +### EncryptedField + +Safely storing boolean fields with the `EncryptedField` API, rather than +the `EncryptedRow` API, is possible but requires a bit more glue code. + +#### Util::boolToChr() and Util::chrToBool() CipherSweet provides a congruent method for compacting a nullable boolean into one character -### The Compound Transformation +#### The Compound Transformation The first rule for protect boolean fields is to **never create a blind index on a boolean field in isolation.** @@ -31,7 +40,7 @@ on a boolean field in isolation.** Instead, consider using the `Compound` transformation to combine multiple values together. -### Example Snippet +#### Example Snippet This code assumes an abstract `$dbh` object that supports an API that looks like this: From f9337d4bda5d21ebc681c29188750b19bd5c4dfe Mon Sep 17 00:00:00 2001 From: Paragon Initiative Enterprises Date: Thu, 21 Jun 2018 14:23:58 -0400 Subject: [PATCH 5/6] Change API to include index name rather than an integer --- src/EncryptedRow.php | 13 +++++++++---- tests/EncryptedRowTest.php | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/EncryptedRow.php b/src/EncryptedRow.php index 15e8734..083d7d2 100644 --- a/src/EncryptedRow.php +++ b/src/EncryptedRow.php @@ -170,7 +170,7 @@ public function createCompoundIndex( * object, calculated from the input array. * * @param array $row - * @return array> + * @return array> * * @throws ArrayKeyException * @throws Exception\CryptoOperationException @@ -180,12 +180,17 @@ public function getAllBlindIndexes(array $row) { $return = []; foreach ($this->blindIndexes as $column => $blindIndexes) { + /** @var BlindIndex $blindIndex */ foreach ($blindIndexes as $blindIndex) { - $return[] = $this->calcBlindIndex($row, $column, $blindIndex); + $return[$blindIndex->getName()] = $this->calcBlindIndex($row, $column, $blindIndex); } } + /** + * @var string $name + * @var CompoundIndex $compoundIndex + */ foreach ($this->compoundIndexes as $name => $compoundIndex) { - $return[] = $this->calcCompoundIndex($row, $compoundIndex); + $return[$name] = $this->calcCompoundIndex($row, $compoundIndex); } return $return; } @@ -266,7 +271,7 @@ public function encryptRow(array $row) * Calling encryptRow() and getAllBlindIndexes() is equivalent. * * @param array $row - * @return array{0: array, 1: array>} + * @return array{0: array, 1: array>} * * @throws ArrayKeyException * @throws Exception\CryptoOperationException diff --git a/tests/EncryptedRowTest.php b/tests/EncryptedRowTest.php index 6c13199..f23ecd4 100644 --- a/tests/EncryptedRowTest.php +++ b/tests/EncryptedRowTest.php @@ -127,8 +127,8 @@ public function testGetAllIndexes() $eF = $this->getExampleRow($this->fipsEngine, true); $indexes = $eF->getAllBlindIndexes($row); - $this->assertEquals('abd9497d226601a2', $indexes[0]['value']); - $this->assertEquals('9c3d53214ab71d7f', $indexes[1]['value']); + $this->assertEquals('abd9497d226601a2', $indexes['contact_ssn_last_four']['value']); + $this->assertEquals('9c3d53214ab71d7f', $indexes['contact_ssnlast4_hivstatus']['value']); } /** From 9a30ecf3b94b8eb724f224c7218299ab6eda7005 Mon Sep 17 00:00:00 2001 From: Paragon Initiative Enterprises Date: Thu, 21 Jun 2018 14:30:36 -0400 Subject: [PATCH 6/6] Document new EncryptedRow API --- docs/README.md | 89 ++++++++++++++++++++++++++++++++++++ docs/solutions/01-boolean.md | 52 +++++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/docs/README.md b/docs/README.md index b0d6f70..e884a31 100644 --- a/docs/README.md +++ b/docs/README.md @@ -102,6 +102,8 @@ $engine = new CipherSweet($provider); Once you have an engine in play, you can start defining encrypted fields and defining one or more **blind index** to be used for fast search operations. +### EncryptedField + This will primarily involve the `EncryptedField` class (as well as one or more instances of `BlindIndex`), mostly: @@ -264,6 +266,93 @@ array(2) { */ ``` +### EncryptedRow + +An alternative approach for datasets with multiple encrypted rows and/or +[encrypted boolean fields](https://github.com/paragonie/ciphersweet/blob/master/docs/solutions/01-boolean.md) +is the `EncryptedRow` API, which looks like this: + +```php +addTextField('ssn') + ->addBooleanField('hivstatus'); + +// Add a normal Blind Index on one field: +$row->addBlindIndex( + 'ssn', + new BlindIndex( + 'contact_ssn_last_four', + [new LastFourDigits()], + 32 // 32 bits = 4 bytes + ) +); + +// Create/add a compound blind index on multiple fields: +$row->addCompoundIndex( + ( + new CompoundIndex( + 'contact_ssnlast4_hivstatus', + ['ssn', 'hivstatus'], + 32, // 32 bits = 4 bytes + true // fast hash + ) + )->addTransform('ssn', new LastFourDigits()) +); + +// Notice: You're passing an entire array at once, not a string +$prepared = $row->prepareRowForStorage([ + 'extraneous' => true, + 'ssn' => '123-45-6789', + 'hivstatus' => false +]); + +var_dump($prepared); +/* +array(2) { + [0]=> + array(3) { + ["extraneous"]=> + bool(true) + ["ssn"]=> + string(73) "nacl:wVMElYqnHrGB4hU118MTuANZXWHZjbsd0uK2N0Exz72mrV8sLrI_oU94vgsWlWJc84-u" + ["hivstatus"]=> + string(61) "nacl:ctWDJBn-NgeWc2mqEWfakvxkG7qCmIKfPpnA7jXHdbZ2CPgnZF0Yzwg=" + } + [1]=> + array(2) { + ["contact_ssn_last_four"]=> + array(2) { + ["type"]=> + string(13) "3dywyifwujcu2" + ["value"]=> + string(8) "805815e4" + } + ["contact_ssnlast4_hivstatus"]=> + array(2) { + ["type"]=> + string(13) "nqtcc56kcf4qg" + ["value"]=> + string(8) "cbfd03c0" + } + } +} +*/ +``` + +With the `EncryptedRow` API, you can encrypt a subset of all of the fields +in a row, and create compound blind indexes based on multiple pieces of +data in the dataset rather than a single field, without writing a ton of +glue code. + ## Using CipherSweet with a Database CipherSweet is database-agnostic, so you'll need to write some code that diff --git a/docs/solutions/01-boolean.md b/docs/solutions/01-boolean.md index c5ff51e..98fdb7f 100644 --- a/docs/solutions/01-boolean.md +++ b/docs/solutions/01-boolean.md @@ -20,7 +20,59 @@ minimize data leaks. ### EncryptedRow +The simplest solution is to use `EncryptedRow` instead of `EncryptedField`. +Instead of operating on naked string data, `EncryptedRow` operates on a +one-dimensional associative array. Fields will be encrypted in-place and +compound blind indexes (i.e. a blind index constructed of multiple fields +at once) are much easier to use. + +For example: + +```php +addTextField('ssn') + ->addBooleanField('hivstatus'); + +// Add a normal Blind Index on one field: +$row->addBlindIndex( + 'ssn', + new BlindIndex( + 'contact_ssn_last_four', + [new LastFourDigits()], + 32 // 32 bits = 4 bytes + ) +); + +// Create/add a compound blind index on multiple fields: +$row->addCompoundIndex( + ( + new CompoundIndex( + 'contact_ssnlast4_hivstatus', + ['ssn', 'hivstatus'], + 32, // 32 bits = 4 bytes + true // fast hash + ) + )->addTransform('ssn', new LastFourDigits()) +); +``` + +In the above example, since the `contact_ssnlast4_hivstatus` blind index +depends on the last 4 digits of the contact's social security number AND +the boolean hivstatus field, it has a keyspace larger than 1 bit, and +thus leaks less information via hash collisions. ### EncryptedField