Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add structural test vectors for transaction formats #85

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Add structural test vectors for transaction formats #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
83255e6
Fix bugs in value balance generation and serialization
str4d Mar 11, 2022
7acac09
Add basic structural test vectors for v5 transactions
str4d Mar 11, 2022
65f611a
Add basic structural test vectors for legacy transactions
str4d Mar 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ zip_0143 = "zcash_test_vectors.zip_0143:main"
zip_0243 = "zcash_test_vectors.zip_0243:main"
zip_0244 = "zcash_test_vectors.zip_0244:main"

# Transaction format test vectors
transaction_legacy = "zcash_test_vectors.transaction:legacy_test_vectors"
transaction_v5 = "zcash_test_vectors.transaction:v5_test_vectors"

# Transparent test vectors
bip_0032 = "zcash_test_vectors.transparent.bip_0032:main"
zip_0316 = "zcash_test_vectors.transparent.zip_0316:main"
Expand Down
1 change: 1 addition & 0 deletions regenerate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ tv_scripts=(
sapling_note_encryption
sapling_signatures
sapling_zip32
transaction_v5
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is transaction_legacy not included?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like that's in the third commit.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@LarryRuane it should be included here as well.

unified_address
unified_full_viewing_keys
unified_incoming_viewing_keys
Expand Down
34 changes: 34 additions & 0 deletions test-vectors/json/transaction_legacy.json
View file
Open in desktop

Large diffs are not rendered by default.

14 changes: 14 additions & 0 deletions test-vectors/json/transaction_v5.json
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions test-vectors/json/zip_0243.json
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions test-vectors/json/zip_0244.json
View file
Open in desktop

Large diffs are not rendered by default.

760 changes: 760 additions & 0 deletions test-vectors/rust/transaction_legacy.rs
View file
Open in desktop

Large diffs are not rendered by default.

336 changes: 336 additions & 0 deletions test-vectors/rust/transaction_v5.rs
View file
Open in desktop

Large diffs are not rendered by default.

16 changes: 8 additions & 8 deletions test-vectors/rust/zip_0243.rs
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions test-vectors/rust/zip_0244.rs
View file
Open in desktop

Large diffs are not rendered by default.

34 changes: 34 additions & 0 deletions test-vectors/zcash/transaction_legacy.json
View file
Open in desktop

Large diffs are not rendered by default.

14 changes: 14 additions & 0 deletions test-vectors/zcash/transaction_v5.json
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions test-vectors/zcash/zip_0243.json
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions test-vectors/zcash/zip_0244.json
View file
Open in desktop

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions zcash_test_vectors/output.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@ def tv_option_vec_bytes_rust(name, value, pad):
else:
print('%s%s: None,' % (pad, name))

def tv_bool_rust(name, value, pad):
print('%s%s: %s,' % (pad, name, 'true' if value else 'false'))

def tv_int_rust(name, value, pad):
print('%s%s: %d,' % (pad, name, value))

Expand Down Expand Up @@ -148,6 +151,8 @@ def tv_part_rust(name, value, config, indent=3):
tv_bytes_rust(name, value, pad)
elif config['rust_type'].startswith('Option<'):
tv_option_int_rust(name, value, pad)
elif type(value) == bool:
tv_bool_rust(name, value, pad)
elif type(value) == int:
tv_int_rust(name, value, pad)
elif type(value) == list:
Expand Down
172 changes: 166 additions & 6 deletions zcash_test_vectors/transaction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,11 @@ def __init__(self, rand, version):
self.nLockTime = rand.u32()
self.nExpiryHeight = rand.u32() % TX_EXPIRY_HEIGHT_THRESHOLD
if self.nVersion >= SAPLING_TX_VERSION:
self.valueBalance = rand.u64() % (MAX_MONEY + 1)
valueBalanceRand = rand.u64()
if valueBalanceRand & 0x8000000000000000:
self.valueBalance = (valueBalanceRand % (MAX_MONEY + 1)) - MAX_MONEY
else:
self.valueBalance = valueBalanceRand % (MAX_MONEY + 1)

self.vShieldedSpends = []
self.vShieldedOutputs = []
Expand Down Expand Up @@ -390,7 +394,7 @@ def __bytes__(self):
ret += struct.pack('<I', self.nExpiryHeight)

if isSaplingV4:
ret += struct.pack('<Q', self.valueBalance)
ret += struct.pack('<q', self.valueBalance)
ret += write_compact_size(len(self.vShieldedSpends))
for desc in self.vShieldedSpends:
ret += bytes(desc)
Expand Down Expand Up @@ -455,7 +459,11 @@ def __init__(self, rand, consensus_branch_id):
self.vSpendsSapling.append(spend)
for _ in range(rand.u8() % 3):
self.vOutputsSapling.append(OutputDescription(rand))
self.valueBalanceSapling = rand.u64() % (MAX_MONEY + 1)
valueBalanceRand = rand.u64()
if valueBalanceRand & 0x8000000000000000:
self.valueBalanceSapling = (valueBalanceRand % (MAX_MONEY + 1)) - MAX_MONEY
else:
self.valueBalanceSapling = valueBalanceRand % (MAX_MONEY + 1)
self.bindingSigSapling = RedJubjubSignature(rand)
else:
# If valueBalanceSapling is not present in the serialized transaction, then
Expand All @@ -471,7 +479,11 @@ def __init__(self, rand, consensus_branch_id):
if is_coinbase:
# set enableSpendsOrchard = 0
self.flagsOrchard &= 2
self.valueBalanceOrchard = rand.u64() % (MAX_MONEY + 1)
valueBalanceRand = rand.u64()
if valueBalanceRand & 0x8000000000000000:
self.valueBalanceOrchard = (valueBalanceRand % (MAX_MONEY + 1)) - MAX_MONEY
else:
self.valueBalanceOrchard = valueBalanceRand % (MAX_MONEY + 1)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Three times and you factor out:

def signed_amount(rand):
    valueRand = rand.u64()
    if valueRand & 0x8000000000000000:
        return (valueRand % (MAX_MONEY + 1)) - MAX_MONEY
    else:
        return valueRand % (MAX_MONEY + 1)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe?

return (valueRand % (MAX_MONEY + 1)) - (MAX_MONEY if valueRand & 0x8000000000000000 else 0)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like @daira's suggestion here: it's explicit as to the switch that you're making at the top level.

self.anchorOrchard = PallasBase(leos2ip(rand.b(32)))
self.proofsOrchard = rand.b(rand.u8() + 32) # Proof will always contain at least one element
self.bindingSigOrchard = RedPallasSignature(rand)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Below here: version_bytes is confusingly named because it doesn't return bytes.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, ZIP 225 has been updated to specify that all fields are little-endian, and so that TODO on line 504 can be removed.

Expand Down Expand Up @@ -517,7 +529,7 @@ def __bytes__(self):
for desc in self.vOutputsSapling:
ret += desc.bytes_v5()
if hasSapling:
ret += struct.pack('<Q', self.valueBalanceSapling)
ret += struct.pack('<q', self.valueBalanceSapling)
if len(self.vSpendsSapling) > 0:
ret += bytes(self.anchorSapling)
# Not explicitly gated in the protocol spec, but if the gate
Expand All @@ -539,7 +551,7 @@ def __bytes__(self):
for desc in self.vActionsOrchard:
ret += bytes(desc) # Excludes spendAuthSig
ret += struct.pack('B', self.flagsOrchard)
ret += struct.pack('<Q', self.valueBalanceOrchard)
ret += struct.pack('<q', self.valueBalanceOrchard)
ret += bytes(self.anchorOrchard)
ret += write_compact_size(len(self.proofsOrchard))
ret += self.proofsOrchard
Expand All @@ -563,3 +575,151 @@ def __getattr__(self, item):

def __bytes__(self):
return bytes(self.inner)


def legacy_test_vectors():
from hashlib import sha256

from .output import render_args, render_tv
from .rand import Rand
from .zip_0244 import txid_digest

args = render_args()

from random import Random
rng = Random(0xabad533d)
def randbytes(l):
ret = []
while len(ret) < l:
ret.append(rng.randrange(0, 256))
return bytes(ret)
rand = Rand(randbytes)

test_vectors = []
while len(test_vectors) < 30:
# Generate transactions with versions prior to ZIP 225.
tx = LegacyTransaction(rand, rand.u8() % NU5_TX_VERSION)
Copy link
Contributor

@daira daira Mar 28, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

0 was never a valid version number. I see that line 327 above will replace a version other than 3 (Overwinter) and 4 (Sapling) with a random version:

            self.nVersion = rand.u32() & ((1 << 31) - 1)

but I think that versions 1 and 2 (as version numbers that were actually used) should have high probability of occurrence, and 0 should be excluded. One option is to keep this line as it is, and only replace 0 with a random number in [1, 231) in the LegacyTransaction constructor.

tx_bytes = bytes(tx)
txid = sha256(sha256(tx_bytes).digest()).digest()

has_sprout = tx.nVersion >= 2 and len(tx.vJoinSplit) > 0
has_sapling = tx.nVersion == SAPLING_TX_VERSION and not (len(tx.vShieldedSpends) == 0 and len(tx.vShieldedOutputs) == 0)

test_vectors.append({
'tx': tx_bytes,
'txid': txid,
'fOverwintered': tx.fOverwintered,
'version': tx.nVersion,
'nVersionGroupId': tx.nVersionGroupId if tx.fOverwintered else None,
'tx_in_count': len(tx.vin),
'tx_out_count': len(tx.vout),
'lock_time': tx.nLockTime,
'nExpiryHeight': tx.nExpiryHeight if tx.fOverwintered else None,
'valueBalanceSapling': tx.valueBalance if tx.nVersion == SAPLING_TX_VERSION else None,
'nSpendsSapling': len(tx.vShieldedSpends) if tx.nVersion == SAPLING_TX_VERSION else None,
'nOutputsSapling': len(tx.vShieldedOutputs) if tx.nVersion == SAPLING_TX_VERSION else None,
'nJoinSplit': len(tx.vJoinSplit) if tx.nVersion > 2 else None,
'joinSplitPubKey': bytes(tx.joinSplitPubKey) if has_sprout else None,
'joinSplitSig': bytes(tx.joinSplitSig) if has_sprout else None,
'bindingSigSapling': bytes(tx.bindingSig) if has_sapling else None,
})

render_tv(
args,
'transaction_legacy',
(
('tx', {'rust_type': 'Vec<u8>', 'bitcoin_flavoured': False}),
('txid', '[u8; 32]'),
('fOverwintered', 'bool'),
('version', 'u32'),
('nVersionGroupId', 'Option<u32>'),
('tx_in_count', 'usize'),
('tx_out_count', 'usize'),
('lock_time', 'u32'),
('nExpiryHeight', 'Option<u32>'),
('valueBalanceSapling', 'Option<i64>'),
('nSpendsSapling', 'Option<usize>'),
('nOutputsSapling', 'Option<usize>'),
('nJoinSplit', 'usize'),
('joinSplitPubKey', 'Option<[u8; 32]>'),
('joinSplitSig', 'Option<[u8; 64]>'),
('bindingSigSapling', 'Option<[u8; 64]>'),
),
test_vectors,
)


def v5_test_vectors():
from .output import render_args, render_tv
from .rand import Rand
from .zip_0244 import txid_digest

args = render_args()

from random import Random
rng = Random(0xabad533d)
def randbytes(l):
ret = []
while len(ret) < l:
ret.append(rng.randrange(0, 256))
return bytes(ret)
rand = Rand(randbytes)

test_vectors = []
while len(test_vectors) < 10:
tx = TransactionV5(rand, rand.u32())
txid = txid_digest(tx)

has_sapling = len(tx.vSpendsSapling) + len(tx.vOutputsSapling) > 0
has_orchard = len(tx.vActionsOrchard) > 0

test_vectors.append({
'tx': bytes(tx),
'txid': txid,
'version': NU5_TX_VERSION,
'nVersionGroupId': tx.nVersionGroupId,
'nConsensusBranchId': tx.nConsensusBranchId,
'lock_time': tx.nLockTime,
'nExpiryHeight': tx.nExpiryHeight,
'tx_in_count': len(tx.vin),
'tx_out_count': len(tx.vout),
'nSpendsSapling': len(tx.vSpendsSapling),
'nOutputsSapling': len(tx.vOutputsSapling),
'valueBalanceSapling': tx.valueBalanceSapling,
'anchorSapling': bytes(tx.anchorSapling) if has_sapling else None,
'bindingSigSapling': bytes(tx.bindingSigSapling) if has_sapling else None,
'nActionsOrchard': len(tx.vActionsOrchard),
'flagsOrchard': tx.flagsOrchard if has_orchard else None,
'valueBalanceOrchard': tx.valueBalanceOrchard,
'anchorOrchard': bytes(tx.anchorOrchard) if has_orchard else None,
'proofsOrchard': tx.proofsOrchard if has_orchard else None,
'bindingSigOrchard': bytes(tx.bindingSigOrchard) if has_orchard else None,
})

render_tv(
args,
'transaction_v5',
(
('tx', {'rust_type': 'Vec<u8>', 'bitcoin_flavoured': False}),
('txid', '[u8; 32]'),
('version', 'u32'),
('nVersionGroupId', 'u32'),
('nConsensusBranchId', 'u32'),
('lock_time', 'u32'),
('nExpiryHeight', 'u32'),
('tx_in_count', 'usize'),
('tx_out_count', 'usize'),
('nSpendsSapling', 'usize'),
('nOutputsSapling', 'usize'),
('valueBalanceSapling', 'i64'),
('anchorSapling', 'Option<[u8; 32]>'),
('bindingSigSapling', 'Option<[u8; 64]>'),
('nActionsOrchard', 'usize'),
('flagsOrchard', 'Option<u8>'),
('valueBalanceOrchard', 'i64'),
('anchorOrchard', 'Option<[u8; 32]>'),
('proofsOrchard', {'rust_type': 'Option<Vec<u8>>', 'bitcoin_flavoured': False}),
('bindingSigOrchard', 'Option<[u8; 64]>'),
),
test_vectors,
)
2 changes: 1 addition & 1 deletion zcash_test_vectors/zip_0243.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def signature_hash(scriptCode, tx, nIn, nHashType, amount, consensusBranchId):
digest.update(hashShieldedOutputs)
digest.update(struct.pack('<I', tx.nLockTime))
digest.update(struct.pack('<I', tx.nExpiryHeight))
digest.update(struct.pack('<Q', tx.valueBalance))
digest.update(struct.pack('<q', tx.valueBalance))
digest.update(struct.pack('<I', nHashType))

if nIn != NOT_AN_INPUT:
Expand Down
4 changes: 2 additions & 2 deletions zcash_test_vectors/zip_0244.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def sapling_digest(tx):
if len(tx.vSpendsSapling) + len(tx.vOutputsSapling) > 0:
digest.update(sapling_spends_digest(tx))
digest.update(sapling_outputs_digest(tx))
digest.update(struct.pack('<Q', tx.valueBalanceSapling))
digest.update(struct.pack('<q', tx.valueBalanceSapling))

return digest.digest()

Expand Down Expand Up @@ -136,7 +136,7 @@ def orchard_digest(tx):
digest.update(orchard_actions_memos_digest(tx))
digest.update(orchard_actions_noncompact_digest(tx))
digest.update(struct.pack('<B', tx.flagsOrchard))
digest.update(struct.pack('<Q', tx.valueBalanceOrchard))
digest.update(struct.pack('<q', tx.valueBalanceOrchard))
digest.update(bytes(tx.anchorOrchard))

return digest.digest()
Expand Down