Add SmartBFT consensus support (hyperledger#3781)

* add SMaRtBFT consensus support Signed-off-by: Parameswaran Selvam <[email protected]> * Upgraded smartbft consensus library Signed-off-by: Parameswaran Selvam <[email protected]> * smartbft mock interfaces Signed-off-by: Parameswaran Selvam <[email protected]> * smartbft consensus unit tests Signed-off-by: Yacov Manevich [email protected] Signed-off-by: Yoav Tock [email protected] Signed-off-by: Parameswaran Selvam <[email protected]> * Validate block validation policy matches consenter set Change-Id: Ia0f926ddb984cc3008542328a85d41af56be36d3 Signed-off-by: Yacov Manevich <[email protected]> * bft configtxgen updates Signed-off-by: Parameswaran Selvam <[email protected]> * integration test updates Signed-off-by: Parameswaran Selvam <[email protected]> * update to orderer main with debug statements Signed-off-by: Parameswaran Selvam <[email protected]> * BFT chains are up and running Change-Id: I59bf40ff90be2299aa1733cc672ea88f94984b4f Signed-off-by: Yacov Manevich <[email protected]> * WIP first integration test DO NOT MERGE - the signature verification in the peer has been temporarily disabled Signed-off-by: andrew-coleman <[email protected]> * Only append block signature if needed Change-Id: I13ebe069fe4f8fb9b7a3d4671668a5da85cab8ea Signed-off-by: Yacov Manevich <[email protected]> * Use identifier headers instead of signature headers and properly encode/decode signatures Change-Id: Ifdd7e9a2a690dc6328bfec60ad22efae9a42fa7c Signed-off-by: Yacov Manevich <[email protected]> * integration test for update config transaction Signed-off-by: Parameswaran Selvam <[email protected]> * Properly encode BFT policy when updating consenters Change-Id: I9eb09e2874f4c4fbea77d99aa0528204412c0820 Signed-off-by: Yacov Manevich <[email protected]> * block puller failing while block data identity referencing Signed-off-by: Parameswaran Selvam <[email protected]> * Fix signature verification Change the BlockPuller to use the protoutil.BlockVerifierFunc generator which works for BFT blocks. Signed-off-by: andrew-coleman <[email protected]> * reconfig with node removal failing Signed-off-by: Parameswaran Selvam <[email protected]> * Add “smartbft multiple nodes view change” test Signed-off-by: andrew-coleman <[email protected]> * integration test iteratively add and remove nodes Signed-off-by: Parameswaran Selvam <[email protected]> * updated orderer rejoin integration test scenario Signed-off-by: Parameswaran Selvam <[email protected]> * fixed lint erros and upgrade consensus module Signed-off-by: Parameswaran Selvam <[email protected]> * Fix unit tests Most of the test build failures were due to changing the signature verifier from an interface to a function generated by a HOF. This meant that the mocks in the tests had to be rewritten. There are still some test failures in the smartbft package that I suspect haven’t worked in this version of Fabric. They will be addressed in another commit. Signed-off-by: andrew-coleman <[email protected]> * Fix ConfigTx ConsenterMapping integration test Signed-off-by: andrew-coleman <[email protected]> * Fix raft integration tests The Raft channel_participation integration tests were failing because the new logic in initializeMultichannelRegistrar() for initialising without system channel assumes that the consenter type is BFT unless it is told otherwise in the local config. This commit adds the consenter type to the orderer config template Signed-off-by: andrew-coleman <[email protected]> * Fix linter error Fixes CI failure in BasicChecks phase Signed-off-by: andrew-coleman <[email protected]> * Fix failures in smartbft unit tests Many of these were failing due to comparing error message that contain serialized proto structures. Ideally, these shouldn’t be relied on in tests. Signed-off-by: andrew-coleman <[email protected]> * Add smartbft integration test to CI Signed-off-by: andrew-coleman <[email protected]> * Fix configtx integration test Signed-off-by: andrew-coleman <[email protected]> * Align log expectation in test to actual log printed by the BFT library Signed-off-by: Yacov Manevich <[email protected]> * Fix autonomous synchronization integration test Signed-off-by: Yacov Manevich <[email protected]> * Make smartbft node addition and removal pass Signed-off-by: Yacov Manevich <[email protected]> * code cleanup and updated quorum calculation Signed-off-by: Parameswaran Selvam <[email protected]> * go modules conflict Signed-off-by: Parameswaran Selvam <[email protected]> * Removed commneted code Signed-off-by: Parameswaran Selvam <[email protected]> * addressed first set of review comments Signed-off-by: Parameswaran Selvam <[email protected]> * Updated detectSelfID to use consentermapping ecert Signed-off-by: Parameswaran Selvam <[email protected]> * BFTchain uses new communication service Signed-off-by: Parameswaran Selvam <[email protected]> * Configure ingress at reconfig and include intermediate TLS CA certs Signed-off-by: Yacov Manevich <[email protected]> * Sanitize e-certs in communication mapping and when looking up ids Signed-off-by: Yacov Manevich <[email protected]> * Vendor latest consensus library Signed-off-by: Yacov Manevich <[email protected]> * Fix iterated removal integration test Signed-off-by: Yacov Manevich <[email protected]> * Fix Raft integration tests Signed-off-by: Yacov Manevich <[email protected]> * Verify requests received from followers Signed-off-by: Yacov Manevich <[email protected]> * Remove redundant print Signed-off-by: Yacov Manevich <[email protected]> * Follower detects rogue genesis block, uses BFT signature verification Signed-off-by: Yacov Manevich <[email protected]> * Sanitize certs in UT Signed-off-by: Yacov Manevich <[email protected]> * Fix UT Signed-off-by: Yacov Manevich <[email protected]> * Fix integration test Signed-off-by: Yacov Manevich <[email protected]> * Fix UTs, fix some ITs Signed-off-by: Yacov Manevich <[email protected]> * add err check Signed-off-by: Yacov Manevich <[email protected]> * Rebuild verifier when encountering config block Signed-off-by: Yacov Manevich <[email protected]> * Remove print, elaborate comment Signed-off-by: Yacov Manevich <[email protected]> * Resolve module conflicts and rebase on top of main Signed-off-by: Yacov Manevich <[email protected]> --------- Signed-off-by: Parameswaran Selvam <[email protected]> Signed-off-by: Yacov Manevich [email protected] Signed-off-by: Yoav Tock [email protected] Signed-off-by: Yacov Manevich <[email protected]> Signed-off-by: andrew-coleman <[email protected]> Signed-off-by: Yacov Manevich <[email protected]> Co-authored-by: Yacov Manevich <[email protected]> Co-authored-by: andrew-coleman <[email protected]> Co-authored-by: Yacov Manevich <[email protected]>
hyperledger-labs · Feb 20, 2023 · cf9030b · cf9030b
1 parent 969040c
commit cf9030b
Show file tree

Hide file tree

Showing 167 changed files with 20,304 additions and 1,036 deletions.
diff --git a/.github/workflows/verify-build.yml b/.github/workflows/verify-build.yml
@@ -54,7 +54,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        INTEGRATION_TEST_SUITE: ["raft","pvtdata","pvtdatapurge","ledger","lifecycle","e2e","discovery gossip devmode pluggable","gateway idemix pkcs11 configtx configtxlator","sbe nwo msp"]
+        INTEGRATION_TEST_SUITE: ["raft","pvtdata","pvtdatapurge","ledger","lifecycle","e2e smartbft","discovery gossip devmode pluggable","gateway idemix pkcs11 configtx configtxlator","sbe nwo msp"]
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/setup-go@v3

diff --git a/common/channelconfig/util.go b/common/channelconfig/util.go
@@ -16,6 +16,7 @@ import (
 	mspprotos "github.com/hyperledger/fabric-protos-go/msp"
 	ab "github.com/hyperledger/fabric-protos-go/orderer"
 	"github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
+	"github.com/hyperledger/fabric-protos-go/orderer/smartbft"
 	pb "github.com/hyperledger/fabric-protos-go/peer"
 	"github.com/hyperledger/fabric/bccsp"
 	"github.com/hyperledger/fabric/protoutil"
@@ -327,3 +328,12 @@ func MarshalEtcdRaftMetadata(md *etcdraft.ConfigMetadata) ([]byte, error) {
 	}
 	return proto.Marshal(copyMd)
 }
+
+// MarshalBFTOptions serializes smartbft options.
+func MarshalBFTOptions(op *smartbft.Options) ([]byte, error) {
+	if copyMd, ok := proto.Clone(op).(*smartbft.Options); ok {
+		return proto.Marshal(copyMd)
+	} else {
+		return nil, errors.New("consenter options type mismatch")
+	}
+}
diff --git a/common/crypto/sanitize.go b/common/crypto/sanitize.go
@@ -0,0 +1,117 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package crypto
+
+import (
+	"crypto/ecdsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/asn1"
+	"encoding/pem"
+	"math/big"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/hyperledger/fabric-protos-go/msp"
+	"github.com/hyperledger/fabric/bccsp/utils"
+	"github.com/pkg/errors"
+)
+
+// SanitizeIdentity sanitizes the signature scheme of the identity
+func SanitizeIdentity(identity []byte) ([]byte, error) {
+	sID := &msp.SerializedIdentity{}
+	if err := proto.Unmarshal(identity, sID); err != nil {
+		return nil, errors.Wrapf(err, "failed unmarshaling identity %s", string(identity))
+	}
+
+	finalPEM, err := SanitizeX509Cert(sID.IdBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	sID.IdBytes = finalPEM
+
+	return proto.Marshal(sID)
+}
+
+func SanitizeX509Cert(initialPEM []byte) ([]byte, error) {
+	der, _ := pem.Decode(initialPEM)
+	if der == nil {
+		return nil, errors.Errorf("failed to PEM decode identity bytes: %s", string(initialPEM))
+	}
+	cert, err := x509.ParseCertificate(der.Bytes)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed parsing certificate %s", string(initialPEM))
+	}
+
+	r, s, err := utils.UnmarshalECDSASignature(cert.Signature)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed unmarshaling ECDSA signature on identity: %s", string(initialPEM))
+	}
+
+	// We assume that the consenter and the CA use the same signature scheme.
+	curveOrderUsedByCryptoGen := cert.PublicKey.(*ecdsa.PublicKey).Curve.Params().N
+	halfOrder := new(big.Int).Rsh(curveOrderUsedByCryptoGen, 1)
+	// Low S, nothing to do here!
+	if s.Cmp(halfOrder) != 1 {
+		return initialPEM, nil
+	}
+	// Else it's high-S, so shift it below half the order.
+	s.Sub(curveOrderUsedByCryptoGen, s)
+
+	var newCert certificate
+	_, err = asn1.Unmarshal(cert.Raw, &newCert)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed unmarshaling certificate")
+	}
+
+	newSig, err := utils.MarshalECDSASignature(r, s)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed marshaling ECDSA signature")
+	}
+	newCert.SignatureValue = asn1.BitString{Bytes: newSig, BitLength: len(newSig) * 8}
+
+	newCert.Raw = nil
+	newRaw, err := asn1.Marshal(newCert)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed marshaling new certificate")
+	}
+
+	finalPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: newRaw})
+	return finalPEM, nil
+}
+
+type certificate struct {
+	Raw                asn1.RawContent
+	TBSCertificate     tbsCertificate
+	SignatureAlgorithm pkix.AlgorithmIdentifier
+	SignatureValue     asn1.BitString
+}
+
+type tbsCertificate struct {
+	Raw                asn1.RawContent
+	Version            int `asn1:"optional,explicit,default:0,tag:0"`
+	SerialNumber       *big.Int
+	SignatureAlgorithm pkix.AlgorithmIdentifier
+	Issuer             asn1.RawValue
+	Validity           validity
+	Subject            asn1.RawValue
+	PublicKey          publicKeyInfo
+	UniqueId           asn1.BitString   `asn1:"optional,tag:1"`
+	SubjectUniqueId    asn1.BitString   `asn1:"optional,tag:2"`
+	Extensions         []pkix.Extension `asn1:"optional,explicit,tag:3"`
+}
+
+type validity struct {
+	NotBefore, NotAfter time.Time
+}
+
+type publicKeyInfo struct {
+	Raw       asn1.RawContent
+	Algorithm pkix.AlgorithmIdentifier
+	PublicKey asn1.BitString
+}
diff --git a/common/policies/bft.go b/common/policies/bft.go
@@ -0,0 +1,57 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package policies
+
+import (
+	"math"
+
+	cb "github.com/hyperledger/fabric-protos-go/common"
+	mspa "github.com/hyperledger/fabric-protos-go/msp"
+	"github.com/hyperledger/fabric/common/policydsl"
+	"github.com/hyperledger/fabric/protoutil"
+)
+
+const (
+	BlockValidationPolicyKey = "BlockValidation"
+)
+
+func EncodeBFTBlockVerificationPolicy(consenterProtos []*cb.Consenter, ordererGroup *cb.ConfigGroup) {
+	n := len(consenterProtos)
+	f := (n - 1) / 3
+
+	var identities []*mspa.MSPPrincipal
+	var pols []*cb.SignaturePolicy
+	for i, consenter := range consenterProtos {
+		pols = append(pols, &cb.SignaturePolicy{
+			Type: &cb.SignaturePolicy_SignedBy{
+				SignedBy: int32(i),
+			},
+		})
+		identities = append(identities, &mspa.MSPPrincipal{
+			PrincipalClassification: mspa.MSPPrincipal_IDENTITY,
+			Principal:               protoutil.MarshalOrPanic(&mspa.SerializedIdentity{Mspid: consenter.MspId, IdBytes: consenter.Identity}),
+		})
+	}
+
+	quorumSize := ComputeBFTQuorum(n, f)
+	sp := &cb.SignaturePolicyEnvelope{
+		Rule:       policydsl.NOutOf(int32(quorumSize), pols),
+		Identities: identities,
+	}
+	ordererGroup.Policies[BlockValidationPolicyKey] = &cb.ConfigPolicy{
+		// Inherit modification policy
+		ModPolicy: ordererGroup.Policies[BlockValidationPolicyKey].ModPolicy,
+		Policy: &cb.Policy{
+			Type:  int32(cb.Policy_SIGNATURE),
+			Value: protoutil.MarshalOrPanic(sp),
+		},
+	}
+}
+
+func ComputeBFTQuorum(totalNodes, faultyNodes int) int {
+	return int(math.Ceil(float64(totalNodes+faultyNodes+1) / 2))
+}
diff --git a/docs/source/metrics_reference.rst b/docs/source/metrics_reference.rst
@@ -61,6 +61,16 @@ The following orderer metrics are exported for consumption by Prometheus.
 |                                              |           |                                                            +-----------+--------------------------------------------------------------------+
 |                                              |           |                                                            | channel   |                                                                    |
 +----------------------------------------------+-----------+------------------------------------------------------------+-----------+--------------------------------------------------------------------+
+| consensus_BFT_cluster_size                   | gauge     | Number of nodes in this channel.                           | channel   |                                                                    |
++----------------------------------------------+-----------+------------------------------------------------------------+-----------+--------------------------------------------------------------------+
+| consensus_BFT_committed_block_number         | gauge     | The number of the latest committed block.                  | channel   |                                                                    |
++----------------------------------------------+-----------+------------------------------------------------------------+-----------+--------------------------------------------------------------------+
+| consensus_BFT_is_leader                      | gauge     | The leadership status of the current node according to the | channel   |                                                                    |
+|                                              |           | latest committed block: 1 if it is the leader else 0.      |           |                                                                    |
++----------------------------------------------+-----------+------------------------------------------------------------+-----------+--------------------------------------------------------------------+
+| consensus_BFT_leader_id                      | gauge     | The id of the current leader according to the latest       | channel   |                                                                    |
+|                                              |           | committed block.                                           |           |                                                                    |
++----------------------------------------------+-----------+------------------------------------------------------------+-----------+--------------------------------------------------------------------+
 | consensus_etcdraft_active_nodes              | gauge     | Number of active nodes in this channel.                    | channel   |                                                                    |
 +----------------------------------------------+-----------+------------------------------------------------------------+-----------+--------------------------------------------------------------------+
 | consensus_etcdraft_cluster_size              | gauge     | Number of nodes in this channel.                           | channel   |                                                                    |
@@ -214,6 +224,16 @@ associated with the metric.
 +---------------------------------------------------------------------------+-----------+------------------------------------------------------------+
 | cluster.comm.msg_send_time.%{host}.%{channel}                             | histogram | The time it takes to send a message in seconds.            |
 +---------------------------------------------------------------------------+-----------+------------------------------------------------------------+
+| consensus.BFT.cluster_size.%{channel}                                     | gauge     | Number of nodes in this channel.                           |
++---------------------------------------------------------------------------+-----------+------------------------------------------------------------+
+| consensus.BFT.committed_block_number.%{channel}                           | gauge     | The number of the latest committed block.                  |
++---------------------------------------------------------------------------+-----------+------------------------------------------------------------+
+| consensus.BFT.is_leader.%{channel}                                        | gauge     | The leadership status of the current node according to the |
+|                                                                           |           | latest committed block: 1 if it is the leader else 0.      |
++---------------------------------------------------------------------------+-----------+------------------------------------------------------------+
+| consensus.BFT.leader_id.%{channel}                                        | gauge     | The id of the current leader according to the latest       |
+|                                                                           |           | committed block.                                           |
++---------------------------------------------------------------------------+-----------+------------------------------------------------------------+
 | consensus.etcdraft.active_nodes.%{channel}                                | gauge     | Number of active nodes in this channel.                    |
 +---------------------------------------------------------------------------+-----------+------------------------------------------------------------+
 | consensus.etcdraft.cluster_size.%{channel}                                | gauge     | Number of nodes in this channel.                           |

diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ require (
 	code.cloudfoundry.org/clock v1.0.0
 	github.com/IBM/idemix v0.0.0-20220112103229-701e7610d405
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
+	github.com/SmartBFT-Go/consensus v0.0.0-20230212211744-e5a79afcea81
 	github.com/VictoriaMetrics/fastcache v1.9.0
 	github.com/bits-and-blooms/bitset v1.2.1
 	github.com/cheggaaa/pb v1.0.29
@@ -37,7 +38,7 @@ require (
 	go.etcd.io/etcd/client/pkg/v3 v3.5.1
 	go.etcd.io/etcd/raft/v3 v3.5.1
 	go.etcd.io/etcd/server/v3 v3.5.1
-	go.uber.org/zap v1.17.0
+	go.uber.org/zap v1.19.0
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
 	golang.org/x/tools v0.1.12
 	google.golang.org/grpc v1.47.0
@@ -103,6 +104,7 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
 	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect

diff --git a/go.sum b/go.sum
@@ -57,6 +57,8 @@ github.com/Microsoft/hcsshim v0.8.25/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/SmartBFT-Go/consensus v0.0.0-20230212211744-e5a79afcea81 h1:yiyJRAf/rsEu3Sl0ATWu1zREfyaj01i9VsPbGiXzZZw=
+github.com/SmartBFT-Go/consensus v0.0.0-20230212211744-e5a79afcea81/go.mod h1:ZOD/ZiAdH9HpqdsJLlUTlbzYBr/qYEzyYx7wClbrH+w=
 github.com/VictoriaMetrics/fastcache v1.9.0 h1:oMwsS6c8abz98B7ytAewQ7M1ZN/Im/iwKoE1euaFvhs=
 github.com/VictoriaMetrics/fastcache v1.9.0/go.mod h1:otoTS3xu+6IzF/qByjqzjp3rTuzM3Qf0ScU1UTj97iU=
 github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE=
@@ -84,6 +86,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -715,6 +719,7 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
 go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
@@ -727,8 +732,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
-go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE=
+go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -843,6 +849,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

diff --git a/integration/chaincode/simple/chaincode.go b/integration/chaincode/simple/chaincode.go
@@ -81,6 +81,8 @@ func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface) pb.Response {
 		return t.mspid(args)
 	case "event":
 		return t.event(stub, args)
+	case "issue":
+		return t.issue(stub, args)
 	default:
 		return shim.Error(`Invalid invoke function name. Expecting "invoke", "delete", "query", "respond", "mspid", or "event"`)
 	}
@@ -243,3 +245,32 @@ func (t *SimpleChaincode) event(stub shim.ChaincodeStubInterface, args []string)
 
 	return shim.Success(nil)
 }
+
+// Issue asset holding. Can only be done once.
+func (t *SimpleChaincode) issue(stub shim.ChaincodeStubInterface, args []string) pb.Response {
+	fmt.Printf("Entry: issue\n")
+
+	if len(args) != 2 {
+		return shim.Error("Incorrect number of arguments. Expecting 2")
+	}
+
+	A := args[0]
+	val, err := stub.GetState(A)
+	if !(val == nil && err == nil) {
+		return shim.Error(fmt.Sprintf("Asset already exists: %s", A))
+	}
+
+	Aval, err := strconv.Atoi(args[1])
+	if err != nil {
+		return shim.Error("Expecting integer value for asset holding")
+	}
+
+	fmt.Printf("Issue: %s = %d\n", A, Aval)
+
+	err = stub.PutState(A, []byte(strconv.Itoa(Aval)))
+	if err != nil {
+		return shim.Error(err.Error())
+	}
+
+	return shim.Success(nil)
+}
diff --git a/integration/channelparticipation/channel_participation.go b/integration/channelparticipation/channel_participation.go
@@ -42,11 +42,12 @@ func Join(n *nwo.Network, o *nwo.Orderer, channel string, block *common.Block, e
 	if n.TLSEnabled {
 		client = authClient
 	}
-	body := doBody(client, req)
-	c := &ChannelInfo{}
-	err = json.Unmarshal(body, c)
-	Expect(err).NotTo(HaveOccurred())
-	Expect(*c).To(Equal(expectedChannelInfo))
+
+	doBody(client, req)
+
+	Eventually(func() ChannelInfo {
+		return ListOne(n, o, channel)
+	}, n.EventuallyTimeout).Should(Equal(expectedChannelInfo))
 }
 
 func GenerateJoinRequest(url, channel string, blockBytes []byte) *http.Request {