Merge branch 'staging' into feat/repair-cancel

.github/workflows/build-&-push-normal-wasm.yaml

@@ -0,0 +1,60 @@
+name: Build & Push Normal Wasm
+
+concurrency:
+  group: "build-release-wasm-${{ github.ref }}"
+  cancel-in-progress: true
+
+on:
+  push:
+    # branches: [ master, staging, qa ]
+    tags:
+      - 'v*.*.*'
+  # pull_request:
+  workflow_dispatch:
+
+jobs:
+  build-wasm:
+    name: Build-wasm
+    runs-on: [self-hosted, arc-runner]
+    outputs:
+      build_version: ${{ steps.wasm_build.outputs.BUILD_TAG }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Build wasm binary using go version 1.22.5
+        id: wasm_build
+        run: |
+          docker run --rm -v $PWD:/gosdk -w /gosdk golang:1.22.5 sh -c "git config --global --add safe.directory /gosdk; make wasm-build"
+          BUILD_TAG=zcn-$(git describe --tags --exact-match)-normal.wasm
+          BUILD_TAG="${BUILD_TAG:0:4}${BUILD_TAG:5}"
+          mv zcn.wasm $BUILD_TAG
+          echo "BUILD_TAG=$BUILD_TAG" >>$GITHUB_OUTPUT
+
+      - name: 'Upload Artifact'
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.wasm_build.outputs.BUILD_TAG }}
+          path: ${{ steps.wasm_build.outputs.BUILD_TAG }}
+
+      - name: Install AWS cli
+        run: |
+          sudo apt update -y
+          sudo apt install awscli -y
+          
+      - name: Copy the wasm binary to aws s3.
+        run: |
+          export AWS_ACCESS_KEY_ID=${{ secrets.WEBAPP_STATIC_ACCESS_KEY }}
+          export AWS_SECRET_ACCESS_KEY=${{ secrets.WEBAPP_STATIC_SECRET_ACCESS_KEY }}
+          export AWS_DEFAULT_REGION=${{ secrets.WEBAPP_STATIC_REGION }}
+          gzip -9 ${{ steps.wasm_build.outputs.BUILD_TAG }} && mv ${{ steps.wasm_build.outputs.BUILD_TAG }}.gz ${{ steps.wasm_build.outputs.BUILD_TAG }}
+          aws s3 cp ${{ steps.wasm_build.outputs.BUILD_TAG }} s3://${{ secrets.WEBAPP_STATIC_BUCKET }}/wasm/${{ steps.wasm_build.outputs.BUILD_TAG }} --content-encoding gzip
+
+      - name: Invalidate wasm binary
+        run: |
+          export AWS_ACCESS_KEY_ID=${{ secrets.WEBAPP_STATIC_ACCESS_KEY }}
+          export AWS_SECRET_ACCESS_KEY=${{ secrets.WEBAPP_STATIC_SECRET_ACCESS_KEY }}
+          export AWS_DEFAULT_REGION=${{ secrets.WEBAPP_STATIC_REGION }}
+          aws cloudfront create-invalidation --distribution-id ${{ secrets.WEBAPP_STATIC_DISTRIBUTION_ID }} --paths '/wasm/*'

.github/workflows/build-release-wasm.yaml

@@ -0,0 +1,48 @@
+name: Build-and-release-wasm
+
+concurrency:
+  group: "build-release-wasm-${{ github.ref }}"
+  cancel-in-progress: true
+
+on:
+  push:
+    # branches: [ master, staging, qa ]
+    tags:
+      - 'v*.*.*'
+  # pull_request:
+  workflow_dispatch:
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GOSDK }}
+
+jobs:
+  build-wasm:
+    name: Build-wasm
+    runs-on: [self-hosted, arc-runner]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # - name: Set up Go 1.23
+      #   uses: actions/setup-go@v3
+      #   with:
+      #     go-version: 1.23
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install build-essential nghttp2 libnghttp2-dev libssl-dev wget
+
+      - name: Build
+        run: |
+          docker run --rm -v $PWD:/gosdk -w /gosdk golang:1.23 make wasm-build
+          ls -lha
+          CURRENT_TAG=$(git describe --tags --exact-match)
+          mv zcn.wasm zcn-${CURRENT_TAG}-normal.wasm
+          ls -lha
+
+      # - name: 'Upload Artifact'
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: zcn.wasm
+      #     path: zcn.wasm

core/client/init_node.go

@@ -18,14 +18,10 @@ import (
 )
 
 var (
-	logging    logger.Logger
+	logging    = logger.GetLogger()
 	nodeClient *Node
 )
 
-func init() {
-	logging.Init(logger.DEBUG, "0chain-core")
-}
-
 // Node Maintains central states of SDK (client's context, network).
 // Initialized through [Init] function.
 // Use client.GetNode() to get its instance after Init is called.

core/logger/logger.go

@@ -3,6 +3,8 @@ package logger
 
 import (
 	"fmt"
+	"github.com/0chain/gosdk/core/version"
+	"gopkg.in/natefinch/lumberjack.v2"
 	"io"
 	"log"
 	"os"
@@ -142,3 +144,39 @@ func (l *Logger) Close() {
 		}
 	}
 }
+
+// Initialize common logger
+var logging Logger
+
+func GetLogger() *Logger {
+	return &logging
+}
+
+func CloseLog() {
+	logging.Close()
+}
+
+func init() {
+	logging.Init(DEBUG, "0chain-core-sdk")
+}
+
+// SetLogLevel set the log level.
+// lvl - 0 disabled; higher number (upto 4) more verbosity
+func SetLogLevel(lvl int) {
+	logging.SetLevel(lvl)
+}
+
+// SetLogFile - sets file path to write log
+// verbose - true - console output; false - no console output
+func SetLogFile(logFile string, verbose bool) {
+	ioWriter := &lumberjack.Logger{
+		Filename:   logFile,
+		MaxSize:    100, // MB
+		MaxBackups: 5,   // number of backups
+		MaxAge:     28,  //days
+		LocalTime:  false,
+		Compress:   false, // disabled by default
+	}
+	logging.SetLogFile(ioWriter, verbose)
+	logging.Info("******* Wallet SDK Version:", version.VERSIONSTR, " ******* (SetLogFile)")
+}

core/transaction/entity.go

@@ -23,7 +23,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 )
 
-var Logger logger.Logger
+var Logger = logger.GetLogger()
 
 const STORAGE_SCADDRESS = "6dba10422e368813802877a85039d3985d96760ed844092319743fb3a76712d7"
 const MINERSC_SCADDRESS = "6dba10422e368813802877a85039d3985d96760ed844092319743fb3a76712d9"
@@ -270,7 +270,7 @@ func (t *Transaction) VerifySigWith(pubkey string, verifyHandler VerifyFunc) (bo
 }
 
 func SendTransactionSync(txn *Transaction, miners []string) error {
-	const requestTimeout = 30 * time.Second // Timeout for each request
+	const requestTimeout = 3 * time.Second // Timeout for each request
 
 	fails := make(chan error, len(miners))
 	var wg sync.WaitGroup

zboxcore/fileref/fileref.go

@@ -106,6 +106,7 @@ type Ref struct {
 	AllocationRoot      string           `json:"allocation_root" mapstructure:"allocation_root"`
 	CreatedAt           common.Timestamp `json:"created_at" mapstructure:"created_at"`
 	UpdatedAt           common.Timestamp `json:"updated_at" mapstructure:"updated_at"`
+	NumFiles            int              `json:"num_files" mapstructure:"num_files"`
 }
 
 // GetReferenceLookup returns the lookup hash for a given allocationID and path

zboxcore/logger/logger.go

@@ -3,11 +3,5 @@ package logger
 
 import "github.com/0chain/gosdk/core/logger"
 
-var defaultLogLevel = logger.DEBUG
-
 // Logger global logger instance
-var Logger logger.Logger
-
-func init() {
-	Logger.Init(defaultLogLevel, "0box-sdk")
-}
+var Logger = logger.GetLogger()

zboxcore/sdk/listworker.go

@@ -69,6 +69,7 @@ type ListResult struct {
 	ThumbnailSize       int64  `json:"thumbnail_size"`
 	ActualThumbnailHash string `json:"actual_thumbnail_hash"`
 	ActualThumbnailSize int64  `json:"actual_thumbnail_size"`
+	NumFiles            int    `json:"num_files"`
 
 	CreatedAt      common.Timestamp `json:"created_at"`
 	UpdatedAt      common.Timestamp `json:"updated_at"`
@@ -271,6 +272,9 @@ func (req *ListRequest) GetListFromBlobbers() (*ListResult, error) {
 		result.ThumbnailSize = ti.ref.ThumbnailSize
 		result.ActualThumbnailHash = ti.ref.ActualThumbnailHash
 		result.ActualThumbnailSize = ti.ref.ActualThumbnailSize
+		if result.Path == "/" && req.storageVersion == StorageV2 {
+			result.NumFiles = ti.ref.NumFiles
+		}
 
 		if ti.ref.ActualSize > 0 {
 			result.ActualNumBlocks = (ti.ref.ActualSize + CHUNK_SIZE - 1) / CHUNK_SIZE

zboxcore/sdk/repairworker.go

@@ -455,9 +455,17 @@ func (r *RepairRequest) iterateDirV2(ctx context.Context) {
 				}
 				diff.tgtRef, diff.tgtEOF = <-diff.tgtChan
 			} else if diff.tgtRef.Path < srcRef.Path {
-				deleteMask = deleteMask.Or(diff.mask)
+				delMask := diff.mask
+				op := OperationRequest{
+					OperationType: constants.FileOperationDelete,
+					RemotePath:    diff.tgtRef.Path,
+					Mask:          &delMask,
+				}
+				ops = append(ops, op)
 				toNextRef = false
 				diff.tgtRef, diff.tgtEOF = <-diff.tgtChan
+			} else {
+				uploadMask = uploadMask.Or(diff.mask)
 			}
 		}
 		if deleteMask.CountOnes() > 0 {

zboxcore/sdk/rollback.go

@@ -316,6 +316,10 @@ func (a *Allocation) CheckAllocStatus() (AllocStatus, []BlobberStatus, error) {
 		return Broken, blobberRes, common.NewError("check_alloc_status_failed", markerError.Error())
 	}
 
+	if a.StorageVersion == StorageV2 {
+		return a.checkStatusV2(markerChan, blobberRes)
+	}
+
 	versionMap := make(map[string][]*RollbackBlobber)
 
 	var (
@@ -472,3 +476,83 @@ func (a *Allocation) RollbackWithMask(mask zboxutil.Uint128) {
 
 	wg.Wait()
 }
+
+func (a *Allocation) checkStatusV2(markerChan chan *RollbackBlobber, blobStatus []BlobberStatus) (AllocStatus, []BlobberStatus, error) {
+	var (
+		latestVersionMap   = make(map[string][]*RollbackBlobber)
+		allVersionMap      = make(map[string][]*RollbackBlobber)
+		consensusVersion   string
+		allVersionConensus string
+	)
+
+	for rb := range markerChan {
+		if rb == nil || rb.lpm.LatestWM == nil {
+			continue
+		}
+		version := rb.lpm.LatestWM.AllocationRoot
+		latestVersionMap[version] = append(latestVersionMap[version], rb)
+		if len(latestVersionMap[version]) > a.DataShards {
+			consensusVersion = version
+		}
+		allVersionMap[version] = append(allVersionMap[version], rb)
+		if rb.lpm.PrevWM != nil && rb.lpm.PrevWM.AllocationRoot != version {
+			allVersionMap[rb.lpm.PrevWM.AllocationRoot] = append(allVersionMap[rb.lpm.PrevWM.AllocationRoot], rb)
+			if len(allVersionMap[rb.lpm.PrevWM.AllocationRoot]) >= a.DataShards {
+				allVersionConensus = rb.lpm.PrevWM.AllocationRoot
+			}
+		}
+		if len(allVersionMap[version]) >= a.DataShards {
+			allVersionConensus = version
+		}
+	}
+
+	if consensusVersion != "" {
+		a.allocationRoot = consensusVersion
+		return Commit, blobStatus, nil
+	}
+
+	if allVersionConensus == "" {
+		if len(latestVersionMap) == 0 {
+			return Commit, blobStatus, nil
+		}
+		return Broken, blobStatus, nil
+	}
+
+	if len(latestVersionMap[allVersionConensus]) >= a.DataShards {
+		a.allocationRoot = allVersionConensus
+		return Repair, blobStatus, nil
+	}
+	l.Logger.Info("Rolling back to previous version")
+	fullConsensus := len(allVersionMap[allVersionConensus]) - len(latestVersionMap[allVersionConensus])
+	consensusThresh := a.DataShards - len(latestVersionMap[allVersionConensus])
+	var successCnt int32
+	wg := &sync.WaitGroup{}
+
+	for _, rb := range allVersionMap[allVersionConensus] {
+		if rb.lpm.LatestWM.AllocationRoot == allVersionConensus {
+			continue
+		}
+		wg.Add(1)
+		go func(rb *RollbackBlobber) {
+			defer wg.Done()
+			err := rb.processRollback(context.TODO(), a.Tx)
+			if err != nil {
+				rb.commitResult = ErrorCommitResult(err.Error())
+				l.Logger.Error("error during rollback", zap.Error(err))
+			} else {
+				atomic.AddInt32(&successCnt, 1)
+				rb.commitResult = SuccessCommitResult()
+			}
+		}(rb)
+	}
+	wg.Wait()
+	if successCnt < int32(consensusThresh) {
+		return Broken, blobStatus, common.NewError("rollback_failed", "Rollback failed")
+	}
+	a.allocationRoot = allVersionConensus
+	if successCnt == int32(fullConsensus) {
+		return Repair, blobStatus, nil
+	}
+
+	return Commit, blobStatus, nil
+}

zboxcore/sdk/sdk.go

@@ -97,7 +97,7 @@ func SetLogFile(logFile string, verbose bool) {
 
 // GetLogger retrieves logger instance
 func GetLogger() *logger.Logger {
-	return &l.Logger
+	return l.Logger
 }
 
 type BackPool struct {

zcncore/wallet_base.go

@@ -79,11 +79,11 @@ const (
 )
 
 var defaultLogLevel = logger.DEBUG
-var logging logger.Logger
+var logging = logger.GetLogger()
 
 // GetLogger returns the logger instance
 func GetLogger() *logger.Logger {
-	return &logging
+	return logging
 }
 
 // CloseLog closes log file
@@ -141,10 +141,6 @@ type AuthCallback interface {
 	OnSetupComplete(status int, err string)
 }
 
-func init() {
-	logging.Init(defaultLogLevel, "0chain-core-sdk")
-}
-
 func checkSdkInit() error {
 	_, err := client.GetNode()
 	if err != nil {