diff --git a/.github/workflows/build-&-push-normal-wasm.yaml b/.github/workflows/build-&-push-normal-wasm.yaml new file mode 100644 index 000000000..417b38c21 --- /dev/null +++ b/.github/workflows/build-&-push-normal-wasm.yaml @@ -0,0 +1,60 @@ +name: Build & Push Normal Wasm + +concurrency: + group: "build-release-wasm-${{ github.ref }}" + cancel-in-progress: true + +on: + push: + # branches: [ master, staging, qa ] + tags: + - 'v*.*.*' + # pull_request: + workflow_dispatch: + +jobs: + build-wasm: + name: Build-wasm + runs-on: [self-hosted, arc-runner] + outputs: + build_version: ${{ steps.wasm_build.outputs.BUILD_TAG }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Build wasm binary using go version 1.22.5 + id: wasm_build + run: | + docker run --rm -v $PWD:/gosdk -w /gosdk golang:1.22.5 sh -c "git config --global --add safe.directory /gosdk; make wasm-build" + BUILD_TAG=zcn-$(git describe --tags --exact-match)-normal.wasm + BUILD_TAG="${BUILD_TAG:0:4}${BUILD_TAG:5}" + mv zcn.wasm $BUILD_TAG + echo "BUILD_TAG=$BUILD_TAG" >>$GITHUB_OUTPUT + + - name: 'Upload Artifact' + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.wasm_build.outputs.BUILD_TAG }} + path: ${{ steps.wasm_build.outputs.BUILD_TAG }} + + - name: Install AWS cli + run: | + sudo apt update -y + sudo apt install awscli -y + + - name: Copy the wasm binary to aws s3. + run: | + export AWS_ACCESS_KEY_ID=${{ secrets.WEBAPP_STATIC_ACCESS_KEY }} + export AWS_SECRET_ACCESS_KEY=${{ secrets.WEBAPP_STATIC_SECRET_ACCESS_KEY }} + export AWS_DEFAULT_REGION=${{ secrets.WEBAPP_STATIC_REGION }} + gzip -9 ${{ steps.wasm_build.outputs.BUILD_TAG }} && mv ${{ steps.wasm_build.outputs.BUILD_TAG }}.gz ${{ steps.wasm_build.outputs.BUILD_TAG }} + aws s3 cp ${{ steps.wasm_build.outputs.BUILD_TAG }} s3://${{ secrets.WEBAPP_STATIC_BUCKET }}/wasm/${{ steps.wasm_build.outputs.BUILD_TAG }} --content-encoding gzip + + - name: Invalidate wasm binary + run: | + export AWS_ACCESS_KEY_ID=${{ secrets.WEBAPP_STATIC_ACCESS_KEY }} + export AWS_SECRET_ACCESS_KEY=${{ secrets.WEBAPP_STATIC_SECRET_ACCESS_KEY }} + export AWS_DEFAULT_REGION=${{ secrets.WEBAPP_STATIC_REGION }} + aws cloudfront create-invalidation --distribution-id ${{ secrets.WEBAPP_STATIC_DISTRIBUTION_ID }} --paths '/wasm/*' diff --git a/.github/workflows/build-release-wasm.yaml b/.github/workflows/build-release-wasm.yaml new file mode 100644 index 000000000..125022d24 --- /dev/null +++ b/.github/workflows/build-release-wasm.yaml @@ -0,0 +1,48 @@ +name: Build-and-release-wasm + +concurrency: + group: "build-release-wasm-${{ github.ref }}" + cancel-in-progress: true + +on: + push: + # branches: [ master, staging, qa ] + tags: + - 'v*.*.*' + # pull_request: + workflow_dispatch: + +env: + GITHUB_TOKEN: ${{ secrets.GOSDK }} + +jobs: + build-wasm: + name: Build-wasm + runs-on: [self-hosted, arc-runner] + steps: + - name: Checkout + uses: actions/checkout@v4 + + # - name: Set up Go 1.23 + # uses: actions/setup-go@v3 + # with: + # go-version: 1.23 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get -y install build-essential nghttp2 libnghttp2-dev libssl-dev wget + + - name: Build + run: | + docker run --rm -v $PWD:/gosdk -w /gosdk golang:1.23 make wasm-build + ls -lha + CURRENT_TAG=$(git describe --tags --exact-match) + mv zcn.wasm zcn-${CURRENT_TAG}-normal.wasm + ls -lha + + # - name: 'Upload Artifact' + # uses: actions/upload-artifact@v3 + # with: + # name: zcn.wasm + # path: zcn.wasm \ No newline at end of file diff --git a/core/client/init_node.go b/core/client/init_node.go index f7a2ad901..444b33834 100644 --- a/core/client/init_node.go +++ b/core/client/init_node.go @@ -18,14 +18,10 @@ import ( ) var ( - logging logger.Logger + logging = logger.GetLogger() nodeClient *Node ) -func init() { - logging.Init(logger.DEBUG, "0chain-core") -} - // Node Maintains central states of SDK (client's context, network). // Initialized through [Init] function. // Use client.GetNode() to get its instance after Init is called. diff --git a/core/logger/logger.go b/core/logger/logger.go index fbe517017..4af6e8692 100644 --- a/core/logger/logger.go +++ b/core/logger/logger.go @@ -3,6 +3,8 @@ package logger import ( "fmt" + "github.com/0chain/gosdk/core/version" + "gopkg.in/natefinch/lumberjack.v2" "io" "log" "os" @@ -142,3 +144,39 @@ func (l *Logger) Close() { } } } + +// Initialize common logger +var logging Logger + +func GetLogger() *Logger { + return &logging +} + +func CloseLog() { + logging.Close() +} + +func init() { + logging.Init(DEBUG, "0chain-core-sdk") +} + +// SetLogLevel set the log level. +// lvl - 0 disabled; higher number (upto 4) more verbosity +func SetLogLevel(lvl int) { + logging.SetLevel(lvl) +} + +// SetLogFile - sets file path to write log +// verbose - true - console output; false - no console output +func SetLogFile(logFile string, verbose bool) { + ioWriter := &lumberjack.Logger{ + Filename: logFile, + MaxSize: 100, // MB + MaxBackups: 5, // number of backups + MaxAge: 28, //days + LocalTime: false, + Compress: false, // disabled by default + } + logging.SetLogFile(ioWriter, verbose) + logging.Info("******* Wallet SDK Version:", version.VERSIONSTR, " ******* (SetLogFile)") +} diff --git a/core/transaction/entity.go b/core/transaction/entity.go index eee839b5a..995a45954 100644 --- a/core/transaction/entity.go +++ b/core/transaction/entity.go @@ -23,7 +23,7 @@ import ( lru "github.com/hashicorp/golang-lru" ) -var Logger logger.Logger +var Logger = logger.GetLogger() const STORAGE_SCADDRESS = "6dba10422e368813802877a85039d3985d96760ed844092319743fb3a76712d7" const MINERSC_SCADDRESS = "6dba10422e368813802877a85039d3985d96760ed844092319743fb3a76712d9" @@ -270,7 +270,7 @@ func (t *Transaction) VerifySigWith(pubkey string, verifyHandler VerifyFunc) (bo } func SendTransactionSync(txn *Transaction, miners []string) error { - const requestTimeout = 30 * time.Second // Timeout for each request + const requestTimeout = 3 * time.Second // Timeout for each request fails := make(chan error, len(miners)) var wg sync.WaitGroup diff --git a/zboxcore/fileref/fileref.go b/zboxcore/fileref/fileref.go index a1c082d5e..3f226777a 100644 --- a/zboxcore/fileref/fileref.go +++ b/zboxcore/fileref/fileref.go @@ -106,6 +106,7 @@ type Ref struct { AllocationRoot string `json:"allocation_root" mapstructure:"allocation_root"` CreatedAt common.Timestamp `json:"created_at" mapstructure:"created_at"` UpdatedAt common.Timestamp `json:"updated_at" mapstructure:"updated_at"` + NumFiles int `json:"num_files" mapstructure:"num_files"` } // GetReferenceLookup returns the lookup hash for a given allocationID and path diff --git a/zboxcore/logger/logger.go b/zboxcore/logger/logger.go index 37a89ee44..64134eb35 100644 --- a/zboxcore/logger/logger.go +++ b/zboxcore/logger/logger.go @@ -3,11 +3,5 @@ package logger import "github.com/0chain/gosdk/core/logger" -var defaultLogLevel = logger.DEBUG - // Logger global logger instance -var Logger logger.Logger - -func init() { - Logger.Init(defaultLogLevel, "0box-sdk") -} +var Logger = logger.GetLogger() diff --git a/zboxcore/sdk/listworker.go b/zboxcore/sdk/listworker.go index 3fde92b6a..656adb05a 100644 --- a/zboxcore/sdk/listworker.go +++ b/zboxcore/sdk/listworker.go @@ -69,6 +69,7 @@ type ListResult struct { ThumbnailSize int64 `json:"thumbnail_size"` ActualThumbnailHash string `json:"actual_thumbnail_hash"` ActualThumbnailSize int64 `json:"actual_thumbnail_size"` + NumFiles int `json:"num_files"` CreatedAt common.Timestamp `json:"created_at"` UpdatedAt common.Timestamp `json:"updated_at"` @@ -271,6 +272,9 @@ func (req *ListRequest) GetListFromBlobbers() (*ListResult, error) { result.ThumbnailSize = ti.ref.ThumbnailSize result.ActualThumbnailHash = ti.ref.ActualThumbnailHash result.ActualThumbnailSize = ti.ref.ActualThumbnailSize + if result.Path == "/" && req.storageVersion == StorageV2 { + result.NumFiles = ti.ref.NumFiles + } if ti.ref.ActualSize > 0 { result.ActualNumBlocks = (ti.ref.ActualSize + CHUNK_SIZE - 1) / CHUNK_SIZE diff --git a/zboxcore/sdk/repairworker.go b/zboxcore/sdk/repairworker.go index 7df6337c0..f9891d649 100644 --- a/zboxcore/sdk/repairworker.go +++ b/zboxcore/sdk/repairworker.go @@ -455,9 +455,17 @@ func (r *RepairRequest) iterateDirV2(ctx context.Context) { } diff.tgtRef, diff.tgtEOF = <-diff.tgtChan } else if diff.tgtRef.Path < srcRef.Path { - deleteMask = deleteMask.Or(diff.mask) + delMask := diff.mask + op := OperationRequest{ + OperationType: constants.FileOperationDelete, + RemotePath: diff.tgtRef.Path, + Mask: &delMask, + } + ops = append(ops, op) toNextRef = false diff.tgtRef, diff.tgtEOF = <-diff.tgtChan + } else { + uploadMask = uploadMask.Or(diff.mask) } } if deleteMask.CountOnes() > 0 { diff --git a/zboxcore/sdk/rollback.go b/zboxcore/sdk/rollback.go index f623dbc63..6d1607c44 100644 --- a/zboxcore/sdk/rollback.go +++ b/zboxcore/sdk/rollback.go @@ -316,6 +316,10 @@ func (a *Allocation) CheckAllocStatus() (AllocStatus, []BlobberStatus, error) { return Broken, blobberRes, common.NewError("check_alloc_status_failed", markerError.Error()) } + if a.StorageVersion == StorageV2 { + return a.checkStatusV2(markerChan, blobberRes) + } + versionMap := make(map[string][]*RollbackBlobber) var ( @@ -472,3 +476,83 @@ func (a *Allocation) RollbackWithMask(mask zboxutil.Uint128) { wg.Wait() } + +func (a *Allocation) checkStatusV2(markerChan chan *RollbackBlobber, blobStatus []BlobberStatus) (AllocStatus, []BlobberStatus, error) { + var ( + latestVersionMap = make(map[string][]*RollbackBlobber) + allVersionMap = make(map[string][]*RollbackBlobber) + consensusVersion string + allVersionConensus string + ) + + for rb := range markerChan { + if rb == nil || rb.lpm.LatestWM == nil { + continue + } + version := rb.lpm.LatestWM.AllocationRoot + latestVersionMap[version] = append(latestVersionMap[version], rb) + if len(latestVersionMap[version]) > a.DataShards { + consensusVersion = version + } + allVersionMap[version] = append(allVersionMap[version], rb) + if rb.lpm.PrevWM != nil && rb.lpm.PrevWM.AllocationRoot != version { + allVersionMap[rb.lpm.PrevWM.AllocationRoot] = append(allVersionMap[rb.lpm.PrevWM.AllocationRoot], rb) + if len(allVersionMap[rb.lpm.PrevWM.AllocationRoot]) >= a.DataShards { + allVersionConensus = rb.lpm.PrevWM.AllocationRoot + } + } + if len(allVersionMap[version]) >= a.DataShards { + allVersionConensus = version + } + } + + if consensusVersion != "" { + a.allocationRoot = consensusVersion + return Commit, blobStatus, nil + } + + if allVersionConensus == "" { + if len(latestVersionMap) == 0 { + return Commit, blobStatus, nil + } + return Broken, blobStatus, nil + } + + if len(latestVersionMap[allVersionConensus]) >= a.DataShards { + a.allocationRoot = allVersionConensus + return Repair, blobStatus, nil + } + l.Logger.Info("Rolling back to previous version") + fullConsensus := len(allVersionMap[allVersionConensus]) - len(latestVersionMap[allVersionConensus]) + consensusThresh := a.DataShards - len(latestVersionMap[allVersionConensus]) + var successCnt int32 + wg := &sync.WaitGroup{} + + for _, rb := range allVersionMap[allVersionConensus] { + if rb.lpm.LatestWM.AllocationRoot == allVersionConensus { + continue + } + wg.Add(1) + go func(rb *RollbackBlobber) { + defer wg.Done() + err := rb.processRollback(context.TODO(), a.Tx) + if err != nil { + rb.commitResult = ErrorCommitResult(err.Error()) + l.Logger.Error("error during rollback", zap.Error(err)) + } else { + atomic.AddInt32(&successCnt, 1) + rb.commitResult = SuccessCommitResult() + } + }(rb) + } + wg.Wait() + if successCnt < int32(consensusThresh) { + return Broken, blobStatus, common.NewError("rollback_failed", "Rollback failed") + } + a.allocationRoot = allVersionConensus + if successCnt == int32(fullConsensus) { + return Repair, blobStatus, nil + } + + return Commit, blobStatus, nil +} diff --git a/zboxcore/sdk/sdk.go b/zboxcore/sdk/sdk.go index 260cd57cc..adb6eddbe 100644 --- a/zboxcore/sdk/sdk.go +++ b/zboxcore/sdk/sdk.go @@ -97,7 +97,7 @@ func SetLogFile(logFile string, verbose bool) { // GetLogger retrieves logger instance func GetLogger() *logger.Logger { - return &l.Logger + return l.Logger } type BackPool struct { diff --git a/zcncore/wallet_base.go b/zcncore/wallet_base.go index f4ead5968..89111ee59 100644 --- a/zcncore/wallet_base.go +++ b/zcncore/wallet_base.go @@ -79,11 +79,11 @@ const ( ) var defaultLogLevel = logger.DEBUG -var logging logger.Logger +var logging = logger.GetLogger() // GetLogger returns the logger instance func GetLogger() *logger.Logger { - return &logging + return logging } // CloseLog closes log file @@ -141,10 +141,6 @@ type AuthCallback interface { OnSetupComplete(status int, err string) } -func init() { - logging.Init(defaultLogLevel, "0chain-core-sdk") -} - func checkSdkInit() error { _, err := client.GetNode() if err != nil {