From 903e6500de87e9a0b1d04a2e788ca7dc46c541a8 Mon Sep 17 00:00:00 2001 From: Recepxx34 <163409185+Recepxx34@users.noreply.github.com> Date: Wed, 20 Nov 2024 22:56:20 +0300 Subject: [PATCH 1/2] * Visual Studio Code Server * * By using the software, you agree to * the Visual Studio Code Server License Terms (https://aka.ms/vscode-server-license) and * the Microsoft Privacy Statement (https://privacy.microsoft.com/en-US/privacystatement). --- service/userproof_service/service.go | 266 --------------------------- 1 file changed, 266 deletions(-) diff --git a/service/userproof_service/service.go b/service/userproof_service/service.go index c7df0376..e69de29b 100644 --- a/service/userproof_service/service.go +++ b/service/userproof_service/service.go @@ -1,266 +0,0 @@ -package userproof_service - -import ( - "encoding/hex" - "encoding/json" - "fmt" - "gate-zkmerkle-proof/config" - "gate-zkmerkle-proof/global" - "gate-zkmerkle-proof/utils" - "github.com/consensys/gnark-crypto/ecc/bn254/fr/poseidon" - zk_smt "github.com/gatechain/gate-zk-smt" - "gorm.io/driver/mysql" - "gorm.io/gorm" - "gorm.io/gorm/logger" - "io/ioutil" - "log" - "os" - "time" -) - -func HandleUserData(userProofConfig *config.Config) []utils.AccountInfo { - startTime := time.Now().UnixMilli() - accounts, _, err := utils.ReadUserAssets(userProofConfig.UserDataFile) - if err != nil { - panic(err.Error()) - } - - endTime := time.Now().UnixMilli() - fmt.Println("handle user data cost ", endTime-startTime, " ms") - return accounts -} - -type AccountLeave struct { - hash []byte - index uint32 -} - -func ComputeAccountRootHash(userProofConfig *config.Config) { - accountTree, err := utils.NewAccountTree("memory", "") - if err != nil { - panic(err.Error()) - } - accounts, _, err := utils.ReadUserAssets(userProofConfig.UserDataFile) - if err != nil { - panic(err.Error()) - } - startTime := time.Now().UnixMilli() - totalOpsNumber := len(accounts) - fmt.Println("total ops number is ", totalOpsNumber) - chs := make(chan AccountLeave, 1000) - workers := 32 - results := make(chan bool, workers) - averageAccounts := (totalOpsNumber + workers - 1) / workers - actualWorkers := 0 - for i := 0; i < workers; i++ { - srcAccountIndex := i * averageAccounts - destAccountIndex := (i + 1) * averageAccounts - if destAccountIndex > totalOpsNumber { - destAccountIndex = totalOpsNumber - } - go CalculateAccountHash(accounts[srcAccountIndex:destAccountIndex], chs, results) - if destAccountIndex == totalOpsNumber { - actualWorkers = i + 1 - break - } - } - fmt.Println("actual workers is ", actualWorkers) - quit := make(chan bool, 1) - go CalculateAccountTreeRoot(chs, &accountTree, quit) - - for i := 0; i < actualWorkers; i++ { - <-results - } - close(chs) - <-quit - endTime := time.Now().UnixMilli() - fmt.Println("user account tree generation cost ", endTime-startTime, " ms") - fmt.Printf("account tree root %x\n", accountTree.Root()) -} - -func CalculateAccountHash(accounts []utils.AccountInfo, chs chan<- AccountLeave, res chan<- bool) { - poseidonHasher := poseidon.NewPoseidon() - for i := 0; i < len(accounts); i++ { - chs <- AccountLeave{ - hash: utils.AccountInfoToHash(&accounts[i], &poseidonHasher), - index: accounts[i].AccountIndex, - } - } - res <- true -} - -func CalculateAccountTreeRoot(accountLeaves <-chan AccountLeave, accountTree *zk_smt.SparseMerkleTree, quit chan<- bool) { - num := 0 - for accountLeaf := range accountLeaves { - (*accountTree).Set(uint64(accountLeaf.index), accountLeaf.hash) - num++ - if num%100000 == 0 { - fmt.Println("for now, already set ", num, " accounts in tree") - } - } - quit <- true -} - -func Handler() { - global.Cfg = &config.Config{} - jsonFile, err := ioutil.ReadFile("./config/config.json") - if err != nil { - panic(fmt.Sprintf("load config err : %s", err.Error())) - } - err = json.Unmarshal(jsonFile, global.Cfg) - if err != nil { - panic(err.Error()) - } - - userProofConfig := global.Cfg - accountTree, err := utils.NewAccountTree(userProofConfig.TreeDB.Driver, userProofConfig.TreeDB.Option.Addr) - accounts := HandleUserData(userProofConfig) - fmt.Println("num", len(accounts)) - userProofModel := OpenUserProofTable(userProofConfig) - latestAccountIndex, err := userProofModel.GetLatestAccountIndex() - if err != nil && err != utils.DbErrNotFound { - panic(err.Error()) - } - if err == nil { - latestAccountIndex += 1 - } - accountTreeRoot := hex.EncodeToString(accountTree.Root()) - jobs := make(chan Job, 1000) - nums := make(chan int, 1) - results := make(chan *UserProof, 1000) - for i := 0; i < 1; i++ { - go worker(jobs, results, nums, accountTreeRoot) - } - quit := make(chan int, 1) - for i := 0; i < 1; i++ { - go WriteDB(results, userProofModel, quit, latestAccountIndex) - } - for i := int(latestAccountIndex); i < len(accounts); i++ { - leaf, err := accountTree.Get(uint64(i), nil) - if err != nil { - panic(err.Error()) - } - proof, err := accountTree.GetProof(uint64(accounts[i].AccountIndex)) - if err != nil { - panic(err.Error()) - } - jobs <- Job{ - account: &accounts[i], - proof: proof, - leaf: leaf, - } - } - close(jobs) - totalCounts := int(latestAccountIndex) - for i := 0; i < 1; i++ { - num := <-nums - totalCounts += num - fmt.Println("totalCounts", totalCounts) - } - if totalCounts != len(accounts) { - fmt.Println("totalCounts actual:expected", totalCounts, len(accounts)) - panic("mismatch num") - } - close(results) - for i := 0; i < 1; i++ { - <-quit - } - fmt.Println("userproof service run finished...") -} - -func WriteDB(results <-chan *UserProof, userProofModel UserProofModel, quit chan<- int, latestAccountIndex uint32) { - index := 0 - proofs := make([]UserProof, 100) - num := int(latestAccountIndex) - for proof := range results { - proofs[index] = *proof - index += 1 - if index%100 == 0 { - error := userProofModel.CreateUserProofs(proofs) - if error != nil { - panic(error.Error()) - } - num += 100 - if num%100000 == 0 { - fmt.Println("write ", num, "proof to db") - } - index = 0 - } - } - proofs = proofs[:index] - if index > 0 { - fmt.Println("write ", len(proofs), "proofs to db") - userProofModel.CreateUserProofs(proofs) - num += index - } - fmt.Println("total write ", num) - quit <- 0 -} - -type Job struct { - account *utils.AccountInfo - proof [][]byte - leaf []byte -} - -func worker(jobs <-chan Job, results chan<- *UserProof, nums chan<- int, root string) { - num := 0 - for job := range jobs { - userProof := ConvertAccount(job.account, job.leaf, job.proof, root) - results <- userProof - num += 1 - } - nums <- num -} - -func ConvertAccount(account *utils.AccountInfo, leafHash []byte, proof [][]byte, root string) *UserProof { - var userProof UserProof - var userConfig UserConfig - userProof.AccountIndex = account.AccountIndex - userProof.AccountId = hex.EncodeToString(account.AccountId) - userProof.AccountLeafHash = hex.EncodeToString(leafHash) - proofSerial, err := json.Marshal(proof) - userProof.Proof = string(proofSerial) - assets, err := json.Marshal(account.Assets) - if err != nil { - panic(err.Error()) - } - userProof.Assets = string(assets) - userProof.TotalDebt = account.TotalDebt.String() - userProof.TotalEquity = account.TotalEquity.String() - - userConfig.Arrangement = account.AccountIndex - userConfig.UniqueIdentification = hex.EncodeToString(account.AccountId) - userConfig.MerkleProofEncode = proof - userConfig.TreeRootHash = root - userConfig.AssetDetails = account.Assets - userConfig.TotalAssetDebt = account.TotalDebt - userConfig.TotalAssetEquity = account.TotalEquity - configSerial, err := json.Marshal(userConfig) - if err != nil { - panic(err.Error()) - } - userProof.Config = string(configSerial) - return &userProof -} - -func OpenUserProofTable(userConfig *config.Config) UserProofModel { - newLogger := logger.New( - log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer - logger.Config{ - SlowThreshold: 60 * time.Second, // Slow SQL threshold - LogLevel: logger.Silent, // Log level - IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger - Colorful: false, // Disable color - }, - ) - db, err := gorm.Open(mysql.Open(userConfig.MysqlDataSource), &gorm.Config{ - Logger: newLogger, - }) - if err != nil { - panic(err.Error()) - } - userProofTable := NewUserProofModel(db, userConfig.DbSuffix) - userProofTable.CreateUserProofTable() - return userProofTable -} From 28a68dc543d1a728c4115045e69019fa3a70ba1b Mon Sep 17 00:00:00 2001 From: Recepxx34 <163409185+Recepxx34@users.noreply.github.com> Date: Wed, 20 Nov 2024 23:01:43 +0300 Subject: [PATCH 2/2] package userproof_service import ( "encoding/hex" "encoding/json" "fmt" "gate-zkmerkle-proof/config" "gate-zkmerkle-proof/global" "gate-zkmerkle-proof/utils" "github.com/consensys/gnark-crypto/ecc/bn254/fr/poseidon" zk_smt "github.com/gatechain/gate-zk-smt" "gorm.io/driver/mysql" "gorm.io/gorm" "gorm.io/gorm/logger" "io/ioutil" "log" "os" "time" ) func HandleUserData(userProofConfig *config.Config) []utils.AccountInfo { startTime := time.Now().UnixMilli() accounts, _, err := utils.ReadUserAssets(userProofConfig.UserDataFile) if err != nil { panic(err.Error()) } endTime := time.Now().UnixMilli() fmt.Println("handle user data cost ", endTime-startTime, " ms") return accounts } type AccountLeave struct { hash []byte index uint32 } func ComputeAccountRootHash(userProofConfig *config.Config) { accountTree, err := utils.NewAccountTree("memory", "") if err != nil { panic(err.Error()) } accounts, _, err := utils.ReadUserAssets(userProofConfig.UserDataFile) if err != nil { panic(err.Error()) } startTime := time.Now().UnixMilli() totalOpsNumber := len(accounts) fmt.Println("total ops number is ", totalOpsNumber) chs := make(chan AccountLeave, 1000) workers := 32 results := make(chan bool, workers) averageAccounts := (totalOpsNumber + workers - 1) / workers actualWorkers := 0 for i := 0; i < workers; i++ { srcAccountIndex := i * averageAccounts destAccountIndex := (i + 1) * averageAccounts if destAccountIndex > totalOpsNumber { destAccountIndex = totalOpsNumber } go CalculateAccountHash(accounts[srcAccountIndex:destAccountIndex], chs, results) if destAccountIndex == totalOpsNumber { actualWorkers = i + 1 break } } fmt.Println("actual workers is ", actualWorkers) quit := make(chan bool, 1) go CalculateAccountTreeRoot(chs, &accountTree, quit) for i := 0; i < actualWorkers; i++ { <-results } close(chs) <-quit endTime := time.Now().UnixMilli() fmt.Println("user account tree generation cost ", endTime-startTime, " ms") fmt.Printf("account tree root %x\n", accountTree.Root()) } func CalculateAccountHash(accounts []utils.AccountInfo, chs chan<- AccountLeave, res chan<- bool) { poseidonHasher := poseidon.NewPoseidon() for i := 0; i < len(accounts); i++ { chs <- AccountLeave{ hash: utils.AccountInfoToHash(&accounts[i], &poseidonHasher), index: accounts[i].AccountIndex, } } res <- true } func CalculateAccountTreeRoot(accountLeaves <-chan AccountLeave, accountTree *zk_smt.SparseMerkleTree, quit chan<- bool) { num := 0 for accountLeaf := range accountLeaves { (*accountTree).Set(uint64(accountLeaf.index), accountLeaf.hash) num++ if num%100000 == 0 { fmt.Println("for now, already set ", num, " accounts in tree") } } quit <- true } func Handler() { global.Cfg = &config.Config{} jsonFile, err := ioutil.ReadFile("./config/config.json") if err != nil { panic(fmt.Sprintf("load config err : %s", err.Error())) } err = json.Unmarshal(jsonFile, global.Cfg) if err != nil { panic(err.Error()) } userProofConfig := global.Cfg accountTree, err := utils.NewAccountTree(userProofConfig.TreeDB.Driver, userProofConfig.TreeDB.Option.Addr) accounts := HandleUserData(userProofConfig) fmt.Println("num", len(accounts)) userProofModel := OpenUserProofTable(userProofConfig) latestAccountIndex, err := userProofModel.GetLatestAccountIndex() if err != nil && err != utils.DbErrNotFound { panic(err.Error()) } if err == nil { latestAccountIndex += 1 } accountTreeRoot := hex.EncodeToString(accountTree.Root()) jobs := make(chan Job, 1000) nums := make(chan int, 1) results := make(chan *UserProof, 1000) for i := 0; i < 1; i++ { go worker(jobs, results, nums, accountTreeRoot) } quit := make(chan int, 1) for i := 0; i < 1; i++ { go WriteDB(results, userProofModel, quit, latestAccountIndex) } for i := int(latestAccountIndex); i < len(accounts); i++ { leaf, err := accountTree.Get(uint64(i), nil) if err != nil { panic(err.Error()) } proof, err := accountTree.GetProof(uint64(accounts[i].AccountIndex)) if err != nil { panic(err.Error()) } jobs <- Job{ account: &accounts[i], proof: proof, leaf: leaf, } } close(jobs) totalCounts := int(latestAccountIndex) for i := 0; i < 1; i++ { num := <-nums totalCounts += num fmt.Println("totalCounts", totalCounts) } if totalCounts != len(accounts) { fmt.Println("totalCounts actual:expected", totalCounts, len(accounts)) panic("mismatch num") } close(results) for i := 0; i < 1; i++ { <-quit } fmt.Println("userproof service run finished...") } func WriteDB(results <-chan *UserProof, userProofModel UserProofModel, quit chan<- int, latestAccountIndex uint32) { index := 0 proofs := make([]UserProof, 100) num := int(latestAccountIndex) for proof := range results { proofs[index] = *proof index += 1 if index%100 == 0 { error := userProofModel.CreateUserProofs(proofs) if error != nil { panic(error.Error()) } num += 100 if num%100000 == 0 { fmt.Println("write ", num, "proof to db") } index = 0 } } proofs = proofs[:index] if index > 0 { fmt.Println("write ", len(proofs), "proofs to db") userProofModel.CreateUserProofs(proofs) num += index } fmt.Println("total write ", num) quit <- 0 } type Job struct { account *utils.AccountInfo proof [][]byte leaf []byte } func worker(jobs <-chan Job, results chan<- *UserProof, nums chan<- int, root string) { num := 0 for job := range jobs { userProof := ConvertAccount(job.account, job.leaf, job.proof, root) results <- userProof num += 1 } nums <- num } func ConvertAccount(account *utils.AccountInfo, leafHash []byte, proof [][]byte, root string) *UserProof { var userProof UserProof var userConfig UserConfig userProof.AccountIndex = account.AccountIndex userProof.AccountId = hex.EncodeToString(account.AccountId) userProof.AccountLeafHash = hex.EncodeToString(leafHash) proofSerial, err := json.Marshal(proof) userProof.Proof = string(proofSerial) assets, err := json.Marshal(account.Assets) if err != nil { panic(err.Error()) } userProof.Assets = string(assets) userProof.TotalDebt = account.TotalDebt.String() userProof.TotalEquity = account.TotalEquity.String() userConfig.Arrangement = account.AccountIndex userConfig.UniqueIdentification = hex.EncodeToString(account.AccountId) userConfig.MerkleProofEncode = proof userConfig.TreeRootHash = root userConfig.AssetDetails = account.Assets userConfig.TotalAssetDebt = account.TotalDebt userConfig.TotalAssetEquity = account.TotalEquity configSerial, err := json.Marshal(userConfig) if err != nil { panic(err.Error()) } userProof.Config = string(configSerial) return &userProof } func OpenUserProofTable(userConfig *config.Config) UserProofModel { newLogger := logger.New( log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer logger.Config{ SlowThreshold: 60 * time.Second, // Slow SQL threshold LogLevel: logger.Silent, // Log level IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger Colorful: false, // Disable color }, ) db, err := gorm.Open(mysql.Open(userConfig.MysqlDataSource), &gorm.Config{ Logger: newLogger, }) if err != nil { panic(err.Error()) } userProofTable := NewUserProofModel(db, userConfig.DbSuffix) userProofTable.CreateUserProofTable() return userProofTable } --- .vscode/extensions.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .vscode/extensions.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..763e3132 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "mechatroner.rainbow-csv" + ] +} \ No newline at end of file