Merge pull request #1217 from ApolloFoundation/stage

Apollo Blockchain 1.51.0 Release (sharding update)
ApolloFoundation · Jun 29, 2022 · 47b2174 · 47b2174
2 parents 83b1332 + 2450e0a
commit 47b2174
Show file tree

Hide file tree

Showing 69 changed files with 1,713 additions and 1,080 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,27 @@
+# Changelog
+
+All notable changes to this project will be documented in this file. Dates are displayed in UTC.
+
+### [1.51.0](https://github.com/ApolloFoundation/Apollo/releases/tag/1.51.0)
+
+> 24 June 2022
+
+Significant sharding speedup and robustness upgrade, general refactoring and tests fixes
+
+- Fix PhasingPollServiceTest [`c393038`](https://github.com/ApolloFoundation/Apollo/commit/c393038d2e6b335596278ee09b24b654bafa9c89)
+- APL-2142 - Speedup sharding tn2 [`#1213`](https://github.com/ApolloFoundation/Apollo/pull/1213)
+- APL-2127 - Speedup account export [`#1208`](https://github.com/ApolloFoundation/Apollo/pull/1208)
+- Fixed smc tests [`#1205`](https://github.com/ApolloFoundation/Apollo/pull/1205)
+- Fix active phasings fetch for shard export [`#1192`](https://github.com/ApolloFoundation/Apollo/pull/1192)
+- Fix dgs/phasing/fts tests, add active phasings consistency check [`d5c1b99`](https://github.com/ApolloFoundation/Apollo/commit/d5c1b99b56a42a47f4c34d63383b3acdc185b843)
+- Add CHANGELOG
+
+
+### [1.50.36](https://github.com/ApolloFoundation/Apollo/releases/tag/1.50.36)
+
+> 29 April 2022
+
+Updater's certificates cleanup and update
+
+- Updater certificates update [`#1215`](https://github.com/ApolloFoundation/Apollo/pull/1215)
+- Cleanup unused update certificates [`a3d6816`](https://github.com/ApolloFoundation/Apollo/commit/a3d6816dc738c3e0a60991cc37bc295716061d76)
diff --git a/README.md b/README.md
@@ -85,7 +85,7 @@ OS name: "linux", version: "5.9.12-200.fc33.x86_64", arch: "amd64", family: "uni
 
 ### Installation artefacts
 
-The final artifact that is ready to install and run located in the ___apl-exec/target___ directory and has name like  ___apollo-blockchain-1.50.36-NoOS-NoArch.zip___.
+The final artifact that is ready to install and run located in the ___apl-exec/target___ directory and has name like  ___apollo-blockchain-1.51.0-NoOS-NoArch.zip___.
 
 Unzip it to some location and run by scripts in ___ApolloWallet/apollo-blockchain/bin___ directory.
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.50.36
+1.51.0
diff --git a/apl-api/pom.xml b/apl-api/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>com.apollocurrency</groupId>
     <artifactId>apollo-blockchain</artifactId>
-    <version>1.50.36</version>
+    <version>1.51.0</version>
   </parent>
 
   <artifactId>apl-api</artifactId>

diff --git a/apl-api2/pom.xml b/apl-api2/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>com.apollocurrency</groupId>
     <artifactId>apollo-blockchain</artifactId>
-    <version>1.50.36</version>
+    <version>1.51.0</version>
   </parent>
 
   <artifactId>apl-api2</artifactId>

diff --git a/apl-bom/pom.xml b/apl-bom/pom.xml
@@ -5,7 +5,7 @@
 
   <groupId>com.apollocurrency</groupId>
   <artifactId>apl-bom</artifactId>
-  <version>1.50.36</version>
+  <version>1.51.0</version>
   <packaging>pom</packaging>
   <name>apl-bom</name>
 

diff --git a/apl-conf/pom.xml b/apl-conf/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.apollocurrency</groupId>
     <artifactId>apollo-blockchain</artifactId>
-    <version>1.50.36</version>
+    <version>1.51.0</version>
   </parent>
   <artifactId>apl-conf</artifactId>
   <name>apl-conf</name>

diff --git a/apl-conf/src/main/resources/conf-tn15/chains.json b/apl-conf/src/main/resources/conf-tn15/chains.json
@@ -50,7 +50,34 @@
         },
         "shardingSettings": {
           "enabled": true,
-          "frequency": 2000
+          "frequency": 100000
+        },
+        "smcSettings": {
+          "masterAccountPublicKey": "dd8a4621988974e669d115b5e0d48234eed145ff2ac94ec1012805459470fe1e",
+          "fuelLimitMinValue": 10000000,
+          "fuelLimitMaxValue": 30000000000,
+          "fuelPriceMinValue": 10000,
+          "fuelPriceMaxValue": 100000000
+        }
+      },
+      {
+        "height": 600000,
+        "maxNumberOfTransactions": 255,
+        "maxArbitraryMessageLength": 160,
+        "maxNumberOfChildAccounts": 10,
+        "blockTime": 2,
+        "maxBlockTimeLimit": 3,
+        "minBlockTimeLimit": 1,
+        "maxBalance": 30000000000,
+        "consensusSettings": {
+          "adaptiveForgingSettings": {
+            "enabled": "true",
+            "adaptiveBlockTime": 10
+          }
+        },
+        "shardingSettings": {
+          "enabled": true,
+          "frequency": 10000
         },
         "smcSettings": {
           "masterAccountPublicKey": "dd8a4621988974e669d115b5e0d48234eed145ff2ac94ec1012805459470fe1e",

diff --git a/apl-conf/src/main/resources/conf-tn2/chains.json b/apl-conf/src/main/resources/conf-tn2/chains.json
@@ -22,8 +22,8 @@
       "dexReopenPendingOrdersHeight": 0,
       "dexExpiredContractWithFinishedPhasingHeightAndStep3": 0,
       "failedTransactionsAcceptanceHeight": 0,
-      "transactionV2Height": 421500,
-      "smartContractTransactionsHeight": 421500
+      "transactionV2Height": 0,
+      "smartContractTransactionsHeight": 0
     },
     "blockchainProperties": [
       {
@@ -85,6 +85,57 @@
             }
           ]
         }
+      },
+      {
+        "height": 201000,
+        "maxNumberOfTransactions": 255,
+        "maxArbitraryMessageLength": 160,
+        "maxNumberOfChildAccounts": 10,
+        "blockTime": 2,
+        "maxBlockTimeLimit": 3,
+        "minBlockTimeLimit": 1,
+        "maxBalance": 30000000000,
+        "consensusSettings": {
+          "adaptiveForgingSettings": {
+            "enabled": "true",
+            "adaptiveBlockTime": 10
+          }
+        },
+        "shardingSettings": {
+          "enabled": true,
+          "frequency": 200
+        },
+        "smcSettings": {
+          "masterAccountPublicKey": "f9c6cd2cebff46137f3ea611683026b94f4bd1b39abbab480478e10766b2c608",
+          "fuelLimitMinValue": 10000000,
+          "fuelLimitMaxValue": 30000000000,
+          "fuelPriceMinValue": 10000,
+          "fuelPriceMaxValue": 100000000
+        },
+        "transactionFeeSettings": {
+          "feeRates": [
+            {
+              "type": 1,
+              "subType": 1,
+              "baseFee": "1000"
+            },
+            {
+              "type": 1,
+              "subType": 6,
+              "baseFee": "5"
+            },
+            {
+              "type": 1,
+              "subType": 7,
+              "baseFee": "5"
+            },
+            {
+              "type": 1,
+              "subType": 8,
+              "baseFee": "5"
+            }
+          ]
+        }
       }
     ]
   }

diff --git a/apl-core/pom.xml b/apl-core/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>com.apollocurrency</groupId>
     <artifactId>apollo-blockchain</artifactId>
-    <version>1.50.36</version>
+    <version>1.51.0</version>
   </parent>
   <artifactId>apl-core</artifactId>
   <name>apl-core</name>
@@ -345,13 +345,6 @@
     </dependency>
   </dependencies>
 
-  <repositories>
-    <repository>
-      <id>binary-repo</id>
-      <url>https://dl.bintray.com/ethereum/maven/</url>
-    </repository>
-  </repositories>
-
   <build>
     <plugins>
       <plugin>

diff --git a/...re/src/main/java/com/apollocurrency/aplwallet/apl/core/app/runnable/GetMoreBlocksJob.java b/...re/src/main/java/com/apollocurrency/aplwallet/apl/core/app/runnable/GetMoreBlocksJob.java
@@ -16,6 +16,7 @@
 import com.apollocurrency.aplwallet.apl.core.model.Block;
 import com.apollocurrency.aplwallet.apl.core.model.BlockchainProcessorState;
 import com.apollocurrency.aplwallet.apl.core.model.PeerBlock;
+import com.apollocurrency.aplwallet.apl.core.model.Transaction;
 import com.apollocurrency.aplwallet.apl.core.peer.Peer;
 import com.apollocurrency.aplwallet.apl.core.peer.PeerNotConnectedException;
 import com.apollocurrency.aplwallet.apl.core.peer.PeerState;
@@ -575,10 +576,12 @@ private void processFork(final Peer peer, final List<Block> forkBlocks, final Bl
         }
 
         if (pushedForkBlocks == 0) {
-            log.debug("Didn't accept any blocks, pushing back my previous blocks");
+            log.debug("Didn't accept any blocks, pushing back my {} previous blocks", myPoppedOffBlocks.size());
             for (int i = myPoppedOffBlocks.size() - 1; i >= 0; i--) {
                 Block block = myPoppedOffBlocks.remove(i);
                 try {
+                    // reset tx fail statuses for our block to push it again
+                    block.getTransactions().forEach(Transaction::resetFail);
                     blockchainProcessor.pushBlock(block);
                 } catch (BlockchainProcessor.BlockNotAcceptedException | AplBlockException e) {
                     log.error("Popped off block no longer acceptable: " + blockSerializer.getJSONObject(block).toJSONString(), e);

diff --git a/...om/apollocurrency/aplwallet/apl/core/dao/state/account/AccountGuaranteedBalanceTable.java b/...om/apollocurrency/aplwallet/apl/core/dao/state/account/AccountGuaranteedBalanceTable.java
@@ -15,6 +15,7 @@
 import com.apollocurrency.aplwallet.apl.util.annotation.DatabaseSpecificDml;
 import com.apollocurrency.aplwallet.apl.util.annotation.DmlMarker;
 import com.apollocurrency.aplwallet.apl.util.injectable.PropertiesHolder;
+import lombok.extern.slf4j.Slf4j;
 
 import javax.enterprise.event.Event;
 import javax.inject.Inject;
@@ -30,6 +31,7 @@
 import java.util.stream.IntStream;
 
 @Singleton
+@Slf4j
 public class AccountGuaranteedBalanceTable extends DerivedDbTable<AccountGuaranteedBalance> {
 
     private static final String TABLE_NAME = "account_guaranteed_balance";
@@ -152,6 +154,7 @@ public void addToGuaranteedBalanceATM(long accountId, long amountATM, int blockc
         if (amountATM <= 0) {
             return;
         }
+        log.info("Add to account {} guaranteed balance {} amount at height {}", accountId, amountATM, blockchainHeight);
         TransactionalDataSource dataSource = databaseManager.getDataSource();
         try (Connection con = dataSource.getConnection();
              PreparedStatement pstmtSelect = con.prepareStatement("SELECT additions FROM account_guaranteed_balance "
@@ -167,7 +170,11 @@ public void addToGuaranteedBalanceATM(long accountId, long amountATM, int blockc
             try (ResultSet rs = pstmtSelect.executeQuery()) {
                 long additions = amountATM;
                 if (rs.next()) {
-                    additions = Math.addExact(additions, rs.getLong(ADDITIONS_COLUMN_NAME));
+
+                    long existingAdditions = rs.getLong(ADDITIONS_COLUMN_NAME);
+                    additions = Math.addExact(additions, existingAdditions);
+                    log.debug("Add existing guaranteed balance additions for account {} at height {}, additions {}, sum {}", accountId
+                    , blockchainHeight, existingAdditions, additions);
                 }
                 pstmtUpdate.setLong(1, accountId);
                 pstmtUpdate.setLong(2, additions);

diff --git a/...e/src/main/java/com/apollocurrency/aplwallet/apl/core/dao/state/account/AccountTable.java b/...e/src/main/java/com/apollocurrency/aplwallet/apl/core/dao/state/account/AccountTable.java
@@ -1,8 +1,9 @@
 /*
- * Copyright © 2018-2019 Apollo Foundation
+ * Copyright © 2018-2022 Apollo Foundation
  */
 package com.apollocurrency.aplwallet.apl.core.dao.state.account;
 
+import com.apollocurrency.aplwallet.apl.core.dao.state.derived.MinMaxValue;
 import com.apollocurrency.aplwallet.apl.core.dao.state.derived.VersionedDeletableEntityDbTable;
 import com.apollocurrency.aplwallet.apl.core.dao.state.keyfactory.DbKey;
 import com.apollocurrency.aplwallet.apl.core.db.DatabaseManager;
@@ -22,7 +23,6 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.List;
-import java.util.Optional;
 
 import static com.apollocurrency.aplwallet.apl.core.utils.CollectionUtil.toList;
 
@@ -205,16 +205,13 @@ public List<Account> getRecentAccounts(int limit) {
 
     }
 
-    private Optional<Account> selectLastExisting(Connection connection, long id) throws SQLException {
-        try (PreparedStatement pstm = connection.prepareStatement("SELECT * FROM " + table + " WHERE id = ? ORDER BY db_id DESC LIMIT 1")) {
-            pstm.setLong(1, id);
-            try (ResultSet rs = pstm.executeQuery()) {
-                if (rs.next()) {
-                    return Optional.of(load(connection, rs, accountDbKeyFactory.newKey(id)));
-                } else {
-                    return Optional.empty();
-                }
-            }
-        }
+    /**
+     * Retrieve {@link MinMaxValue} for account id assuming no duplicates before the given height (trim operation executed)
+     * <p>Note: DB_ID based {@link MinMaxValue} is ignored, because sorted set of unique account ids acts the same </p>
+     * @param height blockchain height which acts as an upper bound for {@link MinMaxValue} retrieval
+     */
+    @Override
+    public MinMaxValue getMinMaxValue(int height) {
+        return super.getMinMaxValue(height, "id");
     }
 }
diff --git a/...c/main/java/com/apollocurrency/aplwallet/apl/core/dao/state/phasing/PhasingPollTable.java b/...c/main/java/com/apollocurrency/aplwallet/apl/core/dao/state/phasing/PhasingPollTable.java
@@ -205,19 +205,25 @@ public int getAllPhasedTransactionsCount() throws SQLException {
         }
     }
 
-    public List<TransactionDbInfo> getActivePhasedTransactionDbIds(int height) throws SQLException {
+    public List<TransactionDbInfo> getActivePhasedTransactionDbIdsAfterHeight(int height) throws SQLException {
         List<TransactionDbInfo> excludeInfos = new ArrayList<>();
         TransactionalDataSource dataSource = databaseManager.getDataSource();
+        int blockTimestamp = blockTimestamp(height);
+        if (blockTimestamp == 0) {
+            throw new IllegalStateException("Block with height " + height + " was not found or has unacceptable timestamp");
+        }
         try (Connection conn = dataSource.getConnection();
              PreparedStatement pstmt = conn.prepareStatement(
                  "SELECT db_id, id FROM transaction WHERE id IN " +
-                     "(SELECT id FROM phasing_poll WHERE height < ? AND id not in " +
-                     "(SELECT id FROM phasing_poll_result WHERE height <= ?))")) {
+                     "(SELECT id FROM phasing_poll WHERE height <= ? AND (finish_height > ? " +
+                     " OR finish_height = -1) AND (finish_time > ? OR finish_time = -1))")) {
             pstmt.setInt(1, height);
             pstmt.setInt(2, height);
+            pstmt.setInt(3, blockTimestamp);
             try (ResultSet rs = pstmt.executeQuery()) {
                 while (rs.next()) {
-                    excludeInfos.add(new TransactionDbInfo(rs.getLong("db_id"), rs.getLong("id")));
+                    long id = rs.getLong("id");
+                    excludeInfos.add(new TransactionDbInfo(rs.getLong("db_id"), id));
                 }
             }
         }
@@ -261,14 +267,19 @@ public void trim(int height) {
         }
     }
 
+    /**
+     * Return all finished phasings before height (exclusive, because it's required trim behavior)
+     * @param height height of the blockchain before which finished polls should be searched
+     * @return lazy db iterator of the resulting finished polls
+     */
     private DbIterator<PhasingPoll> getAllFinishedPolls(int height) {
         Connection con = null;
         try {
-            int blockTimestamp = blockTimestamp(height);
             con = databaseManager.getDataSource().getConnection();
+            int blockTimestamp = blockTimestamp(height - 1);
             String query = "SELECT * FROM phasing_poll WHERE finish_height < ? and finish_height <> -1";
             if (blockTimestamp != 0) {
-                query += " or finish_time < ? and finish_time <> -1";
+                query += " or finish_time <= ? and finish_time <> -1";
             }
             PreparedStatement pstmt = con.prepareStatement(query);
             pstmt.setInt(1, height);

diff --git a/...src/main/java/com/apollocurrency/aplwallet/apl/core/entity/state/phasing/PhasingPoll.java b/...src/main/java/com/apollocurrency/aplwallet/apl/core/entity/state/phasing/PhasingPoll.java
@@ -7,11 +7,13 @@
 import com.apollocurrency.aplwallet.apl.core.app.VoteWeighting;
 import com.apollocurrency.aplwallet.apl.core.entity.state.poll.AbstractPoll;
 import com.apollocurrency.aplwallet.apl.crypto.Convert;
+import lombok.ToString;
 
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 
+@ToString(callSuper = true)
 public class PhasingPoll extends AbstractPoll {
 
     private final long quorum;

diff --git a/...ava/com/apollocurrency/aplwallet/apl/core/service/blockchain/BlockchainProcessorImpl.java b/...ava/com/apollocurrency/aplwallet/apl/core/service/blockchain/BlockchainProcessorImpl.java
@@ -376,6 +376,8 @@ public void processPeerBlock(JSONObject request) throws AplException {
                         log.info("Last block {} was replaced by {} at height {}", lastBlock.getStringId(), peerBlock.getStringId(), blockchain.getHeight());
                     } catch (BlockNotAcceptedException | AplBlockException e) {
                         log.info("Replacement block failed to be accepted, pushing back our last block");
+                        // reset tx fail statuses for our block to push it again
+                        lastBlock.getTransactions().forEach(Transaction::resetFail);
                         pushBlock(lastBlock);
                         transactionProcessor.processLater(peerBlock.getTransactions());
                     }

diff --git a/...main/java/com/apollocurrency/aplwallet/apl/core/service/state/PhasingPollServiceImpl.java b/...main/java/com/apollocurrency/aplwallet/apl/core/service/state/PhasingPollServiceImpl.java
@@ -349,7 +349,7 @@ public PhasingVote getVote(long phasedTransactionId, long voterId) {
     @Override
     public List<TransactionDbInfo> getActivePhasedTransactionDbInfoAtHeight(int height) {
         try {
-            return phasingPollTable.getActivePhasedTransactionDbIds(height);
+            return phasingPollTable.getActivePhasedTransactionDbIdsAfterHeight(height);
         } catch (SQLException e) {
             throw new RuntimeException(e.toString(), e);
         }