dockerization

.deploy.env

@@ -1,2 +1,2 @@
 PROJECT_SECRET=projects/flare-network-staging/secrets/data-provider-4_flare-price-provider_accountPrivateKey/versions/latest
-rpcUrl=wss://songbird-api.flare.network/ext/bc/C/ws
+RPC_URL=wss://songbird-api.flare.network/ext/bc/C/ws

.dev.env

@@ -0,0 +1,9 @@
+# include env files (evi1m3 extension)
+DOTENV_INCLUDE = .env; .deploy.env
+
+# Set to production when deploying to production
+NODE_ENV=development
+
+rpcUrl=ws://127.0.0.1:9650/ext/bc/C/ws
+
+accountPrivateKey=0x8cb860f6f44c45d43a1914b0c9da321dd5f9bc4e19e70686df2a1fca3d92beb8

.env

@@ -0,0 +1,2 @@
+# Set to production when deploying to production
+NODE_ENV=production

Dockerfile

@@ -13,4 +13,4 @@ RUN yarn build
 
 EXPOSE 3000
 
-CMD node dist/DataProvider.js
+CMD node dist/DataProvider.js -c configs/config.json

configs/config-scdev.json

@@ -2,8 +2,8 @@
     "accountPrivateKey": "0xc5e8f61d1ab959b397eecc0a37a6517b8e67a0e7cf1f4bce5591f3ed80199122",
     "rpcUrl": "http://127.0.0.1:9650/ext/bc/C/rpc",
     "priceSubmitterContractAddress": "0x1000000000000000000000000000000000000003",
-    "submitOffset": 170000,
-    "revealOffset": 1000,
+    "submitOffset": 165000,
+    "revealOffset": 8000,
     "whitelist": false,
     "trusted": true,
     "gasPrice": "3000000000000",

lib/DataProvider.ts

@@ -7,7 +7,9 @@ import { PriceSubmitter } from '../typechain-web3-v1/PriceSubmitter';
 import { VoterWhitelister } from '../typechain-web3-v1/VoterWhitelister';
 import { DataProviderConfiguration } from './Configuration';
 import { DataProviderData } from './DataProviderData';
+import { DotEnvExt } from './DotEnvExt';
 import { EpochSettings } from './EpochSettings';
+import { fetchSecret } from './GoogleSecret';
 import { PriceInfo } from './PriceInfo';
 import * as impl from './PriceProviderImpl';
 import { bigNumberToMillis, getContract, getLogger, getProvider, getWeb3, getWeb3Contract, getWeb3Wallet, priceHash, waitFinalize3Factory } from './utils';
@@ -35,8 +37,10 @@ let conf: DataProviderConfiguration = JSON.parse(fs.readFileSync(args['config'])
 const logger = getLogger();
 
 const provider = getProvider(conf.rpcUrl);
-const web3 = getWeb3(conf.rpcUrl) as Web3;
-const account = getWeb3Wallet(web3, conf.accountPrivateKey);
+//const web3 = getWeb3(conf.rpcUrl) as Web3;
+//const account = getWeb3Wallet(web3, conf.accountPrivateKey);
+let web3 : any
+let account : any
 
 let priceSubmitterWeb3Contract: PriceSubmitter;
 let priceSubmitterContract: any;
@@ -371,7 +375,56 @@ function setupEvents() {
     })
 }
 
-async function runDataProvider() {
+async function runDataProvider() 
+{
+    DotEnvExt()
+
+    const configData : string = ""
+    let accountPrivateKey : string =""
+
+    if( process.env.NODE_ENV === "production")
+    {
+        logger.info(`Starting Flare Price Provider`)
+
+        if( process.env.PROJECT_SECRET===undefined )
+        {
+            logger.info(`   * account read from .env`)
+            accountPrivateKey = (conf.accountPrivateKey as string)
+        }
+        else
+        {
+            logger.info(`   * account read from secret`)
+            accountPrivateKey = (await fetchSecret( process.env.PROJECT_SECRET as string ) as string)
+        }
+    }
+    else
+    {
+        logger.info(`Starting Flare Price Provider [developer mode]`)
+        logger.info(`   * account read from .env`)
+
+        accountPrivateKey = (conf.accountPrivateKey as string)
+    }
+
+    // rpcUrl from conf
+    let rpcUrl : string = ""
+    if( process.env.RPC_URL!==undefined )
+    {
+        // rpcUrl from .env if it exsists
+        rpcUrl = (process.env.RPC_URL as string)
+        logger.info(`   * rpcUrl from .env '${rpcUrl}'`)
+    }
+    else
+    {
+       rpcUrl = (conf.rpcUrl as string)
+       logger.info(`   * rpcUrl from conf '${rpcUrl}'`)
+    }
+
+
+    web3 = getWeb3(rpcUrl) as Web3;
+    account = getWeb3Wallet(web3, accountPrivateKey );
+
+
+
 
     priceSubmitterWeb3Contract = await getWeb3Contract(web3, conf.priceSubmitterContractAddress, "PriceSubmitter");
     priceSubmitterContract = await getContract(provider, conf.priceSubmitterContractAddress, "PriceSubmitter");

lib/DotEnvExt.ts

@@ -0,0 +1,30 @@
+import dotenv from "dotenv";
+
+function dotenvInclude()
+{
+    const include = process.env.DOTENV_INCLUDE
+
+    if( include===null || include===undefined ) return
+
+    for(const inc of include.split( ';' ) )
+    {
+        console.log( `[dotenv include '${inc}']` )
+        dotenv.config( { path: inc } );
+    }
+}
+
+export function DotEnvExt()
+{
+    // initialize configuration
+    if( process.env.DOTENV==="DEV" )
+    {
+        console.log( "[loading development env (.dev.env)]")
+        dotenv.config( { path: ".dev.env" } );
+    }
+    else
+    {
+        dotenv.config();
+    }
+
+    dotenvInclude()
+}

lib/GoogleSecret.ts

@@ -0,0 +1,31 @@
+import { SecretManagerServiceClient } from "@google-cloud/secret-manager";
+
+export async function fetchSecret(name: string)
+{
+    // console.log( `      * fetchSecret '${name}'`)
+
+    try {
+
+        const client = new SecretManagerServiceClient();
+        const [version] = await client.accessSecretVersion({name});
+
+        const payload = version.payload?.data?.toString();
+
+        if( !payload )
+        {
+            // throw new Error(`Failed to fetch secret "${name}"`);
+            console.log( `      *** ERROR: failed to fetch secret "${name}"` )
+
+            return "";
+        }
+
+        // console.log( `secret is "${payload}"` )
+
+        return payload
+    }
+    catch( error ) {
+        console.log( `      *** ERROR: failed to fetch secret "${name}"   ${error}` )
+    }
+
+    return undefined
+}

package.json

@@ -5,10 +5,12 @@
   "author": "flare",
   "license": "MIT",
   "dependencies": {
+    "@google-cloud/secret-manager": "3.10.1",
     "@types/glob": "7.1.4",
     "ccxt": "1.50.74",
     "ccxws": "0.43.1",
     "ethers": "5.1.4",
+    "dotenv": "10.0.0",
     "glob": "7.1.7",
     "random-number-csprng": "1.0.2",
     "ts-node": "9.1.1",

scripts/docker-build.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
-docker build --rm --pull -f "./Dockerfile" -t "flarenetwork/flare-price-collector:latest" "."
+docker build --rm --pull -f "./Dockerfile" -t "flarenetwork/flare-price-provider:latest" "."
 
-docker push flarenetwork/flare-price-collector:latest
+docker push flarenetwork/flare-price-provider:latest

scripts/docker-compose.yml

@@ -2,9 +2,11 @@ version: '3.6'
 
 services:
 
-  heartbeat-daemon:
-    image: evi1m3/heartbeat-daemon:latest
+  flare-price-provider:
+    image: flarenetwork/flare-price-provider:latest
     restart: always
 
+    env_file: .deploy.env    
+
     ports:
       - 3000:3000

scripts/docker-deploy.sh

@@ -1,14 +1,14 @@
 #!/bin/bash
 USER=ubuntu
 
-# 4
-SERVER=34.141.58.90
-# 2
+# data-provider-2
 #SERVER=34.107.95.240
+# data-provider-4
+SERVER=34.141.58.90
 
 BRANCH=master
 
-WORKDIR="heartbeat-daemon"
+WORKDIR="flare-price-provider"
 
 
 RED='\033[0;31m'
@@ -21,19 +21,25 @@ NCNORMAL="${NC}$(tput sgr0)"
 
 echo -e "${GREENBOLD}[1/3] Setup server $SERVER${NC}"
 # Enable docker to run witout sudo (on server)
-ssh -n $USER@$SERVER "sudo apt install -y docker-compose"
-ssh -n $USER@$SERVER "sudo groupadd docker; sudo usermod -aG docker $USER; newgrp docker"
+#ssh -n $USER@$SERVER "sudo apt install -y docker-compose"
+#ssh -n $USER@$SERVER "sudo groupadd docker; sudo usermod -aG docker $USER; newgrp docker"
 
 # Create install folder if they do not exist (on server)
 ssh -n $USER@$SERVER "mkdir -p $WORKDIR"
 
 # Copy files to server
 echo -e "${GREENBOLD}[2/3] Copying files on server $SERVER${NC}"
+echo -e "   docker-compose.yml"
 scp scripts/docker-compose.yml $USER@$SERVER:$WORKDIR || { echo 'scp failed' ; exit 1; }
+echo -e "   .deploy.env"
+scp .deploy.env $USER@$SERVER:$WORKDIR || { echo 'scp failed' ; exit 1; }
 
 # Run yarn in $WORKDIR 
 echo -e "${GREENBOLD}[3/3] Installation done - starting docker${NC}"
-ssh -n $USER@$SERVER "cd $WORKDIR; docker-compose up"
+echo -e "   docker pull and restart"
+ssh -n $USER@$SERVER "cd $WORKDIR; docker-compose pull; docker-compose restart"
+echo -e "   docker up"
+ssh -n $USER@$SERVER "cd $WORKDIR; docker-compose up -d"
 
 # Restart the app as a service.
 echo -e "${GREENBOLD}Done.${NC}"