diff --git a/2miner-monitoring b/2miner-monitoring new file mode 100755 index 0000000..19bf61c Binary files /dev/null and b/2miner-monitoring differ diff --git a/README.md b/README.md index aa26d50..88dee04 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,73 @@ -# 2miner-monitoring-go +# Minotor This go application is an upgrade of the 2miner-monitoring. His goal is to be able to crawl more data than the original 2miner-monitoring. -If you want to monitore only yourslef, you should considerate 2miner-monitoring. +If you want to monitore only yourself, you should considerate 2miner-monitoring. This application harvest data from 2miners API (https://apidoc.2miners.com/#/) It send the data to an elasticsearch and on my case grafana is used to retrieves visualisation -## How to run it +## Important Url -### Config file +API => https://load01.ether-source.fr:8999 + +Grafana => https://load01.ether-source.fr + +## How it works + +### The Backend + +Gin is used to create a http server with many handler +Gocron is used to proc all call at a given time to harvest data + +### The FrontEnd + +There is a grafana to display everything. No registration is needed for viewer only. +If you wanna do your own dashboard/alert, you need to register. + +### Service connected + +- 2miners +- Hiveos +- Etherscan +- Hashrate.no +- Coingecko -You have ton configure the config.yaml file (default is config/config.yaml). All field is currently mandatory. +#### 2miners -```yaml ---- -elasticsearch_user: "es_user" -elasticsearch_password: "es_password" -elasticsearch_hosts: ["host1:port", "host2:port"...] -api_token_etherscan: "etherscan_api_token " -log_level: INFO # CRITICAL => ERROR => WARNING => INFO => DEBUG -ca_path: "ssl/chain.pem" -2miners_url: "https://eth.2miners.com/api" +To un/subscribe to the monitoring, you have only to do an api call +````go + server.GET("/subscribe/:wallet", handlers.SuscribeWallet) + server.GET("/unsubscribe/:wallet", handlers.UnSuscribeWallet) +```` -miner_listing: ADDR #ALL is used to get all miners from 2miners, ADDR is used for adress variable -adress: - - "0x........................................" - - "0x........................................" - - "0x........................................" - - "0x........................................" +After that, the server will collectd data from the given wallet -redis_host: 127.0.0.1 -redis_port: 6379 -redis_password: "" -redis_lifetime: 10 -factor: 0.000001 -ether_factor: 0.000000000000000001 -gas_factor: 0.000000001 +#### HiveOS + +To have the Hiveos, the client have to allow **minotor** permission to the user **minotor_LFDM** + +#### The Others + +the others services are called on demand or periodically to populate the data + +## How to run it + +### Config file + +You have ton configure the yaml files(adresses, cards and config). All field is currently mandatory. -``` ### Install requirement +GO >= 1.17, Elasticsearch (cluster is nice to have), grafana, redis. +Haproxy is a must + ### Run the binary +The binary is the brain of the process, everything is handle by him. Technically you don't need ES + REDIS + GRAFANA, that's just a must and tooling. ### TroobleShooting diff --git a/config/config.go b/config/config.go index edf5f27..9543be7 100644 --- a/config/config.go +++ b/config/config.go @@ -41,12 +41,12 @@ type Config struct { APIAdress string `yaml:"api_adress"` AdressFilePath string `yaml:"adress_file_path"` LockPath string `yaml:"lock_path"` + CardsConfigFile string `yaml:"cards_config_file"` CoinList []string `yaml:"coin_list"` HiveosUrl string `yaml:"hiveos_api_url"` MinotorHiveOsUser string `yaml:"hiveos_minotor_user"` MinotorHiveOsPass string `yaml:"hiveos_minotor_password"` - MinotorHiveosToken string `yaml:"hiveos_minotor_token"` - + MinotorHiveosToken string `yaml:"hiveos_minotor_token"` } func LoadYamlConfig(ConfigFilePath string) { diff --git a/config/hashrateNo.go b/config/hashrateNo.go new file mode 100644 index 0000000..dc2a274 --- /dev/null +++ b/config/hashrateNo.go @@ -0,0 +1,23 @@ +package config + +import ( + "fmt" + "gopkg.in/yaml.v2" + "io/ioutil" +) + +var Cards *CardsCrawl + +type CardsCrawl struct { + CardsList []string `yaml:"cards"` +} + +func LoadCardYamlConfig() { + t := CardsCrawl{} + data, err := ioutil.ReadFile(Cfg.CardsConfigFile) + if err != nil { + fmt.Println(err.Error()) + } + err = yaml.Unmarshal(data, &t) + Cards = &t +} diff --git a/data/hashrateNo.go b/data/hashrateNo.go new file mode 100644 index 0000000..b26a86e --- /dev/null +++ b/data/hashrateNo.go @@ -0,0 +1,15 @@ +package data + +type Card struct { + HashratePrimary float64 `json:"hashrate_primary"` + Income float64 `json:"income"` + HashrateAlt float64 `json:"hashrate_alt,omitempty"` + CoinName string `json:"coin_name"` + Timestamp string `json:"@timestamp"` + Card string `json:"card"` + CoinPrimary string `json:"primary_coin"` + UnitPrimary string `json:"primary_unit"` + UnitAlt string `json:"alt_unit,omitempty"` + CoinAlt string `json:"alt_coin,omitempty"` + Conso int `json:"conso"` +} diff --git a/engine/orchestrator.go b/engine/orchestrator.go index 8b29108..1024f79 100644 --- a/engine/orchestrator.go +++ b/engine/orchestrator.go @@ -114,3 +114,12 @@ func GetHiveosWorker() { fmt.Println(resp) } + +func ScrapHashrateNo() { + url := fmt.Sprintf("%s:%d/hashrateNo", config.Cfg.APIAdress, config.Cfg.APIFrontPort) + resp, err := http.Get(url) + utils.HandleHttpError(err) + defer resp.Body.Close() + + fmt.Println(resp) +} diff --git a/es/tools.go b/es/tools.go index e7ef3ca..86484a6 100644 --- a/es/tools.go +++ b/es/tools.go @@ -118,9 +118,9 @@ func Bulk(index, data string) { res esutil.BulkIndexerResponseItem, err error, ) { if err != nil { - log.Printf("ERROR: %s on index %s with data %s", err, index, data) + log.Fatalf("ERROR: %s on index %s with data %s", err, index, data) } else { - log.Printf("ERROR: %s: %s on index %s with data %s", res.Error.Type, res.Error.Reason, index, data) + log.Fatalf("ERROR: %s: %s on index %s with data %s", res.Error.Type, res.Error.Reason, index, data) } }, }) diff --git a/go.mod b/go.mod index 63346b5..67bd797 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/gin-gonic/gin v1.7.7 github.com/go-co-op/gocron v1.11.0 github.com/go-redis/redis v6.15.9+incompatible + github.com/gocolly/colly v1.2.0 github.com/gofrs/flock v0.8.1 github.com/nanmu42/etherscan-api v1.6.0 github.com/superoo7/go-gecko v1.0.0 @@ -16,12 +17,20 @@ require ( ) require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/antchfx/htmlquery v1.2.4 // indirect + github.com/antchfx/xmlquery v1.3.9 // indirect + github.com/antchfx/xpath v1.2.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.13.0 // indirect github.com/go-playground/universal-translator v0.17.0 // indirect github.com/go-playground/validator/v10 v10.4.1 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/json-iterator/go v1.1.9 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect github.com/leodido/go-urn v1.2.0 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect @@ -29,9 +38,14 @@ require ( github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/gomega v1.18.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/temoto/robotstxt v1.1.2 // indirect github.com/ugorji/go/codec v1.1.7 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect + golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + golang.org/x/text v0.3.6 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.26.0 // indirect ) diff --git a/go.sum b/go.sum index 4d358ca..2c3e914 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,13 @@ +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494= +github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc= +github.com/antchfx/xmlquery v1.3.9 h1:Y+zyMdiUZ4fasTQTkDb3DflOXP7+obcYEh80SISBmnQ= +github.com/antchfx/xmlquery v1.3.9/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks= +github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= +github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -28,9 +38,16 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -55,6 +72,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -84,6 +103,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -92,6 +113,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/superoo7/go-gecko v1.0.0 h1:Xa1hZu2AYSA20eVMEd4etY0fcJoEI5deja1mdRmqlpI= github.com/superoo7/go-gecko v1.0.0/go.mod h1:6AMYHL2wP2EN8AB9msPM76Lbo8L/MQOknYjvak5coaY= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= @@ -105,11 +128,15 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -144,6 +171,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/handlers/HashrateNo.go b/handlers/HashrateNo.go new file mode 100644 index 0000000..3c80643 --- /dev/null +++ b/handlers/HashrateNo.go @@ -0,0 +1,12 @@ +package handlers + +import ( + "2miner-monitoring/thirdapp" + "github.com/gin-gonic/gin" +) + +func ScrapHashrateNo(c *gin.Context) { + code, status := thirdapp.RunCrawler() + c.String(code, status) + +} diff --git a/main.go b/main.go index b17da1a..e31622c 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( func main() { cliFilled := cli.Cli() config.LoadYamlConfig(cliFilled.FilePathConfig) + config.LoadCardYamlConfig() //log2miner.InitLogger("2miner.log2miner") redis.InitRedis() go func() { @@ -33,7 +34,8 @@ func main() { s.Every(1).Minutes().Do(engine.GetHiveosWorker) s.Every(1).Minutes().Do(engine.GetLastEthBlock) s.Every(1).Minutes().Do(engine.HarvestCoinPrice) - + s.Every(1).Minutes().Do(engine.ScrapHashrateNo) + s.Every(10).Seconds().Do(engine.HarvestFactory, "data") s.Every(1).Second().Do(engine.HarvestFactory, "workers") diff --git a/sample/adress.yml b/sample/adress.yml new file mode 100644 index 0000000..1bba4f0 --- /dev/null +++ b/sample/adress.yml @@ -0,0 +1,7 @@ +#Sample of adress config file +adress: + - 0x6e80e1e29efedj4951j25fdf198fjjd95f0d55je + - 0x989e5m50dfe878jm76e8de102557je51e094f820 + - 0x995e566121Eem744mjEDjEDej6f2855F54868Dm2 + - 0x54dj0mee029me72658d5m0f102dd12mf8m7e78e0 + - 0x52m488419mee0e0mmj02e45255171m55em854dej diff --git a/sample/cards.yml b/sample/cards.yml new file mode 100644 index 0000000..02c093f --- /dev/null +++ b/sample/cards.yml @@ -0,0 +1,6 @@ +# sample of cards config file +cards: + - 3060 + - 3070 + - 3080 + - RX580 \ No newline at end of file diff --git a/sample/config.yml b/sample/config.yml new file mode 100644 index 0000000..5a0bb8a --- /dev/null +++ b/sample/config.yml @@ -0,0 +1,46 @@ +# Sample of main config file + +elasticsearch_user: "" +elasticsearch_password: "" +elasticsearch_hosts: [""] +elasticsearch_port: 9200 +api_token_etherscan: "" +log_level: INFO # CRITICAL => ERROR => WARNING => INFO => DEBUG +ca_path: "" +2miners_url: "https://eth.2miners.com/api" + +miner_listing: ADDR #ALL is used to get all miners from 2miners, ADDR is used for adress variable +adress_file_path: "/etc/minotor/adress.yml" +lock_path: ".minotor.lock" + +redis_host: 127.0.0.1 +redis_port: +redis_password: +redis_short_lifetime: 10 +redis_mid_lifetime: 600 +redis_long_lifetime: 36000 + + +factor: 0.000001 +ether_factor: 0.000000000000000001 +gas_factor: 0.000000001 + +api_adress: +api_port: # run the api on the port given in api_port +api_front_port: # run the api on the port given in api_port +api_log_file: # if set to "", stdout is used, otherwise use the path given in api_log_file +api_username: # user used to auth yourself on the api +api_password: # password combined with user on the api + +coin_list: + - "ethereum" + - "the-open-network" + - "zelcash" + - "cosmos" + - "osmosis" + - "raptoreum" + +hiveos_api_url: "https://api2.hiveos.farm/api/v2" +hiveos_minotor_user: "" +hiveos_minotor_password: "" +hiveos_minotor_token: "" \ No newline at end of file diff --git a/server/server.go b/server/server.go index ee53c80..260499a 100644 --- a/server/server.go +++ b/server/server.go @@ -59,6 +59,7 @@ func engine() *gin.Engine { hiveosServer.GET("/worker/:worker", handlers.GetHiveosWorker) hiveosServer.GET("/oc/:farmid", handlers.GetHiveosOc) } + server.GET("/hashrateNo", handlers.ScrapHashrateNo) return server } diff --git a/thirdapp/hashrateNo.go b/thirdapp/hashrateNo.go new file mode 100644 index 0000000..84f223a --- /dev/null +++ b/thirdapp/hashrateNo.go @@ -0,0 +1,83 @@ +package thirdapp + +import ( + "2miner-monitoring/config" + "2miner-monitoring/data" + "2miner-monitoring/es" + "encoding/json" + "fmt" + "github.com/gocolly/colly" + "log" + "regexp" + "strconv" + "strings" + "time" +) + +func DualType(e *colly.HTMLElement) { + var dual = regexp.MustCompile(`([a-zA-Z]{0,}?\+[a-zA-Z]{0,})([0-9]{0,}?\.[0-9]{0,}) ([a-zA-Z]{0,}\/s)([0-9]{0,}?\.[0-9]{0,}) ([a-zA-Z]{0,}\/s)([0-9]{0,})w([0-9]{0,}?\.[0-9]{0,}) ([a-zA-Z]{0,})([0-9]{0,}?\.[0-9]{0,}) ([a-zA-Z]{0,}\/w)([0-9]{0,}?\.[0-9]{0,}) ([a-zA-Z]{0,}\/w)\$([0-9]{0,}?\.[0-9]{0,})\$([0-9]{0,}?\.[0-9]{0,}) ([0-9]{0,}) ([a-zA-Z]{0,})([0-9]{0,}) ([a-zA-Z]{0,})`) + if dual.MatchString(e.Text) { + DualCrawled := data.Card{} + coin := dual.ReplaceAllString(e.Text, `$1`) + coinsName := strings.Split(coin, "+") + DualCrawled.CoinPrimary = coinsName[0] + DualCrawled.CoinAlt = coinsName[1] + DualCrawled.CoinName = coin + DualCrawled.Conso, _ = strconv.Atoi(dual.ReplaceAllString(e.Text, `$6`)) + DualCrawled.Income, _ = strconv.ParseFloat(dual.ReplaceAllString(e.Text, `$13`), 64) + DualCrawled.HashrateAlt, _ = strconv.ParseFloat(dual.ReplaceAllString(e.Text, `$4`), 64) + DualCrawled.HashratePrimary, _ = strconv.ParseFloat(dual.ReplaceAllString(e.Text, `$2`), 64) + DualCrawled.UnitPrimary = dual.ReplaceAllString(e.Text, `$3`) + DualCrawled.UnitAlt = dual.ReplaceAllString(e.Text, `$5`) + DualCrawled.Card = e.Request.URL.Path[1:] + DualCrawled.Timestamp = time.Now().Format(time.RFC3339) + JsonDualCrawled, _ := json.Marshal(DualCrawled) + es.Bulk("2miners-hashrate_no", string(JsonDualCrawled)) + } +} + +func Singletype(e *colly.HTMLElement) { + var single = regexp.MustCompile(`([a-zA-Z]{0,})([0-9]{0,}?\.[0-9]{0,}) ([a-zA-Z]{0,}\/s)([0-9]{0,})w([0-9]{0,}?\.[0-9]{0,}) ([a-zA-Z]{0,})([0-9]{0,}?\.[0-9]{0,}) ([a-zA-Z]{0,}\/w)\$([0-9]{0,}?\.[0-9]{0,})\$([0-9]{0,}?\.[0-9]{0,}) ([0-9]{0,}) ([a-zA-Z]{0,})([0-9]{0,}) ([a-zA-Z]{0,})`) + if single.MatchString(e.Text) { + SingleCrawled := data.Card{} + coin := single.ReplaceAllString(e.Text, `$1`) + SingleCrawled.CoinPrimary = coin + SingleCrawled.CoinName = coin + SingleCrawled.HashratePrimary, _ = strconv.ParseFloat(single.ReplaceAllString(e.Text, `$2`), 64) + SingleCrawled.UnitPrimary = single.ReplaceAllString(e.Text, `$3`) + SingleCrawled.Conso, _ = strconv.Atoi(single.ReplaceAllString(e.Text, `$4`)) + SingleCrawled.Income, _ = strconv.ParseFloat(single.ReplaceAllString(e.Text, `$9`), 64) + SingleCrawled.Card = e.Request.URL.Path[1:] + SingleCrawled.Timestamp = time.Now().Format(time.RFC3339) + JsonSingleCrawled, _ := json.Marshal(SingleCrawled) + es.Bulk("2miners-hashrate_no", string(JsonSingleCrawled)) + } +} + +func DispatchType(e *colly.HTMLElement) { + DualType(e) + Singletype(e) +} + +func RunCrawler() (int, string) { + c := colly.NewCollector( + colly.AllowedDomains("www.hashrate.no", "hashrate.no"), + ) + c.OnError(func(_ *colly.Response, err error) { + log.Println("Houston nous avons un problème : ", err) + }) + c.OnHTML("tr", func(e *colly.HTMLElement) { + DispatchType(e) + }) + c.OnRequest(func(r *colly.Request) { + log.Printf("harvesting : %s", r.URL.String()) + }) + c.OnScraped(func(s *colly.Response) { + log.Printf("harvested : %s", s.Request.URL.String()) + }) + for _, elem := range config.Cards.CardsList { + url := fmt.Sprintf("https://www.hashrate.no/%s", elem) + c.Visit(url) + } + return 200, "Card Stats harvested" +}