Inital commit.

.gitignore

@@ -0,0 +1,9 @@
+output
+*.cnf
+.idea
+tmp
+main
+mysql-monitor-*
+
+*.sh
+vendor

.travis.yml

@@ -0,0 +1,55 @@
+language: go
+matrix:
+  include:
+  - go: 1.x
+  - go: 1.11.x
+  - go: 1.12.x
+    env:
+    - LATEST=true
+    - GO111MODULE=on
+  - go: tip
+  allow_failures:
+  - go: tip
+env:
+- GO111MODULE=on
+before_install:
+- go get github.com/mitchellh/gox
+install: true
+script:
+- go mod tidy
+- go mod vendor
+- go test -mod=vendor ./... --race
+- if [ "${LATEST}" = "true" ]; then gox -mod=vendor -output="mysql-monitor-${TRAVIS_BRANCH}-{{.OS}}-{{.Arch}}"
+  -ldflags "-X main.Commit=`git rev-parse --short HEAD` -X main.Version=${TRAVIS_BRANCH}"
+  -verbose github.com/messagebird/mysql-monitor/cmd; fi
+- if [ "${LATEST}" = "true" ]; then ls -l mysql-monitor-${TRAVIS_BRANCH}-* | awk '{printf "%s\n",
+  $9}' | while read line; do tar cfJ $line.tar.xz $line; done; fi
+deploy:
+  skip_cleanup: true
+  provider: releases
+  api_key:
+    secure: cR46TePDc+4BHoMRaapWkvC+WC1q3acq+DHsC26wvWOmr+Anf1p8CRtWMj/HvuP450Qkk3w/EIdvC6iPT47yUpUYiJiXaYScM2qScW/ZVPcuOLEgaL4O3kSmG36iENNXV+OwTAuqwUtoNcJtOALwt/OH2Xkqc03MUI1N99H485QORguc0IolQ5M0XYB37ULia0+wcHyo6O0rNeDN4Rycf2Cdd2faRiaNSFUaYUupye1RKmOtFRnqwxYoA98paKU7PpOuo3XzEwMp+yAXqgu1Z7V0X0KB3fskzCAJylZA6IBXzMdVmWdtd9+rrUbC2yScVk1jGFqw90Knw1i2LmnX1WIPDpxiBJYO/Akfni4A51R3hyNfwcyLfd5QNH4AcHoH6B8mUrsFDg58vM1PaAJfOcDGS9b5QfvswI4f9rvmxAi3buZHzaiWbDFOvMt+YdJru2UWjADqr8mv78X+r+TVbHhHdLg/QwDXsScn7aCPx39QPpr5qPm6PKvIJk+gcWT1G6/KzizoRj/doLjZg0UZi3WIdkqX44IrthyaFACSLswE2itbs1tTgrANCt8OaeGciMR6GJ7xFREKm1CStlLO1jjbBO9UpJ/LMFa4Q7HPXzwBSbr63gfSdIsCzmLNI7/z2fv5Wf5L783WXY+E+sSSWMU41l35rTOdIkLZpUSJb7c=
+  file:
+  - mysql-monitor-${TRAVIS_BRANCH}-darwin-386.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-darwin-amd64.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-freebsd-386.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-freebsd-amd64.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-freebsd-arm.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-linux-386.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-linux-amd64.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-linux-arm.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-linux-mips.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-linux-mips64.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-linux-mips64le.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-linux-mipsle.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-linux-s390x.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-netbsd-386.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-netbsd-amd64.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-netbsd-arm.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-openbsd-386.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-openbsd-amd64.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-windows-386.exe.tar.xz
+  - mysql-monitor-${TRAVIS_BRANCH}-windows-amd64.exe.tar.xz
+  on:
+    tags: true
+    condition: $LATEST = true

LICENSE

@@ -0,0 +1,28 @@
+Copyright (c) 2019 Messagebird. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of MessageBird. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

README.md

@@ -0,0 +1,106 @@
+# Mysql Monitor
+
+This tool is basically Atop with some mysql queries included such as:
+- process list 
+- threads
+- engine innodb status
+- slave status
+- (more can be added if needed)
+
+Which can be either saved inside a slqite3 db(experimental)
+(Some features here are still missing, e.g export from sqite to file). Or to .json.gz/.txt.gz files on disk
+
+
+## Saving .gz's to disk
+
+To use this mode, pass the `--not-in-sqlite` parameter.
+
+When the monitor is ran to save .gz's to disk (inside the /tmp/no-sql folder), you can expect the following file structure:
+```
+drwxr-xr-x    - ogkevin 25 Apr 11:18 tmp/no-sql
+drwxr-xr-x    - ogkevin 16 Apr 11:08 ├── mysql
+drwxr-xr-x    - ogkevin 29 Apr 12:04 │  ├── engine-innodb
+drwxr-xr-x    - ogkevin 29 Apr 12:05 │  │  └── 19-04-29
+drwxr-xr-x    - ogkevin 29 Apr 12:05 │  │     └── 1000
+.rw-r--r-- 1.4k ogkevin 29 Apr 12:05 │  │        ├── 2019-04-29T100550Z.txt.gz
+.rw-r--r-- 1.4k ogkevin 29 Apr 12:05 │  │        └── 2019-04-29T100600Z.txt.gz
+drwxr-xr-x    - ogkevin 29 Apr 12:04 │  ├── process-list
+drwxr-xr-x    - ogkevin 29 Apr 12:05 │  │  └── 19-04-29
+drwxr-xr-x    - ogkevin 29 Apr 12:05 │  │     └── 1000
+.rw-r--r--  249 ogkevin 29 Apr 12:05 │  │        ├── 2019-04-29T100550Z.json.gz
+.rw-r--r--  281 ogkevin 29 Apr 12:05 │  │        └── 2019-04-29T100600Z.json.gz
+drwxr-xr-x    - ogkevin 29 Apr 12:04 │  ├── ps_threads
+drwxr-xr-x    - ogkevin 29 Apr 12:05 │  │  └── 19-04-29
+drwxr-xr-x    - ogkevin 29 Apr 12:05 │  │     └── 1000
+.rw-r--r--  868 ogkevin 29 Apr 12:05 │  │        ├── 2019-04-29T100550Z.json.gz
+.rw-r--r--  834 ogkevin 29 Apr 12:05 │  │        └── 2019-04-29T100600Z.json.gz
+drwxr-xr-x    - ogkevin 29 Apr 12:04 │  └── slave-status
+drwxr-xr-x    - ogkevin 29 Apr 12:05 │     └── 19-04-29
+drwxr-xr-x    - ogkevin 29 Apr 12:05 │        └── 1000
+.rw-r--r--   26 ogkevin 29 Apr 12:05 │           ├── 2019-04-29T100550Z.json.gz
+.rw-r--r--   26 ogkevin 29 Apr 12:05 │           └── 2019-04-29T100600Z.json.gz
+drwxr-xr-x    - ogkevin 25 Apr 11:06 └── system
+drwxr-xr-x    - ogkevin 29 Apr 12:04    ├── ps
+drwxr-xr-x    - ogkevin 29 Apr 12:05    │  └── 19-04-29
+drwxr-xr-x    - ogkevin 29 Apr 12:05    │     └── 1000
+.rw-r--r--  294 ogkevin 29 Apr 12:05    │        ├── 2019-04-29T100550Z.txt.gz
+.rw-r--r--  291 ogkevin 29 Apr 12:05    │        └── 2019-04-29T100600Z.txt.gz
+drwxr-xr-x    - ogkevin 29 Apr 12:04    └── top
+drwxr-xr-x    - ogkevin 29 Apr 12:03       └── 19-04-29
+drwxr-xr-x    - ogkevin 29 Apr 12:06          └── 1000
+.rw-r--r--  422 ogkevin 29 Apr 12:05             ├── 2019-04-29T100550Z.txt.gz
+.rw-r--r--  421 ogkevin 29 Apr 12:06             └── 2019-04-29T100600Z.txt.gz
+```
+
+This is the file output by running the monitor with a 10 sec interval for 2 cycles.
+
+You will get 2 main dirs which are `mysql` and `system`. Which contains mysql and system logs respectively.
+Each dir will then be split in dirs for each log type. These dirs will then be split by date. The date dirs will then be
+split by the hour and inside the hour dirs the actual logs files will be located. This should make the quest to find
+logs of a specific time and date easier. 
+
+## Saving to sqlite
+
+*This mode uses more disk space*
+
+The idea behind this mode is that it will store everything to sqlite and exposes this via an api. This way you could
+build on top of this api. e.g. a dashboard containing all logs of all mysql servers that you're running.
+
+The API spec can be found at [swagger.json](./docs/swagger.json) or by
+visiting the `/swagger` endpoint on `localhost:port`
+
+What this api can do:
+- List all cycles
+- Get all logs of each type per cycle id.
+
+# Commands and args
+## Monitor command (m)
+|Argument|Description|
+|---|---|
+|--interval| How often the monitor should collect logs.|
+|--log-retention| How long these logs should be saved. The monitor will delete files that are eligible for deletion every 60 sec|
+|--mysql-credential-path|The path containing the credentials to connect to the mysql server. For more info on this see [mysql cred section](#mysql-credential-file)|
+|--output-dir|The path where the sqlite db/log files containing the data should be saved (default: "./mysql-monitor-db.sqlite")|
+|--exclude-\<type>|Exclude \<type> from being logged.|
+|--not-in-sqlite|Use logging to disk instead of sqlite see [Saving .gz's to disk](#saving-.gz's-to-disk)|
+
+## Get command (g)
+This command is only needed if using the "save in sqlite" mode.
+
+This command allows one to "query" the logs from the sqlite db. 
+
+The syntax is `mysql-monitor get <category> <type> <cycle-id>`
+
+To get a cycle id run: `mysql-monitor get cycles`
+
+# Mysql credential file
+The contents of the mysql credential file should contain:
+
+```
+[client]
+user=root
+password=root
+host=127.0.0.1
+```
+
+Connecting via socket is also supported.

cmd/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"fmt"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+	"github.com/messagebird/mysql-monitor/internal/commands"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+)
+
+func setupLogging(lvl string) {
+	logLvl, err := logrus.ParseLevel(lvl)
+	if err != nil {
+		logrus.WithError(err).Warning("could not parse log level")
+	} else {
+		logrus.SetLevel(logLvl)
+	}
+
+	logrus.SetOutput(os.Stdout)
+	logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true, DisableLevelTruncation: true, QuoteEmptyFields: true})
+}
+
+var Version string
+var Commit string
+
+func main() {
+	app := cli.NewApp()
+	app.Name = "mysql monitor"
+	app.Usage = "monitors mysql systems"
+	app.Description = "monitors mysql systems"
+	app.Version = fmt.Sprintf("Version: %s\n\t Commit: %s", Version, Commit)
+	app.Before = func(ctx *cli.Context) error {
+		setupLogging(ctx.String("level"))
+
+		if ctx.Bool("trace") {
+			go http.ListenAndServe(":6060", http.DefaultServeMux)
+		}
+
+		return nil
+	}
+
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:  "level",
+			Usage: "log level",
+			Value: "info",
+		},
+		cli.BoolFlag{
+			Name: "trace",
+			Usage: "if golang pprof should be enabled",
+		},
+	}
+
+	app.Commands = []cli.Command{
+		{
+			Name:    "monitor",
+			Aliases: []string{"m"},
+			Usage:   "runs the monitor",
+			Flags:   commands.GetMonitorCommandFlags(),
+			Action:  commands.Monitor,
+		},
+		commands.BuildGetCommand(),
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		logrus.WithError(err).Fatal(err.Error())
+	}
+}

docker-compose.yml

@@ -0,0 +1,19 @@
+version: "3"
+services:
+  mysql:
+    image: mysql:5.7
+    environment:
+      - MYSQL_USER=mysql
+      - MYSQL_PASSWORD=mysql
+      - MYSQL_ROOT_PASSWORD=root
+    ports:
+      - 3301:3306
+  monitor:
+    image: golang
+    ports:
+      - 9090:9090
+      - 6060:6060
+    volumes:
+      - ./:/project
+    working_dir: /project
+    command: 'go run cmd/main.go --trace --level debug m -c dev-docker.cnf -n 1s -o tmp/no-sql/ -r 1h --not-in-sqlite'