diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 975245e..8f1f380 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,6 +22,9 @@ jobs: pg11: ${{ steps.filter.outputs.pg11 }} pg12: ${{ steps.filter.outputs.pg12 }} pg13: ${{ steps.filter.outputs.pg13 }} + pg14: ${{ steps.filter.outputs.pg14 }} + pg15: ${{ steps.filter.outputs.pg15 }} + pg16: ${{ steps.filter.outputs.pg16 }} steps: - name: Checkout 📦 @@ -42,6 +45,12 @@ jobs: - '12/**' pg13: - '13/**' + pg14: + - '14/**' + pg15: + - '15/**' + pg16: + - '16/**' # Job for Docker Build and Push build: @@ -62,6 +71,9 @@ jobs: pg11=${{ needs.changes.outputs.pg11 }} pg12=${{ needs.changes.outputs.pg12 }} pg13=${{ needs.changes.outputs.pg13 }} + pg14=${{ needs.changes.outputs.pg14 }} + pg15=${{ needs.changes.outputs.pg15 }} + pg16=${{ needs.changes.outputs.pg16 }} if [[ $pg9 == true ]]; then VERSION="9.6" @@ -73,6 +85,12 @@ jobs: VERSION="12" elif [[ $pg13 == true ]]; then VERSION="13" + elif [[ $pg14 == true ]]; then + VERSION="14" + elif [[ $pg15 == true ]]; then + VERSION="15" + elif [[ $pg15 == true ]]; then + VERSION="16" fi TAGS="${DOCKER_IMAGE}:${VERSION}" echo ::set-output name=subdir::${VERSION} diff --git a/10/README.md b/10/README.md index 970dc07..83db1f3 100644 --- a/10/README.md +++ b/10/README.md @@ -185,4 +185,4 @@ Further changes: ## Credits * Tim Sutton (tim@kartoza.com) for https://github.com/kartoza/docker-pg-backup - Consulted Oct 2018 -* Just van den Broecke (https://justobjects.nl) - this version - 2018-2021 +* Just van den Broecke (https://justobjects.nl) - this version - 2018 diff --git a/10/build.sh b/10/build.sh index 0e7f637..702b7b1 100755 --- a/10/build.sh +++ b/10/build.sh @@ -1,3 +1,2 @@ #!/bin/bash -# Local build. docker build -t justb4/pgbackup:10 . diff --git a/11/build.sh b/11/build.sh index f343696..bd9b103 100755 --- a/11/build.sh +++ b/11/build.sh @@ -1,3 +1,2 @@ #!/bin/bash -# docker build -t justb4/pgbackup:11 . diff --git a/12/build.sh b/12/build.sh index 00adc1c..ff1ef64 100755 --- a/12/build.sh +++ b/12/build.sh @@ -1,3 +1,2 @@ #!/bin/bash -# docker build -t justb4/pgbackup:12 . diff --git a/13/build.sh b/13/build.sh index 5a90d5f..648a9df 100755 --- a/13/build.sh +++ b/13/build.sh @@ -1,3 +1,2 @@ #!/bin/bash -# docker build -t justb4/pgbackup:13 . diff --git a/14/Dockerfile b/14/Dockerfile new file mode 100644 index 0000000..b92d353 --- /dev/null +++ b/14/Dockerfile @@ -0,0 +1,11 @@ +FROM postgres:14-alpine + +LABEL maintainer="Just van den Broecke " + +RUN apk add --no-cache --update gettext python3 py3-pip && pip3 install click docker && mkdir /pgbackup + +ENV PGB_SCHEDULE 0 23 * * * + +ADD docker/* /pgbackup/ + +CMD ["/pgbackup/entrypoint.sh"] diff --git a/14/README.md b/14/README.md new file mode 100644 index 0000000..0cd6060 --- /dev/null +++ b/14/README.md @@ -0,0 +1,188 @@ +# Docker PG Backup + +Inspired by: https://github.com/kartoza/docker-pg-backup + +A Docker container that runs automated scheduled PostgreSQL/PostGIS backups for all +PostgreSQL-based Docker Containers in its network that have the Label `"pgbackup.enable=true"`. + +It should work with +the following PostgreSQL/PostGIS Docker images: + +* [mdillon/postgis](https://hub.docker.com/r/mdillon/postgis/) +* [Kartoza docker postgis](https://github.com/kartoza//docker-postgis) +* [Standard PostgreSQL Docker image](https://hub.docker.com/_/postgres/) + +Any other PostgreSQL/PostGIS image may work as long as its Container has the `POSTGRES_` environment +variables set (see below). + +By default it will create a backup once per night (at 23:00) in a +nicely ordered directory by container-name/year/month, but you can specify your own schedule. + +* Docker hub at: https://registry.hub.docker.com/u/justb4/pgbackup/ +* Github at: https://github.com/justb4/docker-pgbackup + +## Getting the image + +There are various ways to get the image onto your system: + +The preferred way (but using most bandwidth for the initial image) is to +get our docker trusted build like this: + + +``` +docker pull justb4/pgbackup:14 + +``` + +We highly suggest that you use a tagged image as +latest may change and may not successfully back up your database. Use the same or +greater version of postgis as the database you are backing up. +To build the image yourself: + +``` +docker build -t justb4/pgbackup . +``` + +If you do not wish to do a local checkout first then build directly from github. + +``` +git clone git://github.com/justb4/docker-pgbackup +``` + +## Environment Variables + +* `PGB_SCHEDULE`, crontab schedule line, if not set, defaults to : `0 23 * * *` + +## Run Backups + +To create a running container do: + +``` +docker run --name="pgbackup"\ + -v backup:/backup -v /var/run/docker.sock:/var/run/docker.sock \ + -i -d justb4/pgbackup:14 +``` + +In this example a local dir (`./backup`) is mounted inti which the actual backups will be +stored. + +Best is to use docker-compose, below the as used +for testing, with a schedule that backs up once a minute. + + +``` +# Test for pgbackup with sample db +version: "3" + +services: + db: + image: mdillon/postgis:14-alpine + container_name: pg_db_14 + labels: + - "pgbackup.enable=true" + environment: + - POSTGRES_DB=testdb + - POSTGRES_USER=testuser + - POSTGRES_PASSWORD=testpass + + dbbackup: + image: justb4/pgbackup:14 + container_name: pg_backup_14 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./backup:/backup + environment: + - PGB_SCHEDULE=*/1 * * * * + +``` + +Then run using: + +``` +docker-compose up -d +``` + +It is advised to use explicit DB-container-naming, as backups will be stored in +subdirectories (`year/month/-ymd-hm.sql.gz`). + +## Explicit Backups + +You can also run backups (and restores) explicitly, by calling `exec` on the `justb4/pgbackup` +container, assuming `pgbackup` here. + +Backup all DBs containers: + +``` +docker exec -it pgbackup /pgbackup/backup-all.sh + +``` + +Or you can backup a single DB container: + +``` +docker exec -it pgbackup /pgbackup/backup.sh + + +# example +docker exec -it pgbackup /pgbackup/backup.sh pgdb /backup/mybackup.sql.gz + +``` + +## List Backups + +You can list all backups available in the container: + +``` +docker exec -it pgbackup /pgbackup/list-backups.sh + +``` + +## Restoring Backups + +This Docker Image also provides restore utilities. + +You can `bash` into the `justb4/pgbackup` container and run `restore.sh` or other commands +from there. The following steps are needed: + +* if not already present copy your backup file, assuming `/backup/mybackup.sql.gz` here, into the `pgbackup` container mounted volume +* figure out the name of your `justb4/pgbackup` container, assuming `pgbackup` here +* figure out the name of your target DB container, assuming `pgdb` here +* `bash` into the container: `docker exec -it pgbackup /bin/bash` +* execute restore: `/pgbackup/restore.sh /backup/mybackup.sql.gz pgdb` + +You could also `exec` directly. Below an example: + +``` +docker exec -it pgbackup /pgbackup/restore.sh pgdb /backup/2018/10/pgdb-181013-1050.sql.gz + +``` + +## Design and diffs with kartoza/pg-backup + +Main difference is that `justb4/pgbackup` uses the Docker API to search within its Docker Network for +Containers that have the Label `"pgbackup.enable=true"`. Using Labels in conjunction with the Docker API +is found in many modern Docker-based services, like e.g. Traefik and Kubernetes. + +Each Container to be backed up is then further inspected to get the PostgreSQL credentials +needed to connect with PG tools like `psql`. The Container name will be the PG Hostname +(TODO: figure out IP address via Docker API, +such that single backup/restores can be run commandline). + +This has the following advantages: + +* loose coupling, easy to setup +* one `pgbackup` Container can backup multiple PostgreSQL Containers +* no need to configure `pgbackup` with all PG credentials + +Further changes: + +* works with multiple Docker images for both PostgreSQL and PostGIS (mdillon and kartoza) +* using smaller `postgres:-alpine` as base image (i.s.o. `kartoza/postgis`) +* schedule via env var `PGB_SCHEDULE` +* dumps in SQL gzip format (more portable among PG versions) but may become option in futre +* includes restore command to restore backup file in a named container + +## Credits + +* Tim Sutton (tim@kartoza.com) for https://github.com/kartoza/docker-pg-backup - Consulted Oct 2018 +* Just van den Broecke (https://justobjects.nl) - this version - 2018 diff --git a/14/build.sh b/14/build.sh new file mode 100755 index 0000000..3bcadde --- /dev/null +++ b/14/build.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker build -t justb4/pgbackup:14 . diff --git a/14/docker/backup-all.sh b/14/docker/backup-all.sh new file mode 100755 index 0000000..4b6f571 --- /dev/null +++ b/14/docker/backup-all.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +echo "START Backup" + +pushd /pgbackup + python3 pgbackup.py --backupdir /backup backup-all +popd + +echo "END Backup" diff --git a/14/docker/backup.sh b/14/docker/backup.sh new file mode 100755 index 0000000..b19d49f --- /dev/null +++ b/14/docker/backup.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +echo "START Backup single DB" + +# stop on errors +set -e + +# check that we have right number of arguments +if [[ ! $# -eq 2 ]] +then + echo 'usage:' + echo ' /pgbackup/backup.sh ' + echo '' + echo 'to get a list of available backups, run:' + echo ' /pgbackup/list-backups.sh' + exit 1 +fi + +# set the container name variable +CONTAINERNAME=$1 + +# set the backupfile variable +BACKUPFILE=$2 + +echo "Backing up ${CONTAINERNAME} to ${BACKUPFILE}" + +pushd /pgbackup + python3 pgbackup.py --filepath ${BACKUPFILE} --containername ${CONTAINERNAME} backup +popd + +echo "END Backup" diff --git a/14/docker/cronfile-template b/14/docker/cronfile-template new file mode 100755 index 0000000..ae4807c --- /dev/null +++ b/14/docker/cronfile-template @@ -0,0 +1,4 @@ +# Schedule supplied via env var +${PGB_SCHEDULE} /pgbackup/backup-all.sh 2>&1 + +# We need a blank line here for it to be a valid cron file diff --git a/14/docker/entrypoint.sh b/14/docker/entrypoint.sh new file mode 100755 index 0000000..56129d9 --- /dev/null +++ b/14/docker/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +PGB_SCHEDULE=${PGB_SCHEDULE:='0 23 * * *'} + +pushd /pgbackup + +envsubst < cronfile-template > backup-cron + +# Now launch cron in then foreground. +echo "Launching /usr/sbin/crond in foregound with schedule:" +cat backup-cron +crontab backup-cron +/usr/sbin/crond -f -S -l 0 diff --git a/14/docker/list-backups.sh b/14/docker/list-backups.sh new file mode 100755 index 0000000..7d3202d --- /dev/null +++ b/14/docker/list-backups.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "listing available backups" +echo "-------------------------" +find /backup -type f -name '*.sql.gz' diff --git a/14/docker/pgbackup.py b/14/docker/pgbackup.py new file mode 100644 index 0000000..bc17e3a --- /dev/null +++ b/14/docker/pgbackup.py @@ -0,0 +1,156 @@ +import time +import click +import docker +import subprocess + +client = docker.from_env() +backup_dir = None +container_name = None +file_path = None + +def echo(s): + click.echo(s) + +def get_enabled_containers(): + filters = {'label': 'pgbackup.enable=true'} + + containers = client.containers.list(filters=filters) + return containers + +def get_enabled_container(container_name): + containers = get_enabled_containers() + found_container = None + for container in containers: + if container.name == container_name: + found_container = container + return found_container + +def get_pg_creds(container): + pg_creds = dict() + pg_creds['PGHOST'] = container.name + pg_creds['PGPORT'] = '5432' + + env_list = container.attrs['Config']['Env'] + for env_kv in env_list: + # Matches POSTGRES_DB and POSTGRES_DBNAME + if 'POSTGRES_DB' in env_kv: + pg_creds['PGDB'] = env_kv.split('=')[1] + if 'POSTGRES_USER' in env_kv: + pg_creds['PGUSER'] = env_kv.split('=')[1] + # Matches POSTGRES_PASS and POSTGRES_PASSWORD + if 'POSTGRES_PASS' in env_kv: + pg_creds['PGPASSWORD'] = env_kv.split('=')[1] + + return pg_creds + + +def make_backup_cmd(pg_creds, file_path): + cmd_template = 'export PGPASSWORD=%s; pg_dump -h %s -U %s %s | gzip > %s' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], pg_creds['PGHOST'], + pg_creds['PGUSER'], pg_creds['PGDB'], + file_path) + return cmd + + +def make_dropdb_cmd(pg_creds): + cmd_template = 'export PGPASSWORD=%s; dropdb -h %s -U %s %s' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], pg_creds['PGHOST'], + pg_creds['PGUSER'], pg_creds['PGDB'] + ) + return cmd + + +def make_createdb_cmd(pg_creds): + cmd_template = 'export PGPASSWORD=%s; createdb -h %s -U %s -O %s %s' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], pg_creds['PGHOST'], pg_creds['PGUSER'], + pg_creds['PGUSER'], pg_creds['PGDB'] + ) + return cmd + + +def make_restore_cmd(pg_creds, file_path): + cmd_template = 'export PGPASSWORD=%s; gunzip -c %s | psql -h %s -U %s %s ' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], file_path, pg_creds['PGHOST'], + pg_creds['PGUSER'], pg_creds['PGDB'] + ) + return cmd + + +def execute_cmd(cmd): + echo('executing cmd=%s' % cmd) + exit_code = subprocess.call(cmd, shell=True) + echo('execute done, exit_code=%d' % exit_code) + + +@click.group() +@click.option('--backupdir', help='Backup root directory') +@click.option('--containername', required=False, help='Optional container name for backup single or restore') +@click.option('--filepath', required=False, help='Optional backup file for backup single or restore') +def cli(backupdir, containername, filepath): + global backup_dir + global container_name + global file_path + + backup_dir = backupdir + container_name = containername + file_path = filepath + echo('backupdir=%s' % backup_dir) + + +@cli.command() +def backup_all(): + echo('START backup_all') + containers = get_enabled_containers() + echo('Found %d containers with backup enabled' % len(containers)) + for container in containers: + echo('Backing up %s ...' % container.name) + now = time.localtime() + pg_creds = get_pg_creds(container) + echo('creds: %s' % str(pg_creds)) + + dir_path = '%s/%s' % (backup_dir, time.strftime("%Y/%m", now)) + file_path = '%s/%s-%s.sql.gz' % (dir_path, container.name, time.strftime("%y%m%d-%H%M", now)) + + execute_cmd('mkdir -p %s' % dir_path) + + backup_cmd = make_backup_cmd(pg_creds, file_path) + echo('backup_cmd=%s' % backup_cmd) + execute_cmd(backup_cmd) + + echo('END backup_all') + + +@cli.command() +def backup(): + echo('backup single for container %s' % container_name) + container = get_enabled_container(container_name) + pg_creds = get_pg_creds(container) + + backup_cmd = make_backup_cmd(pg_creds, file_path) + execute_cmd(backup_cmd) + + +@cli.command() +def restore(): + echo('restore single for container %s' % container_name) + + container = get_enabled_container(container_name) + pg_creds = get_pg_creds(container) + + # Three steps: drop existing DB, create new DB and restore + dropdb_cmd = make_dropdb_cmd(pg_creds) + execute_cmd(dropdb_cmd) + + createdb_cmd = make_createdb_cmd(pg_creds) + execute_cmd(createdb_cmd) + + restore_cmd = make_restore_cmd(pg_creds, file_path) + execute_cmd(restore_cmd) + + +if __name__ == '__main__': + cli() diff --git a/14/docker/restore.sh b/14/docker/restore.sh new file mode 100755 index 0000000..5cef866 --- /dev/null +++ b/14/docker/restore.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +echo "START Restore" + +# stop on errors +set -e + +# check that we have right number of arguments +if [[ ! $# -eq 2 ]] +then + echo 'usage:' + echo ' /pgbackup/restore.sh ' + echo '' + echo 'to get a list of available backups, run:' + echo ' /pgbackup/list-backups.sh' + exit 1 +fi + +# set the container name variable +CONTAINERNAME=$1 + +# set the backupfile variable +BACKUPFILE=$2 + +# check that the file exists +if [ ! -f ${BACKUPFILE} ] +then + echo "backup file not found" + echo 'to get a list of available backups, run:' + echo ' /pgbackup/list-backups.sh' + exit 1 +fi + + +echo "Restoring to ${CONTAINERNAME} from ${BACKUPFILE}" + +pushd /pgbackup + python3 pgbackup.py --filepath ${BACKUPFILE} --containername ${CONTAINERNAME} restore +popd + +echo "END Restore" diff --git a/14/test/README.md b/14/test/README.md new file mode 100644 index 0000000..f619ebf --- /dev/null +++ b/14/test/README.md @@ -0,0 +1,49 @@ +# Docker PG Backup Test + +Inspired by: https://github.com/kartoza/docker-pg-backup + +Uses a simple [docker-compose.yml](docker-compose.yml) to start a PostGIS DB +plus a `pgbackup` container. + +## Running + +Invoke [start.sh](start.sh). Optionally look at logs: + +``` +docker logs --follow pg_backup_client +``` + +As the backup runs every minute, it could be that the `pg_backup_db` container is not yet +up completely thus PG DB not ready. + +Invoke [stop.sh](stop.sh) to stop. Look under `./backup` for backup files. + +## TIP - explicit backup + +Simply `bash` into `pgbackup` container. + +``` + $ docker exec -it pg_backup_client bash + $ /backup.sh +``` + +## TIP - explicit restore + +Copy `.sql.gz` into the `./backup` directory. +Simply `bash` into `pgbackup` container. + +``` + $ docker exec -it pg_backup_client bash + $ /restore.sh /backup/.sql.gz +``` + +## TIP - connect with psql + +Simply `bash` into `pgbackup` container, using the env settings file. + +``` + $ docker exec -it pg_backup_client bash + $ source /pgb-env.sh + $ psql +``` + diff --git a/14/test/backup/README.md b/14/test/backup/README.md new file mode 100644 index 0000000..836581d --- /dev/null +++ b/14/test/backup/README.md @@ -0,0 +1 @@ +test dir for backups diff --git a/14/test/docker-compose.yml b/14/test/docker-compose.yml new file mode 100644 index 0000000..32ddce5 --- /dev/null +++ b/14/test/docker-compose.yml @@ -0,0 +1,22 @@ +# Test for pgbackup with sample db +version: "3" + +services: + db: + image: mdillon/postgis:14-alpine + container_name: pg_db_14 + labels: + - "pgbackup.enable=true" + environment: + - POSTGRES_DB=testdb + - POSTGRES_USER=testuser + - POSTGRES_PASSWORD=testpass + + dbbackup: + image: justb4/pgbackup:14 + container_name: pg_backup_14 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./backup:/backup + environment: + - PGB_SCHEDULE=*/1 * * * * diff --git a/14/test/start.sh b/14/test/start.sh new file mode 100755 index 0000000..a693034 --- /dev/null +++ b/14/test/start.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Stop and remove possibly old containers +docker-compose stop +docker-compose rm -f + +# Finally run +docker-compose up -d + + diff --git a/14/test/stop.sh b/14/test/stop.sh new file mode 100755 index 0000000..9193033 --- /dev/null +++ b/14/test/stop.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# Stop and remove possibly old containers +docker-compose stop +docker-compose rm -f diff --git a/15/Dockerfile b/15/Dockerfile new file mode 100644 index 0000000..43030a3 --- /dev/null +++ b/15/Dockerfile @@ -0,0 +1,11 @@ +FROM postgres:15-alpine + +LABEL maintainer="Just van den Broecke " + +RUN apk add --no-cache --update gettext python3 py3-pip && pip3 install click docker && mkdir /pgbackup + +ENV PGB_SCHEDULE 0 23 * * * + +ADD docker/* /pgbackup/ + +CMD ["/pgbackup/entrypoint.sh"] diff --git a/15/README.md b/15/README.md new file mode 100644 index 0000000..c5f8a04 --- /dev/null +++ b/15/README.md @@ -0,0 +1,188 @@ +# Docker PG Backup + +Inspired by: https://github.com/kartoza/docker-pg-backup + +A Docker container that runs automated scheduled PostgreSQL/PostGIS backups for all +PostgreSQL-based Docker Containers in its network that have the Label `"pgbackup.enable=true"`. + +It should work with +the following PostgreSQL/PostGIS Docker images: + +* [mdillon/postgis](https://hub.docker.com/r/mdillon/postgis/) +* [Kartoza docker postgis](https://github.com/kartoza//docker-postgis) +* [Standard PostgreSQL Docker image](https://hub.docker.com/_/postgres/) + +Any other PostgreSQL/PostGIS image may work as long as its Container has the `POSTGRES_` environment +variables set (see below). + +By default it will create a backup once per night (at 23:00) in a +nicely ordered directory by container-name/year/month, but you can specify your own schedule. + +* Docker hub at: https://registry.hub.docker.com/u/justb4/pgbackup/ +* Github at: https://github.com/justb4/docker-pgbackup + +## Getting the image + +There are various ways to get the image onto your system: + +The preferred way (but using most bandwidth for the initial image) is to +get our docker trusted build like this: + + +``` +docker pull justb4/pgbackup:15 + +``` + +We highly suggest that you use a tagged image as +latest may change and may not successfully back up your database. Use the same or +greater version of postgis as the database you are backing up. +To build the image yourself: + +``` +docker build -t justb4/pgbackup . +``` + +If you do not wish to do a local checkout first then build directly from github. + +``` +git clone git://github.com/justb4/docker-pgbackup +``` + +## Environment Variables + +* `PGB_SCHEDULE`, crontab schedule line, if not set, defaults to : `0 23 * * *` + +## Run Backups + +To create a running container do: + +``` +docker run --name="pgbackup"\ + -v backup:/backup -v /var/run/docker.sock:/var/run/docker.sock \ + -i -d justb4/pgbackup:15 +``` + +In this example a local dir (`./backup`) is mounted inti which the actual backups will be +stored. + +Best is to use docker-compose, below the as used +for testing, with a schedule that backs up once a minute. + + +``` +# Test for pgbackup with sample db +version: "3" + +services: + db: + image: mdillon/postgis:15-alpine + container_name: pg_db_15 + labels: + - "pgbackup.enable=true" + environment: + - POSTGRES_DB=testdb + - POSTGRES_USER=testuser + - POSTGRES_PASSWORD=testpass + + dbbackup: + image: justb4/pgbackup:15 + container_name: pg_backup_15 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./backup:/backup + environment: + - PGB_SCHEDULE=*/1 * * * * + +``` + +Then run using: + +``` +docker-compose up -d +``` + +It is advised to use explicit DB-container-naming, as backups will be stored in +subdirectories (`year/month/-ymd-hm.sql.gz`). + +## Explicit Backups + +You can also run backups (and restores) explicitly, by calling `exec` on the `justb4/pgbackup` +container, assuming `pgbackup` here. + +Backup all DBs containers: + +``` +docker exec -it pgbackup /pgbackup/backup-all.sh + +``` + +Or you can backup a single DB container: + +``` +docker exec -it pgbackup /pgbackup/backup.sh + + +# example +docker exec -it pgbackup /pgbackup/backup.sh pgdb /backup/mybackup.sql.gz + +``` + +## List Backups + +You can list all backups available in the container: + +``` +docker exec -it pgbackup /pgbackup/list-backups.sh + +``` + +## Restoring Backups + +This Docker Image also provides restore utilities. + +You can `bash` into the `justb4/pgbackup` container and run `restore.sh` or other commands +from there. The following steps are needed: + +* if not already present copy your backup file, assuming `/backup/mybackup.sql.gz` here, into the `pgbackup` container mounted volume +* figure out the name of your `justb4/pgbackup` container, assuming `pgbackup` here +* figure out the name of your target DB container, assuming `pgdb` here +* `bash` into the container: `docker exec -it pgbackup /bin/bash` +* execute restore: `/pgbackup/restore.sh /backup/mybackup.sql.gz pgdb` + +You could also `exec` directly. Below an example: + +``` +docker exec -it pgbackup /pgbackup/restore.sh pgdb /backup/2018/10/pgdb-181013-1050.sql.gz + +``` + +## Design and diffs with kartoza/pg-backup + +Main difference is that `justb4/pgbackup` uses the Docker API to search within its Docker Network for +Containers that have the Label `"pgbackup.enable=true"`. Using Labels in conjunction with the Docker API +is found in many modern Docker-based services, like e.g. Traefik and Kubernetes. + +Each Container to be backed up is then further inspected to get the PostgreSQL credentials +needed to connect with PG tools like `psql`. The Container name will be the PG Hostname +(TODO: figure out IP address via Docker API, +such that single backup/restores can be run commandline). + +This has the following advantages: + +* loose coupling, easy to setup +* one `pgbackup` Container can backup multiple PostgreSQL Containers +* no need to configure `pgbackup` with all PG credentials + +Further changes: + +* works with multiple Docker images for both PostgreSQL and PostGIS (mdillon and kartoza) +* using smaller `postgres:-alpine` as base image (i.s.o. `kartoza/postgis`) +* schedule via env var `PGB_SCHEDULE` +* dumps in SQL gzip format (more portable among PG versions) but may become option in futre +* includes restore command to restore backup file in a named container + +## Credits + +* Tim Sutton (tim@kartoza.com) for https://github.com/kartoza/docker-pg-backup - Consulted Oct 2018 +* Just van den Broecke (https://justobjects.nl) - this version - 2018 diff --git a/15/build.sh b/15/build.sh new file mode 100755 index 0000000..83a4c3e --- /dev/null +++ b/15/build.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker build -t justb4/pgbackup:15 . diff --git a/15/docker/backup-all.sh b/15/docker/backup-all.sh new file mode 100755 index 0000000..4b6f571 --- /dev/null +++ b/15/docker/backup-all.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +echo "START Backup" + +pushd /pgbackup + python3 pgbackup.py --backupdir /backup backup-all +popd + +echo "END Backup" diff --git a/15/docker/backup.sh b/15/docker/backup.sh new file mode 100755 index 0000000..b19d49f --- /dev/null +++ b/15/docker/backup.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +echo "START Backup single DB" + +# stop on errors +set -e + +# check that we have right number of arguments +if [[ ! $# -eq 2 ]] +then + echo 'usage:' + echo ' /pgbackup/backup.sh ' + echo '' + echo 'to get a list of available backups, run:' + echo ' /pgbackup/list-backups.sh' + exit 1 +fi + +# set the container name variable +CONTAINERNAME=$1 + +# set the backupfile variable +BACKUPFILE=$2 + +echo "Backing up ${CONTAINERNAME} to ${BACKUPFILE}" + +pushd /pgbackup + python3 pgbackup.py --filepath ${BACKUPFILE} --containername ${CONTAINERNAME} backup +popd + +echo "END Backup" diff --git a/15/docker/cronfile-template b/15/docker/cronfile-template new file mode 100755 index 0000000..ae4807c --- /dev/null +++ b/15/docker/cronfile-template @@ -0,0 +1,4 @@ +# Schedule supplied via env var +${PGB_SCHEDULE} /pgbackup/backup-all.sh 2>&1 + +# We need a blank line here for it to be a valid cron file diff --git a/15/docker/entrypoint.sh b/15/docker/entrypoint.sh new file mode 100755 index 0000000..56129d9 --- /dev/null +++ b/15/docker/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +PGB_SCHEDULE=${PGB_SCHEDULE:='0 23 * * *'} + +pushd /pgbackup + +envsubst < cronfile-template > backup-cron + +# Now launch cron in then foreground. +echo "Launching /usr/sbin/crond in foregound with schedule:" +cat backup-cron +crontab backup-cron +/usr/sbin/crond -f -S -l 0 diff --git a/15/docker/list-backups.sh b/15/docker/list-backups.sh new file mode 100755 index 0000000..7d3202d --- /dev/null +++ b/15/docker/list-backups.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "listing available backups" +echo "-------------------------" +find /backup -type f -name '*.sql.gz' diff --git a/15/docker/pgbackup.py b/15/docker/pgbackup.py new file mode 100644 index 0000000..bc17e3a --- /dev/null +++ b/15/docker/pgbackup.py @@ -0,0 +1,156 @@ +import time +import click +import docker +import subprocess + +client = docker.from_env() +backup_dir = None +container_name = None +file_path = None + +def echo(s): + click.echo(s) + +def get_enabled_containers(): + filters = {'label': 'pgbackup.enable=true'} + + containers = client.containers.list(filters=filters) + return containers + +def get_enabled_container(container_name): + containers = get_enabled_containers() + found_container = None + for container in containers: + if container.name == container_name: + found_container = container + return found_container + +def get_pg_creds(container): + pg_creds = dict() + pg_creds['PGHOST'] = container.name + pg_creds['PGPORT'] = '5432' + + env_list = container.attrs['Config']['Env'] + for env_kv in env_list: + # Matches POSTGRES_DB and POSTGRES_DBNAME + if 'POSTGRES_DB' in env_kv: + pg_creds['PGDB'] = env_kv.split('=')[1] + if 'POSTGRES_USER' in env_kv: + pg_creds['PGUSER'] = env_kv.split('=')[1] + # Matches POSTGRES_PASS and POSTGRES_PASSWORD + if 'POSTGRES_PASS' in env_kv: + pg_creds['PGPASSWORD'] = env_kv.split('=')[1] + + return pg_creds + + +def make_backup_cmd(pg_creds, file_path): + cmd_template = 'export PGPASSWORD=%s; pg_dump -h %s -U %s %s | gzip > %s' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], pg_creds['PGHOST'], + pg_creds['PGUSER'], pg_creds['PGDB'], + file_path) + return cmd + + +def make_dropdb_cmd(pg_creds): + cmd_template = 'export PGPASSWORD=%s; dropdb -h %s -U %s %s' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], pg_creds['PGHOST'], + pg_creds['PGUSER'], pg_creds['PGDB'] + ) + return cmd + + +def make_createdb_cmd(pg_creds): + cmd_template = 'export PGPASSWORD=%s; createdb -h %s -U %s -O %s %s' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], pg_creds['PGHOST'], pg_creds['PGUSER'], + pg_creds['PGUSER'], pg_creds['PGDB'] + ) + return cmd + + +def make_restore_cmd(pg_creds, file_path): + cmd_template = 'export PGPASSWORD=%s; gunzip -c %s | psql -h %s -U %s %s ' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], file_path, pg_creds['PGHOST'], + pg_creds['PGUSER'], pg_creds['PGDB'] + ) + return cmd + + +def execute_cmd(cmd): + echo('executing cmd=%s' % cmd) + exit_code = subprocess.call(cmd, shell=True) + echo('execute done, exit_code=%d' % exit_code) + + +@click.group() +@click.option('--backupdir', help='Backup root directory') +@click.option('--containername', required=False, help='Optional container name for backup single or restore') +@click.option('--filepath', required=False, help='Optional backup file for backup single or restore') +def cli(backupdir, containername, filepath): + global backup_dir + global container_name + global file_path + + backup_dir = backupdir + container_name = containername + file_path = filepath + echo('backupdir=%s' % backup_dir) + + +@cli.command() +def backup_all(): + echo('START backup_all') + containers = get_enabled_containers() + echo('Found %d containers with backup enabled' % len(containers)) + for container in containers: + echo('Backing up %s ...' % container.name) + now = time.localtime() + pg_creds = get_pg_creds(container) + echo('creds: %s' % str(pg_creds)) + + dir_path = '%s/%s' % (backup_dir, time.strftime("%Y/%m", now)) + file_path = '%s/%s-%s.sql.gz' % (dir_path, container.name, time.strftime("%y%m%d-%H%M", now)) + + execute_cmd('mkdir -p %s' % dir_path) + + backup_cmd = make_backup_cmd(pg_creds, file_path) + echo('backup_cmd=%s' % backup_cmd) + execute_cmd(backup_cmd) + + echo('END backup_all') + + +@cli.command() +def backup(): + echo('backup single for container %s' % container_name) + container = get_enabled_container(container_name) + pg_creds = get_pg_creds(container) + + backup_cmd = make_backup_cmd(pg_creds, file_path) + execute_cmd(backup_cmd) + + +@cli.command() +def restore(): + echo('restore single for container %s' % container_name) + + container = get_enabled_container(container_name) + pg_creds = get_pg_creds(container) + + # Three steps: drop existing DB, create new DB and restore + dropdb_cmd = make_dropdb_cmd(pg_creds) + execute_cmd(dropdb_cmd) + + createdb_cmd = make_createdb_cmd(pg_creds) + execute_cmd(createdb_cmd) + + restore_cmd = make_restore_cmd(pg_creds, file_path) + execute_cmd(restore_cmd) + + +if __name__ == '__main__': + cli() diff --git a/15/docker/restore.sh b/15/docker/restore.sh new file mode 100755 index 0000000..5cef866 --- /dev/null +++ b/15/docker/restore.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +echo "START Restore" + +# stop on errors +set -e + +# check that we have right number of arguments +if [[ ! $# -eq 2 ]] +then + echo 'usage:' + echo ' /pgbackup/restore.sh ' + echo '' + echo 'to get a list of available backups, run:' + echo ' /pgbackup/list-backups.sh' + exit 1 +fi + +# set the container name variable +CONTAINERNAME=$1 + +# set the backupfile variable +BACKUPFILE=$2 + +# check that the file exists +if [ ! -f ${BACKUPFILE} ] +then + echo "backup file not found" + echo 'to get a list of available backups, run:' + echo ' /pgbackup/list-backups.sh' + exit 1 +fi + + +echo "Restoring to ${CONTAINERNAME} from ${BACKUPFILE}" + +pushd /pgbackup + python3 pgbackup.py --filepath ${BACKUPFILE} --containername ${CONTAINERNAME} restore +popd + +echo "END Restore" diff --git a/15/test/README.md b/15/test/README.md new file mode 100644 index 0000000..f619ebf --- /dev/null +++ b/15/test/README.md @@ -0,0 +1,49 @@ +# Docker PG Backup Test + +Inspired by: https://github.com/kartoza/docker-pg-backup + +Uses a simple [docker-compose.yml](docker-compose.yml) to start a PostGIS DB +plus a `pgbackup` container. + +## Running + +Invoke [start.sh](start.sh). Optionally look at logs: + +``` +docker logs --follow pg_backup_client +``` + +As the backup runs every minute, it could be that the `pg_backup_db` container is not yet +up completely thus PG DB not ready. + +Invoke [stop.sh](stop.sh) to stop. Look under `./backup` for backup files. + +## TIP - explicit backup + +Simply `bash` into `pgbackup` container. + +``` + $ docker exec -it pg_backup_client bash + $ /backup.sh +``` + +## TIP - explicit restore + +Copy `.sql.gz` into the `./backup` directory. +Simply `bash` into `pgbackup` container. + +``` + $ docker exec -it pg_backup_client bash + $ /restore.sh /backup/.sql.gz +``` + +## TIP - connect with psql + +Simply `bash` into `pgbackup` container, using the env settings file. + +``` + $ docker exec -it pg_backup_client bash + $ source /pgb-env.sh + $ psql +``` + diff --git a/15/test/backup/README.md b/15/test/backup/README.md new file mode 100644 index 0000000..836581d --- /dev/null +++ b/15/test/backup/README.md @@ -0,0 +1 @@ +test dir for backups diff --git a/15/test/docker-compose.yml b/15/test/docker-compose.yml new file mode 100644 index 0000000..691d495 --- /dev/null +++ b/15/test/docker-compose.yml @@ -0,0 +1,22 @@ +# Test for pgbackup with sample db +version: "3" + +services: + db: + image: mdillon/postgis:15-alpine + container_name: pg_db_15 + labels: + - "pgbackup.enable=true" + environment: + - POSTGRES_DB=testdb + - POSTGRES_USER=testuser + - POSTGRES_PASSWORD=testpass + + dbbackup: + image: justb4/pgbackup:15 + container_name: pg_backup_15 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./backup:/backup + environment: + - PGB_SCHEDULE=*/1 * * * * diff --git a/15/test/start.sh b/15/test/start.sh new file mode 100755 index 0000000..a693034 --- /dev/null +++ b/15/test/start.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Stop and remove possibly old containers +docker-compose stop +docker-compose rm -f + +# Finally run +docker-compose up -d + + diff --git a/15/test/stop.sh b/15/test/stop.sh new file mode 100755 index 0000000..9193033 --- /dev/null +++ b/15/test/stop.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# Stop and remove possibly old containers +docker-compose stop +docker-compose rm -f diff --git a/16/Dockerfile b/16/Dockerfile new file mode 100644 index 0000000..e1edde5 --- /dev/null +++ b/16/Dockerfile @@ -0,0 +1,11 @@ +FROM postgres:16-alpine + +LABEL maintainer="Just van den Broecke " + +RUN apk add --no-cache --update gettext python3 py3-pip && pip3 install click docker && mkdir /pgbackup + +ENV PGB_SCHEDULE 0 23 * * * + +ADD docker/* /pgbackup/ + +CMD ["/pgbackup/entrypoint.sh"] diff --git a/16/README.md b/16/README.md new file mode 100644 index 0000000..24a60a0 --- /dev/null +++ b/16/README.md @@ -0,0 +1,188 @@ +# Docker PG Backup + +Inspired by: https://github.com/kartoza/docker-pg-backup + +A Docker container that runs automated scheduled PostgreSQL/PostGIS backups for all +PostgreSQL-based Docker Containers in its network that have the Label `"pgbackup.enable=true"`. + +It should work with +the following PostgreSQL/PostGIS Docker images: + +* [mdillon/postgis](https://hub.docker.com/r/mdillon/postgis/) +* [Kartoza docker postgis](https://github.com/kartoza//docker-postgis) +* [Standard PostgreSQL Docker image](https://hub.docker.com/_/postgres/) + +Any other PostgreSQL/PostGIS image may work as long as its Container has the `POSTGRES_` environment +variables set (see below). + +By default it will create a backup once per night (at 23:00) in a +nicely ordered directory by container-name/year/month, but you can specify your own schedule. + +* Docker hub at: https://registry.hub.docker.com/u/justb4/pgbackup/ +* Github at: https://github.com/justb4/docker-pgbackup + +## Getting the image + +There are various ways to get the image onto your system: + +The preferred way (but using most bandwidth for the initial image) is to +get our docker trusted build like this: + + +``` +docker pull justb4/pgbackup:16 + +``` + +We highly suggest that you use a tagged image as +latest may change and may not successfully back up your database. Use the same or +greater version of postgis as the database you are backing up. +To build the image yourself: + +``` +docker build -t justb4/pgbackup . +``` + +If you do not wish to do a local checkout first then build directly from github. + +``` +git clone git://github.com/justb4/docker-pgbackup +``` + +## Environment Variables + +* `PGB_SCHEDULE`, crontab schedule line, if not set, defaults to : `0 23 * * *` + +## Run Backups + +To create a running container do: + +``` +docker run --name="pgbackup"\ + -v backup:/backup -v /var/run/docker.sock:/var/run/docker.sock \ + -i -d justb4/pgbackup:16 +``` + +In this example a local dir (`./backup`) is mounted inti which the actual backups will be +stored. + +Best is to use docker-compose, below the as used +for testing, with a schedule that backs up once a minute. + + +``` +# Test for pgbackup with sample db +version: "3" + +services: + db: + image: mdillon/postgis:16-alpine + container_name: pg_db_16 + labels: + - "pgbackup.enable=true" + environment: + - POSTGRES_DB=testdb + - POSTGRES_USER=testuser + - POSTGRES_PASSWORD=testpass + + dbbackup: + image: justb4/pgbackup:16 + container_name: pg_backup_16 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./backup:/backup + environment: + - PGB_SCHEDULE=*/1 * * * * + +``` + +Then run using: + +``` +docker-compose up -d +``` + +It is advised to use explicit DB-container-naming, as backups will be stored in +subdirectories (`year/month/-ymd-hm.sql.gz`). + +## Explicit Backups + +You can also run backups (and restores) explicitly, by calling `exec` on the `justb4/pgbackup` +container, assuming `pgbackup` here. + +Backup all DBs containers: + +``` +docker exec -it pgbackup /pgbackup/backup-all.sh + +``` + +Or you can backup a single DB container: + +``` +docker exec -it pgbackup /pgbackup/backup.sh + + +# example +docker exec -it pgbackup /pgbackup/backup.sh pgdb /backup/mybackup.sql.gz + +``` + +## List Backups + +You can list all backups available in the container: + +``` +docker exec -it pgbackup /pgbackup/list-backups.sh + +``` + +## Restoring Backups + +This Docker Image also provides restore utilities. + +You can `bash` into the `justb4/pgbackup` container and run `restore.sh` or other commands +from there. The following steps are needed: + +* if not already present copy your backup file, assuming `/backup/mybackup.sql.gz` here, into the `pgbackup` container mounted volume +* figure out the name of your `justb4/pgbackup` container, assuming `pgbackup` here +* figure out the name of your target DB container, assuming `pgdb` here +* `bash` into the container: `docker exec -it pgbackup /bin/bash` +* execute restore: `/pgbackup/restore.sh /backup/mybackup.sql.gz pgdb` + +You could also `exec` directly. Below an example: + +``` +docker exec -it pgbackup /pgbackup/restore.sh pgdb /backup/2018/10/pgdb-181013-1050.sql.gz + +``` + +## Design and diffs with kartoza/pg-backup + +Main difference is that `justb4/pgbackup` uses the Docker API to search within its Docker Network for +Containers that have the Label `"pgbackup.enable=true"`. Using Labels in conjunction with the Docker API +is found in many modern Docker-based services, like e.g. Traefik and Kubernetes. + +Each Container to be backed up is then further inspected to get the PostgreSQL credentials +needed to connect with PG tools like `psql`. The Container name will be the PG Hostname +(TODO: figure out IP address via Docker API, +such that single backup/restores can be run commandline). + +This has the following advantages: + +* loose coupling, easy to setup +* one `pgbackup` Container can backup multiple PostgreSQL Containers +* no need to configure `pgbackup` with all PG credentials + +Further changes: + +* works with multiple Docker images for both PostgreSQL and PostGIS (mdillon and kartoza) +* using smaller `postgres:-alpine` as base image (i.s.o. `kartoza/postgis`) +* schedule via env var `PGB_SCHEDULE` +* dumps in SQL gzip format (more portable among PG versions) but may become option in futre +* includes restore command to restore backup file in a named container + +## Credits + +* Tim Sutton (tim@kartoza.com) for https://github.com/kartoza/docker-pg-backup - Consulted Oct 2018 +* Just van den Broecke (https://justobjects.nl) - this version - 2018 diff --git a/16/build.sh b/16/build.sh new file mode 100755 index 0000000..8d6578a --- /dev/null +++ b/16/build.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker build -t justb4/pgbackup:16 . diff --git a/16/docker/backup-all.sh b/16/docker/backup-all.sh new file mode 100755 index 0000000..4b6f571 --- /dev/null +++ b/16/docker/backup-all.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +echo "START Backup" + +pushd /pgbackup + python3 pgbackup.py --backupdir /backup backup-all +popd + +echo "END Backup" diff --git a/16/docker/backup.sh b/16/docker/backup.sh new file mode 100755 index 0000000..b19d49f --- /dev/null +++ b/16/docker/backup.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +echo "START Backup single DB" + +# stop on errors +set -e + +# check that we have right number of arguments +if [[ ! $# -eq 2 ]] +then + echo 'usage:' + echo ' /pgbackup/backup.sh ' + echo '' + echo 'to get a list of available backups, run:' + echo ' /pgbackup/list-backups.sh' + exit 1 +fi + +# set the container name variable +CONTAINERNAME=$1 + +# set the backupfile variable +BACKUPFILE=$2 + +echo "Backing up ${CONTAINERNAME} to ${BACKUPFILE}" + +pushd /pgbackup + python3 pgbackup.py --filepath ${BACKUPFILE} --containername ${CONTAINERNAME} backup +popd + +echo "END Backup" diff --git a/16/docker/cronfile-template b/16/docker/cronfile-template new file mode 100755 index 0000000..ae4807c --- /dev/null +++ b/16/docker/cronfile-template @@ -0,0 +1,4 @@ +# Schedule supplied via env var +${PGB_SCHEDULE} /pgbackup/backup-all.sh 2>&1 + +# We need a blank line here for it to be a valid cron file diff --git a/16/docker/entrypoint.sh b/16/docker/entrypoint.sh new file mode 100755 index 0000000..56129d9 --- /dev/null +++ b/16/docker/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +PGB_SCHEDULE=${PGB_SCHEDULE:='0 23 * * *'} + +pushd /pgbackup + +envsubst < cronfile-template > backup-cron + +# Now launch cron in then foreground. +echo "Launching /usr/sbin/crond in foregound with schedule:" +cat backup-cron +crontab backup-cron +/usr/sbin/crond -f -S -l 0 diff --git a/16/docker/list-backups.sh b/16/docker/list-backups.sh new file mode 100755 index 0000000..7d3202d --- /dev/null +++ b/16/docker/list-backups.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "listing available backups" +echo "-------------------------" +find /backup -type f -name '*.sql.gz' diff --git a/16/docker/pgbackup.py b/16/docker/pgbackup.py new file mode 100644 index 0000000..bc17e3a --- /dev/null +++ b/16/docker/pgbackup.py @@ -0,0 +1,156 @@ +import time +import click +import docker +import subprocess + +client = docker.from_env() +backup_dir = None +container_name = None +file_path = None + +def echo(s): + click.echo(s) + +def get_enabled_containers(): + filters = {'label': 'pgbackup.enable=true'} + + containers = client.containers.list(filters=filters) + return containers + +def get_enabled_container(container_name): + containers = get_enabled_containers() + found_container = None + for container in containers: + if container.name == container_name: + found_container = container + return found_container + +def get_pg_creds(container): + pg_creds = dict() + pg_creds['PGHOST'] = container.name + pg_creds['PGPORT'] = '5432' + + env_list = container.attrs['Config']['Env'] + for env_kv in env_list: + # Matches POSTGRES_DB and POSTGRES_DBNAME + if 'POSTGRES_DB' in env_kv: + pg_creds['PGDB'] = env_kv.split('=')[1] + if 'POSTGRES_USER' in env_kv: + pg_creds['PGUSER'] = env_kv.split('=')[1] + # Matches POSTGRES_PASS and POSTGRES_PASSWORD + if 'POSTGRES_PASS' in env_kv: + pg_creds['PGPASSWORD'] = env_kv.split('=')[1] + + return pg_creds + + +def make_backup_cmd(pg_creds, file_path): + cmd_template = 'export PGPASSWORD=%s; pg_dump -h %s -U %s %s | gzip > %s' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], pg_creds['PGHOST'], + pg_creds['PGUSER'], pg_creds['PGDB'], + file_path) + return cmd + + +def make_dropdb_cmd(pg_creds): + cmd_template = 'export PGPASSWORD=%s; dropdb -h %s -U %s %s' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], pg_creds['PGHOST'], + pg_creds['PGUSER'], pg_creds['PGDB'] + ) + return cmd + + +def make_createdb_cmd(pg_creds): + cmd_template = 'export PGPASSWORD=%s; createdb -h %s -U %s -O %s %s' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], pg_creds['PGHOST'], pg_creds['PGUSER'], + pg_creds['PGUSER'], pg_creds['PGDB'] + ) + return cmd + + +def make_restore_cmd(pg_creds, file_path): + cmd_template = 'export PGPASSWORD=%s; gunzip -c %s | psql -h %s -U %s %s ' + cmd = cmd_template % ( + pg_creds['PGPASSWORD'], file_path, pg_creds['PGHOST'], + pg_creds['PGUSER'], pg_creds['PGDB'] + ) + return cmd + + +def execute_cmd(cmd): + echo('executing cmd=%s' % cmd) + exit_code = subprocess.call(cmd, shell=True) + echo('execute done, exit_code=%d' % exit_code) + + +@click.group() +@click.option('--backupdir', help='Backup root directory') +@click.option('--containername', required=False, help='Optional container name for backup single or restore') +@click.option('--filepath', required=False, help='Optional backup file for backup single or restore') +def cli(backupdir, containername, filepath): + global backup_dir + global container_name + global file_path + + backup_dir = backupdir + container_name = containername + file_path = filepath + echo('backupdir=%s' % backup_dir) + + +@cli.command() +def backup_all(): + echo('START backup_all') + containers = get_enabled_containers() + echo('Found %d containers with backup enabled' % len(containers)) + for container in containers: + echo('Backing up %s ...' % container.name) + now = time.localtime() + pg_creds = get_pg_creds(container) + echo('creds: %s' % str(pg_creds)) + + dir_path = '%s/%s' % (backup_dir, time.strftime("%Y/%m", now)) + file_path = '%s/%s-%s.sql.gz' % (dir_path, container.name, time.strftime("%y%m%d-%H%M", now)) + + execute_cmd('mkdir -p %s' % dir_path) + + backup_cmd = make_backup_cmd(pg_creds, file_path) + echo('backup_cmd=%s' % backup_cmd) + execute_cmd(backup_cmd) + + echo('END backup_all') + + +@cli.command() +def backup(): + echo('backup single for container %s' % container_name) + container = get_enabled_container(container_name) + pg_creds = get_pg_creds(container) + + backup_cmd = make_backup_cmd(pg_creds, file_path) + execute_cmd(backup_cmd) + + +@cli.command() +def restore(): + echo('restore single for container %s' % container_name) + + container = get_enabled_container(container_name) + pg_creds = get_pg_creds(container) + + # Three steps: drop existing DB, create new DB and restore + dropdb_cmd = make_dropdb_cmd(pg_creds) + execute_cmd(dropdb_cmd) + + createdb_cmd = make_createdb_cmd(pg_creds) + execute_cmd(createdb_cmd) + + restore_cmd = make_restore_cmd(pg_creds, file_path) + execute_cmd(restore_cmd) + + +if __name__ == '__main__': + cli() diff --git a/16/docker/restore.sh b/16/docker/restore.sh new file mode 100755 index 0000000..5cef866 --- /dev/null +++ b/16/docker/restore.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +echo "START Restore" + +# stop on errors +set -e + +# check that we have right number of arguments +if [[ ! $# -eq 2 ]] +then + echo 'usage:' + echo ' /pgbackup/restore.sh ' + echo '' + echo 'to get a list of available backups, run:' + echo ' /pgbackup/list-backups.sh' + exit 1 +fi + +# set the container name variable +CONTAINERNAME=$1 + +# set the backupfile variable +BACKUPFILE=$2 + +# check that the file exists +if [ ! -f ${BACKUPFILE} ] +then + echo "backup file not found" + echo 'to get a list of available backups, run:' + echo ' /pgbackup/list-backups.sh' + exit 1 +fi + + +echo "Restoring to ${CONTAINERNAME} from ${BACKUPFILE}" + +pushd /pgbackup + python3 pgbackup.py --filepath ${BACKUPFILE} --containername ${CONTAINERNAME} restore +popd + +echo "END Restore" diff --git a/16/test/README.md b/16/test/README.md new file mode 100644 index 0000000..f619ebf --- /dev/null +++ b/16/test/README.md @@ -0,0 +1,49 @@ +# Docker PG Backup Test + +Inspired by: https://github.com/kartoza/docker-pg-backup + +Uses a simple [docker-compose.yml](docker-compose.yml) to start a PostGIS DB +plus a `pgbackup` container. + +## Running + +Invoke [start.sh](start.sh). Optionally look at logs: + +``` +docker logs --follow pg_backup_client +``` + +As the backup runs every minute, it could be that the `pg_backup_db` container is not yet +up completely thus PG DB not ready. + +Invoke [stop.sh](stop.sh) to stop. Look under `./backup` for backup files. + +## TIP - explicit backup + +Simply `bash` into `pgbackup` container. + +``` + $ docker exec -it pg_backup_client bash + $ /backup.sh +``` + +## TIP - explicit restore + +Copy `.sql.gz` into the `./backup` directory. +Simply `bash` into `pgbackup` container. + +``` + $ docker exec -it pg_backup_client bash + $ /restore.sh /backup/.sql.gz +``` + +## TIP - connect with psql + +Simply `bash` into `pgbackup` container, using the env settings file. + +``` + $ docker exec -it pg_backup_client bash + $ source /pgb-env.sh + $ psql +``` + diff --git a/16/test/backup/README.md b/16/test/backup/README.md new file mode 100644 index 0000000..836581d --- /dev/null +++ b/16/test/backup/README.md @@ -0,0 +1 @@ +test dir for backups diff --git a/16/test/docker-compose.yml b/16/test/docker-compose.yml new file mode 100644 index 0000000..8ea89af --- /dev/null +++ b/16/test/docker-compose.yml @@ -0,0 +1,22 @@ +# Test for pgbackup with sample db +version: "3" + +services: + db: + image: mdillon/postgis:16-alpine + container_name: pg_db_16 + labels: + - "pgbackup.enable=true" + environment: + - POSTGRES_DB=testdb + - POSTGRES_USER=testuser + - POSTGRES_PASSWORD=testpass + + dbbackup: + image: justb4/pgbackup:16 + container_name: pg_backup_16 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./backup:/backup + environment: + - PGB_SCHEDULE=*/1 * * * * diff --git a/16/test/start.sh b/16/test/start.sh new file mode 100755 index 0000000..a693034 --- /dev/null +++ b/16/test/start.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Stop and remove possibly old containers +docker-compose stop +docker-compose rm -f + +# Finally run +docker-compose up -d + + diff --git a/16/test/stop.sh b/16/test/stop.sh new file mode 100755 index 0000000..9193033 --- /dev/null +++ b/16/test/stop.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# Stop and remove possibly old containers +docker-compose stop +docker-compose rm -f diff --git a/versions.sh b/versions.sh index ac1e96b..91ed103 100755 --- a/versions.sh +++ b/versions.sh @@ -1,3 +1,3 @@ #!/bin/bash -versions="9.6 10 11 12 13" +versions="9.6 10 11 12 13 14 15 16"