From 813cb41322828299acbcb76359c61e60df04db4e Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Thu, 29 Aug 2019 16:54:17 +0200 Subject: [PATCH 01/12] first shot on a c2c/ms_perfs integration --- qgis4/ansible/roles/msperfs/defaults/main.yml | 8 +++ qgis4/ansible/roles/msperfs/tasks/main.yml | 59 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 qgis4/ansible/roles/msperfs/defaults/main.yml create mode 100644 qgis4/ansible/roles/msperfs/tasks/main.yml diff --git a/qgis4/ansible/roles/msperfs/defaults/main.yml b/qgis4/ansible/roles/msperfs/defaults/main.yml new file mode 100644 index 0000000..e79961e --- /dev/null +++ b/qgis4/ansible/roles/msperfs/defaults/main.yml @@ -0,0 +1,8 @@ +ms_perfs: + ms_perfs: + repo_url: https://github.com/camptocamp/ms_perfs.git + # which version (branch name, tag, SHA-1 ref ...) to clone + version: master + docker_osm: + repo_url: https://github.com/pvalsecc/docker-osm.git + version: develop diff --git a/qgis4/ansible/roles/msperfs/tasks/main.yml b/qgis4/ansible/roles/msperfs/tasks/main.yml new file mode 100644 index 0000000..1c33dce --- /dev/null +++ b/qgis4/ansible/roles/msperfs/tasks/main.yml @@ -0,0 +1,59 @@ +- name: ensures python-docker is installed + apt: + name: python-docker + state: present + +- name: clone the ms_perfs repository + git: + repo: "{{ ms_perfs.ms_perfs.repo_url }}" + dest: "/home/{{ user }}/ms_perfs" + version: "{{ ms_perfs.ms_perfs.version }}" + register: ms_perfs_git + +- name: give the ms_perfs repository to {{ user }} + file: + path: "/home/{{ user }}/ms_perfs" + recurse: yes + owner: "{{ user }}" + group: "{{ user }}" + when: ms_perfs_git.changed + +- name: clone the docker-osm repository + git: + repo: "{{ ms_perfs.docker_osm.repo_url }}" + dest: "/home/{{ user }}/docker-osm" + version: "{{ ms_perfs.docker_osm.version }}" + register: ms_perfs_docker_osm_git + +- name: give the docker-osm repository to {{ user }} + file: + path: "/home/{{ user }}/docker-osm" + recurse: yes + owner: "{{ user }}" + group: "{{ user }}" + when: ms_perfs_docker_osm_git.changed + +- name: check if OSM data from switzerland have been downloaded + stat: + path: "/home/{{ user }}/docker-osm/settings/country.pbf" + register: switzerland_data + +- name: download OSM data from switzerland + command: /usr/bin/python pbf_downloader.py switzerland + args: + chdir: "/home/{{ user }}/docker-osm" + when: switzerland_data.stat.exists == false + +- name: give the data to {{ user }} + file: + path: "/home/{{ user }}/docker-osm" + recurse: yes + owner: "{{ user }}" + group: "{{ user }}" + when: switzerland_data.stat.exists == false + +- name: Launch the docker composition + docker_service: # will be docker_compose in ansible >= 2.8 + project_src: "/home/{{ user }}/docker-osm" + build: yes + pull: yes From 777427da3ae1034b719d71aa28082cecc558853b Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Fri, 30 Aug 2019 15:29:51 +0200 Subject: [PATCH 02/12] source OSM data into a docker postgresql --- qgis4/ansible/playbook.yml | 2 + qgis4/ansible/roles/msperfs/defaults/main.yml | 4 +- qgis4/ansible/roles/msperfs/tasks/main.yml | 47 +++++++++++++++---- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/qgis4/ansible/playbook.yml b/qgis4/ansible/playbook.yml index d85879d..4909eb0 100644 --- a/qgis4/ansible/playbook.yml +++ b/qgis4/ansible/playbook.yml @@ -18,3 +18,5 @@ tags: clean - role: dashboard tags: dashboard + - role: msperfs + tags: msperfs diff --git a/qgis4/ansible/roles/msperfs/defaults/main.yml b/qgis4/ansible/roles/msperfs/defaults/main.yml index e79961e..a914f8a 100644 --- a/qgis4/ansible/roles/msperfs/defaults/main.yml +++ b/qgis4/ansible/roles/msperfs/defaults/main.yml @@ -4,5 +4,5 @@ ms_perfs: # which version (branch name, tag, SHA-1 ref ...) to clone version: master docker_osm: - repo_url: https://github.com/pvalsecc/docker-osm.git - version: develop + repo_url: https://github.com/pmauduit/docker-osm.git + version: create-extension-postgis diff --git a/qgis4/ansible/roles/msperfs/tasks/main.yml b/qgis4/ansible/roles/msperfs/tasks/main.yml index 1c33dce..bb4bbea 100644 --- a/qgis4/ansible/roles/msperfs/tasks/main.yml +++ b/qgis4/ansible/roles/msperfs/tasks/main.yml @@ -1,7 +1,7 @@ -- name: ensures python-docker is installed +- name: ensures python-psycopg2 is available apt: - name: python-docker - state: present + name: python-psycopg2 + update_cache: yes - name: clone the ms_perfs repository git: @@ -21,13 +21,13 @@ - name: clone the docker-osm repository git: repo: "{{ ms_perfs.docker_osm.repo_url }}" - dest: "/home/{{ user }}/docker-osm" + dest: "/home/{{ user }}/ms_perfs/docker-osm" version: "{{ ms_perfs.docker_osm.version }}" register: ms_perfs_docker_osm_git - name: give the docker-osm repository to {{ user }} file: - path: "/home/{{ user }}/docker-osm" + path: "/home/{{ user }}/ms_perfs/docker-osm" recurse: yes owner: "{{ user }}" group: "{{ user }}" @@ -35,25 +35,52 @@ - name: check if OSM data from switzerland have been downloaded stat: - path: "/home/{{ user }}/docker-osm/settings/country.pbf" + path: "/home/{{ user }}/ms_perfs/docker-osm/settings/country.pbf" register: switzerland_data - name: download OSM data from switzerland command: /usr/bin/python pbf_downloader.py switzerland args: - chdir: "/home/{{ user }}/docker-osm" + chdir: "/home/{{ user }}/ms_perfs/docker-osm" when: switzerland_data.stat.exists == false - name: give the data to {{ user }} file: - path: "/home/{{ user }}/docker-osm" + path: "/home/{{ user }}/ms_perfs/docker-osm" recurse: yes owner: "{{ user }}" group: "{{ user }}" when: switzerland_data.stat.exists == false -- name: Launch the docker composition +- name: Launch the docker composition from docker-osm docker_service: # will be docker_compose in ansible >= 2.8 - project_src: "/home/{{ user }}/docker-osm" + project_src: "/home/{{ user }}/ms_perfs/docker-osm" build: yes pull: yes + +- name: wait for the docker db container to be up + wait_for: + port: 15432 + delay: 10 + +- name: Verify if the OSM data have already been loaded + postgresql_schema: + login_host: localhost + port: 15432 + name: import + database: gis + login_user: docker + login_password: docker + state: present + check_mode: yes + register: schema_in_osmdb + +- name: Wait for 5 minutes to get data sourced to postgresql + pause: + minutes: 5 + when: schema_in_osmdb.changed + +- name: Stopping the docker composition from docker-osm + docker_service: + project_src: "/home/{{ user }}/ms_perfs/docker-osm" + stopped: true From 47475e94d9b776810aa6ff59de7cc705f6d8afe1 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Fri, 30 Aug 2019 17:21:43 +0200 Subject: [PATCH 03/12] switching branch for ms_perfs --- qgis4/ansible/roles/msperfs/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgis4/ansible/roles/msperfs/defaults/main.yml b/qgis4/ansible/roles/msperfs/defaults/main.yml index a914f8a..adcc946 100644 --- a/qgis4/ansible/roles/msperfs/defaults/main.yml +++ b/qgis4/ansible/roles/msperfs/defaults/main.yml @@ -2,7 +2,7 @@ ms_perfs: ms_perfs: repo_url: https://github.com/camptocamp/ms_perfs.git # which version (branch name, tag, SHA-1 ref ...) to clone - version: master + version: qgis-sysadmin-integration docker_osm: repo_url: https://github.com/pmauduit/docker-osm.git version: create-extension-postgis From 547db5d16f1465f45a2ead3b3db99073622e2348 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Fri, 30 Aug 2019 19:06:26 +0200 Subject: [PATCH 04/12] ms_perfs integration * we need ansible 8 to have postgresql_query * adding a nginx configuration on 8084 to get the results * reworked the "has the data been loaded ?" logic --- qgis4/ansible/requirements.txt | 2 +- qgis4/ansible/roles/msperfs/handlers/main.yml | 4 +++ qgis4/ansible/roles/msperfs/tasks/main.yml | 35 +++++++++++++------ .../roles/msperfs/templates/msperfs_nginx.j2 | 16 +++++++++ 4 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 qgis4/ansible/roles/msperfs/handlers/main.yml create mode 100644 qgis4/ansible/roles/msperfs/templates/msperfs_nginx.j2 diff --git a/qgis4/ansible/requirements.txt b/qgis4/ansible/requirements.txt index 28fdd7e..ae06c17 100644 --- a/qgis4/ansible/requirements.txt +++ b/qgis4/ansible/requirements.txt @@ -1 +1 @@ -ansible==2.7 +ansible==2.8 diff --git a/qgis4/ansible/roles/msperfs/handlers/main.yml b/qgis4/ansible/roles/msperfs/handlers/main.yml new file mode 100644 index 0000000..5463835 --- /dev/null +++ b/qgis4/ansible/roles/msperfs/handlers/main.yml @@ -0,0 +1,4 @@ +- name: restart nginx + service: + name: nginx + state: restarted diff --git a/qgis4/ansible/roles/msperfs/tasks/main.yml b/qgis4/ansible/roles/msperfs/tasks/main.yml index bb4bbea..7d0da0b 100644 --- a/qgis4/ansible/roles/msperfs/tasks/main.yml +++ b/qgis4/ansible/roles/msperfs/tasks/main.yml @@ -52,6 +52,9 @@ group: "{{ user }}" when: switzerland_data.stat.exists == false +# Note: unfortunately, this won't converge, +# because we do need to launch the database container +# to check if OSM datas have been loaded - name: Launch the docker composition from docker-osm docker_service: # will be docker_compose in ansible >= 2.8 project_src: "/home/{{ user }}/ms_perfs/docker-osm" @@ -64,23 +67,35 @@ delay: 10 - name: Verify if the OSM data have already been loaded - postgresql_schema: + postgresql_query: + db: gis login_host: localhost - port: 15432 - name: import - database: gis + login_port: 15432 login_user: docker login_password: docker - state: present - check_mode: yes - register: schema_in_osmdb + query: "SELECT pg_database_size('gis');" + register: query_in_osmdb -- name: Wait for 5 minutes to get data sourced to postgresql +- name: Wait for 10 minutes to get data sourced to postgresql pause: - minutes: 5 - when: schema_in_osmdb.changed + minutes: 10 + when: query_in_osmdb.query_result[0].pg_database_size < 10000000 - name: Stopping the docker composition from docker-osm docker_service: project_src: "/home/{{ user }}/ms_perfs/docker-osm" stopped: true + +- name: Configure Nginx to serve the ms_perfs results + template: + src: msperfs_nginx.j2 + dest: /etc/nginx/sites-available/msperfs + mode: 0644 + notify: restart nginx + +- name: create symlink + file: + src: /etc/nginx/sites-available/msperfs + dest: /etc/nginx/sites-enabled/msperfs + state: link + notify: restart nginx diff --git a/qgis4/ansible/roles/msperfs/templates/msperfs_nginx.j2 b/qgis4/ansible/roles/msperfs/templates/msperfs_nginx.j2 new file mode 100644 index 0000000..5305ac6 --- /dev/null +++ b/qgis4/ansible/roles/msperfs/templates/msperfs_nginx.j2 @@ -0,0 +1,16 @@ +server { + listen 8084; + listen [::]:8084; + + root /home/{{ user }}/ms_perfs/archives; + + index index.html index.htm index.nginx-debian.html; + + server_name _; + + location / { + try_files $uri $uri/ =404; + autoindex on; + } + +} From 1cde9b31877ec0852adae48dc91cf22aa7ba4416 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 2 Sep 2019 11:54:08 +0200 Subject: [PATCH 05/12] Saving work before switching branches --- qgis4/ansible/playbook.yml | 1 + qgis4/ansible/roles/msperfs/tasks/main.yml | 8 +++++--- qgis4/ansible/roles/perfsuite/tasks/main.yml | 6 +++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/qgis4/ansible/playbook.yml b/qgis4/ansible/playbook.yml index 4909eb0..a613c2b 100644 --- a/qgis4/ansible/playbook.yml +++ b/qgis4/ansible/playbook.yml @@ -20,3 +20,4 @@ tags: dashboard - role: msperfs tags: msperfs + diff --git a/qgis4/ansible/roles/msperfs/tasks/main.yml b/qgis4/ansible/roles/msperfs/tasks/main.yml index 7d0da0b..9e3dec4 100644 --- a/qgis4/ansible/roles/msperfs/tasks/main.yml +++ b/qgis4/ansible/roles/msperfs/tasks/main.yml @@ -55,8 +55,10 @@ # Note: unfortunately, this won't converge, # because we do need to launch the database container # to check if OSM datas have been loaded +# TODO: instead, use a file to "flag" if data have already been imported or not + - name: Launch the docker composition from docker-osm - docker_service: # will be docker_compose in ansible >= 2.8 + docker_compose: project_src: "/home/{{ user }}/ms_perfs/docker-osm" build: yes pull: yes @@ -79,10 +81,10 @@ - name: Wait for 10 minutes to get data sourced to postgresql pause: minutes: 10 - when: query_in_osmdb.query_result[0].pg_database_size < 10000000 + when: query_in_osmdb.query_result[0].pg_database_size < 13000000 # 13MB - name: Stopping the docker composition from docker-osm - docker_service: + docker_compose: project_src: "/home/{{ user }}/ms_perfs/docker-osm" stopped: true diff --git a/qgis4/ansible/roles/perfsuite/tasks/main.yml b/qgis4/ansible/roles/perfsuite/tasks/main.yml index c32f9fd..5a1dff1 100644 --- a/qgis4/ansible/roles/perfsuite/tasks/main.yml +++ b/qgis4/ansible/roles/perfsuite/tasks/main.yml @@ -1,6 +1,10 @@ +- name: Include the NodeJS role + include_role: + name: geerlingguy.nodejs + - name: Install prerequisites apt: - name: ['postgresql-client', 'curl', 'virtualenv'] + name: ['postgresql-client', 'curl', 'virtualenv', 'libpq-dev', 'npm', 'libpython3.5-dev'] update_cache: yes - name: Copy aggregate report scripts From 9ebc0a9e02d4bce25f9910b5ca83a1a464d5f32c Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 2 Sep 2019 16:03:50 +0200 Subject: [PATCH 06/12] rework the OSM data loading process --- qgis4/ansible/roles/msperfs/tasks/main.yml | 33 +++++++++++----------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/qgis4/ansible/roles/msperfs/tasks/main.yml b/qgis4/ansible/roles/msperfs/tasks/main.yml index 9e3dec4..689071d 100644 --- a/qgis4/ansible/roles/msperfs/tasks/main.yml +++ b/qgis4/ansible/roles/msperfs/tasks/main.yml @@ -52,41 +52,40 @@ group: "{{ user }}" when: switzerland_data.stat.exists == false -# Note: unfortunately, this won't converge, -# because we do need to launch the database container -# to check if OSM datas have been loaded -# TODO: instead, use a file to "flag" if data have already been imported or not +- name: Check if OSM datas have been already loaded + stat: + path: "/home/{{ user }}/.ms_perfs_data_loaded" + register: data_loaded - name: Launch the docker composition from docker-osm - docker_compose: + docker_service: project_src: "/home/{{ user }}/ms_perfs/docker-osm" build: yes pull: yes + when: not data_loaded.stat.exists - name: wait for the docker db container to be up wait_for: port: 15432 delay: 10 - -- name: Verify if the OSM data have already been loaded - postgresql_query: - db: gis - login_host: localhost - login_port: 15432 - login_user: docker - login_password: docker - query: "SELECT pg_database_size('gis');" - register: query_in_osmdb + when: not data_loaded.stat.exists - name: Wait for 10 minutes to get data sourced to postgresql pause: minutes: 10 - when: query_in_osmdb.query_result[0].pg_database_size < 13000000 # 13MB + when: not data_loaded.stat.exists - name: Stopping the docker composition from docker-osm - docker_compose: + docker_service: project_src: "/home/{{ user }}/ms_perfs/docker-osm" stopped: true + when: not data_loaded.stat.exists + +- name: Create the flag so that OSM data won't be overwritten again + copy: + content: "" + dest: "/home/{{ user }}/.ms_perfs_data_loaded" + when: not data_loaded.stat.exists - name: Configure Nginx to serve the ms_perfs results template: From 3259e84e3744c8c7b12f214dfd7cf8e23a733c1b Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Tue, 3 Sep 2019 18:28:26 +0200 Subject: [PATCH 07/12] Adding a role dependency msperfs -> nginx --- qgis4/ansible/roles/msperfs/meta/main.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 qgis4/ansible/roles/msperfs/meta/main.yml diff --git a/qgis4/ansible/roles/msperfs/meta/main.yml b/qgis4/ansible/roles/msperfs/meta/main.yml new file mode 100644 index 0000000..97f5b7e --- /dev/null +++ b/qgis4/ansible/roles/msperfs/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: +- role: nginx From f44942203faec20fb75cf24a3278846aaf14be63 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Tue, 3 Sep 2019 18:29:10 +0200 Subject: [PATCH 08/12] getting rid of geerlingguy's role --- qgis4/ansible/roles/perfsuite/tasks/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/qgis4/ansible/roles/perfsuite/tasks/main.yml b/qgis4/ansible/roles/perfsuite/tasks/main.yml index 5a1dff1..32a7217 100644 --- a/qgis4/ansible/roles/perfsuite/tasks/main.yml +++ b/qgis4/ansible/roles/perfsuite/tasks/main.yml @@ -1,7 +1,3 @@ -- name: Include the NodeJS role - include_role: - name: geerlingguy.nodejs - - name: Install prerequisites apt: name: ['postgresql-client', 'curl', 'virtualenv', 'libpq-dev', 'npm', 'libpython3.5-dev'] From 3d761161eee5a3b85b329e4114741a8260fd66c4 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 9 Sep 2019 10:54:34 +0200 Subject: [PATCH 09/12] handler unecessary, provided by the nginx role --- qgis4/ansible/roles/msperfs/handlers/main.yml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 qgis4/ansible/roles/msperfs/handlers/main.yml diff --git a/qgis4/ansible/roles/msperfs/handlers/main.yml b/qgis4/ansible/roles/msperfs/handlers/main.yml deleted file mode 100644 index 5463835..0000000 --- a/qgis4/ansible/roles/msperfs/handlers/main.yml +++ /dev/null @@ -1,4 +0,0 @@ -- name: restart nginx - service: - name: nginx - state: restarted From 09923b607b28cb24fb0eb9ca330e693e4be3b1e6 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 9 Sep 2019 10:56:36 +0200 Subject: [PATCH 10/12] waiting for the db to be up is not necessary anymore --- qgis4/ansible/roles/msperfs/tasks/main.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/qgis4/ansible/roles/msperfs/tasks/main.yml b/qgis4/ansible/roles/msperfs/tasks/main.yml index 689071d..abdfb09 100644 --- a/qgis4/ansible/roles/msperfs/tasks/main.yml +++ b/qgis4/ansible/roles/msperfs/tasks/main.yml @@ -64,12 +64,6 @@ pull: yes when: not data_loaded.stat.exists -- name: wait for the docker db container to be up - wait_for: - port: 15432 - delay: 10 - when: not data_loaded.stat.exists - - name: Wait for 10 minutes to get data sourced to postgresql pause: minutes: 10 From 005edfa0488ccb224517a635dc2aa3c6f4685e82 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 9 Sep 2019 10:57:12 +0200 Subject: [PATCH 11/12] Actually adding a cronjob TODO: check if the unprivileged user can use it without having issues with permissions. --- qgis4/ansible/roles/msperfs/tasks/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/qgis4/ansible/roles/msperfs/tasks/main.yml b/qgis4/ansible/roles/msperfs/tasks/main.yml index abdfb09..a9c7698 100644 --- a/qgis4/ansible/roles/msperfs/tasks/main.yml +++ b/qgis4/ansible/roles/msperfs/tasks/main.yml @@ -94,3 +94,13 @@ dest: /etc/nginx/sites-enabled/msperfs state: link notify: restart nginx + +- name: Update crontab to periodically run ms_perfs + cron: + name: msperfs + user: "{{ user }}" + minute: "0" + hour: "2" + weekday: "6" + job: > + cd /home/{{ user }}/ms_perfs/ && make pull && make run > /tmp/msperfs.log 2>&1 From 3e992df3a637f27bcdaff782d4d06771b9dc97b4 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 16 Sep 2019 12:40:24 +0200 Subject: [PATCH 12/12] revert some unrelated modifications * ansible 2.8 was necessary before switching strategy to ensure data were loaded in the underlying docker composition (which was making use of postgresql* ansible modules only available in 2.7) * Due to the previous point, I was in the need of libpq-dev because of the use of the postgresql modules. --- qgis4/ansible/requirements.txt | 2 +- qgis4/ansible/roles/perfsuite/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qgis4/ansible/requirements.txt b/qgis4/ansible/requirements.txt index ae06c17..28fdd7e 100644 --- a/qgis4/ansible/requirements.txt +++ b/qgis4/ansible/requirements.txt @@ -1 +1 @@ -ansible==2.8 +ansible==2.7 diff --git a/qgis4/ansible/roles/perfsuite/tasks/main.yml b/qgis4/ansible/roles/perfsuite/tasks/main.yml index 32a7217..c32f9fd 100644 --- a/qgis4/ansible/roles/perfsuite/tasks/main.yml +++ b/qgis4/ansible/roles/perfsuite/tasks/main.yml @@ -1,6 +1,6 @@ - name: Install prerequisites apt: - name: ['postgresql-client', 'curl', 'virtualenv', 'libpq-dev', 'npm', 'libpython3.5-dev'] + name: ['postgresql-client', 'curl', 'virtualenv'] update_cache: yes - name: Copy aggregate report scripts