diff --git a/Makefile b/Makefile index 9639c0c..d2316f2 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,8 @@ export MATCHID_CONFIG_BUCKET=$(shell echo ${APP_GROUP} | tr '[:upper:]' '[:lower export ES_INDEX=${APP_GROUP} export ES_NODES = 1 # elasticsearch number of nodes export ES_SWARM_NODE_NUMBER = 2 # elasticsearch number of nodes -export ES_MEM = 1024m # elasticsearch : memory of each node +export ES_MEM := 1024m # elasticsearch : memory of each node +export ES_JAVA_OPTS := -Xms${ES_MEM} -Xmx${ES_MEM} # elasticsearch : java options export ES_VERSION = 8.1.3 export ES_DATA = ${BACKEND}/esdata export ES_THREADS = 2 @@ -192,49 +193,77 @@ endif elasticsearch2-stop: @${DC} -f ${DC_FILE}-elasticsearch-huge-remote.yml down -elastisearch-repository-plugin: - @docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch sh -c \ - "echo ${STORAGE_ACCESS_KEY} | bin/elasticsearch-keystore add --stdin --force s3.client.default.access_key" - @docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch sh -c \ - "echo ${STORAGE_SECRET_KEY} | bin/elasticsearch-keystore add --stdin --force s3.client.default.secret_key" - @docker restart ${DC_PREFIX}-elasticsearch - @timeout=${TIMEOUT} ; ret=1 ; until [ "$$timeout" -le 0 -o "$$ret" -eq "0" ] ; do (docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch curl -s --fail -XGET localhost:9200/_cat/indices > /dev/null) ; ret=$$? ; if [ "$$ret" -ne "0" ] ; then echo -en "\rwaiting for elasticsearch to start $$timeout" ; fi ; ((timeout--)); sleep 1 ; done ; echo ; exit $$ret - @touch elastisearch-repository-plugin +elasticsearch-mini: network vm_max + @if [ "$(docker container inspect -f '{{.State.Status}}' ${DC_PREFIX}-elasticsearch)" != "running" ]; then\ + make elasticsearch ES_MEM=256m ES_NODES=1 ES_VERSION=${ES_VERSION} ES_JAVA_OPTS="-Xms256m -Xmx256m";\ + fi; -elasticsearch-repository-config: elastisearch-repository-plugin - @docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \ - curl -s -XPUT "localhost:9200/_snapshot/${APP_GROUP}" -H 'Content-Type: application/json' \ - -d '{"type": "s3","settings": {"bucket": "${REPOSITORY_BUCKET}","client": "default","region": "${SCW_REGION}","endpoint": "${SCW_ENDPOINT}","path_style_access": true,"protocol": "https"}}' \ - | grep -q '"acknowledged":true' && touch elasticsearch-repository-config -elasticsearch-repository-backup: elasticsearch-repository-config - @echo creating snapshot ${ES_BACKUP_NAME} in elasticsearch repository;\ - docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \ - curl -s -XPUT "localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}?wait_for_completion=true" -H 'Content-Type: application/json'\ - -d '{"indices": "${ES_INDEX}", "ignore_unavailable": true, "include_global_state": false}' +elasticsearch-repository-plugin: elasticsearch-mini + @if [ ! -f "elasticsearch-repository-plugin" ]; then\ + echo installing elasticsearch repository plugin;\ + docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch sh -c \ + "echo ${STORAGE_ACCESS_KEY} | bin/elasticsearch-keystore add --stdin --force s3.client.default.access_key";\ + docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch sh -c \ + "echo ${STORAGE_SECRET_KEY} | bin/elasticsearch-keystore add --stdin --force s3.client.default.secret_key";\ + docker restart ${DC_PREFIX}-elasticsearch;\ + timeout=${TIMEOUT} ; ret=1 ; until [ "$$timeout" -le 0 -o "$$ret" -eq "0" ] ; do (docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch curl -s --fail -XGET localhost:9200/_cat/indices > /dev/null) ; ret=$$? ; if [ "$$ret" -ne "0" ] ; then echo -en "\rwaiting for elasticsearch to start $$timeout" ; fi ; ((timeout--)); sleep 1 ; done ;\ + echo; touch elasticsearch-repository-plugin ; exit $$ret;\ + fi; + +elasticsearch-repository-config: elasticsearch-repository-plugin + @if [ ! -f "elasticsearch-repository-config" ]; then\ + echo creating elasticsearch repository ${APP_GROUP} in s3 bucket ${REPOSITORY_BUCKET} && \ + docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \ + curl -s -XPUT "localhost:9200/_snapshot/${APP_GROUP}" -H 'Content-Type: application/json' \ + -d '{"type": "s3","settings": {"bucket": "${REPOSITORY_BUCKET}","client": "default","region": "${SCW_REGION}","endpoint": "${SCW_ENDPOINT}","path_style_access": true,"protocol": "https"}}' \ + | grep -q '"acknowledged":true' && touch elasticsearch-repository-config;\ + fi -elasticsearch-repository-backup-async: elastisearch-repository-config +elasticsearch-repository-backup: elasticsearch-repository-config + @echo creating snapshot ${ES_BACKUP_NAME} in elasticsearch repository ...;\ + (\ + docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \ + curl -s -XPUT "localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}?wait_for_completion=true" -H 'Content-Type: application/json'\ + -d '{"indices": "${ES_INDEX}", "ignore_unavailable": true, "include_global_state": false}' \ + | grep -q '"state":"SUCCESS"' \ + > /dev/null 2>&1\ + ) && echo "snapshot ${ES_BACKUP_NAME} created in elasticsearch repository" && touch elasticsearch-repository-backup + +elasticsearch-repository-backup-async: elasticsearch-repository-config @docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \ curl -s -XPUT "localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}" -H 'Content-Type: application/json'\ -d '{"indices": "${ES_INDEX}", "ignore_unavailable": true, "include_global_state": false}' -elasticsearch-repository-delete: elastisearch-repository-config - @docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \ - curl -s -XDELETE "localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}" +elasticsearch-repository-delete: elasticsearch-repository-config + @(\ + docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \ + curl -s -XDELETE "localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}"\ + > /dev/null 2>&1\ + ) && echo "snapshot ${ES_BACKUP_NAME} deleted from elasticsearch repository" -elasticsearch-repository-restore: elastisearch-repository-config +elasticsearch-repository-list: elasticsearch-repository-config @docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \ - curl -s -XPOST localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}/_restore?wait_for_completion=true -H 'Content-Type: application/json'\ - -d '{"indices": "${ES_INDEX}","ignore_unavailable": true,"include_global_state": false}' - -elasticsearch-repository-check: + curl -s -XGET "localhost:9200/_snapshot/${APP_GROUP}/_all"\ + | jq -r '.snapshots[].snapshot' + +elasticsearch-repository-restore: elasticsearch-repository-config + @echo restoring snapshot ${ES_BACKUP_NAME} from elasticsearch repository;\ + (\ + docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \ + curl -s -XPOST localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}/_restore?wait_for_completion=true -H 'Content-Type: application/json'\ + -d '{"indices": "${ES_INDEX}","ignore_unavailable": true,"include_global_state": false}' \ + > /dev/null 2>&1\ + ) && echo "snapshot ${ES_BACKUP_NAME} restored from elasticsearch repository" && touch elasticsearch-repository-restore + +elasticsearch-repository-check: elasticsearch-repository-config backup-dir @if [ ! -f "${BACKUP_DIR}/${ES_BACKUP_NAME}.check" ]; then\ (\ docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \ - curl -s -XGET "localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}" \ + curl -s -XGET "localhost:9200/_snapshot/${APP_GROUP}/_all" \ | jq -r '.snapshots[].snapshot' | grep -q "${ES_BACKUP_NAME}" \ ) > /dev/null 2>&1 \ - && echo "snapshot found for or ${ES_BACKUP_NAME} in elasticsearch repository" && touch "${BACKUP_DIR}/${ES_BACKUP_NAME}.check" \ + && echo "snapshot found for or ${ES_BACKUP_NAME} in elasticsearch repository" && mkdir -p touch "${BACKUP_DIR}/${ES_BACKUP_NAME}.check" \ || (echo "no snapshot found for ${ES_BACKUP_NAME} in elasticsearch repository")\ fi @@ -276,9 +305,9 @@ endif elasticsearch-dev: elasticsearch -elasticsearch: network vm_max +elasticsearch-cluster: network vm_max @echo docker-compose up matchID elasticsearch with ${ES_NODES} nodes - @cat ${DC_FILE}-elasticsearch.yml | sed "s/%M/${ES_MEM}/g" > ${DC_FILE}-elasticsearch-huge.yml + @cat ${DC_FILE}-elasticsearch.yml > ${DC_FILE}-elasticsearch-huge.yml @(if [ ! -d ${ES_DATA}/node1 ]; then sudo mkdir -p ${ES_DATA}/node1 ; sudo chmod g+rw ${ES_DATA}/node1/.; sudo chown 1000:1000 ${ES_DATA}/node1/.; fi) @(i=$(ES_NODES); while [ $${i} -gt 1 ]; \ do \ @@ -290,6 +319,12 @@ elasticsearch: network vm_max ${DC} -f ${DC_FILE}-elasticsearch-huge.yml up -d @timeout=${TIMEOUT} ; ret=1 ; until [ "$$timeout" -le 0 -o "$$ret" -eq "0" ] ; do (docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch curl -s --fail -XGET localhost:9200/_cat/indices > /dev/null) ; ret=$$? ; if [ "$$ret" -ne "0" ] ; then echo -en "\rwaiting for elasticsearch to start $$timeout" ; fi ; ((timeout--)); sleep 1 ; done ; echo ; exit $$ret +elasticsearch: network vm_max + @echo docker-compose up matchID elasticsearch with ${ES_NODES} nodes + @(if [ ! -d ${ES_DATA}/node1 ]; then sudo mkdir -p ${ES_DATA}/node1 ; sudo chmod g+rw ${ES_DATA}/node1/.; sudo chown 1000:1000 ${ES_DATA}/node1/.; fi) + ${DC} -f ${DC_FILE}-elasticsearch.yml up -d + @timeout=${TIMEOUT} ; ret=1 ; until [ "$$timeout" -le 0 -o "$$ret" -eq "0" ] ; do (docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch curl -s --fail -XGET localhost:9200/_cat/indices > /dev/null) ; ret=$$? ; if [ "$$ret" -ne "0" ] ; then echo -en "\rwaiting for elasticsearch to start $$timeout" ; fi ; ((timeout--)); sleep 1 ; done ; echo ; exit $$ret + elasticsearch2: @echo docker-compose up matchID elasticsearch with ${ES_NODES} nodes @cat ${DC_FILE}-elasticsearch.yml | head -8 > ${DC_FILE}-elasticsearch-huge-remote.yml diff --git a/docker-components/docker-compose-elasticsearch-node.yml b/docker-components/docker-compose-elasticsearch-node.yml index 9de959e..f1da59a 100644 --- a/docker-components/docker-compose-elasticsearch-node.yml +++ b/docker-components/docker-compose-elasticsearch-node.yml @@ -7,7 +7,7 @@ environment: - cluster.name=matchid-cluster - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms%M -Xmx%M" + - ES_JAVA_OPTS - "discovery.zen.ping.unicast.hosts=elasticsearch" - xpack.security.enabled=false ulimits: diff --git a/docker-components/docker-compose-elasticsearch.yml b/docker-components/docker-compose-elasticsearch.yml index 5ab9155..4c1c94b 100644 --- a/docker-components/docker-compose-elasticsearch.yml +++ b/docker-components/docker-compose-elasticsearch.yml @@ -12,13 +12,21 @@ services: - /bin/bash - -c - | - bin/elasticsearch-plugin install --batch repository-s3 + if [ ! -z "$http_proxy" ]; then + host=$${http_proxy##*//}; + host=$${host%%:*}; + port=$${http_proxy##*:}; + (echo -e "cluster.name: \"docker-cluster\"\nnetwork.host: 0.0.0.0\ns3.client.default.proxy.host: $${host}\ns3.client.default.proxy.port: $${port}" > /usr/share/elasticsearch/config/elasticsearch.yml); + fi; exec /usr/local/bin/docker-entrypoint.sh elasticsearch container_name: ${DC_PREFIX}-elasticsearch environment: - cluster.name=matchid-cluster - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms%M -Xmx%M" + - ES_JAVA_OPTS + - http_proxy + - https_proxy + - no_proxy - discovery.type=single-node - xpack.security.enabled=false ulimits: