From b94049baec9b41bdb5525613abb9c8a72e00c293 Mon Sep 17 00:00:00 2001 From: Sebastien DAMAYE Date: Thu, 6 Mar 2025 07:46:16 +0100 Subject: [PATCH] v1.1 - FR#45 FR#48 FR#49 --- deephunter/settings.github | 5 + docs/settings.rst | 34 ++++ qm/scripts/upgrade.sh | 309 ++++++++++++++++++++++++++++++------- qm/templates/about.html | 13 ++ qm/templates/base.html | 10 +- qm/urls.py | 1 + qm/views.py | 37 ++++- 7 files changed, 348 insertions(+), 61 deletions(-) create mode 100755 qm/templates/about.html diff --git a/deephunter/settings.github b/deephunter/settings.github index f601b23..5715c84 100755 --- a/deephunter/settings.github +++ b/deephunter/settings.github @@ -9,6 +9,11 @@ SECRET_KEY = '*****************************************************************' DEBUG = False +# Update settings +UPDATE_ON = "release" # Possible values: commit|release +TEMP_FOLDER = "/data/tmp" +VENV_PATH = "/data/venv" + SHOW_LOGIN_FORM = False ALLOWED_HOSTS = ['deephunter.domain.com'] diff --git a/docs/settings.rst b/docs/settings.rst index 4f719a2..b87620c 100755 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -10,6 +10,40 @@ DEBUG - **Description**: Used to display debug information. Can be set to ``True`` in development environement, but always to ``False`` in a production environment. - **Example**: ``DEBUG = True`` +UPDATE_ON +********* + +- **Type**: String +- **Description**: Choose how you should be notified about updates and what repository should be considered for updates. If you select ``commit``, you will be notified each time there is a new commit, and the upgrade script will download the very latest version of DeepHunter. If you select ``release``, you will only be notified about updates when a new version is released, and the upgrade script will download the latest release, instead of the latest commited code. +- **Possible values**: ``commit`` or ``release`` +- **Example**: + +.. code-block:: py + + UPDATE_ON = "release" + +TEMP_FOLDER +*********** + +- **Type**: String +- **Description**: Used by the upgrade script. This is a temporary location where the current version of deephunter will be saved before upgrading. This can be used for easy rollback in case of error during the update process. +- **Example**: + +.. code-block:: py + + TEMP_FOLDER = "/data/tmp" + +VENV_PATH +********* + +- **Type**: String +- **Description**: Used by the update script. This is your python virtual path. +- **Example**: + +.. code-block:: py + + VENV_PATH = "/data/venv" + SHOW_LOGIN_FORM *************** - **Type**: Boolean diff --git a/qm/scripts/upgrade.sh b/qm/scripts/upgrade.sh index fc8968c..426f9dd 100755 --- a/qm/scripts/upgrade.sh +++ b/qm/scripts/upgrade.sh @@ -1,80 +1,276 @@ #!/bin/bash -### custom settings -TEMP_FOLDER="/data/tmp" -APP_PATH="/data/deephunter" -VENV_PATH="/data/venv" +###################################### +# FUNCTIONS +# -### -### Do not change what is below, unless you understand what you are doing -### +# Function to extract keys from a settings.py file +extract_keys() { + grep -oP "^\s*\K\w+" "$1" +} +# Function that checks if a variable is empty +check_empty() { + if [ -z "$2" ]; then + echo -e "[\033[31mERROR\033[0m] Unable to extract $1 from your settings file." + exit 1 + fi +} -# List of django apps for migrations -APPS=(qm extensions reports) +# Banner +echo '' +echo ' _____ ______ ______ _____ _ _ _ _ _ _ _______ ______ _____ ' +echo ' | __ \| ____| ____| __ \| | | | | | | \ | |__ __| ____| __ \ ' +echo ' | | | | |__ | |__ | |__) | |__| | | | | \| | | | | |__ | |__) |' +echo ' | | | | __| | __| | ___/| __ | | | | . ` | | | | __| | _ / ' +echo ' | |__| | |____| |____| | | | | | |__| | |\ | | | | |____| | \ \ ' +echo ' |_____/|______|______|_| |_| |_|\____/|_| \_| |_| |______|_| \_\' +echo '' +echo ' *** DEEPHUNTER UPGRADE SCRIPT ***' +echo '' -USER=$(grep -oP 'USER_GROUP\s?=\s?"\K[^"]+' $APP_PATH/deephunter/settings.py) -GITHUB_URL=$(grep -oP 'GITHUB_URL\s?=\s?"\K[^"]+' $APP_PATH/deephunter/settings.py) -read -p "About to start upgrade. Press to continue" -echo "[INFO] STARTING UPGRADE..." +###################################### +# PREREQUISITES +# + +error=0 +echo -e "[\033[90mINFO\033[0m] CHECKING PREREQUISITES" + +# Checking that sudo is installed +if which sudo > /dev/null 2>&1; then + echo -e " sudo ................................................. [\033[32mfound\033[0m]" +else + echo -e " sudo ................................................. [\033[31mmissing\033[0m]" + error=1 +fi +# Checking that curl is installed +if which curl > /dev/null 2>&1; then + echo -e " curl ................................................. [\033[32mfound\033[0m]" +else + echo -e " curl ................................................. [\033[31mmissing\033[0m]" + error=1 +fi +# Checking that wget is installed +if which wget > /dev/null 2>&1; then + echo -e " wget ................................................. [\033[32mfound\033[0m]" +else + echo -e " wget ................................................. [\033[31mmissing\033[0m]" + error=1 +fi +# Checking that git is installed +if which git > /dev/null 2>&1; then + echo -e " git .................................................. [\033[32mfound\033[0m]" +else + echo -e " git .................................................. [\033[31mmissing\033[0m]" + error=1 +fi +# Checking that tar is installed +if which tar > /dev/null 2>&1; then + echo -e " tar .................................................. [\033[32mfound\033[0m]" +else + echo -e " tar .................................................. [\033[31mmissing\033[0m]" + error=1 +fi +# Checking that user has sudo +user_groups=$(groups $(whoami)) +if echo "$user_groups" | grep -qw "sudo"; then + echo -e " User has sudo access ................................. [\033[32mOK\033[0m]" +else + echo -e " User has sudo access ................................. [\033[31mfailed\033[0m]" + error=1 +fi + +# Exit on error +if [ $error = 1 ]; then + echo -e "[\033[31mERROR\033[0m] The upgrade script is missing mandatory dependencies. Install these packages first." + exit 1 +fi + +###################################### +# GET SETTINGS +# + +# Search for the settings file on disk +# (assuming it is in the following relative path ./deephunter/deephunter/settings.py) +echo -n -e "[\033[90mINFO\033[0m] LOOKING FOR THE SETTINGS.PY FILE ................ " +SETTINGS_PATH=$(find / -type f -path "*/deephunter/deephunter/*" -name "settings.py" 2>/dev/null) +if [ -z "${SETTINGS_PATH}" ]; then + echo -e "[\033[31mnot found\033[0m]" + exit 1 +else + echo -e "[\033[32mfound\033[0m]" +fi + +# Confirm settings path is correct +echo -n -e "[\033[34mCONFIRM\033[0m] " +read -p "Settings file found in \"$SETTINGS_PATH\". Is it correct (Y/n)? " response +# If no input is provided (just Enter), set response to 'Y' +response=${response:-Y} +# Convert the response to uppercase to handle both 'y' and 'Y' +response=$(echo "$response" | tr '[:lower:]' '[:upper:]') +# Check the response +if [[ "$response" == "Y" || "$response" == "YES" ]]; then + # Extract variables from settings.py + APP_PATH=$(dirname "$SETTINGS_PATH" | sed 's:/deephunter$::') + # Remove all comments from settings and save a copy in tmp + # This is a necessary prerequisite in order to avoid duplicates during extraction + grep -v '^#' $APP_PATH/deephunter/settings.py > /tmp/settings.py + TEMP_FOLDER=$(grep -oP 'TEMP_FOLDER\s?=\s?"\K[^"]+' /tmp/settings.py) + check_empty "TEMP_FOLDER" "$TEMP_FOLDER" + VENV_PATH=$(grep -oP 'VENV_PATH\s?=\s?"\K[^"]+' /tmp/settings.py) + check_empty "VENV_PATH" "$VENV_PATH" + UPDATE_ON=$(grep -oP 'UPDATE_ON\s?=\s?"\K[^"]+' /tmp/settings.py) + check_empty "UPDATE_ON" "$UPDATE_ON" + USER=$(grep -oP 'USER_GROUP\s?=\s?"\K[^"]+' /tmp/settings.py) + check_empty "USER" "$USER" + GITHUB_URL=$(grep -oP 'GITHUB_URL\s?=\s?"\K[^"]+' /tmp/settings.py) + check_empty "GITHUB_URL" "$GITHUB_URL" + # List of django apps for migrations + APPS=(qm extensions reports) +elif [[ "$response" == "N" || "$response" == "NO" ]]; then + exit 1 +else + echo "Invalid response. Please enter Y, YES, N, or NO." +fi + + +###################################### +# DOWNLOAD NEW VERSION (RELEASE OR COMMIT) +# + +# Downloading new version +echo -n -e "[\033[90mINFO\033[0m] DOWNLOADING NEW VERSION OF DEEPHUNTER ........... " +cd /tmp +rm -fR deephunter* +if [ $UPDATE_ON = "release" ]; then + response=$(curl -s "https://api.github.com/repos/sebastiendamaye/deephunter/releases/latest") + url=$(echo $response | grep -o '"tarball_url": *"[^"]*"' | sed 's/"tarball_url": "//' | sed 's/"$//') + wget -q -O deephunter.tar.gz $url + mkdir deephunter + tar xzf deephunter.tar.gz -C deephunter --strip-components=1 +else + rm -fR d + mkdir d + cd d + git clone -q $GITHUB_URL + cd /tmp + mv d/deephunter . + rm -fR d +fi + +echo -e "[\033[32mcomplete\033[0m]" + + +###################################### +# CHECK SETTINGS CONSISTENCY +# + +# Checking settings.py consistency (presence of all keys) +echo -n -e "[\033[90mINFO\033[0m] CHECKING SETTINGS FILE CONSISTENCY .............. " + +# Extract keys from the local and remote settings.py files +LOCAL_KEYS=$(extract_keys /tmp/settings.py) +NEW_KEYS=$(extract_keys "/tmp/deephunter/deephunter/settings.github") + +# Compare the sets of keys (sorted to handle order) and check consistency +if diff <(echo "$LOCAL_KEYS" | sort) <(echo "$NEW_KEYS" | sort) > /dev/null; then + echo -e "[\033[32mOK\033[0m]" +else + echo -e "[\033[31mfailed\033[0m]" + echo -e "[\033[31mERROR\033[0m] There are likely missing variables in your current settings.py file." + # Show the differences between the local and new settings + diff <(echo "$LOCAL_KEYS" | sort) <(echo "$NEW_KEYS" | sort) + exit 1 +fi + +###################################### +# UPGRADE +# # Stop services -echo "[INFO] Stopping services..." +echo -n -e "[\033[90mINFO\033[0m] STOPPING SERVICES ............................... " sudo systemctl stop apache2 sudo systemctl stop celery sudo systemctl stop redis-server +echo -e "[\033[32mdone\033[0m]" # Backup DB (encrypted. Use the same as DB backup in crontab. Backup will be located in the same folder) -echo "[INFO] Starting DB backup..." +echo -n -e "[\033[90mINFO\033[0m] STARTING DB BACKKUP ............................. " source $VENV_PATH/bin/activate cd $APP_PATH $VENV_PATH/bin/python3 manage.py dbbackup --encrypt #leave virtual env deactivate +echo -e "[\033[32mdone\033[0m]" # Backup source -echo "[INFO] Backup application..." +echo -n -e "[\033[90mINFO\033[0m] STARTING APP BACKUP ............................. " rm -fR $TEMP_FOLDER/deephunter mkdir -p $TEMP_FOLDER cp -R $APP_PATH $TEMP_FOLDER +echo -e "[\033[32mdone\033[0m]" -read -p "About to download new version. Press to continue" -rm -fR $APP_PATH -# Download new version -echo "[INFO] Downloading new version from github..." -rm -fR $APP_PATH -cd /data -git clone $GITHUB_URL - -read -p "About to restore prod settings and files. Press to continue" -# restore prod settings and files -echo "[INFO] Restoring migrations folders and settings..." -for app in ${APPS[@]} -do - cp -R $TEMP_FOLDER/deephunter/$app/migrations/ $APP_PATH/$app/ -done -# Restore settings -cp $TEMP_FOLDER/deephunter/deephunter/settings.py $APP_PATH/deephunter/ -# Restore token renewal date -cp $TEMP_FOLDER/deephunter/static/tokendate.txt $APP_PATH/static/ - -# Migrate -read -p "About to start DB migrations. Press to continue" -echo "[INFO] Proceeding with DB migrations..." -source $VENV_PATH/bin/activate -cd $APP_PATH/ -for app in ${APPS[@]} -do - ./manage.py makemigrations $app -done -./manage.py migrate - -# Leave python virtual env -deactivate -echo "[INFO] DB migrations complete" +# Installation of the update +echo -n -e "[\033[34mCONFIRM\033[0m] " +read -p "Proceed with installation of the new version (Y/n)? " response +# If no input is provided (just Enter), set response to 'Y' +response=${response:-Y} +# Convert the response to uppercase to handle both 'y' and 'Y' +response=$(echo "$response" | tr '[:lower:]' '[:upper:]') +# Check the response +if [[ "$response" == "Y" || "$response" == "YES" ]]; then + echo -n -e "[\033[90mINFO\033[0m] INSTALLING UPDATE ............................... " + rm -fR $APP_PATH + cp -R /tmp/deephunter $APP_PATH + echo -e "[\033[32mdone\033[0m]" + + echo -n -e "[\033[90mINFO\033[0m] RESTORING MIGRATIONS FOLDERS AND SETTINGS ....... " + for app in ${APPS[@]} + do + cp -R $TEMP_FOLDER/deephunter/$app/migrations/ $APP_PATH/$app/ 2>/dev/null + done + # Restore settings + cp $TEMP_FOLDER/deephunter/deephunter/settings.py $APP_PATH/deephunter/ + # Restore token renewal date + cp $TEMP_FOLDER/deephunter/static/tokendate.txt $APP_PATH/static/ + echo -e "[\033[32mdone\033[0m]" + +elif [[ "$response" == "N" || "$response" == "NO" ]]; then + exit 0 +else + echo "Invalid response. Please enter Y, YES, N, or NO." +fi + +# DB Migrations +echo -n -e "[\033[34mCONFIRM\033[0m] " +read -p "Proceed with DB migrations (Y/n)? " response +# If no input is provided (just Enter), set response to 'Y' +response=${response:-Y} +# Convert the response to uppercase to handle both 'y' and 'Y' +response=$(echo "$response" | tr '[:lower:]' '[:upper:]') +# Check the response +if [[ "$response" == "Y" || "$response" == "YES" ]]; then + echo -e "[\033[90mINFO\033[0m] PERFORMING DB MIGRATIONS ........................ " + source $VENV_PATH/bin/activate + cd $APP_PATH/ + for app in ${APPS[@]} + do + ./manage.py makemigrations $app + done + ./manage.py migrate + + # Leave python virtual env + deactivate + echo -e "[\033[90mINFO\033[0m] DB MIGRATIONS COMPLETE" + +elif [[ "$response" == "N" || "$response" == "NO" ]]; then + exit 0 +else + echo "Invalid response. Please enter Y, YES, N, or NO." +fi # Restore permissions -echo "[INFO] Restoring permissions..." +echo -n -e "[\033[90mINFO\033[0m] RESTORING PERMISSIONS ........................... " chmod -R 775 $APP_PATH touch $APP_PATH/campaigns.log chmod 666 $APP_PATH/campaigns.log @@ -84,11 +280,18 @@ touch $APP_PATH/static/tokendate.txt chmod 666 $APP_PATH/static/tokendate.txt chown -R $USER $VENV_PATH chmod -R 775 $VENV_PATH +echo -e "[\033[32mdone\033[0m]" # Restart apache2 -echo "[INFO] Restarting services..." +echo -n -e "[\033[90mINFO\033[0m] RESTARTING SERVICES ............................. " sudo systemctl start apache2 sudo systemctl restart redis-server sudo systemctl restart celery +echo -e "[\033[32mdone\033[0m]" + +# cleaning /tmp +echo -n -e "[\033[90mINFO\033[0m] CLEANING TMP DIR ................................ " +rm -fR /tmp/deephunter* /tmp/settings.py +echo -e "[\033[32mdone\033[0m]" -echo "[INFO] UPGRADE COMPLETE" +echo -e "[\033[90mINFO\033[0m] UPGRADE COMPLETE" diff --git a/qm/templates/about.html b/qm/templates/about.html new file mode 100755 index 0000000..7de9e29 --- /dev/null +++ b/qm/templates/about.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block body %} +
+ _____  ______ ______ _____  _    _ _    _ _   _ _______ ______ _____  
+|  __ \|  ____|  ____|  __ \| |  | | |  | | \ | |__   __|  ____|  __ \ 
+| |  | | |__  | |__  | |__) | |__| | |  | |  \| |  | |  | |__  | |__) |
+| |  | |  __| |  __| |  ___/|  __  | |  | | . ` |  | |  |  __| |  _  / 
+| |__| | |____| |____| |    | |  | | |__| | |\  |  | |  | |____| | \ \ 
+|_____/|______|______|_|    |_|  |_|\____/|_| \_|  |_|  |______|_|  \_\
+                                                       Version: {{ version }}
+
+{% endblock %} diff --git a/qm/templates/base.html b/qm/templates/base.html index 135612f..7534bdb 100755 --- a/qm/templates/base.html +++ b/qm/templates/base.html @@ -46,11 +46,17 @@ {% if request.user.is_superuser %}
  • Debug
    • {% endif %} -
  • Log out
    • -
  • Help
    • +
  • Help + +
    • + +
  • Log out
    • diff --git a/qm/urls.py b/qm/urls.py index 08b857c..d0d1370 100755 --- a/qm/urls.py +++ b/qm/urls.py @@ -12,4 +12,5 @@ path('/regen/', views.regen, name='regen'), path('/progress/', views.progress, name='progress'), path('netview', views.netview, name='netview'), + path('about', views.about, name='about'), ] diff --git a/qm/views.py b/qm/views.py index 91673e4..a912a41 100755 --- a/qm/views.py +++ b/qm/views.py @@ -19,6 +19,7 @@ VT_API_KEY = settings.VT_API_KEY CUSTOM_FIELDS = settings.CUSTOM_FIELDS BASE_DIR = settings.BASE_DIR +UPDATE_ON = settings.UPDATE_ON # Params for requests API calls S1_URL = settings.S1_URL @@ -187,7 +188,7 @@ def index(request): tokenexpires = 1000 try: with open('{}/tokendate.txt'.format(STATIC_PATH), 'r') as f: - d = re.search('\d{4}-\d{2}-\d{2}', f.readline()) + d = re.search(r'\d{4}-\d{2}-\d{2}', f.readline()) tokendate = datetime.strptime(d.group(), "%Y-%m-%d") tokenelapseddays = (datetime.today() - tokendate).days tokenexpires = S1_TOKEN_EXPIRATION - tokenelapseddays @@ -197,18 +198,30 @@ def index(request): # Check if new version available try: update_available = False + # remote version - r = requests.get( - 'https://api.github.com/repos/sebastiendamaye/deephunter/releases/latest', - proxies=PROXY - ) - remote_ver = r.json()['name'] + # update on new release only + if UPDATE_ON == 'release': + r = requests.get( + 'https://api.github.com/repos/sebastiendamaye/deephunter/releases/latest', + proxies=PROXY + ) + remote_ver = r.json()['name'] + else: + # update on every new commit + r = requests.get( + 'https://raw.githubusercontent.com/sebastiendamaye/deephunter/refs/heads/main/static/VERSION', + proxies=PROXY + ) + remote_ver = r.text.strip() + # local version with open(f'{STATIC_PATH}/VERSION', 'r') as f: local_ver = f.readline().strip() # compare if local_ver != remote_ver: update_available = True + except: update_available = False @@ -689,3 +702,15 @@ def netview(request): 'debug': debug } return render(request, 'netview.html', context) + +@login_required +def about(request): + + # local version + with open(f'{STATIC_PATH}/VERSION', 'r') as f: + version = f.readline().strip() + + context = { + 'version': version + } + return render(request, 'about.html', context)