From fbe263a51a5bdf145ee4c25a47e153e469fb4258 Mon Sep 17 00:00:00 2001 From: PE39806 <185931318+PE39806@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:20:49 +0000 Subject: [PATCH 1/7] BAI-1570 add new get-used-storage script --- README.md | 1 + backend/src/scripts/getTotalStorageUsed.sh | 168 ++++++++++++++++++ .../src/scripts/getTotalStorageUsedMongo.ts | 32 ++++ 3 files changed, 201 insertions(+) create mode 100755 backend/src/scripts/getTotalStorageUsed.sh create mode 100644 backend/src/scripts/getTotalStorageUsedMongo.ts diff --git a/README.md b/README.md index 271629413..d4f1b606c 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,7 @@ information. - [Img Shields](https://shields.io) - [Othneils's README Template](https://github.com/othneildrew/Best-README-Template) - [Mattermost's Code Contribution Guidelines](https://github.com/mattermost/mattermost-server/blob/master/CONTRIBUTING.md) +- [xwmx's Bash Boilerplate](https://github.com/xwmx/bash-boilerplate) diff --git a/backend/src/scripts/getTotalStorageUsed.sh b/backend/src/scripts/getTotalStorageUsed.sh new file mode 100755 index 000000000..44d7e1a1f --- /dev/null +++ b/backend/src/scripts/getTotalStorageUsed.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Created using Bash Boilerplate: https://github.com/xwmx/bash-boilerplate + +# Treat unset variables and parameters other than the special parameters '@' or '*' as an error when performing parameter expansion +set -o nounset +# Exit immediately if a pipeline returns non-zero +set -o errexit +# Print a helpful message if a pipeline with non-zero exit code causes the script to exit as described above +trap 'echo "Aborting due to errexit on line $LINENO. Exit code: $?" >&2' ERR +# Allow the above trap be inherited by all functions in the script +set -o errtrace + +# Set $IFS to only newline and tab - see http://www.dwheeler.com/essays/filenames-in-shell.html +IFS=$'\n\t' + +_ME="$(basename "${0}")" + +# Exit with status 1 after executing the specified command with output redirected to standard error +_exit_1() { + { + printf "%s " "$(tput setaf 1)!$(tput sgr0)" + "${@}" + } 1>&2 + exit 1 +} + +# Check required program(s) are installed +if ! command -v jq > /dev/null 2>&1; then + _exit_1 printf "Error: jq is not installed\\n" +fi + +_print_help() { + cat <] + ${_ME} -h | --help + +Example: + ${_ME} -b -r json -o usage/file-storage.json + +Options: + -h, --help Show this screen. + -b, --bytes Display the storage usage in bytes. If unset, values are formatted for human readability. + -o, --output Write the results to the specified file. If unset, results are printed to the terminal. + -r, --reporting-format + Set the output format to one of ((c|console)|(j|json)). Defaults to console. +HEREDOC +} + +# Option parsing +_PRINT_HELP=0 + +_INCLUDE_DELETED=0 +_REPORTING_FORMAT="console" +_NORMALISED_REPORTING_FORMAT= +_FORMAT_BYTES=1 +_OUTPUT_TO_FILE=0 +_OUTPUT_FILE= + +# Given a flag (e.g., -e | --example) return the value or exit 1 if value is blank or appears to be another option +__get_option_value() { + local __arg="${1:-}" + local __val="${2:-}" + + if [[ -n "${__val:-}" ]] && [[ ! "${__val:-}" =~ ^- ]]; then + printf "%s\\n" "${__val}" + else + _exit_1 printf "%s requires a valid argument.\\n" "${__arg}" + fi +} + +# Parse the CLI args +while ((${#})); do + __arg="${1:-}" + __val="${2:-}" + + case "${__arg}" in + -h|--help) + _PRINT_HELP=1 + ;; + -b|--bytes) + _FORMAT_BYTES=0 + ;; + -o|--output) + _OUTPUT_TO_FILE=1 + _OUTPUT_FILE="$(__get_option_value "${__arg}" "${__val:-}")" + shift + ;; + -r|--reporting-format) + _REPORTING_FORMAT="$(__get_option_value "${__arg}" "${__val:-}")" + shift + ;; + -*) + _exit_1 printf "Unexpected option: %s\\n" "${__arg}" + ;; + esac + + shift +done + +# Parse the _REPORTING_FORMAT value +_REPORTING_FORMAT=$(echo "$_REPORTING_FORMAT" | awk '{print tolower($0)}') +case "${_REPORTING_FORMAT}" in + c|console) + _NORMALISED_REPORTING_FORMAT=0 + ;; + j|json) + _NORMALISED_REPORTING_FORMAT=1 + ;; + *) + _exit_1 printf "Unexpected --reporting-format option: %s\\nShould be one of ((c|console)|(j|json))\\n" "${_REPORTING_FORMAT}" + ;; +esac + +# Main script functionality +_get_storage() { + # Get the actual mongo values + tsx_output=$(npm run script -- getTotalStorageUsedMongo) + + # Parse the returned values + if ((_FORMAT_BYTES)); then + unformatted_result=$(echo "${tsx_output}" | tail -n 1) + # Values always end in B for bytes (e.g. GB) + extraction_regex_suffix="[^B]*B" + else + unformatted_result=$(echo "${tsx_output}" | tail -n 2 | head -n 1) + # Values are always space or close parentheses delimited + extraction_regex_suffix="[^ )]*" + fi + # grep for the value, then cut on the = to get just the value without the text prefix + existing_bytes=$(echo "${unformatted_result}" | grep -o "existing=${extraction_regex_suffix}" | cut -d '=' -f2-) + deleted_bytes=$(echo "${unformatted_result}" | grep -o "deleted=${extraction_regex_suffix}" | cut -d '=' -f2-) + total_bytes=$(echo "${unformatted_result}" | grep -o "total=${extraction_regex_suffix}" | cut -d '=' -f2-) + + # Format the result string + if [ $_NORMALISED_REPORTING_FORMAT -eq 1 ]; then + # Use jq to safely (escape key characters etc) create a JSON string + result_string=$(jq -n --arg eb "${existing_bytes}" --arg db "${deleted_bytes}" --arg tb "${total_bytes}" '{existing: $eb, deleted: $db, total: $tb}') + elif [ $_NORMALISED_REPORTING_FORMAT -eq 0 ]; then + result_string="existing: ${existing_bytes}\\ndeleted: ${deleted_bytes}\\ntotal: ${total_bytes}" + else + # This shouldn't happen, but better safe than sorry + _exit_1 printf "Got _NORMALISED_REPORTING_FORMAT=%s\\n" "${_NORMALISED_REPORTING_FORMAT}" + fi + + # Write the output + if ((_OUTPUT_TO_FILE)); then + # Make sure parent directory exists + mkdir -p "$(dirname "$_OUTPUT_FILE")" + printf "%b" "${result_string}" > "${_OUTPUT_FILE}" + else + printf "%b\\n" "${result_string}" + fi +} + +_main() { + # Avoid complex option parsing when only one program option is expected + if ((_PRINT_HELP)); then + _print_help + else + _get_storage "$@" + fi +} + +# Call `_main` after everything has been defined +_main "$@" diff --git a/backend/src/scripts/getTotalStorageUsedMongo.ts b/backend/src/scripts/getTotalStorageUsedMongo.ts new file mode 100644 index 000000000..d0e2af55d --- /dev/null +++ b/backend/src/scripts/getTotalStorageUsedMongo.ts @@ -0,0 +1,32 @@ +import prettyBytes from 'pretty-bytes' + +import FileModel from '../models/File.js' +import log from '../services/log.js' +import { connectToMongoose, disconnectFromMongoose } from '../utils/database.js' + +// Helper file for getTotalStorageUsed.sh + +async function main() { + await connectToMongoose() + + const existingResults = await FileModel.find({}) + const deletedResults = await (FileModel as any).findDeleted() + + await disconnectFromMongoose() + + const totalExistingBytes = existingResults.reduce((sum, fileModel) => sum + fileModel.size, 0) + const totalDeletedBytes = deletedResults.reduce((sum, fileModel) => sum + fileModel.size, 0) + + const totalBytes = { + existing: totalExistingBytes, + deleted: totalDeletedBytes, + total: totalExistingBytes + totalDeletedBytes, + } + const totalFormattedBytes = Object.fromEntries( + Object.entries(totalBytes).map(([key, value]) => [key, prettyBytes(value)]), + ) + + log.info(totalBytes, 'Storage used:') + log.info(totalFormattedBytes, 'Formatted storage used:') +} +main() From 6bd9e1d1a585530cc541043cddc4a38091f5ff30 Mon Sep 17 00:00:00 2001 From: PE39806 <185931318+PE39806@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:25:40 +0000 Subject: [PATCH 2/7] BAI-1570 add comments --- backend/src/scripts/getTotalStorageUsedMongo.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/scripts/getTotalStorageUsedMongo.ts b/backend/src/scripts/getTotalStorageUsedMongo.ts index d0e2af55d..842ac33f7 100644 --- a/backend/src/scripts/getTotalStorageUsedMongo.ts +++ b/backend/src/scripts/getTotalStorageUsedMongo.ts @@ -5,11 +5,13 @@ import log from '../services/log.js' import { connectToMongoose, disconnectFromMongoose } from '../utils/database.js' // Helper file for getTotalStorageUsed.sh +// TS handles Mongoose connections more easily than just using a BASH script async function main() { await connectToMongoose() const existingResults = await FileModel.find({}) + // mongoose-delete plugin doesn't have correct typing so cast to any const deletedResults = await (FileModel as any).findDeleted() await disconnectFromMongoose() @@ -22,11 +24,14 @@ async function main() { deleted: totalDeletedBytes, total: totalExistingBytes + totalDeletedBytes, } + // Copy of totalBytes with human readable values const totalFormattedBytes = Object.fromEntries( Object.entries(totalBytes).map(([key, value]) => [key, prettyBytes(value)]), ) + // Print results log.info(totalBytes, 'Storage used:') log.info(totalFormattedBytes, 'Formatted storage used:') } + main() From cb60d15d48416712875961e64d02638543969d1b Mon Sep 17 00:00:00 2001 From: PE39806 <185931318+PE39806@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:32:46 +0000 Subject: [PATCH 3/7] BAI-1570 make json return a number rather than a string when just returning the unformatted byte count --- backend/src/scripts/getTotalStorageUsed.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/src/scripts/getTotalStorageUsed.sh b/backend/src/scripts/getTotalStorageUsed.sh index 44d7e1a1f..f55dee8ac 100755 --- a/backend/src/scripts/getTotalStorageUsed.sh +++ b/backend/src/scripts/getTotalStorageUsed.sh @@ -45,7 +45,7 @@ Options: -b, --bytes Display the storage usage in bytes. If unset, values are formatted for human readability. -o, --output Write the results to the specified file. If unset, results are printed to the terminal. -r, --reporting-format - Set the output format to one of ((c|console)|(j|json)). Defaults to console. + Set the output format to one of ((c|console)|(j|json)). Defaults to console. When set as json, if <-b|--bytes> is passed then the values are numbers otherwise the values are strings. HEREDOC } @@ -137,7 +137,11 @@ _get_storage() { # Format the result string if [ $_NORMALISED_REPORTING_FORMAT -eq 1 ]; then # Use jq to safely (escape key characters etc) create a JSON string - result_string=$(jq -n --arg eb "${existing_bytes}" --arg db "${deleted_bytes}" --arg tb "${total_bytes}" '{existing: $eb, deleted: $db, total: $tb}') + if ((_FORMAT_BYTES)); then + result_string=$(jq -n --arg eb "${existing_bytes}" --arg db "${deleted_bytes}" --arg tb "${total_bytes}" '{existing: $eb, deleted: $db, total: $tb}') + else + result_string=$(jq -n --arg eb "${existing_bytes}" --arg db "${deleted_bytes}" --arg tb "${total_bytes}" '{existing: ($eb)|tonumber, deleted: ($db)|tonumber, total: ($tb)|tonumber}') + fi elif [ $_NORMALISED_REPORTING_FORMAT -eq 0 ]; then result_string="existing: ${existing_bytes}\\ndeleted: ${deleted_bytes}\\ntotal: ${total_bytes}" else From 3e720246f9e6e8063c2b05bb690ca15903a8cced Mon Sep 17 00:00:00 2001 From: PE39806 <185931318+PE39806@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:42:34 +0000 Subject: [PATCH 4/7] BAI-1570 improve help text --- backend/src/scripts/getTotalStorageUsed.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/scripts/getTotalStorageUsed.sh b/backend/src/scripts/getTotalStorageUsed.sh index f55dee8ac..a3f137617 100755 --- a/backend/src/scripts/getTotalStorageUsed.sh +++ b/backend/src/scripts/getTotalStorageUsed.sh @@ -43,9 +43,9 @@ Example: Options: -h, --help Show this screen. -b, --bytes Display the storage usage in bytes. If unset, values are formatted for human readability. - -o, --output Write the results to the specified file. If unset, results are printed to the terminal. + -o, --output Write the results to the specified . If unset, results are printed to the terminal. -r, --reporting-format - Set the output format to one of ((c|console)|(j|json)). Defaults to console. When set as json, if <-b|--bytes> is passed then the values are numbers otherwise the values are strings. + Set the output to one of (c[onsole]|j[son]). Defaults to (console). When set as (json), if [-b|--bytes] is also passed then the values are numbers otherwise the values are strings. HEREDOC } From 32dd732c87e961d26af8ee50f07cab83a44b6acc Mon Sep 17 00:00:00 2001 From: PE39806 <185931318+PE39806@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:50:58 +0000 Subject: [PATCH 5/7] BAI-1570 copy -r help section to error message --- backend/src/scripts/getTotalStorageUsed.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/scripts/getTotalStorageUsed.sh b/backend/src/scripts/getTotalStorageUsed.sh index a3f137617..f9e5957c4 100755 --- a/backend/src/scripts/getTotalStorageUsed.sh +++ b/backend/src/scripts/getTotalStorageUsed.sh @@ -110,7 +110,7 @@ case "${_REPORTING_FORMAT}" in _NORMALISED_REPORTING_FORMAT=1 ;; *) - _exit_1 printf "Unexpected --reporting-format option: %s\\nShould be one of ((c|console)|(j|json))\\n" "${_REPORTING_FORMAT}" + _exit_1 printf "Unexpected --reporting-format option: %s\\nShould be one of (c[onsole]|j[son])\\n" "${_REPORTING_FORMAT}" ;; esac From b25e959cea4f61a4865b89c06f2d1ad6eca59904 Mon Sep 17 00:00:00 2001 From: PE39806 <185931318+PE39806@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:32:01 +0000 Subject: [PATCH 6/7] BAI-1570 remove bash wrapper script for simplicity --- backend/src/scripts/getTotalStorageUsed.sh | 172 ------------------ .../src/scripts/getTotalStorageUsedMongo.ts | 5 +- 2 files changed, 1 insertion(+), 176 deletions(-) delete mode 100755 backend/src/scripts/getTotalStorageUsed.sh diff --git a/backend/src/scripts/getTotalStorageUsed.sh b/backend/src/scripts/getTotalStorageUsed.sh deleted file mode 100755 index f9e5957c4..000000000 --- a/backend/src/scripts/getTotalStorageUsed.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bash -# Created using Bash Boilerplate: https://github.com/xwmx/bash-boilerplate - -# Treat unset variables and parameters other than the special parameters '@' or '*' as an error when performing parameter expansion -set -o nounset -# Exit immediately if a pipeline returns non-zero -set -o errexit -# Print a helpful message if a pipeline with non-zero exit code causes the script to exit as described above -trap 'echo "Aborting due to errexit on line $LINENO. Exit code: $?" >&2' ERR -# Allow the above trap be inherited by all functions in the script -set -o errtrace - -# Set $IFS to only newline and tab - see http://www.dwheeler.com/essays/filenames-in-shell.html -IFS=$'\n\t' - -_ME="$(basename "${0}")" - -# Exit with status 1 after executing the specified command with output redirected to standard error -_exit_1() { - { - printf "%s " "$(tput setaf 1)!$(tput sgr0)" - "${@}" - } 1>&2 - exit 1 -} - -# Check required program(s) are installed -if ! command -v jq > /dev/null 2>&1; then - _exit_1 printf "Error: jq is not installed\\n" -fi - -_print_help() { - cat <] - ${_ME} -h | --help - -Example: - ${_ME} -b -r json -o usage/file-storage.json - -Options: - -h, --help Show this screen. - -b, --bytes Display the storage usage in bytes. If unset, values are formatted for human readability. - -o, --output Write the results to the specified . If unset, results are printed to the terminal. - -r, --reporting-format - Set the output to one of (c[onsole]|j[son]). Defaults to (console). When set as (json), if [-b|--bytes] is also passed then the values are numbers otherwise the values are strings. -HEREDOC -} - -# Option parsing -_PRINT_HELP=0 - -_INCLUDE_DELETED=0 -_REPORTING_FORMAT="console" -_NORMALISED_REPORTING_FORMAT= -_FORMAT_BYTES=1 -_OUTPUT_TO_FILE=0 -_OUTPUT_FILE= - -# Given a flag (e.g., -e | --example) return the value or exit 1 if value is blank or appears to be another option -__get_option_value() { - local __arg="${1:-}" - local __val="${2:-}" - - if [[ -n "${__val:-}" ]] && [[ ! "${__val:-}" =~ ^- ]]; then - printf "%s\\n" "${__val}" - else - _exit_1 printf "%s requires a valid argument.\\n" "${__arg}" - fi -} - -# Parse the CLI args -while ((${#})); do - __arg="${1:-}" - __val="${2:-}" - - case "${__arg}" in - -h|--help) - _PRINT_HELP=1 - ;; - -b|--bytes) - _FORMAT_BYTES=0 - ;; - -o|--output) - _OUTPUT_TO_FILE=1 - _OUTPUT_FILE="$(__get_option_value "${__arg}" "${__val:-}")" - shift - ;; - -r|--reporting-format) - _REPORTING_FORMAT="$(__get_option_value "${__arg}" "${__val:-}")" - shift - ;; - -*) - _exit_1 printf "Unexpected option: %s\\n" "${__arg}" - ;; - esac - - shift -done - -# Parse the _REPORTING_FORMAT value -_REPORTING_FORMAT=$(echo "$_REPORTING_FORMAT" | awk '{print tolower($0)}') -case "${_REPORTING_FORMAT}" in - c|console) - _NORMALISED_REPORTING_FORMAT=0 - ;; - j|json) - _NORMALISED_REPORTING_FORMAT=1 - ;; - *) - _exit_1 printf "Unexpected --reporting-format option: %s\\nShould be one of (c[onsole]|j[son])\\n" "${_REPORTING_FORMAT}" - ;; -esac - -# Main script functionality -_get_storage() { - # Get the actual mongo values - tsx_output=$(npm run script -- getTotalStorageUsedMongo) - - # Parse the returned values - if ((_FORMAT_BYTES)); then - unformatted_result=$(echo "${tsx_output}" | tail -n 1) - # Values always end in B for bytes (e.g. GB) - extraction_regex_suffix="[^B]*B" - else - unformatted_result=$(echo "${tsx_output}" | tail -n 2 | head -n 1) - # Values are always space or close parentheses delimited - extraction_regex_suffix="[^ )]*" - fi - # grep for the value, then cut on the = to get just the value without the text prefix - existing_bytes=$(echo "${unformatted_result}" | grep -o "existing=${extraction_regex_suffix}" | cut -d '=' -f2-) - deleted_bytes=$(echo "${unformatted_result}" | grep -o "deleted=${extraction_regex_suffix}" | cut -d '=' -f2-) - total_bytes=$(echo "${unformatted_result}" | grep -o "total=${extraction_regex_suffix}" | cut -d '=' -f2-) - - # Format the result string - if [ $_NORMALISED_REPORTING_FORMAT -eq 1 ]; then - # Use jq to safely (escape key characters etc) create a JSON string - if ((_FORMAT_BYTES)); then - result_string=$(jq -n --arg eb "${existing_bytes}" --arg db "${deleted_bytes}" --arg tb "${total_bytes}" '{existing: $eb, deleted: $db, total: $tb}') - else - result_string=$(jq -n --arg eb "${existing_bytes}" --arg db "${deleted_bytes}" --arg tb "${total_bytes}" '{existing: ($eb)|tonumber, deleted: ($db)|tonumber, total: ($tb)|tonumber}') - fi - elif [ $_NORMALISED_REPORTING_FORMAT -eq 0 ]; then - result_string="existing: ${existing_bytes}\\ndeleted: ${deleted_bytes}\\ntotal: ${total_bytes}" - else - # This shouldn't happen, but better safe than sorry - _exit_1 printf "Got _NORMALISED_REPORTING_FORMAT=%s\\n" "${_NORMALISED_REPORTING_FORMAT}" - fi - - # Write the output - if ((_OUTPUT_TO_FILE)); then - # Make sure parent directory exists - mkdir -p "$(dirname "$_OUTPUT_FILE")" - printf "%b" "${result_string}" > "${_OUTPUT_FILE}" - else - printf "%b\\n" "${result_string}" - fi -} - -_main() { - # Avoid complex option parsing when only one program option is expected - if ((_PRINT_HELP)); then - _print_help - else - _get_storage "$@" - fi -} - -# Call `_main` after everything has been defined -_main "$@" diff --git a/backend/src/scripts/getTotalStorageUsedMongo.ts b/backend/src/scripts/getTotalStorageUsedMongo.ts index 842ac33f7..eea018f89 100644 --- a/backend/src/scripts/getTotalStorageUsedMongo.ts +++ b/backend/src/scripts/getTotalStorageUsedMongo.ts @@ -4,9 +4,6 @@ import FileModel from '../models/File.js' import log from '../services/log.js' import { connectToMongoose, disconnectFromMongoose } from '../utils/database.js' -// Helper file for getTotalStorageUsed.sh -// TS handles Mongoose connections more easily than just using a BASH script - async function main() { await connectToMongoose() @@ -14,7 +11,7 @@ async function main() { // mongoose-delete plugin doesn't have correct typing so cast to any const deletedResults = await (FileModel as any).findDeleted() - await disconnectFromMongoose() + setTimeout(disconnectFromMongoose, 50) const totalExistingBytes = existingResults.reduce((sum, fileModel) => sum + fileModel.size, 0) const totalDeletedBytes = deletedResults.reduce((sum, fileModel) => sum + fileModel.size, 0) From 6f2449f06aabc95848fce1939e63da2430ab511c Mon Sep 17 00:00:00 2001 From: PE39806 <185931318+PE39806@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:11:54 +0000 Subject: [PATCH 7/7] BAI-1570 remove unused acknowledgement line from readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d4f1b606c..271629413 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,6 @@ information. - [Img Shields](https://shields.io) - [Othneils's README Template](https://github.com/othneildrew/Best-README-Template) - [Mattermost's Code Contribution Guidelines](https://github.com/mattermost/mattermost-server/blob/master/CONTRIBUTING.md) -- [xwmx's Bash Boilerplate](https://github.com/xwmx/bash-boilerplate)