diff --git a/.github/ISSUE_TEMPLATE/data-request.md b/.github/ISSUE_TEMPLATE/data-request.md new file mode 100644 index 0000000..d168aed --- /dev/null +++ b/.github/ISSUE_TEMPLATE/data-request.md @@ -0,0 +1,35 @@ +--- +name: Data Request +about: Use this template to submit a request for data processing +labels: data request + +--- + +# Zephyr Data Request + +## Instructions + +1. Give this issue a meaningful title. +2. Replace `PASTE_YOUR_REQUEST_HERE` with your request that you copied to clipboard: +```json +PASTE_YOUR_REQUEST_HERE +``` +3. Click on "Preview" to verify that the JSON format of your request is correctly formatted and displayed within a code block. +4. Click on "Submit new issue". + +Zephyr Data Request will now process your data based on the request you provided here. Once the processing is successful, it will post a link in this issue. +The processed data will be available under that link for up to **7 days**. + +If you encounter any problems, please open a discussion in the [C2SM forum](https://github.com/C2SM/Tasks-Support/discussions). + +## Status Labels + +Labels reflect the current state of your request: + +![Static Badge](https://img.shields.io/badge/submitted-yellow) - Your request is currently under processing. Please wait for further updates. + +![Static Badge](https://img.shields.io/badge/completed-green) - Your request has been successfully processed. You can download your data using the provided link. + +![Static Badge](https://img.shields.io/badge/failed-red) - Unfortunately, your request could not be processed. Please refer to the log files in the zip file at the download link for more details. + +![Static Badge](https://img.shields.io/badge/aborted-lightgray) - Your request was aborted. This might be due to a timeout. Please try again or contact support if the problem persists. diff --git a/.github/workflows/close_PR_and_delete_branch.yml b/.github/workflows/close_PR_and_delete_branch.yml new file mode 100644 index 0000000..e23a81d --- /dev/null +++ b/.github/workflows/close_PR_and_delete_branch.yml @@ -0,0 +1,54 @@ +name: Close PR and delete branch +on: + issue_comment: + types: [created] + +jobs: + CloseFinishedPR: + if: github.event.issue.pull_request && contains(github.event.issue.labels.*.name, 'auto-generated') + runs-on: ubuntu-latest + steps: + - name: Install jq + run: sudo apt-get install jq + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set env variables + run: | + echo "PR_DELETE=$(jq -r '.PR_DELETE' keywords.json)" >> $GITHUB_ENV + - name: Close PR + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const prComment = context.payload.comment.body; + if (!prComment.includes(process.env.PR_DELETE)) { + console.log(`The PR comment does not contain the string "${process.env.PR_DELETE}".`); + return; + } + await github.pulls.update({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + state: 'closed' + }); + - name: Delete branch + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const prComment = context.payload.comment.body; + if (!prComment.includes(process.env.PR_DELETE)) { + console.log(`The PR comment does not contain the string "${process.env.PR_DELETE}".`); + return; + } + const pr = await github.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + const branchName = pr.data.head.ref; + await github.git.deleteRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: 'heads/' + branchName + }); diff --git a/.github/workflows/close_old_data_request_issues.yml b/.github/workflows/close_old_data_request_issues.yml new file mode 100644 index 0000000..3dea582 --- /dev/null +++ b/.github/workflows/close_old_data_request_issues.yml @@ -0,0 +1,36 @@ +name: Close old data request issues +on: + schedule: + - cron: '0 0 * * *' # Run this workflow every day at midnight + workflow_dispatch: + +env: + DAYS: 7 +jobs: + closeOldIssues: + runs-on: ubuntu-latest + steps: + - name: Close old data request issues + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const max_age = process.env.DAYS * 24 * 60 * 60 * 1000; + const now = new Date(); + const issues = await github.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + labels: 'data request', + state: 'open' + }); + for (const issue of issues.data) { + const createdAt = new Date(issue.created_at); + if (now - createdAt > max_age) { + await github.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + state: 'closed' + }); + } + } diff --git a/.github/workflows/convert_issue_to_PR.yml b/.github/workflows/convert_issue_to_PR.yml new file mode 100644 index 0000000..fa379b4 --- /dev/null +++ b/.github/workflows/convert_issue_to_PR.yml @@ -0,0 +1,82 @@ +name: Forward request to PR +on: + issues: + types: [labeled] + +jobs: + createPullRequest: + if: github.event.label.name == 'data request' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Create new branch and commit + run: | + git config --global user.name 'GitHub Action' + git config --global user.email 'action@github.com' + git checkout -b issue-${{ github.event.issue.number }}/request + echo "This is a new file" > newfile.txt + git add newfile.txt + git commit -m "Add new file for issue #${{ github.event.issue.number }}" + git push origin issue-${{ github.event.issue.number }}/request + + - name: Create Pull Request + id: create_pr + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const pr = await github.pulls.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `Request #${context.issue.number}`, + head: `issue-${context.issue.number}/request`, + base: 'main', + body: `Request triggered by #${context.issue.number}` + }); + return pr.data.number; + - name: Extract JSON + id: extract + run: | + BODY='${{ github.event.issue.body }}' + JSON=$(echo "$BODY" | sed -n '/```json/,/```/p' | sed '/```json/d' | sed '/```/d') + echo "$JSON" > json.txt + - name: Add issue comment to PR + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const fs = require('fs'); + const issueComment = context.payload.issue.body; + const prNumber = ${{ steps.create_pr.outputs.result }}; + const JSON = fs.readFileSync('json.txt', 'utf8'); + github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: `submit request ${JSON}` + }); + - name: Add label to issue + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + github.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: ['submitted'] + }); + - name: Add label to PR + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const prNumber = ${{ steps.create_pr.outputs.result }}; + github.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + labels: ['auto-generated'] + }); diff --git a/.github/workflows/forward_PR_comment_to_issue.yml b/.github/workflows/forward_PR_comment_to_issue.yml new file mode 100644 index 0000000..0602a39 --- /dev/null +++ b/.github/workflows/forward_PR_comment_to_issue.yml @@ -0,0 +1,40 @@ +name: Forward Download Link to Issue +on: + issue_comment: + types: [created] + +jobs: + forwardDownloadLinkToIssue: + if: github.event.issue.pull_request && contains(github.event.issue.labels.*.name, 'auto-generated') + runs-on: ubuntu-latest + steps: + - name: Install jq + run: sudo apt-get install jq + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set env variables + run: | + echo "DOWNLOAD_LINK=$(jq -r '.DOWNLOAD_LINK' keywords.json)" >> $GITHUB_ENV + - name: Forward comment to issue + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const prComment = context.payload.comment.body; + if (!prComment.includes(process.env.DOWNLOAD_LINK)) { + console.log(`The PR comment does not contain the string "${process.env.DOWNLOAD_LINK}".`); + return; + } + const pr = await github.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + const branchName = pr.data.head.ref; + const issueNumber = branchName.split('-')[1].split('/')[0]; + github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + body: `Your data is ready for up to 7 days: ${prComment}` + }); diff --git a/.github/workflows/report_PR_status_as_label.yml b/.github/workflows/report_PR_status_as_label.yml new file mode 100644 index 0000000..834a9d2 --- /dev/null +++ b/.github/workflows/report_PR_status_as_label.yml @@ -0,0 +1,62 @@ +name: Report PR status as label +on: + issue_comment: + types: [created] + +jobs: + reportPRStatusAsLabel: + if: github.event.issue.pull_request && contains(github.event.issue.labels.*.name, 'auto-generated') + runs-on: ubuntu-latest + steps: + - name: Install jq + run: sudo apt-get install jq + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set env variables + run: | + echo "PR_FAIL=$(jq -r '.PR_FAIL' keywords.json)" >> $GITHUB_ENV + echo "PR_SUCCESS=$(jq -r '.PR_SUCCESS' keywords.json)" >> $GITHUB_ENV + echo "PR_ABORT=$(jq -r '.PR_ABORT' keywords.json)" >> $GITHUB_ENV + - name: Update label + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const pr = await github.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + const branchName = pr.data.head.ref; + const issueNumber = branchName.split('-')[1].split('/')[0]; + if (context.payload.comment.body.includes(process.env.PR_FAIL) || + context.payload.comment.body.includes(process.env.PR_SUCCESS) || + context.payload.comment.body.includes(process.env.PR_ABORT)) { + await github.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + name: 'submitted' + }); + } if (context.payload.comment.body.includes(process.env.PR_FAIL)) { + await github.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + labels: ['failed'] + }); + } if (context.payload.comment.body.includes(process.env.PR_SUCCESS)){ + await github.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + labels: ['completed'] + }); + } if (context.payload.comment.body.includes(process.env.PR_ABORT)){ + await github.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + labels: ['aborted'] + }); + } diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9cd4692 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Center for Climate Systems Modeling + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..9084fa2 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.1.0 diff --git a/jenkins/Cleanup b/jenkins/Cleanup new file mode 100644 index 0000000..bbc440d --- /dev/null +++ b/jenkins/Cleanup @@ -0,0 +1,23 @@ +def https_public_root = '/net/co2/c2sm-data/jenkins/extpar-request' + +pipeline { + agent { + node { + label 'atmos' + } + } + stages { + stage('Cleanup HTTPS-server') { + steps { + sh """ + python3 src/cleanup.py --path ${https_public_root} --threshold 7 --exclude ${https_public_root}/file_index + """ + } + } + } + post { + always { + deleteDir() + } + } +} diff --git a/jenkins/RemoteExtraction b/jenkins/RemoteExtraction new file mode 100644 index 0000000..48b76fc --- /dev/null +++ b/jenkins/RemoteExtraction @@ -0,0 +1,99 @@ +def https_public_root = '/net/co2/c2sm-data/jenkins/extpar-request/' + +pipeline { + agent { + node { + label 'atmos' + } + } + options { + timeout(time: 3, unit: 'HOURS') + } + stages { + stage('Create archive') { + steps { + sh """ + mkdir -p ${WORKSPACE}/output/logs + """ + } + } + stage('Create Hash from Build ID') { + steps { + sh """ + python3 src/hash.py --build-id ${BUILD_ID} --hash-file ${WORKSPACE}/hash.txt + """ + } + } + stage('Validate User Input') { + steps { + sh """ + python3 src/validate_user_input.py --comment_body '${env.ghprbCommentBody}' || + (echo "The request you submitted is not valid! \n Please check for typos or wrong format of JSON" > ${WORKSPACE}/output/logs/pipeline.log && + exit 1) + """ + } + } + stage('Clone Zephyr') { + steps { + sh """ + git clone -b ${zephyr_version} git@github.com:C2SM/zephyr.git + """ + } + } + stage('Setup Python Environment') { + steps { + sh """ + python3 -m venv zephyr_venv + source zephyr_venv/bin/activate + pip install -r zephyr/tools/requirements/requirements.txt + cd zephyr + pip install -e . + """ + } + } + stage('Process Data') { + steps { + script { + env.VERSION = readFile('VERSION').trim() + } + sh """ + source zephyr_venv/bin/activate + cd zephyr/run + python launch_zephyr.py --json_path=${WORKSPACE}/config.json --output_directory=${WORKSPACE}/output/data --github_issue_id=${ghprbPullId} --zephyr_request_version=${VERSION} + """ + } + } + } + post { + success { + sh "mkdir -p zephyr/run/log && touch zephyr/run/log/ignore" + sh "cp zephyr/run/log/* ${WORKSPACE}/output/logs" + sh "zip -r output.zip output" + sh "python3 src/copy_zip.py --zip-file output.zip --destination ${https_public_root} --hash-file ${WORKSPACE}/hash.txt" + withCredentials([string(credentialsId: 'd976fe24-cabf-479e-854f-587c152644bc', variable: 'GITHUB_AUTH_TOKEN')]) { + sh "python3 src/report.py --auth_token ${GITHUB_AUTH_TOKEN} --issue_id ${ghprbPullId} --hash-file ${WORKSPACE}/hash.txt --keyword-file ${WORKSPACE}/keywords.json" + } + deleteDir() + } + failure { + sh "mkdir -p zephyr/run/log && touch zephyr/run/log/ignore" + sh "cp zephyr/run/log/* ${WORKSPACE}/output/logs" + sh "zip -r output.zip output" + sh "python3 src/copy_zip.py --zip-file output.zip --destination ${https_public_root} --hash-file ${WORKSPACE}/hash.txt" + withCredentials([string(credentialsId: 'd976fe24-cabf-479e-854f-587c152644bc', variable: 'GITHUB_AUTH_TOKEN')]) { + sh "python3 src/report.py --auth_token ${GITHUB_AUTH_TOKEN} --issue_id ${ghprbPullId} --hash-file ${WORKSPACE}/hash.txt --failure --keyword-file ${WORKSPACE}/keywords.json" + } + deleteDir() + } + aborted { + sh "mkdir -p zephyr/run/log && touch zephyr/run/log/ignore" + sh "cp zephyr/run/log/* ${WORKSPACE}/output/logs" + sh "zip -r output.zip output" + sh "python3 src/copy_zip.py --zip-file output.zip --destination ${https_public_root} --hash-file ${WORKSPACE}/hash.txt" + withCredentials([string(credentialsId: 'd976fe24-cabf-479e-854f-587c152644bc', variable: 'GITHUB_AUTH_TOKEN')]) { + sh "python3 src/report.py --auth_token ${GITHUB_AUTH_TOKEN} --issue_id ${ghprbPullId} --hash-file ${WORKSPACE}/hash.txt --keyword-file ${WORKSPACE}/keywords.json --abort" + } + deleteDir() + } + } +} diff --git a/keywords.json b/keywords.json new file mode 100644 index 0000000..b5832cc --- /dev/null +++ b/keywords.json @@ -0,0 +1,7 @@ +{ + "PR_DELETE": "Close and delete please", + "PR_FAIL": "Request failed", + "PR_ABORT": "Request aborted", + "PR_SUCCESS": "Request succesful", + "DOWNLOAD_LINK": "https://data.iac.ethz.ch" +} diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 0000000..97a71dc --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,45 @@ +import os +import shutil +import time +import argparse + +def stage_for_deletion(filepath, dry_run, exclude): + if filepath in exclude: + print(f"Skip {filepath}") + return + if not dry_run: + if os.path.isdir(filepath): + shutil.rmtree(filepath) + elif os.path.isfile(filepath): + os.remove(filepath) + print(f"Delete {filepath}") + +# Create the parser +parser = argparse.ArgumentParser(description="Delete files older than a certain age") + +# Add the arguments +parser.add_argument('--path', type=str, required=True, help='The directory path') +parser.add_argument('--threshold', type=int, required=True, help='The file age threshold (in days)') +parser.add_argument('--dry-run', action='store_true', help='Only print the files that would be deleted, without actually deleting them') +parser.add_argument('--exclude', nargs='*', default=[], help='Directories not to be deleted') + +# Parse the arguments +args = parser.parse_args() + +# The directory to check +directory = args.path + +# The file age threshold (in seconds) +threshold = args.threshold * 24 * 3600 # Convert days to seconds + +# The current time +now = time.time() + +# Check each file in the directory +for filename in os.listdir(directory): + # Get the full path of the file + filepath = os.path.join(directory, filename) + + # If the file is older than the threshold, delete it + if os.path.getmtime(filepath) < now - threshold: + stage_for_deletion(filepath, args.dry_run, args.exclude) diff --git a/src/copy_zip.py b/src/copy_zip.py new file mode 100644 index 0000000..9dc93e5 --- /dev/null +++ b/src/copy_zip.py @@ -0,0 +1,26 @@ +import os +import hashlib +import argparse +import shutil + +# Create the parser +parser = argparse.ArgumentParser(description="Process build ID and HTTPS link") + +# Add the arguments +parser.add_argument('--destination', type=str, required=True, help='The destination folder to store the zip file') +parser.add_argument('--zip-file', type=str, required=True, help='Zip file to share with the user') +parser.add_argument('--hash-file', type=str, required=True, help='Hash file') + +# Parse the arguments +args = parser.parse_args() + +with open(args.hash_file, 'r') as f: + hash = f.read() + +folder = os.path.join(args.destination, hash) +# Create the directory +os.makedirs(folder, exist_ok=True) +print(f"Created directory {folder}") + +# Copy the zip file to the directory +shutil.copy(args.zip_file, folder) diff --git a/src/hash.py b/src/hash.py new file mode 100644 index 0000000..f477775 --- /dev/null +++ b/src/hash.py @@ -0,0 +1,20 @@ +import os +import hashlib +import argparse +import shutil + +# Create the parser +parser = argparse.ArgumentParser(description="Hash Build ID and create a file with the hash value.") + +# Add the arguments +parser.add_argument('--build-id', type=str, required=True, help='The build ID') +parser.add_argument('--hash-file', type=str, required=True, help='Hash file') + +# Parse the arguments +args = parser.parse_args() + +# Compute the SHA256 hash of BUILD_ID +hash = hashlib.sha256(args.build_id.encode()).hexdigest() + +with open(args.hash_file, 'w') as f: + f.write(hash) diff --git a/src/report.py b/src/report.py new file mode 100644 index 0000000..7ac62b0 --- /dev/null +++ b/src/report.py @@ -0,0 +1,63 @@ +import requests +import argparse +import json + +class GitHubRepo: + + def __init__(self, group: str, repo: str, auth_token: str = None) -> None: + self.group: str = group + self.repo: str = repo + self.auth_token: str = auth_token + + def comment(self, issue_id: str, text: str) -> None: + url = f'https://api.github.com/repos/{self.group}/{self.repo}/issues/{issue_id}/comments' + + headers = {'Content-Type': 'application/json'} + if self.auth_token is not None: + headers['Authorization'] = 'token ' + self.auth_token + + requests.post(url, headers=headers, json={'body': text}) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--auth_token', type=str, required=False) + parser.add_argument('--issue_id', type=str, required=True) + parser.add_argument('--hash-file', type=str, required=True) + parser.add_argument('--keyword-file', type=str, required=True) + + # Create a mutually exclusive group for the --abort and --failure arguments + group = parser.add_mutually_exclusive_group() + group.add_argument('--failure', action='store_true') + group.add_argument('--abort', action='store_true') + + args = parser.parse_args() + + repo = GitHubRepo(group='c2sm', + repo='extpar-request', + auth_token=args.auth_token) + + with open(args.hash_file, 'r') as f: + hash = f.read() + + with open(args.keyword_file, 'r') as f: + keywords = json.load(f) + PR_DELETE = keywords['PR_DELETE'] + PR_FAIL = keywords['PR_FAIL'] + PR_ABORT = keywords['PR_ABORT'] + PR_SUCCESS = keywords['PR_SUCCESS'] + + + url = f'https://data.iac.ethz.ch/extpar-request/{hash}' + + repo.comment(issue_id=args.issue_id, text=f'{url}') + + if args.failure: + repo.comment(issue_id=args.issue_id, text=PR_FAIL) + elif args.abort: + repo.comment(issue_id=args.issue_id, text=PR_ABORT) + else: + repo.comment(issue_id=args.issue_id, text=PR_SUCCESS) + + repo.comment(issue_id=args.issue_id, text=PR_DELETE) + diff --git a/src/validate_user_input.py b/src/validate_user_input.py new file mode 100644 index 0000000..92aeab3 --- /dev/null +++ b/src/validate_user_input.py @@ -0,0 +1,26 @@ +import argparse +import json + +def validate_user_input(comment_body): + # Add your validation logic here + print(comment_body) + +def convert_to_json(comment_body): + # Convert the dictionary back to a JSON string + json_str = json.dumps(json.loads(comment_body), indent=4) + + # Write the JSON string to a file + with open('config.json', 'w') as f: + f.write(json_str) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Validate user input") + parser.add_argument("--comment_body", type=str, help="User input from the comment body.") + args = parser.parse_args() + + comment = args.comment_body.replace('\\r\\n', ' ').replace('submit request', '').replace('\\', '') + + # Validate the user input + validate_user_input(comment) + + convert_to_json(comment)