Merge branch 'master' into fix-static-methods

.github/workflows/build-client-container.yml

@@ -19,7 +19,7 @@ jobs:
     steps:
     - name: make date tag
       id: mkdatetag
-      run: echo "::set-output name=dtag::$(date +%Y%m%d-%H%M)"
+      run: echo "dtag=$(date +%Y%m%d-%H%M)" >> $GITHUB_OUTPUT
 
   build:
     runs-on: ubuntu-latest
@@ -29,7 +29,7 @@ jobs:
       fail-fast: False
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Generate tag list
       id: generate-tag-list
@@ -43,26 +43,26 @@ jobs:
           tag_list+=($registry/$docker_repo:release-$TIMESTAMP)
         done
         IFS=,
-        echo "::set-output name=taglist::${tag_list[*]}"
+        echo "taglist=${tag_list[*]}" >> $GITHUB_OUTPUT
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2.7.0
 
     - name: Log in to Docker Hub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2.2.0
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - name: Log in to OSG Harbor
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2.2.0
       with:
         registry: hub.opensciencegrid.org
         username: ${{ secrets.OSG_HARBOR_ROBOT_USER }}
         password: ${{ secrets.OSG_HARBOR_ROBOT_PASSWORD }}
 
     - name: Build and push Client Docker images
-      uses: docker/build-push-action@v2.2.0
+      uses: docker/build-push-action@v4
       with:
         push: true
         tags: "${{ steps.generate-tag-list.outputs.taglist }}"

.github/workflows/build-sw-container.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     if: startsWith(github.repository, 'opensciencegrid/')
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Generate tag list
       id: generate-tag-list
@@ -28,26 +28,26 @@ jobs:
         # This causes the tag_list array to be comma-separated below,
         # which is required for build-push-action
         IFS=,
-        echo "::set-output name=taglist::${tag_list[*]}"
+        echo "taglist=${tag_list[*]}" >> $GITHUB_OUTPUT
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2.7.0
 
     - name: Log in to Docker Hub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2.2.0
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - name: Log in to OSG Harbor
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2.2.0
       with:
         registry: hub.opensciencegrid.org
         username: ${{ secrets.OSG_HARBOR_ROBOT_USER }}
         password: ${{ secrets.OSG_HARBOR_ROBOT_PASSWORD }}
 
     - name: Build and push Docker images
-      uses: docker/build-push-action@v2.2.0
+      uses: docker/build-push-action@v4
       with:
         context: .
         push: true

.github/workflows/validate-caches.yml

@@ -10,11 +10,11 @@ jobs:
     if: startsWith(github.repository, 'opensciencegrid/')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.6
+          python-version: 3.9.15
       - name: Compare StashCache config lists
         run: |
           ./src/tests/diff_cache_configs.py

.github/workflows/validate-code.yml

@@ -6,11 +6,11 @@ jobs:
     name: Validate Topology code
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.6
+          python-version: 3.9.15
       - name: Install packages
         run: |
           sudo apt-get update
@@ -23,10 +23,12 @@ jobs:
       - name: Test API
         run: |
           export TOPOLOGY_CONFIG=$PWD/src/config-ci.py
+          export FLASK_DEBUG=1
           py.test ./src/tests/test_api.py
       - name: Test StashCache
         run: |
           export TOPOLOGY_CONFIG=$PWD/src/config-ci.py
+          export FLASK_DEBUG=1
           py.test ./src/tests/test_stashcache.py
       - name: Test cacher
         run: |

.github/workflows/validate-data.yml

@@ -7,11 +7,11 @@ jobs:
     name: Validate Topology data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.6
+          python-version: 3.9.15
       - name: Install packages
         run: |
           sudo apt-get update

Dockerfile

@@ -1,4 +1,4 @@
-FROM opensciencegrid/software-base:3.6-el8-release
+FROM opensciencegrid/software-base:3.6-al8-release
 
 # Install dependencies (application, Apache)
 RUN \
@@ -7,8 +7,8 @@ RUN \
       gcc \
       git \
       libyaml-devel \
-      python3-devel \
-      python3-pip \
+      python39-devel \
+      python39-pip \
     && yum install -y \
       fetch-crl \
       httpd \
@@ -19,10 +19,12 @@ RUN \
       /usr/bin/pkill \
     && yum clean all && rm -rf /var/cache/yum/*
 
+RUN alternatives --set python3 /usr/bin/python3.9
+
 WORKDIR /app
 
 # Install application dependencies
-COPY requirements-apache.txt src/ ./
+COPY requirements-apache.txt requirements-rootless.txt ./
 RUN pip3 install --no-cache-dir -r requirements-apache.txt
 
 # Create data directory, and gather SSH keys for git
@@ -36,6 +38,8 @@ RUN echo "45 */6 * * * root /usr/sbin/fetch-crl -q -r 21600 -p 10" >  /etc/cron.
     echo "@reboot      root /usr/sbin/fetch-crl -q          -p 10" >> /etc/cron.d/fetch-crl && \
     echo "0 0 * * *    root /usr/bin/pkill -USR1 httpd"            >  /etc/cron.d/httpd
 
+# Install application
+COPY src/ ./
 
 # Set up Apache configuration
 # Remove default SSL config: default certs don't exist on EL8 so the

Dockerfile.client

@@ -1,6 +1,6 @@
 FROM quay.io/centos/centos:stream8
 
-LABEL maintainer="OSG Software <help@opensciencegrid.org>"
+LABEL maintainer="OSG Software <help@osg-htc.org>"
 LABEL name="OSG 3.5 OSG-Notify client"
 
 RUN yum -y install https://repo.opensciencegrid.org/osg/3.5/osg-3.5-el8-release-latest.rpm \

README.md

@@ -146,4 +146,4 @@ for the VO(s) that you support:
 | LHC VOs                     | [GGUS](https://ggus.eu)                          |
 | Anyone else                 | [Helpdesk](https://support.opensciencegrid.org) |
 
-Or email us at help@opensciencegrid.org.
+Or email us at help@osg-htc.org.

bin/next_downtime_id

@@ -4,6 +4,7 @@ set -o pipefail
 cd "$(dirname "$0")"/..
 
 unset GREP_OPTIONS
-egrep -h '^ *ID: *[0-9]{1,9}\>' topology/*/*/*_downtime.yaml |
+# After 18 digits we might start running into integer overflow issues...
+egrep -h '^ *ID: *[0-9]{1,18}\>' topology/*/*/*_downtime.yaml |
   awk '$2 > max {max = $2}; END {print max + 1}'
 

bin/osg-notify

@@ -5,6 +5,7 @@
 A simple tool for generating notification emails to the OSG
 """
 import os
+import re
 import sys
 import getpass
 import smtplib
@@ -116,10 +117,9 @@ def parseargs():
     oparser.add_argument("--oim-owner-filter", dest="owner_vo",
                          help="Filter on resources that list VO(s) as a partial owner")
 
-    oparser.add_argument("--oim-contact-type", default="all", dest="contact_type",
-                         choices=["all", "administrative", "miscellaneous", "security", "submitter", "site"],
-                         help="Filter on contact type e.g. administrative, miscellaneous, security, submitter or site"
-                         "(default: all)", )
+    oparser.add_argument("--oim-contact-type", action="append", dest="contact_type",
+                         choices=["all"] + topology_utils.CONTACT_TYPES,
+                         help="Filter on contact type (default: all)")
     oparser.add_argument("--bypass-dns-check", action="store_true", dest="bypass_dns_check",
                          help="Bypass checking that one of the host's IP addresses matches with the hostanme resolution")
     oparser.add_argument("--allow-non-ascii", action="store_true", dest="allow_non_ascii",
@@ -132,6 +132,10 @@ def parseargs():
     if args.name_filter and args.fqdn_filter:
         oparser.error("Can't specify both --oim-name-filter and --oim-fqdn-filter")
 
+    if not args.contact_type or ("all" in args.contact_type):
+        args.contact_type = ["all"]
+    args.contact_type = set(args.contact_type)  # remove dupes
+
     if args.from_name == 'security':
         args.from_name = 'OSG Security Team'
         args.from_addr = '[email protected]'
@@ -158,7 +162,12 @@ def network_ok(bypass_dns_check):
 
 def replace_smart_quotes_and_dashes(contents):
     # Replace smart quotes and em/en dashes
-    replaced = contents.replace('“','"').replace('”','"').replace("–","-").replace("—","-")
+    regex_sub_map = [(r'[“”]', '"'),
+                     (r'[‘’]', "'"),
+                     (r'[—–]', '-')]
+    replaced = contents
+    for pattern, sub in regex_sub_map:
+        replaced = re.sub(pattern, sub, replaced)
     return replaced
 
 def has_non_printable_ascii_characters(contents):
@@ -181,6 +190,7 @@ def main():
         while attempts > 0:
             try:
                 results = pm.get_vo_contacts(args)
+                break
             except topology_utils.InvalidPathError as exc:
                 print(exc)
                 exit(1)

docker/apache.conf

@@ -1,11 +1,11 @@
 Listen 8080
 Listen 8443
 
-LoadModule wsgi_module /usr/local/lib64/python3.6/site-packages/mod_wsgi/server/mod_wsgi-py36.cpython-36m-x86_64-linux-gnu.so
+LoadModule wsgi_module /usr/local/lib64/python3.9/site-packages/mod_wsgi/server/mod_wsgi-py39.cpython-39-x86_64-linux-gnu.so
 WSGIPythonPath /app
 
 # Run apps in separate processes to stop yaml.CSafeLoader import-time error
-WSGIDaemonProcess topology  home=/app
+WSGIDaemonProcess topology  home=/app processes=5
 WSGIDaemonProcess topomerge home=/app
 
 # vhost for topology, SSL terminated here (for gridsite auth)
@@ -35,6 +35,14 @@ WSGIDaemonProcess topomerge home=/app
 
   WSGIScriptAlias /        /app/topology.wsgi         process-group=topology application-group=topology
 
+  # Enable compression for text files
+  AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/javascript application/json
+
+  # Enable memory caching
+  CacheSocache shmcb
+  CacheSocacheMaxSize 102400
+  CacheEnable socache /
+
 </VirtualHost>
 
 # Separate vhost for map, no SSL required (terminated in traefik)