Merge pull request #149 from GenomicDataInfrastructure/main

chore: promote main changes

.github/workflows/main.yml

@@ -76,7 +76,7 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.28.0
+        uses: aquasecurity/trivy-action@0.29.0
         with:
           image-ref: "${{ steps.tag.outputs.last_tag }}"
           format: "table"

.github/workflows/release.yml

@@ -74,7 +74,7 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.28.0
+        uses: aquasecurity/trivy-action@0.29.0
         with:
           image-ref: "${{ steps.tag.outputs.last_tag }}"
           format: "table"

.github/workflows/test.yml

@@ -15,4 +15,4 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: REUSE Compliance Check
-        uses: fsfe/reuse-action@v4
+        uses: fsfe/reuse-action@v5

ckan/Dockerfile

@@ -6,18 +6,18 @@
 
 FROM docker.io/ckan/ckan-base:2.10.5
 
-RUN  pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected].6#egg=ckanext-gdi-userportal && \
+RUN  pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected].8#egg=ckanext-gdi-userportal && \
     pip3 install -r ${APP_DIR}/src/ckanext-gdi-userportal/requirements.txt
 
-RUN  pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v2.0.0#egg=ckanext-dcat && \
+RUN  pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v2.1.0#egg=ckanext-dcat && \
     pip3 install -r ${APP_DIR}/src/ckanext-dcat/requirements.txt
 
-RUN pip3 install -e git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest && \
+RUN pip3 install -e git+https://github.com/ckan/ckanext-harvest.git@v1.6.0#egg=ckanext-harvest && \
     pip3 install -r ${APP_DIR}/src/ckanext-harvest/requirements.txt
 
 RUN pip3 install -e git+https://github.com/CivityNL/[email protected]#egg=ckanext-scheming[requirements]
 
-RUN  pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected].1#egg=ckanext-fairdatapoint && \
+RUN  pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected].2#egg=ckanext-fairdatapoint && \
     pip3 install -r ${APP_DIR}/src/ckanext-fairdatapoint/requirements.txt
 
 # Create log directories for CKAN harvester

ckan/Dockerfile.dev

@@ -6,15 +6,15 @@
 
 FROM docker.io/ckan/ckan-dev:2.10.5
 
-RUN  pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v2.0.0#egg=ckanext-dcat && \
+RUN  pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v2.1.0#egg=ckanext-dcat && \
     pip3 install -r ${APP_DIR}/src/ckanext-dcat/requirements.txt
 
-RUN pip3 install -e git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest && \
+RUN pip3 install -e git+https://github.com/ckan/ckanext-harvest.git@v1.6.0#egg=ckanext-harvest && \
     pip3 install -r ${APP_DIR}/src/ckanext-harvest/requirements.txt
 
 RUN pip3 install -e git+https://github.com/CivityNL/[email protected]#egg=ckanext-scheming[requirements]    
 
-RUN  pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected].1#egg=ckanext-fairdatapoint && \
+RUN  pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected].2#egg=ckanext-fairdatapoint && \
     pip3 install -r ${APP_DIR}/src/ckanext-fairdatapoint/requirements.txt
 
 # Create log directories for CKAN harvester

ckan/docker-entrypoint.d/upload_vocabulary.sh

@@ -6,7 +6,7 @@
 
 set -e
 
-psql postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:5432/$CKAN_DB <<-EOSQL
+psql $CKAN_SQLALCHEMY_URL <<-EOSQL
 
   SELECT
       EXISTS(SELECT 1 FROM public.term_translation) as table_not_empty

ckan/setup/prerun.py

@@ -7,6 +7,7 @@
 import sys
 import subprocess
 import psycopg2
+
 try:
     from urllib.request import urlopen
     from urllib.error import URLError
@@ -31,12 +32,18 @@ def update_plugins():
     subprocess.check_output(cmd, stderr=subprocess.STDOUT)
     print("[prerun] Plugins set.")
 
+
 def update_database():
 
     sqlalchemy_url = os.environ.get("CKAN_SQLALCHEMY_URL", "")
     print(("[prerun] Setting the SqlAlchemy URL in {}:".format(ckan_ini)))
     print(sqlalchemy_url)
-    cmd = ["ckan", "config-tool", ckan_ini, "sqlalchemy.url = {}".format(sqlalchemy_url)]
+    cmd = [
+        "ckan",
+        "config-tool",
+        ckan_ini,
+        "sqlalchemy.url = {}".format(sqlalchemy_url),
+    ]
     subprocess.check_output(cmd, stderr=subprocess.STDOUT)
     print("[prerun] SqlAlchemy URL set.")
 
@@ -49,14 +56,6 @@ def check_main_db_connection(retry=None):
     return check_db_connection(conn_str, retry)
 
 
-def check_datastore_db_connection(retry=None):
-
-    conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL")
-    if not conn_str:
-        print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db")
-    return check_db_connection(conn_str, retry)
-
-
 def check_db_connection(conn_str, retry=None):
 
     if retry is None:
@@ -86,7 +85,7 @@ def check_solr_connection(retry=None):
         sys.exit(1)
 
     url = os.environ.get("CKAN_SOLR_URL", "")
-    search_url = '{url}/schema/name?wt=json'.format(url=url)
+    search_url = "{url}/schema/name?wt=json".format(url=url)
 
     try:
         connection = urlopen(search_url)
@@ -96,13 +95,14 @@ def check_solr_connection(retry=None):
         time.sleep(10)
         check_solr_connection(retry=retry - 1)
     else:
-        import re                                                                                                                                                      
-        conn_info = connection.read()                                                                                                                                  
-        schema_name = json.loads(conn_info)                                                                                                                            
-        if 'ckan' in schema_name['name']:                                                                                                                              
-            print('[prerun] Succesfully connected to solr and CKAN schema loaded')                                                                                     
-        else:                                                                                                                                                          
-            print('[prerun] Succesfully connected to solr, but CKAN schema not found')
+        import re
+
+        conn_info = connection.read()
+        schema_name = json.loads(conn_info)
+        if "ckan" in schema_name["name"]:
+            print("[prerun] Succesfully connected to solr and CKAN schema loaded")
+        else:
+            print("[prerun] Succesfully connected to solr, but CKAN schema not found")
 
 
 def init_db():
@@ -123,39 +123,12 @@ def init_db():
             raise e
 
 
-def init_datastore_db():
-
-    conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL")
-    if not conn_str:
-        print("[prerun] Skipping datastore initialization")
-        return
-
-    datastore_perms_command = ["ckan", "-c", ckan_ini, "datastore", "set-permissions"]
-
-    connection = psycopg2.connect(conn_str)
-    cursor = connection.cursor()
-
-    print("[prerun] Initializing datastore db - start")
+def init_db_harvest():
+    db_command = ["ckan", "-c", ckan_ini, "db", "upgrade", "-p", "harvest"]
+    print("[prerun] Initializing or upgrading harvest db - start")
     try:
-        datastore_perms = subprocess.Popen(
-            datastore_perms_command, stdout=subprocess.PIPE
-        )
-
-        perms_sql = datastore_perms.stdout.read()
-        # Remove internal pg command as psycopg2 does not like it
-        perms_sql = re.sub(b'\\\\connect "(.*)"', b"", perms_sql)
-        cursor.execute(perms_sql)
-        for notice in connection.notices:
-            print(notice)
-
-        connection.commit()
-
-        print("[prerun] Initializing datastore db - end")
-        print(datastore_perms.stdout.read())
-    except psycopg2.Error as e:
-        print("[prerun] Could not initialize datastore")
-        print(str(e))
-
+        subprocess.check_output(db_command, stderr=subprocess.STDOUT)
+        print("[prerun] Initializing or upgrading harvest db - end")
     except subprocess.CalledProcessError as e:
         if "OperationalError" in e.output:
             print(e.output)
@@ -165,9 +138,6 @@ def init_datastore_db():
         else:
             print(e.output)
             raise e
-    finally:
-        cursor.close()
-        connection.close()
 
 
 def create_sysadmin():
@@ -219,8 +189,6 @@ def create_sysadmin():
         update_database()
         check_main_db_connection()
         init_db()
-        #check_datastore_db_connection()
-        #init_datastore_db()
+        init_db_harvest()
         check_solr_connection()
         create_sysadmin()
-