From 3d7953107559a71e517a579fdcddb79bf6d4a7e8 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Fri, 12 Jan 2024 11:56:15 +0100 Subject: [PATCH 01/87] Added token property and improved exiting client context manager --- .../src/molgenis_emx2_pyclient/client.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index c76513415a..38857c41c5 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -1,6 +1,7 @@ import csv import io import logging +import os from typing import TypeAlias, Literal import pandas as pd @@ -21,7 +22,7 @@ class Client: Use the Client object to log in to a Molgenis EMX2 server and perform operations on the server. """ - def __init__(self, url: str, schema: str = None) -> None: + def __init__(self, url: str, schema: str = None, token: str = None) -> None: """ Initializes a Client object with a server url. """ @@ -34,6 +35,10 @@ def __init__(self, url: str, schema: str = None) -> None: self.session: requests.Session = requests.Session() + if token is None: + token = os.environ.get('MOLGENIS_TOKEN', None) + self._token = token + self.schemas: list = self.get_schemas() self.default_schema: str = self._set_schema(schema) @@ -47,7 +52,8 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): if exc_type or exc_val or exc_tb: print(exc_type, exc_val, exc_tb, sep="\n") - self.signout() + if self.signin_status == 'success': + self.signout() self.session.close() async def __aenter__(self): @@ -57,7 +63,8 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_val, exc_tb): if exc_type or exc_val or exc_tb: print(exc_type, exc_val, exc_tb, sep="\n") - self.signout() + if self.signin_status == 'success': + self.signout() self.session.close() def signin(self, username: str, password: str): @@ -166,6 +173,11 @@ def schema_names(self): """Returns a list of the names of the schemas.""" return [schema['name'] for schema in self.schemas] + @property + def token(self): + """Returns the token by a property to prevent it being modified.""" + return self._token + @property def version(self): """Lists the current EMX2 version on the server""" From 8da29d622f623dddd128cd67c5c0ea6fe24d8862 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Fri, 12 Jan 2024 13:12:35 +0100 Subject: [PATCH 02/87] Started adding token to header in a request --- tools/pyclient/src/molgenis_emx2_pyclient/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index 38857c41c5..5b545f5fda 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -158,7 +158,9 @@ def get_schemas(self): response = self.session.post( url=self.api_graphql, - json={'query': query} + json={'query': query}, + # TODO https://stackoverflow.com/questions/66368895/how-to-authenticate-through-graphiql-playground + headers={'Authorization': "JWT " + self.token} ) if response.status_code == 404: From 2935fd7c22bbfc13b13197325cb7ad2aa1fcfd0e Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Fri, 12 Jan 2024 16:03:02 +0100 Subject: [PATCH 03/87] Added headers in session requests for relevant methods --- .../src/molgenis_emx2_pyclient/client.py | 53 ++++++++++++------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index 5b545f5fda..9f21e3d801 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -1,7 +1,6 @@ import csv import io import logging -import os from typing import TypeAlias, Literal import pandas as pd @@ -10,7 +9,7 @@ from . import graphql_queries as queries from . import utils as utils from .exceptions import NoSuchSchemaException, ServiceUnavailableError, SigninError, ServerNotFoundError, \ - PyclientException, NoSuchTableException, NoContextManagerException, GraphQLException + PyclientException, NoSuchTableException, NoContextManagerException, GraphQLException, InvalidTokenException log = logging.getLogger("Molgenis EMX2 Pyclient") @@ -27,6 +26,8 @@ def __init__(self, url: str, schema: str = None, token: str = None) -> None: Initializes a Client object with a server url. """ self._as_context_manager = False + self._token = token + self.url: str = utils.parse_url(url) self.api_graphql = self.url + "/api/graphql" @@ -35,10 +36,6 @@ def __init__(self, url: str, schema: str = None, token: str = None) -> None: self.session: requests.Session = requests.Session() - if token is None: - token = os.environ.get('MOLGENIS_TOKEN', None) - self._token = token - self.schemas: list = self.get_schemas() self.default_schema: str = self._set_schema(schema) @@ -159,12 +156,16 @@ def get_schemas(self): response = self.session.post( url=self.api_graphql, json={'query': query}, - # TODO https://stackoverflow.com/questions/66368895/how-to-authenticate-through-graphiql-playground - headers={'Authorization': "JWT " + self.token} + headers={'x-molgenis-token': self.token} ) if response.status_code == 404: raise ServerNotFoundError(f"Server with url '{self.url}'") + if response.status_code == 400: + if 'Invalid token' in response.text: + raise InvalidTokenException(f"Invalid token or token expired.") + else: + raise PyclientException("An unknown error occurred when trying to reach this server.") response_json: dict = response.json() schemas = response_json['data']['_schemas'] @@ -285,7 +286,8 @@ def _table_in_schema(self, table: str, schema: str) -> bool: """ response = self.session.post( url=f"{self.url}/{schema}/graphql", - json={'query': queries.list_tables()} + json={'query': queries.list_tables()}, + headers={"x-molgenis-token": self.token} ) schema_tables = [tab['name'] for tab in response.json().get('data').get('_schema').get('tables')] @@ -320,7 +322,8 @@ def save_schema(self, name: str = None, table: str = None, file: str = None, dat response = self.session.post( url=f"{self.url}/{current_schema}/api/csv/{table}", - headers={'Content-Type': 'text/csv'}, + headers={'x-molgenis-token': self.token, + 'Content-Type': 'text/csv'}, data=import_data ) @@ -359,7 +362,8 @@ def delete_records(self, schema: str = None, table: str = None, file: str = None response = self.session.delete( url=f"{self.url}/{current_schema}/api/csv/{table}", - headers={'Content-Type': 'text/csv'}, + headers={'x-molgenis-token': self.token, + 'Content-Type': 'text/csv'}, data=import_data ) @@ -394,7 +398,8 @@ def get(self, schema: str = None, table: str = None, as_df: bool = False) -> lis if not self._table_in_schema(table, current_schema): raise NoSuchTableException(f"Table '{table}' not found in schema '{current_schema}'.") - response = self.session.get(url=f"{self.url}/{current_schema}/api/csv/{table}") + response = self.session.get(url=f"{self.url}/{current_schema}/api/csv/{table}", + headers={'x-molgenis-token': self.token}) if response.status_code != 200: message = f"Failed to retrieve data from '{current_schema}::{table}'." \ @@ -433,7 +438,8 @@ def export(self, schema: str = None, table: str = None, fmt: OutputFormat = 'csv if table is None: # Export the whole schema url = f"{self.url}/{current_schema}/api/excel" - response = self.session.get(url) + response = self.session.get(url=url, + headers={'x-molgenis-token': self.token}) filename = f"{current_schema}.xlsx" with open(filename, "wb") as f: @@ -442,7 +448,8 @@ def export(self, schema: str = None, table: str = None, fmt: OutputFormat = 'csv else: # Export the single table url = f"{self.url}/{current_schema}/api/excel/{table}" - response = self.session.get(url) + response = self.session.get(url=url, + headers={'x-molgenis-token': self.token}) filename = f"{table}.xlsx" with open(filename, "wb") as f: @@ -452,7 +459,8 @@ def export(self, schema: str = None, table: str = None, fmt: OutputFormat = 'csv if fmt == 'csv': if table is None: url = f"{self.url}/{current_schema}/api/zip" - response = self.session.get(url) + response = self.session.get(url=url, + headers={'x-molgenis-token': self.token}) filename = f"{current_schema}.zip" with open(filename, "wb") as f: @@ -461,7 +469,8 @@ def export(self, schema: str = None, table: str = None, fmt: OutputFormat = 'csv else: # Export the single table url = f"{self.url}/{current_schema}/api/csv/{table}" - response = self.session.get(url) + response = self.session.get(url=url, + headers={'x-molgenis-token': self.token}) filename = f"{table}.csv" with open(filename, "wb") as f: @@ -493,7 +502,8 @@ def create_schema(self, name: str = None, response = self.session.post( url=f"{self.url}/api/graphql", - json={'query': query, 'variables': variables} + json={'query': query, 'variables': variables}, + headers={'x-molgenis-token': self.token} ) response_json = response.json() @@ -518,7 +528,8 @@ def delete_schema(self, name: str = None): response = self.session.post( url=f"{self.url}/api/graphql", - json={'query': query, 'variables': variables} + json={'query': query, 'variables': variables}, + headers={'x-molgenis-token': self.token} ) response_json = response.json() @@ -545,7 +556,8 @@ def update_schema(self, name: str = None, description: str = None): response = self.session.post( url=f"{self.url}/api/graphql", - json={'query': query, 'variables': variables} + json={'query': query, 'variables': variables}, + headers={'x-molgenis-token': self.token} ) response_json = response.json() @@ -616,7 +628,8 @@ def get_schema_metadata(self, name: str = None): query = queries.list_schema_meta() response = self.session.post( url=f"{self.url}/{current_schema}/api/graphql", - json={'query': query} + json={'query': query}, + headers={'x-molgenis-token': self.token} ) response_json = response.json() From ad101d610c029eca07d13eabd9e26d11c4268fde Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Fri, 12 Jan 2024 16:03:20 +0100 Subject: [PATCH 04/87] Created an exception in case of an invalid or expired token --- tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py b/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py index bf67a8ff98..e39272051e 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py @@ -36,5 +36,10 @@ class ServiceUnavailableError(PyclientException): class NoContextManagerException(PyclientException): """Thrown when sign in is attempted outside a context manager.""" + class GraphQLException(PyclientException): - """Thrown when a query fails to execute""" \ No newline at end of file + """Thrown when a query fails to execute""" + + +class InvalidTokenException(PyclientException): + """Thrown when a token that is used in a request has expired.""" From 3c2ca1af018d16fca3626bf81d2af9c8dee380af Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Fri, 12 Jan 2024 16:22:41 +0100 Subject: [PATCH 05/87] Moved static and private methods down in the file --- .../src/molgenis_emx2_pyclient/client.py | 210 +++++++++--------- 1 file changed, 105 insertions(+), 105 deletions(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index 9f21e3d801..be640bca55 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -191,108 +191,6 @@ def version(self): ) return response.json().get('data').get('_manifest').get('SpecificationVersion') - @staticmethod - def _prep_data_or_file(file_path: str = None, data: list = None) -> str: - """Prepares the data from memory or loaded from disk for addition or deletion action. - - :param file_path: path to the file to be prepared - :type file_path: str - :param data: data to be prepared - :type data: list - - :returns: prepared data in dataframe format - :rtype: pd.DataFrame - """ - if file_path is None and data is None: - print("No data to import. Specify a file location or a dataset.") - - if file_path is not None: - return utils.read_file(file_path=file_path) - - if data is not None: - return pd.DataFrame(data).to_csv(index=False, quoting=csv.QUOTE_NONNUMERIC, encoding='UTF-8') - - def _set_schema(self, name: str) -> str: - """Sets the default schema to the schema supplied as argument. - Raises NoSuchSchemaException if the schema cannot be found on the server. - - :param name: name of a schema - :type name: str - - :returns: a schema name - :rtype: str - """ - if name not in [*self.schema_names, None]: - raise NoSuchSchemaException(f"Schema '{name}' not found on server.") - self.default_schema = name - - return name - - @staticmethod - def _graphql_validate_response(response_json: dict, mutation: str, fallback_error_message: str): - """Validates a GraphQL response and prints the appropriate message. - - :param response_json: a graphql response from the server - :type response_json: dict - :param mutation: the name of the graphql mutation executed - :type mutation: str - :param fallback_error_message: a fallback error message - :type fallback_error_message: str - - :returns: a success or error message - :rtype: string - """ - response_keys = response_json.keys() - if 'error' not in response_keys and 'data' not in response_keys: - message = fallback_error_message - log.error(message) - print(message) - - elif 'error' in response_keys: - message = response_json.get('error').errors[0].get('message') - log.error(message) - raise GraphQLException(message) - - else: - if response_json.get('data').get(mutation).get('status') == 'SUCCESS': - message = response_json.get('data').get(mutation).get('message') - log.info(message) - print(message) - else: - message = f"Failed to validate response for {mutation}" - log.error(message) - print(message) - - @staticmethod - def _format_optional_params(**kwargs): - """Parses optional keyword arguments to a format suitable for GraphQL queries.""" - keys = kwargs.keys() - args = {key: kwargs[key] for key in keys if (key != 'self') and (key is not None)} - if 'name' in args.keys(): - args['name'] = args.pop('name') - if 'include_demo_data' in args.keys(): - args['includeDemoData'] = args.pop('include_demo_data') - return args - - def _table_in_schema(self, table: str, schema: str) -> bool: - """Checks whether the requested table is present in the schema. - - :param table: the name of the table - :type table: str - :param schema: the name of the schema - :type schema: str - :returns: boolean indicating whether table is present - :rtype: bool - """ - response = self.session.post( - url=f"{self.url}/{schema}/graphql", - json={'query': queries.list_tables()}, - headers={"x-molgenis-token": self.token} - ) - schema_tables = [tab['name'] for tab in - response.json().get('data').get('_schema').get('tables')] - return table in schema_tables - def save_schema(self, name: str = None, table: str = None, file: str = None, data: list = None): """Imports or updates records in a table of a named schema. @@ -507,7 +405,7 @@ def create_schema(self, name: str = None, ) response_json = response.json() - self._graphql_validate_response( + self._validate_graphql_response( response_json=response_json, mutation='createSchema', fallback_error_message=f"Failed to create schema '{name}'" @@ -533,7 +431,7 @@ def delete_schema(self, name: str = None): ) response_json = response.json() - self._graphql_validate_response( + self._validate_graphql_response( response_json=response_json, mutation='deleteSchema', fallback_error_message=f"Failed to delete schema '{name}'" @@ -561,7 +459,7 @@ def update_schema(self, name: str = None, description: str = None): ) response_json = response.json() - self._graphql_validate_response( + self._validate_graphql_response( response_json=response_json, mutation='updateSchema', fallback_error_message=f"Failed to update schema '{name}'" @@ -641,3 +539,105 @@ def get_schema_metadata(self, name: str = None): metadata = response_json.get('data').get('_schema') return metadata + + @staticmethod + def _prep_data_or_file(file_path: str = None, data: list = None) -> str: + """Prepares the data from memory or loaded from disk for addition or deletion action. + + :param file_path: path to the file to be prepared + :type file_path: str + :param data: data to be prepared + :type data: list + + :returns: prepared data in dataframe format + :rtype: pd.DataFrame + """ + if file_path is None and data is None: + print("No data to import. Specify a file location or a dataset.") + + if file_path is not None: + return utils.read_file(file_path=file_path) + + if data is not None: + return pd.DataFrame(data).to_csv(index=False, quoting=csv.QUOTE_NONNUMERIC, encoding='UTF-8') + + def _set_schema(self, name: str) -> str: + """Sets the default schema to the schema supplied as argument. + Raises NoSuchSchemaException if the schema cannot be found on the server. + + :param name: name of a schema + :type name: str + + :returns: a schema name + :rtype: str + """ + if name not in [*self.schema_names, None]: + raise NoSuchSchemaException(f"Schema '{name}' not found on server.") + self.default_schema = name + + return name + + @staticmethod + def _validate_graphql_response(response_json: dict, mutation: str, fallback_error_message: str): + """Validates a GraphQL response and prints the appropriate message. + + :param response_json: a graphql response from the server + :type response_json: dict + :param mutation: the name of the graphql mutation executed + :type mutation: str + :param fallback_error_message: a fallback error message + :type fallback_error_message: str + + :returns: a success or error message + :rtype: string + """ + response_keys = response_json.keys() + if 'error' not in response_keys and 'data' not in response_keys: + message = fallback_error_message + log.error(message) + print(message) + + elif 'error' in response_keys: + message = response_json.get('error').errors[0].get('message') + log.error(message) + raise GraphQLException(message) + + else: + if response_json.get('data').get(mutation).get('status') == 'SUCCESS': + message = response_json.get('data').get(mutation).get('message') + log.info(message) + print(message) + else: + message = f"Failed to validate response for {mutation}" + log.error(message) + print(message) + + @staticmethod + def _format_optional_params(**kwargs): + """Parses optional keyword arguments to a format suitable for GraphQL queries.""" + keys = kwargs.keys() + args = {key: kwargs[key] for key in keys if (key != 'self') and (key is not None)} + if 'name' in args.keys(): + args['name'] = args.pop('name') + if 'include_demo_data' in args.keys(): + args['includeDemoData'] = args.pop('include_demo_data') + return args + + def _table_in_schema(self, table: str, schema: str) -> bool: + """Checks whether the requested table is present in the schema. + + :param table: the name of the table + :type table: str + :param schema: the name of the schema + :type schema: str + :returns: boolean indicating whether table is present + :rtype: bool + """ + response = self.session.post( + url=f"{self.url}/{schema}/graphql", + json={'query': queries.list_tables()}, + headers={"x-molgenis-token": self.token} + ) + schema_tables = [tab['name'] for tab in + response.json().get('data').get('_schema').get('tables')] + return table in schema_tables From 90dbe9254e927de11d518fc4c509a533532cefe1 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Fri, 12 Jan 2024 16:29:45 +0100 Subject: [PATCH 06/87] Updated README.md --- tools/pyclient/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tools/pyclient/README.md b/tools/pyclient/README.md index 1463043290..caf5e955db 100644 --- a/tools/pyclient/README.md +++ b/tools/pyclient/README.md @@ -39,6 +39,31 @@ with Client('https://example.molgeniscloud.org') as client: # Delete a schema from the server client.delete_schema(name='New Schema') +``` +Instead of signing in with a username and password the client can also be used while authorized by a (temporary) token that is generated on the server. +```py +from molgenis_emx2_pyclient import Client + +token = '...' + +with Client('https://example.molgeniscloud.org', token=token) as client: + + # Retrieve signin information + print(client.status) + """ Output: + Host: https://example.molgeniscloud.org + Status: Signed in + Schemas: + CatalogueOntologies + catalogue + ExampleSchema + ... + Version: v10.10.1 + """ + + ... + ... + ``` ## Development From 56ebd138f334945db6a4f1e3527dc897105926d8 Mon Sep 17 00:00:00 2001 From: Ype Zijlstra Date: Tue, 16 Jan 2024 10:47:09 +0100 Subject: [PATCH 07/87] * Improved exception to token that cannot be parsed --- tools/pyclient/src/molgenis_emx2_pyclient/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index be640bca55..bf2b475335 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -162,7 +162,7 @@ def get_schemas(self): if response.status_code == 404: raise ServerNotFoundError(f"Server with url '{self.url}'") if response.status_code == 400: - if 'Invalid token' in response.text: + if ' token' in response.text: raise InvalidTokenException(f"Invalid token or token expired.") else: raise PyclientException("An unknown error occurred when trying to reach this server.") From ebe09a003c273777de5e3eff202f46d413c1de09 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Wed, 17 Jan 2024 11:37:02 +0100 Subject: [PATCH 08/87] * Added method to set the token --- tools/pyclient/src/molgenis_emx2_pyclient/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index bf2b475335..eb2be8ca3f 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -181,6 +181,10 @@ def token(self): """Returns the token by a property to prevent it being modified.""" return self._token + def set_token(self, token: str): + """Sets the token supplied as the argument as the client's token.""" + self._token = token + @property def version(self): """Lists the current EMX2 version on the server""" From bc564edbc82b88ec404b2eb46e80d2b8d6ac7270 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Wed, 17 Jan 2024 14:24:01 +0100 Subject: [PATCH 09/87] * Updated dev script for token use --- tools/pyclient/dev/dev.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tools/pyclient/dev/dev.py b/tools/pyclient/dev/dev.py index fc811de46d..50714cde52 100644 --- a/tools/pyclient/dev/dev.py +++ b/tools/pyclient/dev/dev.py @@ -7,9 +7,9 @@ # STATUS: ongoing # PACKAGES: pandas, python-dotenv # COMMENTS: Designed to interact with the schema "pet store". -# Create a file called '.env' that specify the username and password: -# MG_USERNAME = blabla -# MG_PASSWORD = blabla123 +# Create a file called '.env' that states the molgenis token, to get this token login into the server (UI) as admin. +# Next click on 'Hi admin' and under Manage token give the new token a name and create. Copy this token into .env as described below. +# MG_TOKEN = .... # /////////////////////////////////////////////////////////////////////////////// import asyncio import logging @@ -31,13 +31,9 @@ async def main(): # Load the login details into the environment load_dotenv() - username = os.environ.get('MG_USERNAME') - password = os.environ.get('MG_PASSWORD') - + token = os.environ.get('MG_TOKEN') # Connect to the server and sign in - async with Client('https://emx2.dev.molgenis.org/') as client: - client.signin(username, password) - + async with Client('https://emx2.dev.molgenis.org/', token=token) as client: # Check sign in status print(client.status) @@ -62,9 +58,7 @@ async def main(): client.export(schema='catalogue-demo', table='Cohorts', fmt='csv') # Connect to server with a default schema specified - with Client('https://emx2.dev.molgenis.org/', schema='pet store') as client: - client.signin(username, password) - + with Client('https://emx2.dev.molgenis.org/', schema='pet store', token=token) as client: client.export(fmt='csv') client.export(table='Pet', fmt='csv') client.export(table='Pet', fmt='xlsx') @@ -131,16 +125,14 @@ async def main(): client.delete_records(schema='pet store', table='Tag', file='demodata/Tag.csv') # Connect to server and create, update, and drop schemas - with Client('https://emx2.dev.molgenis.org/') as client: - client.signin(username, password) - + with Client('https://emx2.dev.molgenis.org/', token=token) as client: # Create a schema try: client.create_schema(name='myNewSchema') print(client.schema_names) except GraphQLException as e: print(e) - + # Update the description try: client.update_schema(name='myNewSchema', description='I forgot the description') @@ -148,14 +140,14 @@ async def main(): print(client.schemas) except GraphQLException as e: print(e) - + # Recreate the schema: delete and create try: client.recreate_schema(name='myNewSchema') print(client.schema_names) except GraphQLException as e: print(e) - + # Delete the schema try: client.delete_schema(name='myNewSchema') From d32b5853dc3864537daeacd626f29d046fe56956 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Wed, 17 Jan 2024 14:27:38 +0100 Subject: [PATCH 10/87] * Added PermissionDeniedException --- .../src/molgenis_emx2_pyclient/client.py | 77 +++++++++++-------- .../src/molgenis_emx2_pyclient/exceptions.py | 4 + 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index eb2be8ca3f..3d4434adfb 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -8,8 +8,10 @@ from . import graphql_queries as queries from . import utils as utils -from .exceptions import NoSuchSchemaException, ServiceUnavailableError, SigninError, ServerNotFoundError, \ - PyclientException, NoSuchTableException, NoContextManagerException, GraphQLException, InvalidTokenException +from .exceptions import (NoSuchSchemaException, ServiceUnavailableError, SigninError, ServerNotFoundError, + PyclientException, NoSuchTableException, NoContextManagerException, GraphQLException, + InvalidTokenException, + PermissionDeniedException) log = logging.getLogger("Molgenis EMX2 Pyclient") @@ -21,6 +23,7 @@ class Client: Use the Client object to log in to a Molgenis EMX2 server and perform operations on the server. """ + def __init__(self, url: str, schema: str = None, token: str = None) -> None: """ Initializes a Client object with a server url. @@ -96,7 +99,7 @@ def signin(self, username: str, password: str): f"\nStatus code: {response.status_code}. Reason: '{response.reason}'.") response_json: dict = response.json().get('data', {}).get('signin', {}) - + if response_json.get('status') == 'SUCCESS': self.signin_status = 'success' message = f"User '{self.username}' is signed in to '{self.url}'." @@ -115,7 +118,7 @@ def signin(self, username: str, password: str): log.error(message) raise SigninError(message) self.schemas = self.get_schemas() - + def signout(self): """Signs the client out of the EMX2 server.""" response = self.session.post( @@ -131,7 +134,7 @@ def signout(self): print(f"Unable to sign out of {self.url}.") message = response.json().get('errors')[0].get('message') print(message) - + @property def status(self): """Shows the sign-in status of the user, the server version @@ -139,11 +142,11 @@ def status(self): """ schemas = '\n\t'.join(self.schema_names) message = ( - f"Host: {self.url}\n" - f"User: {self.username}\n" - f"Status: {'Signed in' if self.signin_status == 'success' else 'Signed out'}\n" - f"Schemas: \n\t{schemas}\n" - f"Version: {self.version}\n" + f"Host: {self.url}\n" + f"User: {self.username}\n" + f"Status: {'Signed in' if self.signin_status == 'success' else 'Signed out'}\n" + f"Schemas: \n\t{schemas}\n" + f"Version: {self.version}\n" ) return message @@ -194,7 +197,7 @@ def version(self): json={'query': query} ) return response.json().get('data').get('_manifest').get('SpecificationVersion') - + def save_schema(self, name: str = None, table: str = None, file: str = None, data: list = None): """Imports or updates records in a table of a named schema. @@ -221,16 +224,22 @@ def save_schema(self, name: str = None, table: str = None, file: str = None, dat raise NoSuchTableException(f"Table '{table}' not found in schema '{current_schema}'.") import_data = self._prep_data_or_file(file_path=file, data=data) - + response = self.session.post( url=f"{self.url}/{current_schema}/api/csv/{table}", headers={'x-molgenis-token': self.token, 'Content-Type': 'text/csv'}, data=import_data ) - + if response.status_code == 200: log.info(f"Imported data into {current_schema}::{table}.") + elif response.status_code == 400: + if 'permission denied' in response.text: + raise PermissionDeniedException(f"Transaction failed: permission denied for table {table}.") + else: + errors = '\n'.join([err['message'] for err in response.json().get('errors')]) + log.error(f"Failed to import data into {current_schema}::{table}\n{errors}.") else: errors = '\n'.join([err['message'] for err in response.json().get('errors')]) log.error(f"Failed to import data into {current_schema}::{table}\n{errors}.") @@ -261,20 +270,20 @@ def delete_records(self, schema: str = None, table: str = None, file: str = None raise NoSuchTableException(f"Table '{table}' not found in schema '{current_schema}'.") import_data = self._prep_data_or_file(file_path=file, data=data) - + response = self.session.delete( url=f"{self.url}/{current_schema}/api/csv/{table}", headers={'x-molgenis-token': self.token, 'Content-Type': 'text/csv'}, data=import_data ) - + if response.status_code == 200: log.info(f"Deleted data from {current_schema}::{table}.") else: errors = '\n'.join([err['message'] for err in response.json().get('errors')]) log.error(f"Failed to delete data from {current_schema}::{table}\n{errors}.") - + def get(self, schema: str = None, table: str = None, as_df: bool = False) -> list | pd.DataFrame: """Retrieves data from a schema and returns as a list of dictionaries or as a pandas DataFrame (as pandas is used to parse the response). @@ -293,7 +302,7 @@ def get(self, schema: str = None, table: str = None, as_df: bool = False) -> lis current_schema = schema if current_schema is None: current_schema = self.default_schema - + if current_schema not in self.schema_names: raise NoSuchSchemaException(f"Schema '{current_schema}' not found on server.") @@ -401,13 +410,13 @@ def create_schema(self, name: str = None, query = queries.create_schema() variables = self._format_optional_params(name=name, description=description, template=template, include_demo_data=include_demo_data) - + response = self.session.post( - url=f"{self.url}/api/graphql", - json={'query': query, 'variables': variables}, - headers={'x-molgenis-token': self.token} + url=f"{self.url}/api/graphql", + json={'query': query, 'variables': variables}, + headers={'x-molgenis-token': self.token} ) - + response_json = response.json() self._validate_graphql_response( response_json=response_json, @@ -415,7 +424,7 @@ def create_schema(self, name: str = None, fallback_error_message=f"Failed to create schema '{name}'" ) self.schemas = self.get_schemas() - + def delete_schema(self, name: str = None): """Deletes a schema from the EMX2 server. @@ -433,7 +442,7 @@ def delete_schema(self, name: str = None): json={'query': query, 'variables': variables}, headers={'x-molgenis-token': self.token} ) - + response_json = response.json() self._validate_graphql_response( response_json=response_json, @@ -461,7 +470,7 @@ def update_schema(self, name: str = None, description: str = None): json={'query': query, 'variables': variables}, headers={'x-molgenis-token': self.token} ) - + response_json = response.json() self._validate_graphql_response( response_json=response_json, @@ -469,7 +478,7 @@ def update_schema(self, name: str = None, description: str = None): fallback_error_message=f"Failed to update schema '{name}'" ) self.schemas = self.get_schemas() - + def recreate_schema(self, name: str = None, description: str = None, template: str = None, @@ -494,7 +503,7 @@ def recreate_schema(self, name: str = None, message = f"Schema '{name}' does not exist" log.error(message) raise NoSuchSchemaException(message) - + schema_meta = [db for db in self.schemas if db['name'] == name][0] schema_description = description if description else schema_meta.get('description', None) @@ -513,7 +522,7 @@ def recreate_schema(self, name: str = None, print(message) self.schemas = self.get_schemas() - + def get_schema_metadata(self, name: str = None): """Retrieves a schema's metadata. @@ -526,21 +535,21 @@ def get_schema_metadata(self, name: str = None): current_schema = name if name is not None else self.default_schema if current_schema not in self.schema_names: raise NoSuchSchemaException(f"Schema '{current_schema}' not found on server.") - + query = queries.list_schema_meta() response = self.session.post( - url=f"{self.url}/{current_schema}/api/graphql", - json={'query': query}, - headers={'x-molgenis-token': self.token} + url=f"{self.url}/{current_schema}/api/graphql", + json={'query': query}, + headers={'x-molgenis-token': self.token} ) - + response_json = response.json() if 'id' not in response_json.get('data').get('_schema'): message = f"Unable to retrieve metadata for schema '{current_schema}'" log.error(message) raise GraphQLException(message) - + metadata = response_json.get('data').get('_schema') return metadata diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py b/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py index e39272051e..12b36df425 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py @@ -43,3 +43,7 @@ class GraphQLException(PyclientException): class InvalidTokenException(PyclientException): """Thrown when a token that is used in a request has expired.""" + + +class PermissionDeniedException(PyclientException): + """Thrown when an operation is attempted without sufficient permissions.""" From de9986728ba259f7e18c1b9f4de4290865278400 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Wed, 17 Jan 2024 17:06:44 +0100 Subject: [PATCH 11/87] * Added PermissionDeniedException --- tools/pyclient/dev/dev.py | 55 +++++++++++-------- .../src/molgenis_emx2_pyclient/client.py | 8 ++- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/tools/pyclient/dev/dev.py b/tools/pyclient/dev/dev.py index 50714cde52..b0adfa0ab9 100644 --- a/tools/pyclient/dev/dev.py +++ b/tools/pyclient/dev/dev.py @@ -20,7 +20,7 @@ from tools.pyclient.src.molgenis_emx2_pyclient import Client from tools.pyclient.src.molgenis_emx2_pyclient.exceptions import (NoSuchSchemaException, NoSuchTableException, - GraphQLException) + GraphQLException, PermissionDeniedException) async def main(): @@ -94,35 +94,42 @@ async def main(): }] # Import new data - client.save_schema(name='pet store', table='Tag', data=new_tags) - client.save_schema(name='pet store', table='Pet', data=new_pets) - - # Retrieve records - tags_data = client.get(schema='pet store', table='Tag', as_df=True) - print(tags_data) - pets_data = client.get(schema='pet store', table='Pet', as_df=True) - print(pets_data) - - # Drop records - tags_to_remove = [{'name': row['name']} for row in new_tags if row['name'] == 'canis'] - client.delete_records(schema='pet store', table='Pet', data=new_pets) - client.delete_records(schema='pet store', table='Tag', data=tags_to_remove) + try: + client.save_schema(name='pet store', table='Tag', data=new_tags) + client.save_schema(name='pet store', table='Pet', data=new_pets) + + # Retrieve records + tags_data = client.get(schema='pet store', table='Tag', as_df=True) + print(tags_data) + pets_data = client.get(schema='pet store', table='Pet', as_df=True) + print(pets_data) + + # Drop records + tags_to_remove = [{'name': row['name']} for row in new_tags if row['name'] == 'canis'] + client.delete_records(schema='pet store', table='Pet', data=new_pets) + client.delete_records(schema='pet store', table='Tag', data=tags_to_remove) + except PermissionDeniedException: + print(f"Permission denied for importing or deleting data. " + f"Ensure correct authorization.") # /////////////////////////////////////// # ~ 1b ~ # Check import via the `file` parameter + try: + # Save datasets + pd.DataFrame(new_tags).to_csv('demodata/Tag.csv', index=False) + pd.DataFrame(new_pets).to_csv('demodata/Pet.csv', index=False) - # Save datasets - pd.DataFrame(new_tags).to_csv('demodata/Tag.csv', index=False) - pd.DataFrame(new_pets).to_csv('demodata/Pet.csv', index=False) - - # Import files - client.save_schema(name='pet store', table='Tag', file='demodata/Tag.csv') - client.save_schema(name='pet store', table='Pet', file='demodata/Pet.csv') + # Import files + client.save_schema(name='pet store', table='Tag', file='demodata/Tag.csv') + client.save_schema(name='pet store', table='Pet', file='demodata/Pet.csv') - client.delete_records(schema='pet store', table='Pet', file='demodata/Pet.csv') - client.delete_records(schema='pet store', table='Tag', file='demodata/Tag.csv') + client.delete_records(schema='pet store', table='Pet', file='demodata/Pet.csv') + client.delete_records(schema='pet store', table='Tag', file='demodata/Tag.csv') + except PermissionDeniedException: + print(f"Permission denied for importing or deleting data. " + f"Ensure correct authorization.") # Connect to server and create, update, and drop schemas with Client('https://emx2.dev.molgenis.org/', token=token) as client: @@ -130,7 +137,7 @@ async def main(): try: client.create_schema(name='myNewSchema') print(client.schema_names) - except GraphQLException as e: + except (GraphQLException, PermissionDeniedException) as e: print(e) # Update the description diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index 3d4434adfb..68e00a5841 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -605,14 +605,16 @@ def _validate_graphql_response(response_json: dict, mutation: str, fallback_erro :rtype: string """ response_keys = response_json.keys() - if 'error' not in response_keys and 'data' not in response_keys: + if 'errors' not in response_keys and 'data' not in response_keys: message = fallback_error_message log.error(message) print(message) - elif 'error' in response_keys: - message = response_json.get('error').errors[0].get('message') + elif 'errors' in response_keys: + message = response_json.get('errors')[0].get('message') log.error(message) + if 'permission denied' in message: + raise PermissionDeniedException(message) raise GraphQLException(message) else: From de81801dcdb628ee8560f4f8f4534e79a26b0257 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Fri, 19 Jan 2024 15:20:14 +0100 Subject: [PATCH 12/87] * Updated status method * Updated dev script dates --- tools/pyclient/dev/dev.py | 8 +++++--- tools/pyclient/src/molgenis_emx2_pyclient/client.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/pyclient/dev/dev.py b/tools/pyclient/dev/dev.py index b0adfa0ab9..31ac85db00 100644 --- a/tools/pyclient/dev/dev.py +++ b/tools/pyclient/dev/dev.py @@ -2,13 +2,15 @@ # FILE: dev.py # AUTHOR: David Ruvolo, Ype Zijlstra # CREATED: 2023-05-22 -# MODIFIED: 2023-11-28 +# MODIFIED: 2024-01-19 # PURPOSE: development script for initial testing of the py-client # STATUS: ongoing # PACKAGES: pandas, python-dotenv # COMMENTS: Designed to interact with the schema "pet store". -# Create a file called '.env' that states the molgenis token, to get this token login into the server (UI) as admin. -# Next click on 'Hi admin' and under Manage token give the new token a name and create. Copy this token into .env as described below. +# Create a file called '.env' that states the molgenis token, to get +# this token login into the server (UI) as admin. Next, click on +# 'Hi admin' and under Manage token give the new token a name and +# create. Copy this token into .env as described below. # MG_TOKEN = .... # /////////////////////////////////////////////////////////////////////////////// import asyncio diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index 68e00a5841..d6733dc8da 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -141,12 +141,16 @@ def status(self): and the schemas that the user can interact with. """ schemas = '\n\t'.join(self.schema_names) + host = self.url + user = self.username if self.username else ('token' if self.token else 'anonymous') + status = 'signed in' if self.signin_status == 'success' else ('session-less' if self.token else 'signed out') + version = self.version message = ( - f"Host: {self.url}\n" - f"User: {self.username}\n" - f"Status: {'Signed in' if self.signin_status == 'success' else 'Signed out'}\n" + f"Host: {host}\n" + f"User: {user}\n" + f"Status: {status}\n" f"Schemas: \n\t{schemas}\n" - f"Version: {self.version}\n" + f"Version: {version}\n" ) return message From 815135aebc074077688bcdbadfa502950a42aa58 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Wed, 24 Jan 2024 15:30:38 +0100 Subject: [PATCH 13/87] * Added TokenSigninException when sign in is attempted on client with token and vice versa --- tools/pyclient/src/molgenis_emx2_pyclient/client.py | 7 ++++++- tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index d6733dc8da..5cbcd9e2de 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -11,7 +11,7 @@ from .exceptions import (NoSuchSchemaException, ServiceUnavailableError, SigninError, ServerNotFoundError, PyclientException, NoSuchTableException, NoContextManagerException, GraphQLException, InvalidTokenException, - PermissionDeniedException) + PermissionDeniedException, TokenSigninException) log = logging.getLogger("Molgenis EMX2 Pyclient") @@ -77,6 +77,9 @@ def signin(self, username: str, password: str): """ self.username = username + if self.token is not None: + raise TokenSigninException("Cannot sign in to client authorized with token.") + if not self._as_context_manager: raise NoContextManagerException("Ensure the Client is called as a context manager,\n" "e.g. `with Client(url) as client:`") @@ -190,6 +193,8 @@ def token(self): def set_token(self, token: str): """Sets the token supplied as the argument as the client's token.""" + if self.signin_status == 'success': + raise TokenSigninException("Cannot set a token on a client authorized with sign in.") self._token = token @property diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py b/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py index 12b36df425..ffbe841a9a 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py @@ -47,3 +47,7 @@ class InvalidTokenException(PyclientException): class PermissionDeniedException(PyclientException): """Thrown when an operation is attempted without sufficient permissions.""" + + +class TokenSigninException(PyclientException): + """Thrown when sign in is attempted on a client that is authorized with a token.""" From 73428d22a5912da4318b61d3a963faf0d29b1525 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Wed, 24 Jan 2024 16:15:52 +0100 Subject: [PATCH 14/87] * Improved error messaging for operations with insufficient permissions --- tools/pyclient/README.md | 3 +-- tools/pyclient/dev/dev.py | 6 ++--- .../src/molgenis_emx2_pyclient/client.py | 27 ++++++++++++++++--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/tools/pyclient/README.md b/tools/pyclient/README.md index caf5e955db..b965547a2e 100644 --- a/tools/pyclient/README.md +++ b/tools/pyclient/README.md @@ -52,13 +52,12 @@ with Client('https://example.molgeniscloud.org', token=token) as client: print(client.status) """ Output: Host: https://example.molgeniscloud.org - Status: Signed in Schemas: CatalogueOntologies catalogue ExampleSchema ... - Version: v10.10.1 + Version: v10.32.1 """ ... diff --git a/tools/pyclient/dev/dev.py b/tools/pyclient/dev/dev.py index 31ac85db00..fec8ee3305 100644 --- a/tools/pyclient/dev/dev.py +++ b/tools/pyclient/dev/dev.py @@ -147,21 +147,21 @@ async def main(): client.update_schema(name='myNewSchema', description='I forgot the description') print(client.schema_names) print(client.schemas) - except GraphQLException as e: + except (GraphQLException, NoSuchSchemaException) as e: print(e) # Recreate the schema: delete and create try: client.recreate_schema(name='myNewSchema') print(client.schema_names) - except GraphQLException as e: + except (GraphQLException, NoSuchSchemaException) as e: print(e) # Delete the schema try: client.delete_schema(name='myNewSchema') print(client.schema_names) - except GraphQLException as e: + except (GraphQLException, NoSuchSchemaException) as e: print(e) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index 5cbcd9e2de..5fe54d70d4 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -443,6 +443,16 @@ def delete_schema(self, name: str = None): :returns: a success or error message :rtype: string """ + if name is None: + if self.default_schema is None: + raise KeyError("Supply the schema name.") + name = self.default_schema + else: + if name not in self.schema_names: + message = f"Schema '{name}' not available." + log.error(message) + raise NoSuchSchemaException(message) + query = queries.delete_schema() variables = {'id': name} @@ -471,6 +481,16 @@ def update_schema(self, name: str = None, description: str = None): :returns: a success or error message :rtype: string """ + if name is None: + if self.default_schema is None: + raise KeyError("Supply the schema name.") + name = self.default_schema + else: + if name not in self.schema_names: + message = f"Schema '{name}' not available." + log.error(message) + raise NoSuchSchemaException(message) + query = queries.update_schema() variables = {'name': name, 'description': description} @@ -509,7 +529,7 @@ def recreate_schema(self, name: str = None, :rtype: string """ if name not in self.schema_names: - message = f"Schema '{name}' does not exist" + message = f"Schema '{name}' not available." log.error(message) raise NoSuchSchemaException(message) @@ -621,9 +641,10 @@ def _validate_graphql_response(response_json: dict, mutation: str, fallback_erro elif 'errors' in response_keys: message = response_json.get('errors')[0].get('message') - log.error(message) if 'permission denied' in message: - raise PermissionDeniedException(message) + log.error("Insufficient permissions for this operations.") + raise PermissionDeniedException("Insufficient permissions for this operations.") + log.error(message) raise GraphQLException(message) else: From 25b67f424dad341cebc64fe0488ab9cbd1e1f239 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Wed, 24 Jan 2024 16:53:54 +0100 Subject: [PATCH 15/87] * Updated README.md * Edited other documentation --- docs/molgenis/dev_graphql.md | 23 ++++++++++++----------- docs/molgenis/use_usingapis.md | 14 ++++++++++---- tools/pyclient/README.md | 11 +++++++++-- tools/pyclient/requirements.txt | 2 +- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/docs/molgenis/dev_graphql.md b/docs/molgenis/dev_graphql.md index 20e817dc55..fdbf37a563 100644 --- a/docs/molgenis/dev_graphql.md +++ b/docs/molgenis/dev_graphql.md @@ -1,27 +1,28 @@ # GraphQL in MOLGENIS -Each database in MOLGENIS has a graphql endpoint, that expose a graphql API for the data model of that database. In -addition, at the root there is a generic API. +Each database in MOLGENIS has a GraphQL endpoint that exposes a GraphQL API for the data model of that database. +In addition, at the root there is a generic API. For example: -- https://emx2.dev.molgenis.org/api/graphql - root api -- https://emx2.dev.molgenis.org/pet%20store/api/graphql - api for database 'pet store' +- https://emx2.dev.molgenis.org/api/graphql - root API +- https://emx2.dev.molgenis.org/pet%20store/api/graphql - API for database 'pet store' Full documentation can be found while visiting the graphql-playground app. You can click 'docs' there. -- https://emx2.dev.molgenis.org/apps/graphql-playground/ - playground for 'root' api +- https://emx2.dev.molgenis.org/apps/graphql-playground/ - playground for 'root' API - https://emx2.dev.molgenis.org/pet%20store/graphql-playground/ - example for 'pet store' database ## Functions available on all APIs. -These functionalities are available for both the 'root' api and the database api. +These functionalities are available for both the 'root' API and the database API. ### Sign in -Sign into an existing account. When running in a website then a session will be created. However, when you use a script -you can retrieve a token to authenticate in other calls. Provide these in subsequent request headers as ' -x-molgenis-token'. +Sign in to an existing account. +When running in a website then a session will be created. +However, when you use a script you can retrieve a token to authenticate in other calls. +Provide these in subsequent request headers as `x-molgenis-token`. ```graphql mutation { @@ -180,7 +181,7 @@ To get settings for all users To change settings you can use the 'change' mutation, e.g. -Database or schema settings (depending on database vs schema api): +Database or schema settings (depending on database vs schema API): ```graphql mutation { @@ -585,7 +586,7 @@ Create a new connection: `const query = new QueryEMX2('graphQLEndpoint')` -Where graphQLEndpoint is the endpoint of the api. Usually `'graphql'` +Where graphQLEndpoint is the endpoint of the API. Usually `'graphql'` The way it works is with _function chaining_. diff --git a/docs/molgenis/use_usingapis.md b/docs/molgenis/use_usingapis.md index bc17db9da4..837ab14a77 100644 --- a/docs/molgenis/use_usingapis.md +++ b/docs/molgenis/use_usingapis.md @@ -2,6 +2,16 @@ The EMX2 APIs can be used to various environments to retrieve, add, manipulate or replace data. + + +## Python + +Data management of a Molgenis EMX2 server can be performed through the use of the Python client. +This client can be installed from the [PyPI repository](https://pypi.org/project/molgenis-emx2-pyclient/) and can be applied in a number of operations, such as +- retrieving data from a table +- uploading data to a table + + ## R R is a free software environment for statistical computing and graphics. @@ -86,10 +96,6 @@ Result: ### Define new tables todo -## Python - -todo - ## Command line todo diff --git a/tools/pyclient/README.md b/tools/pyclient/README.md index b965547a2e..5b5a206446 100644 --- a/tools/pyclient/README.md +++ b/tools/pyclient/README.md @@ -1,4 +1,8 @@ -# Installation +The Molgenis EMX2 Pyclient is Python package developed to be used for data management on Molgenis EMX2 servers. + +## Installation +The releases of the package are hosted at [PyPI](https://pypi.org/project/molgenis-emx2-pyclient/). +The recommended way to install the latest version is through `pip`. ```console pip install molgenis_emx2_pyclient @@ -6,7 +10,8 @@ pip install molgenis_emx2_pyclient ## How to use -Within your Python project import the class Client and use it as a context manager +Within your Python project import the class Client and instantiate it as a context manager. +Operations and queries can then be executed from within the context. ```py from molgenis_emx2_pyclient import Client @@ -52,6 +57,8 @@ with Client('https://example.molgeniscloud.org', token=token) as client: print(client.status) """ Output: Host: https://example.molgeniscloud.org + User: token + Status: session-less Schemas: CatalogueOntologies catalogue diff --git a/tools/pyclient/requirements.txt b/tools/pyclient/requirements.txt index 29b3658aab..d95278168c 100644 --- a/tools/pyclient/requirements.txt +++ b/tools/pyclient/requirements.txt @@ -1,3 +1,3 @@ requests~=2.31.0 -pandas~=2.1.1 +pandas~=2.1.3 python-dotenv~=1.0.0 \ No newline at end of file From 8fda1f0e82972fbaf292eb168e85cfb1479c3067 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Fri, 26 Jan 2024 10:27:21 +0100 Subject: [PATCH 16/87] * Improved error catching. * Simplified check for existing schema --- .../src/molgenis_emx2_pyclient/client.py | 66 +++++++------------ 1 file changed, 25 insertions(+), 41 deletions(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index 5fe54d70d4..56645b2ddf 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -172,7 +172,7 @@ def get_schemas(self): if response.status_code == 404: raise ServerNotFoundError(f"Server with url '{self.url}'") if response.status_code == 400: - if ' token' in response.text: + if 'Invalid token or token expired' in response.text: raise InvalidTokenException(f"Invalid token or token expired.") else: raise PyclientException("An unknown error occurred when trying to reach this server.") @@ -227,7 +227,7 @@ def save_schema(self, name: str = None, table: str = None, file: str = None, dat current_schema = self.default_schema if current_schema not in self.schema_names: - raise NoSuchSchemaException(f"Schema '{current_schema}' not found on server.") + raise NoSuchSchemaException(f"Schema '{current_schema}' not available.") if not self._table_in_schema(table, current_schema): raise NoSuchTableException(f"Table '{table}' not found in schema '{current_schema}'.") @@ -273,7 +273,7 @@ def delete_records(self, schema: str = None, table: str = None, file: str = None current_schema = self.default_schema if current_schema not in self.schema_names: - raise NoSuchSchemaException(f"Schema '{current_schema}' not found on server.") + raise NoSuchSchemaException(f"Schema '{current_schema}' not available.") if not self._table_in_schema(table, current_schema): raise NoSuchTableException(f"Table '{table}' not found in schema '{current_schema}'.") @@ -313,7 +313,7 @@ def get(self, schema: str = None, table: str = None, as_df: bool = False) -> lis current_schema = self.default_schema if current_schema not in self.schema_names: - raise NoSuchSchemaException(f"Schema '{current_schema}' not found on server.") + raise NoSuchSchemaException(f"Schema '{current_schema}' not available.") if not self._table_in_schema(table, current_schema): raise NoSuchTableException(f"Table '{table}' not found in schema '{current_schema}'.") @@ -344,12 +344,9 @@ def export(self, schema: str = None, table: str = None, fmt: OutputFormat = 'csv :type fmt: str """ - current_schema = schema - if current_schema is None: - current_schema = self.default_schema - + current_schema = schema if schema is not None else self.default_schema if current_schema not in self.schema_names: - raise NoSuchSchemaException(f"Schema '{current_schema}' not found on server.") + raise NoSuchSchemaException(f"Schema '{current_schema}' not available.") if table is not None and not self._table_in_schema(table, current_schema): raise NoSuchTableException(f"Table '{table}' not found in schema '{current_schema}'.") @@ -443,18 +440,12 @@ def delete_schema(self, name: str = None): :returns: a success or error message :rtype: string """ - if name is None: - if self.default_schema is None: - raise KeyError("Supply the schema name.") - name = self.default_schema - else: - if name not in self.schema_names: - message = f"Schema '{name}' not available." - log.error(message) - raise NoSuchSchemaException(message) + current_schema = name if name is not None else self.default_schema + if current_schema not in self.schema_names: + raise NoSuchSchemaException(f"Schema '{current_schema}' not available.") query = queries.delete_schema() - variables = {'id': name} + variables = {'id': current_schema} response = self.session.post( url=f"{self.url}/api/graphql", @@ -466,7 +457,7 @@ def delete_schema(self, name: str = None): self._validate_graphql_response( response_json=response_json, mutation='deleteSchema', - fallback_error_message=f"Failed to delete schema '{name}'" + fallback_error_message=f"Failed to delete schema '{current_schema}'" ) self.schemas = self.get_schemas() @@ -481,18 +472,12 @@ def update_schema(self, name: str = None, description: str = None): :returns: a success or error message :rtype: string """ - if name is None: - if self.default_schema is None: - raise KeyError("Supply the schema name.") - name = self.default_schema - else: - if name not in self.schema_names: - message = f"Schema '{name}' not available." - log.error(message) - raise NoSuchSchemaException(message) + current_schema = name if name is not None else self.default_schema + if current_schema not in self.schema_names: + raise NoSuchSchemaException(f"Schema '{current_schema}' not available.") query = queries.update_schema() - variables = {'name': name, 'description': description} + variables = {'name': current_schema, 'description': description} response = self.session.post( url=f"{self.url}/api/graphql", @@ -504,7 +489,7 @@ def update_schema(self, name: str = None, description: str = None): self._validate_graphql_response( response_json=response_json, mutation='updateSchema', - fallback_error_message=f"Failed to update schema '{name}'" + fallback_error_message=f"Failed to update schema '{current_schema}'" ) self.schemas = self.get_schemas() @@ -528,25 +513,24 @@ def recreate_schema(self, name: str = None, :returns: a success or error message :rtype: string """ - if name not in self.schema_names: - message = f"Schema '{name}' not available." - log.error(message) - raise NoSuchSchemaException(message) + current_schema = name if name is not None else self.default_schema + if current_schema not in self.schema_names: + raise NoSuchSchemaException(f"Schema '{current_schema}' not available.") - schema_meta = [db for db in self.schemas if db['name'] == name][0] + schema_meta = [db for db in self.schemas if db['name'] == current_schema][0] schema_description = description if description else schema_meta.get('description', None) try: - self.delete_schema(name=name) + self.delete_schema(name=current_schema) self.create_schema( - name=name, + name=current_schema, description=schema_description, template=template, include_demo_data=include_demo_data ) except GraphQLException: - message = f"Failed to recreate '{name}'" + message = f"Failed to recreate '{current_schema}'" log.error(message) print(message) @@ -563,7 +547,7 @@ def get_schema_metadata(self, name: str = None): """ current_schema = name if name is not None else self.default_schema if current_schema not in self.schema_names: - raise NoSuchSchemaException(f"Schema '{current_schema}' not found on server.") + raise NoSuchSchemaException(f"Schema '{current_schema}' not available.") query = queries.list_schema_meta() response = self.session.post( @@ -614,7 +598,7 @@ def _set_schema(self, name: str) -> str: :rtype: str """ if name not in [*self.schema_names, None]: - raise NoSuchSchemaException(f"Schema '{name}' not found on server.") + raise NoSuchSchemaException(f"Schema '{name}' not available.") self.default_schema = name return name From bcf6acfe47f7001d9ac6fa04848a5fab738c8202 Mon Sep 17 00:00:00 2001 From: konstantina-gkp Date: Tue, 30 Jan 2024 10:55:47 +0100 Subject: [PATCH 17/87] fixes: #3225; Initially, if a user entered the name of a biobank, such as Parelsnoer, into the search bar, the directory app would not display the Parelsnoer biobank in the search results if there were no collections matching the search term. With this fix, the directory app shows all biobanks that match the search term, regardless of whether they have any collections that match the search term. --- apps/directory/src/stores/biobanksStore.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/directory/src/stores/biobanksStore.js b/apps/directory/src/stores/biobanksStore.js index 5ed4db480e..52b96c7ce3 100644 --- a/apps/directory/src/stores/biobanksStore.js +++ b/apps/directory/src/stores/biobanksStore.js @@ -129,10 +129,8 @@ export const useBiobanksStore = defineStore("biobanksStore", () => { biobankCards.value = []; const biobankResult = await baseQuery.execute(); - /** only show biobanks that have collections */ - const foundBiobanks = biobankResult.Biobanks - ? biobankResult.Biobanks.filter((biobank) => biobank.collections) - : []; + /** also show biobanks that don't have collections */ + const foundBiobanks = biobankResult.Biobanks || []; biobankCards.value = filterWithdrawn(foundBiobanks); waitingForResponse.value = false; From 03b6aed02ab3200cca90604f48fc7ecc125cfb4d Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Tue, 30 Jan 2024 13:54:59 +0100 Subject: [PATCH 18/87] Fixed log formatting, f-strings and returns to comply with Pylint --- .../src/molgenis_emx2_pyclient/client.py | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index 56645b2ddf..3d4aec9a32 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -7,10 +7,10 @@ import requests from . import graphql_queries as queries -from . import utils as utils -from .exceptions import (NoSuchSchemaException, ServiceUnavailableError, SigninError, ServerNotFoundError, - PyclientException, NoSuchTableException, NoContextManagerException, GraphQLException, - InvalidTokenException, +from . import utils +from .exceptions import (NoSuchSchemaException, ServiceUnavailableError, SigninError, + ServerNotFoundError, PyclientException, NoSuchTableException, + NoContextManagerException, GraphQLException, InvalidTokenException, PermissionDeniedException, TokenSigninException) log = logging.getLogger("Molgenis EMX2 Pyclient") @@ -173,9 +173,8 @@ def get_schemas(self): raise ServerNotFoundError(f"Server with url '{self.url}'") if response.status_code == 400: if 'Invalid token or token expired' in response.text: - raise InvalidTokenException(f"Invalid token or token expired.") - else: - raise PyclientException("An unknown error occurred when trying to reach this server.") + raise InvalidTokenException("Invalid token or token expired.") + raise PyclientException("An unknown error occurred when trying to reach this server.") response_json: dict = response.json() schemas = response_json['data']['_schemas'] @@ -242,16 +241,17 @@ def save_schema(self, name: str = None, table: str = None, file: str = None, dat ) if response.status_code == 200: - log.info(f"Imported data into {current_schema}::{table}.") + log.info("Imported data into %s::%s.", current_schema, table) elif response.status_code == 400: if 'permission denied' in response.text: raise PermissionDeniedException(f"Transaction failed: permission denied for table {table}.") - else: - errors = '\n'.join([err['message'] for err in response.json().get('errors')]) - log.error(f"Failed to import data into {current_schema}::{table}\n{errors}.") + errors = '\n'.join([err['message'] for err in response.json().get('errors')]) + # log.error(f"Failed to import data into {current_schema}::{table}\n{errors}.") + log.error("Failed to import data into %s::%s\n%s", current_schema, table, errors) else: errors = '\n'.join([err['message'] for err in response.json().get('errors')]) - log.error(f"Failed to import data into {current_schema}::{table}\n{errors}.") + # log.error(f"Failed to import data into {current_schema}::{table}\n{errors}.") + log.error("Failed to import data into %s::%s\n%s", current_schema, table, errors) def delete_records(self, schema: str = None, table: str = None, file: str = None, data: list = None): """Deletes records from a table. @@ -288,10 +288,10 @@ def delete_records(self, schema: str = None, table: str = None, file: str = None ) if response.status_code == 200: - log.info(f"Deleted data from {current_schema}::{table}.") + log.info("Deleted data from %s::%s.", current_schema, table) else: errors = '\n'.join([err['message'] for err in response.json().get('errors')]) - log.error(f"Failed to delete data from {current_schema}::{table}\n{errors}.") + log.error("Failed to delete data from %s::%s\n%s.", current_schema, table, errors) def get(self, schema: str = None, table: str = None, as_df: bool = False) -> list | pd.DataFrame: """Retrieves data from a schema and returns as a list of dictionaries or as @@ -359,9 +359,9 @@ def export(self, schema: str = None, table: str = None, fmt: OutputFormat = 'csv headers={'x-molgenis-token': self.token}) filename = f"{current_schema}.xlsx" - with open(filename, "wb") as f: - f.write(response.content) - log.info(f"Exported data from schema {current_schema} to '{filename}'.") + with open(filename, "wb") as file: + file.write(response.content) + log.info("Exported data from schema %s to '%s'.", current_schema, filename) else: # Export the single table url = f"{self.url}/{current_schema}/api/excel/{table}" @@ -369,9 +369,9 @@ def export(self, schema: str = None, table: str = None, fmt: OutputFormat = 'csv headers={'x-molgenis-token': self.token}) filename = f"{table}.xlsx" - with open(filename, "wb") as f: - f.write(response.content) - log.info(f"Exported data from table {table} in schema {current_schema} to '{filename}'.") + with open(filename, "wb") as file: + file.write(response.content) + log.info("Exported data from table %s in schema %s to '%s'.", table, current_schema, filename) if fmt == 'csv': if table is None: @@ -380,9 +380,9 @@ def export(self, schema: str = None, table: str = None, fmt: OutputFormat = 'csv headers={'x-molgenis-token': self.token}) filename = f"{current_schema}.zip" - with open(filename, "wb") as f: - f.write(response.content) - log.info(f"Exported data from schema {current_schema} to '{filename}'.") + with open(filename, "wb") as file: + file.write(response.content) + log.info("Exported data from schema %s to '%s'.", current_schema, filename) else: # Export the single table url = f"{self.url}/{current_schema}/api/csv/{table}" @@ -390,9 +390,9 @@ def export(self, schema: str = None, table: str = None, fmt: OutputFormat = 'csv headers={'x-molgenis-token': self.token}) filename = f"{table}.csv" - with open(filename, "wb") as f: - f.write(response.content) - log.info(f"Exported data from table {table} in schema {current_schema} to '{filename}'.") + with open(filename, "wb") as file: + file.write(response.content) + log.info("Exported data from table %s in schema %s to '%s'.", table, current_schema, filename) def create_schema(self, name: str = None, description: str = None, @@ -567,7 +567,7 @@ def get_schema_metadata(self, name: str = None): return metadata @staticmethod - def _prep_data_or_file(file_path: str = None, data: list = None) -> str: + def _prep_data_or_file(file_path: str = None, data: list = None) -> str | None: """Prepares the data from memory or loaded from disk for addition or deletion action. :param file_path: path to the file to be prepared @@ -578,8 +578,6 @@ def _prep_data_or_file(file_path: str = None, data: list = None) -> str: :returns: prepared data in dataframe format :rtype: pd.DataFrame """ - if file_path is None and data is None: - print("No data to import. Specify a file location or a dataset.") if file_path is not None: return utils.read_file(file_path=file_path) @@ -587,6 +585,9 @@ def _prep_data_or_file(file_path: str = None, data: list = None) -> str: if data is not None: return pd.DataFrame(data).to_csv(index=False, quoting=csv.QUOTE_NONNUMERIC, encoding='UTF-8') + print("No data to import. Specify a file location or a dataset.") + return None + def _set_schema(self, name: str) -> str: """Sets the default schema to the schema supplied as argument. Raises NoSuchSchemaException if the schema cannot be found on the server. From 4c88c9a2b7f71d99e6ed566aae229e1fbae01b27 Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Wed, 31 Jan 2024 16:08:45 +0100 Subject: [PATCH 19/87] Removed default None for argument `table` in save_schema, delete_records and get as it is technically a required argument * Some minor tweaks --- .../pyclient/src/molgenis_emx2_pyclient/client.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index 3d4aec9a32..4a73ff1791 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -40,7 +40,7 @@ def __init__(self, url: str, schema: str = None, token: str = None) -> None: self.session: requests.Session = requests.Session() self.schemas: list = self.get_schemas() - self.default_schema: str = self._set_schema(schema) + self.default_schema: str = self.set_schema(schema) def __str__(self): return self.url @@ -157,7 +157,7 @@ def status(self): ) return message - def get_schemas(self): + def get_schemas(self) -> list[dict]: """Returns the schemas on the database for this user as a list of dictionaries containing for each schema the id, name, label and description. """ @@ -206,7 +206,7 @@ def version(self): ) return response.json().get('data').get('_manifest').get('SpecificationVersion') - def save_schema(self, name: str = None, table: str = None, file: str = None, data: list = None): + def save_schema(self, table: str, name: str = None, file: str = None, data: list = None): """Imports or updates records in a table of a named schema. :param name: name of a schema @@ -253,7 +253,7 @@ def save_schema(self, name: str = None, table: str = None, file: str = None, dat # log.error(f"Failed to import data into {current_schema}::{table}\n{errors}.") log.error("Failed to import data into %s::%s\n%s", current_schema, table, errors) - def delete_records(self, schema: str = None, table: str = None, file: str = None, data: list = None): + def delete_records(self, table: str, schema: str = None, file: str = None, data: list = None): """Deletes records from a table. :param schema: name of a schema @@ -262,7 +262,7 @@ def delete_records(self, schema: str = None, table: str = None, file: str = None :type table: str :param file: location of the file containing records to import or update :type file: str - :param data: a dataset containing records to import or update (list of dictionaries) + :param data: a dataset containing records to delete (list of dictionaries) :type data: list :returns: status message or response @@ -293,7 +293,7 @@ def delete_records(self, schema: str = None, table: str = None, file: str = None errors = '\n'.join([err['message'] for err in response.json().get('errors')]) log.error("Failed to delete data from %s::%s\n%s.", current_schema, table, errors) - def get(self, schema: str = None, table: str = None, as_df: bool = False) -> list | pd.DataFrame: + def get(self, table: str, schema: str = None, as_df: bool = False) -> list | pd.DataFrame: """Retrieves data from a schema and returns as a list of dictionaries or as a pandas DataFrame (as pandas is used to parse the response). @@ -588,7 +588,7 @@ def _prep_data_or_file(file_path: str = None, data: list = None) -> str | None: print("No data to import. Specify a file location or a dataset.") return None - def _set_schema(self, name: str) -> str: + def set_schema(self, name: str) -> str: """Sets the default schema to the schema supplied as argument. Raises NoSuchSchemaException if the schema cannot be found on the server. From 3f5cc8ec2674c442dcfa6839acef4474fac7e1ec Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Wed, 31 Jan 2024 16:09:47 +0100 Subject: [PATCH 20/87] Started writing up documentation for the Pyclient --- docs/molgenis/_sidebar.md | 2 + docs/molgenis/use_tokens.md | 1 + docs/molgenis/use_usingpyclient.md | 193 +++++++++++++++++++++++++++++ 3 files changed, 196 insertions(+) create mode 100644 docs/molgenis/use_tokens.md create mode 100644 docs/molgenis/use_usingpyclient.md diff --git a/docs/molgenis/_sidebar.md b/docs/molgenis/_sidebar.md index 8c61e9f8b0..6b68ad9421 100644 --- a/docs/molgenis/_sidebar.md +++ b/docs/molgenis/_sidebar.md @@ -12,8 +12,10 @@ - [Permissions](use_permissions.md) - [Settings](use_database_settings.md) - [Scripts/Jobs](use_scripts_jobs.md) + - [Tokens](use_tokens.md) - [Linked data](semantics.md) - [Using APIs](use_usingapis.md) + - [Pyclient API](use_usingpyclient.md) - [Admin](use_global_settings.md) - [Analytics](use_analytics.md) - **Installation guide** diff --git a/docs/molgenis/use_tokens.md b/docs/molgenis/use_tokens.md new file mode 100644 index 0000000000..d80e19a4c1 --- /dev/null +++ b/docs/molgenis/use_tokens.md @@ -0,0 +1 @@ +A nice instruction of creating tokens on the website. \ No newline at end of file diff --git a/docs/molgenis/use_usingpyclient.md b/docs/molgenis/use_usingpyclient.md new file mode 100644 index 0000000000..f30936327d --- /dev/null +++ b/docs/molgenis/use_usingpyclient.md @@ -0,0 +1,193 @@ +# Molgenis Pyclient +The MOLGENIS EMX2 Python client allows the user to retrieve, create, update and delete entities on a MOLGENIS EMX2 server using the Python language. + + +## Installation +The releases of the package are hosted at [PyPI](https://pypi.org/project/molgenis-emx2-pyclient/). +The recommended way to install the latest + +```commandline +pip install molgenis_emx2_pyclient +``` + +## Setting up the client +The Python client can be integrated in scripts authorized by either a username/password combination or a temporary token. + +Signing in with a username/password combination requires using the client as context manager: +```python +from molgenis_emx2_pyclient import Client + +username = 'username' +password = '********' + +# Initialize the client as a context manager +with Client('https://example.molgeniscloud.org') as client: + # Apply the 'signin' method with the username and password + client.signin(username, password) + + # Perform other tasks + ... +``` + +Before using the Pyclient with a token, this token should be generated in the UI, see [Tokens](use_tokens.md). +Using the Pyclient with a token requires supplying the token in the initialization of the Client object: +```python +from molgenis_emx2_pyclient import Client + +token = '********************************' + +client = Client(url='https://example.molgeniscloud.org', token=token) + +# Perform other tasks +... + +``` + +Additionally, if the Pyclient is to be used on a particular schema, this schema can be supplied in the initialization of the client, alongside the server URL: +```python +with Client('https://example.molgeniscloud.org', schema='My Schema') as client: +``` +or +```python +client = Client('https://example.molgeniscloud.org', schema='My Schema', token=token) +``` + +## Methods and properties +This section outlines some of the methods that are supported by the Pyclient. + +### schema_names +```python +client.schema_names +``` + +Returns a list of the names of the schemas on the server that are available for the user. + +### status +```python +client.status +``` +The `status` property returns a string with information, including the server URL, the user (if applicable), the sign-in status, the version of MOLGENIS EMX2 running on the server, and the result of `schema_names`. + + +### get_schemas +```python +client.get_schemas() +``` +Retrieves the schemas on the server for the user as a list of dictionaries containing for each schema the id, name, label and description. +This method accepts no arguments. + + +### set_schema +```python +client.set_schema('My Schema') +``` +Sets the default schema for the server. Throws the `NoSuchSchemaException` if the schema is not available. + +| argument | type | description | required | default | +|----------|------|----------------------|----------|---------| +| name | str | the name of a schema | True | | + +### set_token +```python +client.set_token(token='***************') +``` +Sets the client's token in case no token was supplied in the initialization. +Raises the `TokenSigninException` when the client is already signed in with a username/password combination. + +| argument | type | description | required | default | +|----------|------|-------------|----------|---------| +| token | str | the token | True | | + +### get_schema_metadata +```python +client.get_schema_metadata(name='My Schema') +``` +Retrieves a schema's metadata, including the names of the tables, the metadata of the columns in the tables, schema settings, etc. +If no `name` is supplied the metadata for the default schema is queried, if applicable. Else the `NoSuchSchemaException` is thrown. + +| argument | type | description | required | default | +|----------|------|----------------------|----------|---------| +| name | str | the name of a schema | True | | + +### get +```python +client.get(table='Data Table', schema='My Schema', as_df=True) +``` +Retrieves data from a table on a schema and returns the result either as a list of dictionaries or as a pandas DataFrame (as pandas is used to parse the response). + +| argument | type | description | required | default | +|----------|------|--------------------------------------------------------------------------------|----------|---------| +| table | str | the name of a table | True | None | +| schema | str | the name of a schema | False | None | +| as_df | bool | if true: returns data as pandas DataFrame
else as a list of dictionaries | False | False | + + +### save_schema +```python +client.save_schema(table='Data Table', name='My Schema', + file='location/of/data/file.csv', data=[{'col1': value1, ...}, {'col2': value2, ...}, ...]) +``` +Imports or updates records in a table of a named schema. +The data either originates from a file on the disk, or is supplied by the user after, for example, preprocessing. +Either `file` or `data` must be supplied. The data must be compatible with the schema to which it is uploaded. + +| argument | type | description | required | default | +|----------|------|----------------------------------------|----------|---------| +| table | str | the name of a table | True | | +| schema | str | the name of a schema | False | None | +| file | str | the location of a `csv` file with data | False | None | +| data | list | data as a list of dictionaries | False | None | + + +### delete_records +```python +client.delete_records(table='Data Table', name='My Schema', + file='location/of/data/file.csv', data=[{'col1': value1, ...}, {'col2': value2, ...}, ...]) +``` +Deletes data records from a table. +As in the `save_schema` method, the data either originates from disk or the program. + +[//]: # (In order to delete records from a table the data must specify the row values with principal keys + +| argument | type | description | required | default | +|----------|------|----------------------------------------|----------|---------| +| table | str | the name of a table | True | | +| schema | str | the name of a schema | False | None | +| file | str | the location of a `csv` file with data | False | None | +| data | list | data as a list of dictionaries | False | None | + + +### export +```python +client.export(schema='My Schema', table='Data Table', fmt='csv') +``` +Exports data from a schema to a file in the desired format. +If the table is specified, only data from that table is exported, otherwise the export contains all tables on the schema. +If all tables from a schema are exported with given format `csv`, the data is exported as a zip file containing a csv file of each table. + +| argument | type | description | required | default | +|----------|------|-------------------------------------------|----------|---------| +| table | str | the name of a table | False | None | +| schema | str | the name of a schema | False | None | +| fmt | str | the output format, either `csv` or `xlsx` | False | `csv` | + +### create_schema +```python +client.create_schema(name='New Schema', description='My new schema!', template='DATA_CATALOGUE', include_demo_data=True) +``` +Creates a new schema on the server. +If no template is selected, an empty schema is created. + +| argument | type | description | required | default | +|-------------------|------|-------------------------------|----------|---------| +| name | str | the name of the new schema | True | | +| description | str | description of the new schema | False | None | +| template | str | the template for this schema | False | None | +| include_demo_data | bool | whether to include demo data | False | False | + +### delete_schema +TODO + +### update_schema + +### recreate_schema From 2ef152f8d09b9d35ee41dd2d1e16caf724d37ced Mon Sep 17 00:00:00 2001 From: ypezijlstra Date: Wed, 31 Jan 2024 16:12:29 +0100 Subject: [PATCH 21/87] Updated requirements.txt --- tools/pyclient/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pyclient/requirements.txt b/tools/pyclient/requirements.txt index d95278168c..b934710eb4 100644 --- a/tools/pyclient/requirements.txt +++ b/tools/pyclient/requirements.txt @@ -1,3 +1,3 @@ requests~=2.31.0 -pandas~=2.1.3 +pandas~=2.2.0 python-dotenv~=1.0.0 \ No newline at end of file From 435b02da6bce28fba11a3132e8095568902b2f10 Mon Sep 17 00:00:00 2001 From: "K. Joeri van der Velde" <82420+joerivandervelde@users.noreply.github.com> Date: Fri, 2 Feb 2024 10:57:33 +0100 Subject: [PATCH 22/87] Added helper functions for users in schema --- .../java/org/molgenis/emx2/sql/SqlSchema.java | 30 +++++++++++++++++++ .../main/java/org/molgenis/emx2/Schema.java | 6 ++++ 2 files changed, 36 insertions(+) diff --git a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlSchema.java b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlSchema.java index fed3cc9392..e43cdaeb97 100644 --- a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlSchema.java +++ b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlSchema.java @@ -63,6 +63,36 @@ public List getMembers() { } } + @Override + public boolean hasMember(String user) { + for (Member member : this.getMembers()) { + if (member.getUser().equals(user)) { + return true; + } + } + return false; + } + + @Override + public Member getMember(String user) { + for (Member member : this.getMembers()) { + if (member.getUser().equals(user)) { + return member; + } + } + return null; + } + + @Override + public void addUpdateMember(String user, String role) { + if (this.hasMember(user)) { + Member member = this.getMember(user); + member.setRole(role); + } else { + this.addMember(user, role); + } + } + @Override public void removeMembers(List members) { tx(database -> executeRemoveMembers((SqlDatabase) database, getName(), members)); diff --git a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Schema.java b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Schema.java index 7c35881746..ff86d3ead8 100644 --- a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Schema.java +++ b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Schema.java @@ -46,6 +46,12 @@ public interface Schema { List getMembers(); + boolean hasMember(String user); + + Member getMember(String user); + + void addUpdateMember(String user, String role); + void removeMembers(Member... members); void removeMembers(List members); From 2da8210fa7209ec5bf231ac8fba72a7dd931db49 Mon Sep 17 00:00:00 2001 From: "K. Joeri van der Velde" <82420+joerivandervelde@users.noreply.github.com> Date: Fri, 2 Feb 2024 10:59:28 +0100 Subject: [PATCH 23/87] More detailed permissions in profiles --- .../emx2/datamodels/ProfileLoader.java | 14 ++++++++------ .../emx2/datamodels/profiles/Profiles.java | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/ProfileLoader.java b/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/ProfileLoader.java index 8c7fcb5c5a..ac814c7e40 100644 --- a/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/ProfileLoader.java +++ b/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/ProfileLoader.java @@ -54,11 +54,13 @@ void loadInternalImplementation(Schema schema, boolean includeDemoData) { Schema ontoSchema; if (profiles.getOntologiesToFixedSchema() != null) { ontoSchema = createSchema(profiles.getOntologiesToFixedSchema(), schema.getDatabase()); - if (profiles.getSetViewPermission() != null) { - ontoSchema.addMember(profiles.getSetViewPermission(), Privileges.VIEWER.toString()); + if (profiles.getSetFixedSchemaViewPermission() != null) { + ontoSchema.addUpdateMember( + profiles.getSetFixedSchemaViewPermission(), Privileges.VIEWER.toString()); } - if (profiles.getSetEditPermission() != null) { - ontoSchema.addMember(profiles.getSetEditPermission(), Privileges.EDITOR.toString()); + if (profiles.getSetFixedSchemaEditPermission() != null) { + ontoSchema.addUpdateMember( + profiles.getSetFixedSchemaEditPermission(), Privileges.EDITOR.toString()); } } else { ontoSchema = schema; @@ -72,10 +74,10 @@ void loadInternalImplementation(Schema schema, boolean includeDemoData) { // special options: provide specific user/role with View/Edit permissions on imported schema if (profiles.getSetViewPermission() != null) { - schema.addMember(profiles.getSetViewPermission(), Privileges.VIEWER.toString()); + schema.addUpdateMember(profiles.getSetViewPermission(), Privileges.VIEWER.toString()); } if (profiles.getSetEditPermission() != null) { - schema.addMember(profiles.getSetEditPermission(), Privileges.EDITOR.toString()); + schema.addUpdateMember(profiles.getSetEditPermission(), Privileges.EDITOR.toString()); } // optionally, load demo data (i.e. some example records, or specific application data) diff --git a/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/profiles/Profiles.java b/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/profiles/Profiles.java index baf5084135..367a4b90f4 100644 --- a/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/profiles/Profiles.java +++ b/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/profiles/Profiles.java @@ -23,8 +23,26 @@ public class Profiles { private String ontologiesToFixedSchema; private String setViewPermission; private String setEditPermission; + private String setFixedSchemaViewPermission; + private String setFixedSchemaEditPermission; private List firstCreateSchemasIfMissing; + public String getSetFixedSchemaViewPermission() { + return setFixedSchemaViewPermission; + } + + public void setSetFixedSchemaViewPermission(String setFixedSchemaViewPermission) { + this.setFixedSchemaViewPermission = setFixedSchemaViewPermission; + } + + public String getSetFixedSchemaEditPermission() { + return setFixedSchemaEditPermission; + } + + public void setSetFixedSchemaEditPermission(String setFixedSchemaEditPermission) { + this.setFixedSchemaEditPermission = setFixedSchemaEditPermission; + } + public String getSetEditPermission() { return setEditPermission; } From 622524fad89146a668290db21e6d76634f4f441b Mon Sep 17 00:00:00 2001 From: "K. Joeri van der Velde" <82420+joerivandervelde@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:00:22 +0100 Subject: [PATCH 24/87] Update profile permissions for cohort staging --- data/_profiles/CohortStaging.yaml | 3 +-- data/_profiles/DataCatalogue.yaml | 3 ++- data/_profiles/SharedStaging.yaml | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/data/_profiles/CohortStaging.yaml b/data/_profiles/CohortStaging.yaml index a87197a80a..713acf1965 100644 --- a/data/_profiles/CohortStaging.yaml +++ b/data/_profiles/CohortStaging.yaml @@ -10,8 +10,7 @@ settings: # special options ontologiesToFixedSchema: CatalogueOntologies -setViewPermission: anonymous # applies to both the 'main' schema and CatalogueOntologies, required for anonymous viewing of catalogue shared staging -setEditPermission: user +setFixedSchemaViewPermission: anonymous firstCreateSchemasIfMissing: - name: catalogue profile: _profiles/DataCatalogue.yaml diff --git a/data/_profiles/DataCatalogue.yaml b/data/_profiles/DataCatalogue.yaml index d079ef94e1..457d93b188 100644 --- a/data/_profiles/DataCatalogue.yaml +++ b/data/_profiles/DataCatalogue.yaml @@ -9,5 +9,6 @@ demoData: _demodata/applications/datacatalogue settings: _settings/datacatalogue # special options +setViewPermission: anonymous ontologiesToFixedSchema: CatalogueOntologies -setViewPermission: anonymous # applies to both the 'main' schema and CatalogueOntologies, required for anonymous viewing of the catalogue +setFixedSchemaViewPermission: anonymous diff --git a/data/_profiles/SharedStaging.yaml b/data/_profiles/SharedStaging.yaml index 5dd98cfea0..0e6a01b022 100644 --- a/data/_profiles/SharedStaging.yaml +++ b/data/_profiles/SharedStaging.yaml @@ -9,5 +9,6 @@ demoData: _demodata/applications/datacatalogue_sharedstaging settings: _settings/datacatalogue_sharedstaging # special options +setEditPermission: user ontologiesToFixedSchema: CatalogueOntologies -setViewPermission: anonymous # applies to both the 'main' schema and CatalogueOntologies, required for anonymous viewing of catalogue shared staging +setFixedSchemaViewPermission: anonymous From 2532a662b39c714df795e70e73c67665cbd24f82 Mon Sep 17 00:00:00 2001 From: "K. Joeri van der Velde" <82420+joerivandervelde@users.noreply.github.com> Date: Fri, 2 Feb 2024 16:15:30 +0100 Subject: [PATCH 25/87] Revert "Added helper functions for users in schema" This reverts commit 435b02da6bce28fba11a3132e8095568902b2f10. --- .../java/org/molgenis/emx2/sql/SqlSchema.java | 30 ------------------- .../main/java/org/molgenis/emx2/Schema.java | 6 ---- 2 files changed, 36 deletions(-) diff --git a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlSchema.java b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlSchema.java index e43cdaeb97..fed3cc9392 100644 --- a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlSchema.java +++ b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlSchema.java @@ -63,36 +63,6 @@ public List getMembers() { } } - @Override - public boolean hasMember(String user) { - for (Member member : this.getMembers()) { - if (member.getUser().equals(user)) { - return true; - } - } - return false; - } - - @Override - public Member getMember(String user) { - for (Member member : this.getMembers()) { - if (member.getUser().equals(user)) { - return member; - } - } - return null; - } - - @Override - public void addUpdateMember(String user, String role) { - if (this.hasMember(user)) { - Member member = this.getMember(user); - member.setRole(role); - } else { - this.addMember(user, role); - } - } - @Override public void removeMembers(List members) { tx(database -> executeRemoveMembers((SqlDatabase) database, getName(), members)); diff --git a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Schema.java b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Schema.java index ff86d3ead8..7c35881746 100644 --- a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Schema.java +++ b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Schema.java @@ -46,12 +46,6 @@ public interface Schema { List getMembers(); - boolean hasMember(String user); - - Member getMember(String user); - - void addUpdateMember(String user, String role); - void removeMembers(Member... members); void removeMembers(List members); From 3173a9ef93a11d17e43b8dd439096f651973cab6 Mon Sep 17 00:00:00 2001 From: "K. Joeri van der Velde" <82420+joerivandervelde@users.noreply.github.com> Date: Fri, 2 Feb 2024 16:24:08 +0100 Subject: [PATCH 26/87] Add tests for profile parsing and loading --- .../emx2/datamodels/ProfileLoader.java | 8 +-- .../emx2/datamodels/profiles/Profiles.java | 2 +- .../emx2/datamodels/TestProfileLoader.java | 57 +++++++++++++++++++ .../test/resources/TestIncludeProfile.yaml | 4 ++ .../src/test/resources/TestProfile.yaml | 22 +++++++ 5 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 backend/molgenis-emx2-datamodels/src/test/java/org/molgenis/emx2/datamodels/TestProfileLoader.java create mode 100644 backend/molgenis-emx2-datamodels/src/test/resources/TestIncludeProfile.yaml create mode 100644 backend/molgenis-emx2-datamodels/src/test/resources/TestProfile.yaml diff --git a/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/ProfileLoader.java b/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/ProfileLoader.java index ac814c7e40..2b5666023d 100644 --- a/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/ProfileLoader.java +++ b/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/ProfileLoader.java @@ -55,11 +55,11 @@ void loadInternalImplementation(Schema schema, boolean includeDemoData) { if (profiles.getOntologiesToFixedSchema() != null) { ontoSchema = createSchema(profiles.getOntologiesToFixedSchema(), schema.getDatabase()); if (profiles.getSetFixedSchemaViewPermission() != null) { - ontoSchema.addUpdateMember( + ontoSchema.addMember( profiles.getSetFixedSchemaViewPermission(), Privileges.VIEWER.toString()); } if (profiles.getSetFixedSchemaEditPermission() != null) { - ontoSchema.addUpdateMember( + ontoSchema.addMember( profiles.getSetFixedSchemaEditPermission(), Privileges.EDITOR.toString()); } } else { @@ -74,10 +74,10 @@ void loadInternalImplementation(Schema schema, boolean includeDemoData) { // special options: provide specific user/role with View/Edit permissions on imported schema if (profiles.getSetViewPermission() != null) { - schema.addUpdateMember(profiles.getSetViewPermission(), Privileges.VIEWER.toString()); + schema.addMember(profiles.getSetViewPermission(), Privileges.VIEWER.toString()); } if (profiles.getSetEditPermission() != null) { - schema.addUpdateMember(profiles.getSetEditPermission(), Privileges.EDITOR.toString()); + schema.addMember(profiles.getSetEditPermission(), Privileges.EDITOR.toString()); } // optionally, load demo data (i.e. some example records, or specific application data) diff --git a/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/profiles/Profiles.java b/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/profiles/Profiles.java index 367a4b90f4..0aa3cb515c 100644 --- a/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/profiles/Profiles.java +++ b/backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/profiles/Profiles.java @@ -99,7 +99,7 @@ protected void setSettings(String settings) { this.settings = settings; } - protected List getProfileTagsList() { + public List getProfileTagsList() { return profileTagsList; } diff --git a/backend/molgenis-emx2-datamodels/src/test/java/org/molgenis/emx2/datamodels/TestProfileLoader.java b/backend/molgenis-emx2-datamodels/src/test/java/org/molgenis/emx2/datamodels/TestProfileLoader.java new file mode 100644 index 0000000000..15699abc37 --- /dev/null +++ b/backend/molgenis-emx2-datamodels/src/test/java/org/molgenis/emx2/datamodels/TestProfileLoader.java @@ -0,0 +1,57 @@ +package org.molgenis.emx2.datamodels; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.*; +import org.molgenis.emx2.Database; +import org.molgenis.emx2.Schema; +import org.molgenis.emx2.datamodels.profiles.Profiles; +import org.molgenis.emx2.datamodels.profiles.SchemaFromProfile; +import org.molgenis.emx2.sql.TestDatabaseFactory; + +@TestMethodOrder(MethodOrderer.MethodName.class) +@Tag("slow") +public class TestProfileLoader { + + public static final String TEST_PROFILE = "testProfileSchema"; + public static final String TEST_INCLUDE = "testIncludeSchema"; + + static Database database; + + @BeforeAll + public static void setup() { + database = TestDatabaseFactory.getTestDatabase(); + database.dropSchemaIfExists(TEST_PROFILE); + database.dropSchemaIfExists(TEST_INCLUDE); + } + + @Test + void testProfileParser() { + SchemaFromProfile schemaFromProfile = new SchemaFromProfile("TestProfile.yaml"); + Profiles profiles = schemaFromProfile.getProfiles(); + assertEquals("TestProfile", profiles.getName()); + assertEquals("Test profile with all options but small model", profiles.getDescription()); + assertEquals("JRC-CDE", profiles.getProfileTagsList().get(0)); + assertEquals("_demodata/shared-examples", profiles.getDemoDataList().get(0)); + assertEquals("_settings/datacatalogue", profiles.getSettingsList().get(0)); + assertEquals("anonymous", profiles.getSetViewPermission()); + assertEquals("user", profiles.getSetEditPermission()); + assertEquals("TestProfileOntologies", profiles.getOntologiesToFixedSchema()); + assertEquals("anonymous", profiles.getSetFixedSchemaViewPermission()); + assertEquals("user", profiles.getSetFixedSchemaEditPermission()); + assertEquals("testIncludeSchema", profiles.getFirstCreateSchemasIfMissing().get(0).getName()); + } + + @Test + void testProfileLoader() { + Schema testProfileSchema = database.createSchema(TEST_PROFILE); + ProfileLoader profileLoader = new ProfileLoader("TestProfile.yaml"); + profileLoader.load(testProfileSchema, false); + assertEquals(12, testProfileSchema.getTableNames().size()); + assertTrue(testProfileSchema.getTableNames().contains("Individuals")); + assertFalse(testProfileSchema.getTableNames().contains("Distributions")); + Schema testIncludeProfileSchema = database.getSchema(TEST_INCLUDE); + assertFalse(testIncludeProfileSchema.getTableNames().contains("Individuals")); + assertTrue(testIncludeProfileSchema.getTableNames().contains("Distribution")); + } +} diff --git a/backend/molgenis-emx2-datamodels/src/test/resources/TestIncludeProfile.yaml b/backend/molgenis-emx2-datamodels/src/test/resources/TestIncludeProfile.yaml new file mode 100644 index 0000000000..48c9f1b467 --- /dev/null +++ b/backend/molgenis-emx2-datamodels/src/test/resources/TestIncludeProfile.yaml @@ -0,0 +1,4 @@ +--- +name: TestIncludeProfile +description: "Profile to test inclusion into other profile" +profileTags: DCAT, DCAT files add-on diff --git a/backend/molgenis-emx2-datamodels/src/test/resources/TestProfile.yaml b/backend/molgenis-emx2-datamodels/src/test/resources/TestProfile.yaml new file mode 100644 index 0000000000..2280a9906c --- /dev/null +++ b/backend/molgenis-emx2-datamodels/src/test/resources/TestProfile.yaml @@ -0,0 +1,22 @@ +--- +name: TestProfile +description: "Test profile with all options but small model" + +profileTags: JRC-CDE + +demoData: _demodata/shared-examples + +settings: _settings/datacatalogue + +# special options +setViewPermission: anonymous +setEditPermission: user + +ontologiesToFixedSchema: TestProfileOntologies +setFixedSchemaViewPermission: anonymous +setFixedSchemaEditPermission: user + +firstCreateSchemasIfMissing: + - name: testIncludeSchema + profile: TestIncludeProfile.yaml + importDemoData: false From 7533658326d16f39e8293140655080cfe5a577a7 Mon Sep 17 00:00:00 2001 From: "K. Joeri van der Velde" <82420+joerivandervelde@users.noreply.github.com> Date: Fri, 2 Feb 2024 16:33:26 +0100 Subject: [PATCH 27/87] Update documentation --- docs/molgenis/dev_profiles.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/molgenis/dev_profiles.md b/docs/molgenis/dev_profiles.md index 527f03dfd4..d623111608 100644 --- a/docs/molgenis/dev_profiles.md +++ b/docs/molgenis/dev_profiles.md @@ -39,12 +39,14 @@ The most common options for profiles are: In addition, more specialized options can be used: -| Option | Description | -|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ontologiesToFixedSchema | Load any ontologies into a separate schema with the specified name. | -| setViewPermission | Apply view permissions to specified user or role to the imported schema and ontologies. Includes the schema specified by ontologiesToFixedSchema. | -| setEditPermission | Apply editor permissions to specified user or role to the imported schema and ontologies. Includes the schema specified by ontologiesToFixedSchema. | -| firstCreateSchemasIfMissing | Before creating the schema and ontologies specified by this profile, import one more more other profiles first. Required 3 additional properties to be provided: name, profile and importDemoData. Name specified the name under which the other profile should be imported. Profile points to the profile YAML file. Lastly, importDemoData is either true or false and specified whether the demo data for this profile should be imported or not. | +| Option | Description | +|------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| setViewPermission | Apply view permissions to specified user or role to the imported schema. | +| setEditPermission | Apply editor permissions to specified user or role to the imported schema. | +| ontologiesToFixedSchema | Load any ontologies into a separate schema with the specified name. | +| setFixedSchemaViewPermission | Apply view permissions to specified user or role to the the schema specified by ontologiesToFixedSchema. | +| setFixedSchemaEditPermission | Apply editor permissions to specified user or role to the the schema specified by ontologiesToFixedSchema. | +| firstCreateSchemasIfMissing | Before creating the schema and ontologies specified by this profile, import one more more other profiles first. Required 3 additional properties to be provided: name, profile and importDemoData. Name specified the name under which the other profile should be imported. Profile points to the profile YAML file. Lastly, importDemoData is either true or false and specified whether the demo data for this profile should be imported or not. | ## Complete example @@ -59,9 +61,14 @@ demoData: _demodata/applications/datacatalogue_sharedstaging settings: _settings/datacatalogue_sharedstaging # special options -ontologiesToFixedSchema: CatalogueOntologies + setViewPermission: anonymous setEditPermission: user + +ontologiesToFixedSchema: CatalogueOntologies +setFixedSchemaViewPermission: anonymous +setFixedSchemaEditPermission: user + firstCreateSchemasIfMissing: - name: catalogue profile: _profiles/DataCatalogue.yaml From 3a0e1efbe0159d1b91a19d6379623d06570f9d09 Mon Sep 17 00:00:00 2001 From: dtroelofsprins <57137088+dtroelofsprins@users.noreply.github.com> Date: Mon, 5 Feb 2024 13:09:32 +0100 Subject: [PATCH 28/87] fix: change ID to ID in the negotiator request (#3353) --- apps/directory/src/stores/checkoutStore.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/directory/src/stores/checkoutStore.js b/apps/directory/src/stores/checkoutStore.js index 07418b7626..c99a43e3e8 100644 --- a/apps/directory/src/stores/checkoutStore.js +++ b/apps/directory/src/stores/checkoutStore.js @@ -193,8 +193,8 @@ export const useCheckoutStore = defineStore("checkoutStore", () => { for (const collection of collectionSelection) { collections.push( toRaw({ - collectionID: collection.value, - biobankID: biobankIdDictionary.value[biobank], + collectionId: collection.value, + biobankId: biobankIdDictionary.value[biobank], }) ); } From 6222cef0205bd7d3f31582480cafce7b79755198 Mon Sep 17 00:00:00 2001 From: Esther van Enckevort Date: Mon, 5 Feb 2024 16:45:11 +0100 Subject: [PATCH 29/87] chore: add directory to demoset (#3354) * chore: add directory to demoset Add the BBMRI-ERIC directory to the demoset (Trac issue 10794) * Add environment to the deployment configuration and scripts - Added a directory section to values.yaml with an includeDirectoryDemo boolean - Set the value of the environment variable in deployment.yaml - Updated the shell scripts for kubernetes. --- .../src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java | 9 +++++++++ .../src/main/java/org/molgenis/emx2/Constants.java | 1 + ci/create_or_update_k8s-azure.sh | 3 ++- helm-chart/templates/deployment.yaml | 2 ++ helm-chart/values.yaml | 3 +++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java index beb2c1af6e..23982c43e7 100644 --- a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java +++ b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java @@ -3,6 +3,7 @@ import static org.molgenis.emx2.ColumnType.BOOL; import static org.molgenis.emx2.ColumnType.INT; +import org.molgenis.emx2.datamodels.BiobankDirectoryLoader; import org.molgenis.emx2.datamodels.PetStoreLoader; import org.molgenis.emx2.datamodels.ProfileLoader; import org.molgenis.emx2.sql.SqlDatabase; @@ -14,11 +15,15 @@ public class RunMolgenisEmx2 { public static final String CATALOGUE_DEMO = "catalogue-demo"; + public static final String DIRECTORY_DEMO = "directory-demo"; private static Logger logger = LoggerFactory.getLogger(RunMolgenisEmx2.class); public static final boolean INCLUDE_CATALOGUE_DEMO = (Boolean) EnvironmentProperty.getParameter(Constants.MOLGENIS_INCLUDE_CATALOGUE_DEMO, false, BOOL); + public static final boolean INCLUDE_DIRECTORY_DEMO = + (Boolean) + EnvironmentProperty.getParameter(Constants.MOLGENIS_INCLUDE_DIRECTORY_DEMO, false, BOOL); public static final boolean EXCLUDE_PETSTORE_DEMO = (Boolean) EnvironmentProperty.getParameter(Constants.MOLGENIS_EXCLUDE_PETSTORE_DEMO, false, BOOL); @@ -52,6 +57,10 @@ public static void main(String[] args) { Schema schema = db.createSchema(CATALOGUE_DEMO, "from DataCatalogue demo data loader"); new ProfileLoader("_profiles/DataCatalogue.yaml").load(schema, true); } + if (INCLUDE_DIRECTORY_DEMO && db.getSchema(DIRECTORY_DEMO) == null) { + Schema schema = db.createSchema(DIRECTORY_DEMO, "BBMRI-ERIC Directory Demo"); + new BiobankDirectoryLoader().load(schema, true); + } }); // start diff --git a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Constants.java b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Constants.java index a3ca0b921b..89063d9f14 100644 --- a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Constants.java +++ b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Constants.java @@ -57,6 +57,7 @@ public class Constants { public static final String MOLGENIS_OIDC_DISCOVERY_URI = "MOLGENIS_OIDC_DISCOVERY_URI"; public static final String MOLGENIS_OIDC_CALLBACK_URL = "MOLGENIS_OIDC_CALLBACK_URL"; public static final String MOLGENIS_INCLUDE_CATALOGUE_DEMO = "MOLGENIS_INCLUDE_CATALOGUE_DEMO"; + public static final String MOLGENIS_INCLUDE_DIRECTORY_DEMO = "MOLGENIS_INCLUDE_DIRECTORY_DEMO"; public static final String MOLGENIS_EXCLUDE_PETSTORE_DEMO = "MOLGENIS_EXCLUDE_PETSTORE_DEMO"; public static final String MOLGENIS_JWT_SHARED_SECRET = "MOLGENIS_JWT_SHARED_SECRET"; diff --git a/ci/create_or_update_k8s-azure.sh b/ci/create_or_update_k8s-azure.sh index 6f5df66b12..cbe3cb5668 100644 --- a/ci/create_or_update_k8s-azure.sh +++ b/ci/create_or_update_k8s-azure.sh @@ -35,4 +35,5 @@ helm upgrade --install ${NAME} ./helm-chart --namespace ${NAME} \ --set ssrCatalogue.image.repository=$REPO2 \ --set ssrCatalogue.environment.siteTitle="Preview Catalogue" \ --set ssrCatalogue.environment.apiBase=https://${NAME}.dev.molgenis.org/ \ ---set catalogue.includeCatalogueDemo=true +--set catalogue.includeCatalogueDemo=true \ +--set directory.includeDirectoryDemo=true diff --git a/helm-chart/templates/deployment.yaml b/helm-chart/templates/deployment.yaml index e5337a9b34..548fb10ca4 100644 --- a/helm-chart/templates/deployment.yaml +++ b/helm-chart/templates/deployment.yaml @@ -42,6 +42,8 @@ spec: value: jdbc:postgresql://localhost/{{ .Values.database.name }} - name: MOLGENIS_INCLUDE_CATALOGUE_DEMO value: {{ .Values.catalogue.includeCatalogueDemo | quote }} + - name: MOLGENIS_INCLUDE_DIRECTORY_DEMO + value: {{ .Values.directory.includeDirectoryDemo | quote }} {{- if .Values.oidc.enabled }} - name: MOLGENIS_OIDC_CLIENT_ID value: {{ .Values.oidc.client_id }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 3a6540ef42..c092d342f0 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -43,6 +43,9 @@ postgres: catalogue: includeCatalogueDemo: false +directory: + includeDirectoryDemo: false + ssrCatalogue: image: repository: molgenis/ssr-catalogue-snapshot From 5371d254e4446a33801ccd7163d381c9c57cab72 Mon Sep 17 00:00:00 2001 From: Morris Swertz Date: Mon, 5 Feb 2024 17:53:58 +0100 Subject: [PATCH 30/87] fix: design is hidden when values are provided (#3358) --- .../components/content/cohort/GeneralDesign.vue | 13 ++++++++----- .../[catalogue]/cohorts/[cohort]/index.vue | 1 - 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/nuxt3-ssr/components/content/cohort/GeneralDesign.vue b/apps/nuxt3-ssr/components/content/cohort/GeneralDesign.vue index fcce386bf3..dfe3d425e2 100644 --- a/apps/nuxt3-ssr/components/content/cohort/GeneralDesign.vue +++ b/apps/nuxt3-ssr/components/content/cohort/GeneralDesign.vue @@ -99,16 +99,19 @@ function setData() { return dp.title + (dp.doi ? ` (doi: ${dp.doi})` : ""); }), }, - { - label: "Main medical condition", - content: mainMedicalCondition, - type: "ONTOLOGY", - }, { label: "PID", content: cohort?.pid, }, ]; + + if (mainMedicalCondition && mainMedicalCondition.length > 0) { + generalDesign.splice(generalDesign.length - 2, 0, { + label: "Main medical condition", + content: mainMedicalCondition, + type: "ONTOLOGY", + }); + } } diff --git a/apps/nuxt3-ssr/pages/[schema]/ssr-catalogue/[catalogue]/cohorts/[cohort]/index.vue b/apps/nuxt3-ssr/pages/[schema]/ssr-catalogue/[catalogue]/cohorts/[cohort]/index.vue index 002b8aaf04..e579b8c95a 100644 --- a/apps/nuxt3-ssr/pages/[schema]/ssr-catalogue/[catalogue]/cohorts/[cohort]/index.vue +++ b/apps/nuxt3-ssr/pages/[schema]/ssr-catalogue/[catalogue]/cohorts/[cohort]/index.vue @@ -395,7 +395,6 @@ if (route.params.catalogue) { /> Date: Mon, 5 Feb 2024 17:59:39 +0100 Subject: [PATCH 31/87] fix: cohort only view render (#3357) - fix, should check cohort-only flag before dataSources_agg ref --- apps/nuxt3-ssr/components/header/Catalogue.vue | 2 +- .../pages/[schema]/ssr-catalogue/[catalogue]/index.vue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/nuxt3-ssr/components/header/Catalogue.vue b/apps/nuxt3-ssr/components/header/Catalogue.vue index 4d4dcb0cc1..954c2d07b6 100644 --- a/apps/nuxt3-ssr/components/header/Catalogue.vue +++ b/apps/nuxt3-ssr/components/header/Catalogue.vue @@ -27,7 +27,7 @@ if (catalogueRouteParam === "all" || props.catalogue.cohorts_agg?.count > 0) }); if ( (!cohortOnly.value && catalogueRouteParam === "all") || - props.catalogue.dataSources_agg?.count > 0 + (!cohortOnly.value && props.catalogue.dataSources_agg?.count > 0) ) menu.push({ label: "Data sources", diff --git a/apps/nuxt3-ssr/pages/[schema]/ssr-catalogue/[catalogue]/index.vue b/apps/nuxt3-ssr/pages/[schema]/ssr-catalogue/[catalogue]/index.vue index 00469264de..2c9bc2cade 100644 --- a/apps/nuxt3-ssr/pages/[schema]/ssr-catalogue/[catalogue]/index.vue +++ b/apps/nuxt3-ssr/pages/[schema]/ssr-catalogue/[catalogue]/index.vue @@ -246,7 +246,7 @@ const aboutLink = `/${route.params.schema}/ssr-catalogue/${catalogueRouteParam}/ :link="`/${route.params.schema}/ssr-catalogue/${catalogueRouteParam}/cohorts`" /> Date: Tue, 6 Feb 2024 14:12:20 +0100 Subject: [PATCH 33/87] chore: add anonymous access to the directory demo (#3359) Add molgenis_members.csv to set anonymous viewer access to the Directory app by default. --- data/biobank-directory/demo/molgenis_members.csv | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 data/biobank-directory/demo/molgenis_members.csv diff --git a/data/biobank-directory/demo/molgenis_members.csv b/data/biobank-directory/demo/molgenis_members.csv new file mode 100644 index 0000000000..c3aeef1248 --- /dev/null +++ b/data/biobank-directory/demo/molgenis_members.csv @@ -0,0 +1,2 @@ +user,role +anonymous,Viewer From 8dc31f18e07eb970573a17e895ed33a680e4b16d Mon Sep 17 00:00:00 2001 From: Ype Zijlstra Date: Tue, 6 Feb 2024 14:30:40 +0100 Subject: [PATCH 34/87] Added NonExistentTemplateException --- tools/pyclient/src/molgenis_emx2_pyclient/client.py | 5 ++++- tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py index 4a73ff1791..3c206516bf 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/client.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/client.py @@ -11,7 +11,7 @@ from .exceptions import (NoSuchSchemaException, ServiceUnavailableError, SigninError, ServerNotFoundError, PyclientException, NoSuchTableException, NoContextManagerException, GraphQLException, InvalidTokenException, - PermissionDeniedException, TokenSigninException) + PermissionDeniedException, TokenSigninException, NonExistentTemplateException) log = logging.getLogger("Molgenis EMX2 Pyclient") @@ -629,6 +629,9 @@ def _validate_graphql_response(response_json: dict, mutation: str, fallback_erro if 'permission denied' in message: log.error("Insufficient permissions for this operations.") raise PermissionDeniedException("Insufficient permissions for this operations.") + if 'AvailableDataModels' in message: + log.error("Selected template does not exist.") + raise NonExistentTemplateException("Selected template does not exist.") log.error(message) raise GraphQLException(message) diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py b/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py index ffbe841a9a..ebc55c7049 100644 --- a/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py +++ b/tools/pyclient/src/molgenis_emx2_pyclient/exceptions.py @@ -51,3 +51,7 @@ class PermissionDeniedException(PyclientException): class TokenSigninException(PyclientException): """Thrown when sign in is attempted on a client that is authorized with a token.""" + + +class NonExistentTemplateException(PyclientException): + """Thrown when creation of schema with non-existent template is attempted.""" From 85bb0ee2dd719966574e08bfcf47de44d57d1303 Mon Sep 17 00:00:00 2001 From: Ype Zijlstra Date: Tue, 6 Feb 2024 15:02:36 +0100 Subject: [PATCH 35/87] * Finished writing up delete_schema, update_schema, recreate_schema methods * Added exceptions where applicable * Explained and linked permission levels * Changed order of some methods --- docs/molgenis/use_usingpyclient.md | 134 ++++++++++++++++++++++++----- 1 file changed, 111 insertions(+), 23 deletions(-) diff --git a/docs/molgenis/use_usingpyclient.md b/docs/molgenis/use_usingpyclient.md index f30936327d..7e84347ee3 100644 --- a/docs/molgenis/use_usingpyclient.md +++ b/docs/molgenis/use_usingpyclient.md @@ -1,5 +1,5 @@ # Molgenis Pyclient -The MOLGENIS EMX2 Python client allows the user to retrieve, create, update and delete entities on a MOLGENIS EMX2 server using the Python language. +The MOLGENIS EMX2 Python client allows the user to retrieve, create, update and delete entities on a MOLGENIS EMX2 server using scripts or programs written in the Python language. ## Installation @@ -46,6 +46,7 @@ client = Client(url='https://example.molgeniscloud.org', token=token) Additionally, if the Pyclient is to be used on a particular schema, this schema can be supplied in the initialization of the client, alongside the server URL: ```python with Client('https://example.molgeniscloud.org', schema='My Schema') as client: + ... ``` or ```python @@ -54,13 +55,17 @@ client = Client('https://example.molgeniscloud.org', schema='My Schema', token=t ## Methods and properties This section outlines some of the methods that are supported by the Pyclient. +The results of these methods depend on the permission level the user has on the schemas involved in the method. + +The user roles on MOLGENIS EMX2 and their respective permissions are described in the [Permissions](use_permissions.md) section. + ### schema_names ```python client.schema_names ``` -Returns a list of the names of the schemas on the server that are available for the user. +Returns a list of the names of the schemas for which the user has at least _viewer_ permissions. ### status ```python @@ -73,15 +78,15 @@ The `status` property returns a string with information, including the server UR ```python client.get_schemas() ``` -Retrieves the schemas on the server for the user as a list of dictionaries containing for each schema the id, name, label and description. +Retrieves the schemas for which the user has at least _viewer_ permissions as a list of dictionaries containing for each schema the id, name, label and description. This method accepts no arguments. - ### set_schema ```python client.set_schema('My Schema') ``` -Sets the default schema for the server. Throws the `NoSuchSchemaException` if the schema is not available. +Sets the default schema for the server. +Throws the `NoSuchSchemaException` if the user does not have at least _viewer_ permissions or if the schema does not exist. | argument | type | description | required | default | |----------|------|----------------------|----------|---------| @@ -103,17 +108,19 @@ Raises the `TokenSigninException` when the client is already signed in with a us client.get_schema_metadata(name='My Schema') ``` Retrieves a schema's metadata, including the names of the tables, the metadata of the columns in the tables, schema settings, etc. -If no `name` is supplied the metadata for the default schema is queried, if applicable. Else the `NoSuchSchemaException` is thrown. +If no `name` is supplied the metadata for the default schema is queried, if applicable. +Throws the `NoSuchSchemaException` if the user does not have at least _viewer_ permissions or if the schema does not exist. | argument | type | description | required | default | |----------|------|----------------------|----------|---------| -| name | str | the name of a schema | True | | +| name | str | the name of a schema | False | None | ### get ```python client.get(table='Data Table', schema='My Schema', as_df=True) ``` Retrieves data from a table on a schema and returns the result either as a list of dictionaries or as a pandas DataFrame (as pandas is used to parse the response). +Throws the `NoSuchSchemaException` if the user does not have at least _viewer_ permissions or if the schema does not exist. | argument | type | description | required | default | |----------|------|--------------------------------------------------------------------------------|----------|---------| @@ -121,6 +128,21 @@ Retrieves data from a table on a schema and returns the result either as a list | schema | str | the name of a schema | False | None | | as_df | bool | if true: returns data as pandas DataFrame
else as a list of dictionaries | False | False | +### export +```python +client.export(schema='My Schema', table='Data Table', fmt='csv') +``` +Exports data from a schema to a file in the desired format. +If the table is specified, only data from that table is exported, otherwise the export contains all tables on the schema. +If all tables from a schema are exported with given format `csv`, the data is exported as a zip file containing a csv file of each table. +Throws the `NoSuchSchemaException` if the user does not have at least _viewer_ permissions or if the schema does not exist. + +| argument | type | description | required | default | +|----------|------|-------------------------------------------|----------|---------| +| table | str | the name of a table | False | None | +| schema | str | the name of a schema | False | None | +| fmt | str | the output format, either `csv` or `xlsx` | False | `csv` | + ### save_schema ```python @@ -130,6 +152,8 @@ client.save_schema(table='Data Table', name='My Schema', Imports or updates records in a table of a named schema. The data either originates from a file on the disk, or is supplied by the user after, for example, preprocessing. Either `file` or `data` must be supplied. The data must be compatible with the schema to which it is uploaded. +Throws the `PermissionDeniedException` if the user does not have at least _editor_ permissions for this schema. +Throws the `NoSuchSchemaException` if the schema is not found on the server. | argument | type | description | required | default | |----------|------|----------------------------------------|----------|---------| @@ -146,8 +170,10 @@ client.delete_records(table='Data Table', name='My Schema', ``` Deletes data records from a table. As in the `save_schema` method, the data either originates from disk or the program. +Throws the `PermissionDeniedException` if the user does not have at least _editor_ permissions. +Throws the `NoSuchSchemaException` if the schema is not found on the server. -[//]: # (In order to delete records from a table the data must specify the row values with principal keys +[//]: # ((In order to delete records from a table the data must specify the row values with principal keys) | argument | type | description | required | default | |----------|------|----------------------------------------|----------|---------| @@ -157,26 +183,13 @@ As in the `save_schema` method, the data either originates from disk or the prog | data | list | data as a list of dictionaries | False | None | -### export -```python -client.export(schema='My Schema', table='Data Table', fmt='csv') -``` -Exports data from a schema to a file in the desired format. -If the table is specified, only data from that table is exported, otherwise the export contains all tables on the schema. -If all tables from a schema are exported with given format `csv`, the data is exported as a zip file containing a csv file of each table. - -| argument | type | description | required | default | -|----------|------|-------------------------------------------|----------|---------| -| table | str | the name of a table | False | None | -| schema | str | the name of a schema | False | None | -| fmt | str | the output format, either `csv` or `xlsx` | False | `csv` | - ### create_schema ```python client.create_schema(name='New Schema', description='My new schema!', template='DATA_CATALOGUE', include_demo_data=True) ``` Creates a new schema on the server. If no template is selected, an empty schema is created. +Only users with _admin_ privileges are able to perform this action. | argument | type | description | required | default | |-------------------|------|-------------------------------|----------|---------| @@ -185,9 +198,84 @@ If no template is selected, an empty schema is created. | template | str | the template for this schema | False | None | | include_demo_data | bool | whether to include demo data | False | False | +[//]: # (The available templates are) + +[//]: # () +[//]: # (| template | description |) + +[//]: # (|--------------------------------|---------------------------------------------------------------|) + +[//]: # (| PET_STORE | example template |) + +[//]: # (| FAIR_DATA_HUB | see [FAIR Data Point](dev_fairdatapoint.md) |) + +[//]: # (| DATA_CATALOGUE | see [Data Catalogue](../catalogue/cat_cohort-data-manager.md) |) + +[//]: # (| DATA_CATALOGUE_COHORT_STAGING | see [Data Catalogue](../catalogue/cat_cohort-data-manager.md) |) + +[//]: # (| DATA_CATALOGUE_NETWORK_STAGING | see [Data Catalogue](../catalogue/cat_cohort-data-manager.md) |) + +[//]: # (| RD3 | |) + +[//]: # (| JRC_COMMON_DATA_ELEMENTS | | ) + +[//]: # (| FAIR_GENOMES | |) + +[//]: # (| BEACON_V2 | see [Beacon v2](dev_beaconv2.md) |) + +[//]: # (| ERN_DASHBOARD | |) + +[//]: # (| ERN_CRANIO | |) + +[//]: # (| BIOBANK_DIRECTORY | |) + +[//]: # (| SHARED_STAGING | |) + ### delete_schema -TODO +```python +client.delete_schema(name='Old Schema') +``` + +Deletes a schema from the server. +Only users with _admin_ privileges are able to perform this action. +Throws the `PermissionDeniedException` if execution of the method is attempted without _admin_ privileges. +Throws the `NoSuchSchemaException` if the schema is not found on the server. + + +| argument | type | description | required | default | +|-------------------|------|----------------------------------|----------|---------| +| name | str | the name of the schema to delete | True | | + ### update_schema +```python +client.update_schema(name='My Schema', description='The new description of this schema.') +``` +Updates a schema's description. +Only users with _admin_ privileges are able to perform this action. +Throws the `PermissionDeniedException` if execution of the method is attempted without _admin_ privileges. +Throws the `NoSuchSchemaException` if the schema is not found on the server. + +| argument | type | description | required | default | +|-------------------|------|-------------------------|----------|---------| +| name | str | the name of the schema | True | | +| description | str | the updated description | True | | ### recreate_schema +```python +client.recreate_schema(name='My Schema', description='Updated description', + template='DATA_CATALOGUE', include_demo_data=False) +``` +Recreates a schema on the EMX2 server by deleting and subsequently creating it without data on the EMX2 server. + +If no template is selected, an empty schema is created. +Only users with _admin_ privileges are able to perform this action. +Throws the `PermissionDeniedException` if execution of the method is attempted without _admin_ privileges. +Throws the `NoSuchSchemaException` if the schema is not found on the server. + +| argument | type | description | required | default | +|-------------------|------|-------------------------------|----------|---------| +| name | str | the name of the schema | True | | +| description | str | new description of the schema | False | None | +| template | str | the template for this schema | False | None | +| include_demo_data | bool | whether to include demo data | False | False | From ae71d49f19131ed031881a2bfc6aa30090aba215 Mon Sep 17 00:00:00 2001 From: Ype Zijlstra Date: Tue, 6 Feb 2024 15:23:37 +0100 Subject: [PATCH 36/87] Wrote use_tokens.md --- docs/molgenis/use_tokens.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/molgenis/use_tokens.md b/docs/molgenis/use_tokens.md index d80e19a4c1..e0933b13cb 100644 --- a/docs/molgenis/use_tokens.md +++ b/docs/molgenis/use_tokens.md @@ -1 +1,13 @@ -A nice instruction of creating tokens on the website. \ No newline at end of file +# Tokens +MOLGENIS allows the user to create temporary tokens that can be used in scripts and programs that interact with a MOLGENIS EMX2 server. +The token grants the same permissions to schemas on the server as the user has when using the MOLGENIS instance through the website. + +Generating a token is done as follows: + +1. Sign in to the MOLGENIS instance + +2. Click on the username in the upper right corner next to `Sign out` to open the account management dialog + +3. Underneath _Manage tokens_ fill out text field with the name associated with the token and click the `Create token` button + +4. Copy the resulting token from the green dialog box From ef0aff8f5a851b17084d313b00b2de26a69b1069 Mon Sep 17 00:00:00 2001 From: YpeZ Date: Wed, 7 Feb 2024 08:21:57 +0100 Subject: [PATCH 37/87] Update use_usingapis.md --- docs/molgenis/use_usingapis.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/molgenis/use_usingapis.md b/docs/molgenis/use_usingapis.md index 837ab14a77..6fb8bb1c56 100644 --- a/docs/molgenis/use_usingapis.md +++ b/docs/molgenis/use_usingapis.md @@ -1,15 +1,12 @@ # Using APIs -The EMX2 APIs can be used to various environments to retrieve, add, manipulate or replace data. +The EMX2 APIs can be used in various environments to retrieve, add, manipulate or replace data. ## Python -Data management of a Molgenis EMX2 server can be performed through the use of the Python client. -This client can be installed from the [PyPI repository](https://pypi.org/project/molgenis-emx2-pyclient/) and can be applied in a number of operations, such as -- retrieving data from a table -- uploading data to a table +Data management of a Molgenis EMX2 server can be performed through the use of the Python client, see [Pyclient](use_usingpyclient.md). ## R From 2882f02426e3f427cf62963a2733b733063cb7e2 Mon Sep 17 00:00:00 2001 From: BrendaHijmans <42644481+BrendaHijmans@users.noreply.github.com> Date: Wed, 7 Feb 2024 11:51:13 +0100 Subject: [PATCH 38/87] feat: Update Catalogue data model, Publications.doi to hyperlink (#3321) feat: Update model Publications.doi to hyperlink - Update meta data model - Update ui to render Doi as link in de Cohort detail view --- .../components/CatalogueItemList.vue | 37 ++- .../content/cohort/GeneralDesign.vue | 37 ++- apps/nuxt3-ssr/interfaces/types.ts | 4 +- apps/nuxt3-ssr/utils/errorLogger.ts | 10 +- .../applications/datacatalogue/Cohorts.csv | 79 +++--- .../applications/datacatalogue/Databanks.csv | 6 +- .../datacatalogue/Publications.csv | 157 ++++++------ data/_models/shared/DataCatalogue-TODO.csv | 4 +- .../datacatalogue/stagingCohorts/molgenis.csv | 4 +- .../stagingCohortsUMCG/molgenis.csv | 4 +- .../stagingNetworks/molgenis.csv | 4 +- data/datacatalogue/stagingRWE/molgenis.csv | 4 +- data/scripts/molgenis-model-update/run.py | 238 ++++++++++++++++++ .../update/update_3_11.py | 129 ++++++++++ .../molgenis-model-update/util/client.py | 16 +- .../util/zip_handling.py | 12 +- 16 files changed, 579 insertions(+), 166 deletions(-) create mode 100644 data/scripts/molgenis-model-update/run.py create mode 100644 data/scripts/molgenis-model-update/update/update_3_11.py diff --git a/apps/nuxt3-ssr/components/CatalogueItemList.vue b/apps/nuxt3-ssr/components/CatalogueItemList.vue index edd312eb14..8287a71195 100644 --- a/apps/nuxt3-ssr/components/CatalogueItemList.vue +++ b/apps/nuxt3-ssr/components/CatalogueItemList.vue @@ -58,22 +58,41 @@ function showAsFile(item: IDefinitionListItem) { :collapse-all="true" > - -
- - {{ item.label }} - -
- - + + + {{ item.content.label }} + + +
+ + {{ item.label }} + +
+ +

{{ item.content.value }} diff --git a/apps/nuxt3-ssr/components/content/cohort/GeneralDesign.vue b/apps/nuxt3-ssr/components/content/cohort/GeneralDesign.vue index dfe3d425e2..72bb2ec634 100644 --- a/apps/nuxt3-ssr/components/content/cohort/GeneralDesign.vue +++ b/apps/nuxt3-ssr/components/content/cohort/GeneralDesign.vue @@ -1,5 +1,9 @@ diff --git a/apps/nuxt3-ssr/interfaces/types.ts b/apps/nuxt3-ssr/interfaces/types.ts index 6ab0ab8192..c6c33527f9 100644 --- a/apps/nuxt3-ssr/interfaces/types.ts +++ b/apps/nuxt3-ssr/interfaces/types.ts @@ -269,10 +269,12 @@ export interface IMgError { data: { errors: { message: string }[] }; } +export type DefinitionListItemType = "ONTOLOGY" | "LINK"; + export interface IDefinitionListItem { label: string; tooltip?: string; - type?: string; + type?: DefinitionListItemType; content: any; } export interface IOntologyItem { diff --git a/apps/nuxt3-ssr/utils/errorLogger.ts b/apps/nuxt3-ssr/utils/errorLogger.ts index 7eeadc0f66..5b9fafbb66 100644 --- a/apps/nuxt3-ssr/utils/errorLogger.ts +++ b/apps/nuxt3-ssr/utils/errorLogger.ts @@ -7,8 +7,10 @@ export const logError = (error: IMgError, contextMsg?: string) => { console.log(`[ERROR] StatusCode: ${error.statusCode}`); console.log(`[ERROR] Message: ${error.message}`); - console.log("[ERROR] MESSAGES FROM API: "); - error.data.errors.forEach((e: { message: string }, lineNr) => - console.log(` ${lineNr}: ${e.message}`) - ); + if (error.data.errors) { + console.log("[ERROR] MESSAGES FROM API: "); + error.data.errors.forEach((e: { message: string }, lineNr) => + console.log(` ${lineNr}: ${e.message}`) + ); + } }; diff --git a/data/_demodata/applications/datacatalogue/Cohorts.csv b/data/_demodata/applications/datacatalogue/Cohorts.csv index 63c9437915..ad80229f07 100644 --- a/data/_demodata/applications/datacatalogue/Cohorts.csv +++ b/data/_demodata/applications/datacatalogue/Cohorts.csv @@ -1,17 +1,17 @@ id,pid,acronym,name,local name,keywords,website,lead organisation,additional organisations,description,contact email,type,type other,design,design description,design schematic,collection type,logo,number of participants,number of participants with samples,countries,regions,population age groups,inclusion criteria,other inclusion criteria,start year,end year,population disease,population oncology topology,population oncology morphology,release type,release description,linkage options,data holder,data access conditions,data use conditions,data access conditions description,data access fee,design paper,publications,informed consent type,funding statement,acknowledgements,supplementary information -CONSTANCES,,CONSTANCES,Cohorte des Consultants des Centres d’Examens de,,,https://www.constances.fr/,INSERM,,,,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,220000,,France,,Adult (18+ years),,,2012,2019,,,,,,,,,,,,10.1007/s10654-015-0096-4,,,,, -G21,,,Generation 21,Geração,,http://www.geracao21.com/pt/,ISPUP,,,,Birth cohort,,Longitudinal,,,,,8647,,Portugal,,"Newborn (0-1 months),Child (2-12 years),Adolescent (13-17 years)",,,2005,,,,,,,,,,,,,,,,,, +CONSTANCES,,CONSTANCES,Cohorte des Consultants des Centres d’Examens ,,,https://www.constances.fr/,INSERM,,,,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,220000,,France,,Adult (18+ years),,,2012,2019,,,,,,,,,,,,https://doi.org/10.1007/s10654-015-0096-4,,,,, +G21,,,Generation 21,Geraç ,,http://www.geracao21.com/pt/,ISPUP,,,,Birth cohort,,Longitudinal,,,,,8647,,Portugal,,"Newborn (0-1 months),Child (2-12 years),Adolescent (13-17 years)",,,2005,,,,,,,,,,,,,,,,,, mCRC-VHIO,,mCRC-VHIO,metastatic Colorrectar Cancer - VHIO,,,,VHIO,,,xvillalobos@vhio.net,Clinical cohort,,Cross-sectional,,,Retrospective,,116,,,,,,,,,,,,,,,,disease specific research,,,,,,,,, -FORCE-NEN,,FORCE-NEN,InFrastructure fOr Rare Cancers in the nEtherlands – Neuroendocrine Neopl,,,https://www.ikdoemeemetforce.nl/,"UMCU,UMCG","MUMC,NKI,AmsterdamUMC (VUmc),EMC,Maxima MC",Multicenter Clinical Biobank with primary object to determine the value of ctDNA in the diagnosis and treatment of Neuroendocrine Neoplasms.,force-nen@umcutrecht.nl,"Clinical cohort,Biobank",,Longitudinal,,,"Retrospective,Prospective",,,,Netherlands (the),,"Adult (18+ years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years),Aged (65-79 years),Aged (80+ years)",,,2022,,,,,Closed dataset,,,UMCU,disease specific research,"collaboration required,ethics approval required,project specific restriction",,,,,,FORCE is made possible by a grant from the KWF Dutch Cancer Society (13363),, +FORCE-NEN,,FORCE-NEN,InFrastructure fOr Rare Cancers in the nEtherlands – Neuroendocrine Neo,,,https://www.ikdoemeemetforce.nl/,"UMCU,UMCG","MUMC,NKI,AmsterdamUMC (VUmc),EMC,Maxima MC",Multicenter Clinical Biobank with primary object to determine the value of ctDNA in the diagnosis and treatment of Neuroendocrine Neoplasms.,force-nen@umcutrecht.nl,"Clinical cohort,Biobank",,Longitudinal,,,"Retrospective,Prospective",,,,Netherlands (the),,"Adult (18+ years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years),Aged (65-79 years),Aged (80+ years)",,,2022,,,,,Closed dataset,,,UMCU,disease specific research,"collaboration required,ethics approval required,project specific restriction",,,,,,FORCE is made possible by a grant from the KWF Dutch Cancer Society (13363),, CON,CON,CON,Consumer cohort,CON,"consumer, supermarket, data epidemiology, cohort",https://mineindkob.dk/,SSI,,"This project aims to investigate how what we buy, and thus our diet, affects health, by analyzing purchase patterns and health information over time.","frtm@ssi.dk, mineindkob@ssi.dk","Population cohort,Registry",,Longitudinal,,,"Retrospective,Prospective",,407,407,Denmark,,"Adult (18+ years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years),Aged (80+ years)",,,2021,,,,,Other release type,on request,"GS1 data, FRIDA data",,,,,,,,,Through the project HEAP: https://heap-exposome.eu/ grant agreement No. 874662,,Full consent can be found at https://www.mineindkob.dk/da-DK/registration#1 CSC,CSC,CSC,Cervical screening cohort,,"cancer, CIN, LSIL, HSIL, cytology, pathology, invitations, HPV, cervical cancer",https://nkcx.se,KI,,Cervical screening cohort comprising all women resident in Sweden aged 23-64 that are screened for human papillomavirus or cervical abnormalities.,joakim.dillner@ki.se; info@nkcx.se,"Population cohort,Registry,Biobank",,Longitudinal,,,"Retrospective,Prospective",,500000,500000,Sweden,,"Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65-79 years)",,,1968,,,,,Annually,new data from past year is imported annualy in April online,"NKCx, biobank, laboratory experiments (sequencing)",,,,"For research purposes data can be provided pseudonomized, providing there is i) an approved ethical application, ii) an approved application for data (which is to be submitted to the registry)",,,,,SKR (Swedish Municipalities and Regions) funds partly for NKCx.,"It is required to name the data source for instance in the materials and methods section as ""Swedish National Cervical Screening Registry (NKCx)""", -HPV,,HPV,HPV vaccination cohort,,"Community randomized, HPV vaccination, gender-neutral",https://projects.tuni.fi/hpv-rokotiitus/,TUNI,,"Community-randomized HPV vaccination trial cohort: gender-neutral HPV16/18 vaccination arm, girls-only HPV16/18 vaccination and hepatitis B-virus (HBV) vaccination of boys arm, and arm gender-neutral HBV vaccination arm. Total of boys (31,117) and girls (30,139), 9482 (60,7 %) were HPV vaccinated and 6131 (39,3 %) were HBV vaccinated at the age of 14 and followed up at he age of 18, 22, 25 and 28.","tiina.eriksson@tuni.fi, ville.pimenoff@ki.se, karolina.louvanto@tuni.fi, matti.lehtinen@tuni.fi",Clinical trial,,Longitudinal,,,Prospective,,61256,15613,Finland,,"Adolescent (13-17 years),Young adult (18-24 years),Adult (25-44 years)",,,2007,,,,,Other release type,not determined; new data still created,,,,,,,,"10.1258/095646206776253453,10.1002/ijc.31281,10.1002/ijc.32791,10.1016/j.vaccine.2014.12.019,10.1371/journal.pmed.1003588,10.1371/journal.pone.0072088,10.1002/ijc.31119,10.1258/095646206778145550,10.1080/21645515.2016.1183847,10.1002/ijc.32189,10.1002/ijc.32802,10.1016/S1473-3099(20)30873-2,10.1093/infdis/jiaa099,10.1002/ijc.31618,10.1002/ijc.33169,10.1136/bmjopen-2017-015867,10.1002/ijc.27586",,"A total of more than 40 million euros of grants by Prof. Matti Lehtinen from Finnish and Nordic Cancer Cociety, Finnish Academy, EU FP5 prgram, FP6 program, FP7 program, Merck&Co, GlaxoSmithKline Biologicals, IMI, Swedish Cancerfonden, Jane and aatos Erkko Foundation, EU Horizon and Finnish Cancer Society.",Original articles and data owners need to be cited., -LSC,LSC,LSC,Lifestyle cohort,TirolGESUND,intermittent fasting; smoking cessation; DNA methylation; prevention;,https://www.uibk.ac.at/,UIBK,,"Data from 6-month intervention trial looking at the effects of intermittent fasting or smoking cessation in healthy women aged 30-60 with risk factors for cancer or other diseases (elevanted BMI, heavy smoking).",martin.widschwendter@uibk.ac.at,"Population cohort,Clinical trial",,Longitudinal,,,Prospective,,156,156,Austria,,"Adult (25-44 years),Middle-aged (45-64 years)",,,2021,,,,,Other release type,"Data will be embargoed until initial publication of the study results. Personally identifiable data will be under restricted access, controlled by a data access committee. A data access form will need to be filled in providing data use. Use of data is for scientific purposes only and a cooperation agreement has to be signed for use.",,,,,"Data will be embargoed until initial publication of the study results. Personally identifiable data will be under restricted access, controlled by a data access committee. A data access form will need to be filled in providing data use. Use of data is for scientific purposes only and a cooperation agreement has to be signed for use.",,,,,"This study received funding by the European Union’s Horizon 2020 Research and Innovation Program, HEAP under grant agreement No. 874662 and the Land Ti","Any users must acknowledge Prof. Martin Widschwendter, Dr Chiara Herzog, and the EUTOPS Institute and cite the relevant key publications out of the project.", +HPV,,HPV,HPV vaccination cohort,,"Community randomized, HPV vaccination, gender-neutral",https://projects.tuni.fi/hpv-rokotiitus/,TUNI,,"Community-randomized HPV vaccination trial cohort: gender-neutral HPV16/18 vaccination arm, girls-only HPV16/18 vaccination and hepatitis B-virus (HBV) vaccination of boys arm, and arm gender-neutral HBV vaccination arm. Total of boys (31,117) and girls (30,139), 9482 (60,7 %) were HPV vaccinated and 6131 (39,3 %) were HBV vaccinated at the age of 14 and followed up at he age of 18, 22, 25 and 28.","tiina.eriksson@tuni.fi, ville.pimenoff@ki.se, karolina.louvanto@tuni.fi, matti.lehtinen@tuni.fi",Clinical trial,,Longitudinal,,,Prospective,,61256,15613,Finland,,"Adolescent (13-17 years),Young adult (18-24 years),Adult (25-44 years)",,,2007,,,,,Other release type,not determined; new data still created,,,,,,,,,,"A total of more than 40 million euros of grants by Prof. Matti Lehtinen from Finnish and Nordic Cancer Cociety, Finnish Academy, EU FP5 prgram, FP6 program, FP7 program, Merck&Co, GlaxoSmithKline Biologicals, IMI, Swedish Cancerfonden, Jane and aatos Erkko Foundation, EU Horizon and Finnish Cancer Society.",Original articles and data owners need to be cited., +LSC,LSC,LSC,Lifestyle cohort,TirolGESUND,intermittent fasting; smoking cessation; DNA methylation; prevention;,https://www.uibk.ac.at/,UIBK,,"Data from 6-month intervention trial looking at the effects of intermittent fasting or smoking cessation in healthy women aged 30-60 with risk factors for cancer or other diseases (elevanted BMI, heavy smoking).",martin.widschwendter@uibk.ac.at,"Population cohort,Clinical trial",,Longitudinal,,,Prospective,,156,156,Austria,,"Adult (25-44 years),Middle-aged (45-64 years)",,,2021,,,,,Other release type,"Data will be embargoed until initial publication of the study results. Personally identifiable data will be under restricted access, controlled by a data access committee. A data access form will need to be filled in providing data use. Use of data is for scientific purposes only and a cooperation agreement has to be signed for use.",,,,,"Data will be embargoed until initial publication of the study results. Personally identifiable data will be under restricted access, controlled by a data access committee. A data access form will need to be filled in providing data use. Use of data is for scientific purposes only and a cooperation agreement has to be signed for use.",,,,,"This study received funding by the European Union’s Horizon 2020 Research and Innovation Program, HEAP under grant agreement No. 874662 and the Land ","Any users must acknowledge Prof. Martin Widschwendter, Dr Chiara Herzog, and the EUTOPS Institute and cite the relevant key publications out of the project.", BAMSE,,,BAMSE,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, HBCS,,HBCS,Helsinki Birth Cohort Study,,,https://thl.fi/en/web/thlfi-en/research-and-expertwork/projects-and-programmes/helsinki-birth-cohort-study-hbcs-idefix,,,,,Birth cohort,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -MAT,MAT,MAT,Maternity cohort,MAT,"Biobank, maternity cohort, serum",https://www.ppshp.fi/Tutkimus-ja-opetus/Biopankki/Pages/default.aspx,,,"The Finnish Maternity Cohort (FMC) in Northern Finland Biobank Borealis is a collection of serum samples comprising virtually the entire population of Finnish females, who have been pregnant since 1983. FMC is linkable with population-based health registers including the Finnish Cancer Registry and the Medical Birth Register. Linkage of FMC data and samples with pathological archives of Finnish Biobanks provides an opportunity to expand the scope of research into interaction of biological and genomic factors in association with various diseases. Serial prediagnostic serum samples and comprehensive data accessible over three generations provide possibilities for unique study settings.","hanna.ohman@ppshp.fi, ville.pimenoff@oulu.fi, helja-marja.surcel@oulu.fi",Biobank,,Longitudinal,,,Retrospective,,950000,950000,Finland,,"Adolescent (13-17 years),Adult (18+ years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years),Aged (80+ years)",,,1983,2017,,,,Closed dataset,,FMC is linkable with population-based health registers including the Finnish Cancer Registry and the Medical Birth Register. Linkage of FMC data and samples with pathological archives of Finnish Biobanks provides an opportunity to expand the scope of research into interaction of biological and genomic factors in association with various diseases.,,,,,,10.1002/cam4.1222,10.1002/cam4.1222,,European Science Infrastructure Services (https://esis.fi/),European Science Infrastructure Services (https://esis.fi/),European Science Infrastructure Services (https://esis.fi/) -WDC,WDC,WDC,Wearable data collection study,WDC,"wearables, exposome, pregnancy, environmental exposure",,,"UOULU,KI,TUNI","Longitudinal cohort of pregnant and non-pregnant women for exposome estimation from blood, saliva, sweat and personal aerosols.","ville.pimenoff@oulu.fi, ville.pimenoff@ki.se, matti.lehtinen@tuni.fi",Population cohort,,Longitudinal,Longitudinal,,Prospective,,100,100,Finland,,Adult (25-44 years),,,2022,2022,,,,Closed dataset,,,,,,,,10.1097/EE9.0000000000000182,10.1097/EE9.0000000000000182,,Horizon 2020 (HEAP),Contributors need to be cited., -PELAGIE,,PELAGIE,"Perturbateurs Endocriniens: Étude Longitudinale sur les Anomalies de la Grossesse, l’Infertilité et","Perturbateurs Endocriniens: Étude Longitudinale sur les Anomalies de la Grossesse, l’Infertilité et",Pregnancy; Child; environment; growth; development; neurodevelopment; pesticides; solvant; occupation,http://www.pelagie-inserm.fr/,INSERM,,"The PELAGIE mother-child cohort was set up with the aim to study the effect of prenatal exposure to environmental and occupational chemicals on reproduction and child development. The PELAGIE cohort enrolled 3421 pregnant women from Brittany, France between 2002 and 2006. Offsprings were followed at delivery (n=3439) and at 2 (n=1506), 6 (n=1406) and 12 (n=1211) years old. Almost all women provided a urine samples at inclusion. Cord blood samples, maternal hair samples and pieces of placenta were collected for 55%, 60%, and 57% of the participants, respectively. A subgroup of 287 and 559 children performed a more in-depth follow-up including a clinical examination and biological sample collection at age 6 and 12 years old, respectively.",,Population cohort,,Longitudinal,,,Prospective,,3421,3400,France,Bretagne,"Prenatal,Newborn (0-1 months),Child (2-12 years)",,,2002,2006,,,,,Once a follow-up is completed and data have been processed.,No,,,,Send a first request to Cécile Chevrier (cecile.chevrier@inserm.f,,,"10.1289/ehp.1002775,10.1186/1476-069X-12-102,10.1016/j.envint.2015.05.009",,"Depends on the data used (birth, 2, 6, 12 years old). Main funders includes: Inserm, Institut national de Veille Sanitaire, Ministère du travail, Ministère de la recherche, Agence Nationale de la Recherche, Afsset/Anses, DRASS de Bretagne, EU commission, Programme 189 Post Grenelle, Fondation de Fr","We are grateful to the midwives, obstetricians, gynecologists, sonographers, pediatricians, and families who participated in the study.", +MAT,MAT,MAT,Maternity cohort,MAT,"Biobank, maternity cohort, serum",https://www.ppshp.fi/Tutkimus-ja-opetus/Biopankki/Pages/default.aspx,,,"The Finnish Maternity Cohort (FMC) in Northern Finland Biobank Borealis is a collection of serum samples comprising virtually the entire population of Finnish females, who have been pregnant since 1983. FMC is linkable with population-based health registers including the Finnish Cancer Registry and the Medical Birth Register. Linkage of FMC data and samples with pathological archives of Finnish Biobanks provides an opportunity to expand the scope of research into interaction of biological and genomic factors in association with various diseases. Serial prediagnostic serum samples and comprehensive data accessible over three generations provide possibilities for unique study settings.","hanna.ohman@ppshp.fi, ville.pimenoff@oulu.fi, helja-marja.surcel@oulu.fi",Biobank,,Longitudinal,,,Retrospective,,950000,950000,Finland,,"Adolescent (13-17 years),Adult (18+ years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years),Aged (80+ years)",,,1983,2017,,,,Closed dataset,,FMC is linkable with population-based health registers including the Finnish Cancer Registry and the Medical Birth Register. Linkage of FMC data and samples with pathological archives of Finnish Biobanks provides an opportunity to expand the scope of research into interaction of biological and genomic factors in association with various diseases.,,,,,,https://doi.org/10.1002/cam4.1222,https://doi.org/10.1002/cam4.1222,,European Science Infrastructure Services (https://esis.fi/),European Science Infrastructure Services (https://esis.fi/),European Science Infrastructure Services (https://esis.fi/) +WDC,WDC,WDC,Wearable data collection study,WDC,"wearables, exposome, pregnancy, environmental exposure",,,"UOULU,KI,TUNI","Longitudinal cohort of pregnant and non-pregnant women for exposome estimation from blood, saliva, sweat and personal aerosols.","ville.pimenoff@oulu.fi, ville.pimenoff@ki.se, matti.lehtinen@tuni.fi",Population cohort,,Longitudinal,Longitudinal,,Prospective,,100,100,Finland,,Adult (25-44 years),,,2022,2022,,,,Closed dataset,,,,,,,,https://doi.org/10.1097/EE9.0000000000000182,https://doi.org/10.1097/EE9.0000000000000182,,Horizon 2020 (HEAP),Contributors need to be cited., +PELAGIE,,PELAGIE,"Perturbateurs Endocriniens: Étude Longitudinale sur les Anomalies de la Grossesse, l’Infertilit ","Perturbateurs Endocriniens: Étude Longitudinale sur les Anomalies de la Grossesse, l’Infertilit ",Pregnancy; Child; environment; growth; development; neurodevelopment; pesticides; solvant; occupation,http://www.pelagie-inserm.fr/,INSERM,,"The PELAGIE mother-child cohort was set up with the aim to study the effect of prenatal exposure to environmental and occupational chemicals on reproduction and child development. The PELAGIE cohort enrolled 3421 pregnant women from Brittany, France between 2002 and 2006. Offsprings were followed at delivery (n=3439) and at 2 (n=1506), 6 (n=1406) and 12 (n=1211) years old. Almost all women provided a urine samples at inclusion. Cord blood samples, maternal hair samples and pieces of placenta were collected for 55%, 60%, and 57% of the participants, respectively. A subgroup of 287 and 559 children performed a more in-depth follow-up including a clinical examination and biological sample collection at age 6 and 12 years old, respectively.",,Population cohort,,Longitudinal,,,Prospective,,3421,3400,France,Bretagne,"Prenatal,Newborn (0-1 months),Child (2-12 years)",,,2002,2006,,,,,Once a follow-up is completed and data have been processed.,No,,,,Send a first request to Cécile Chevrier (cecile.chevrier@inserm.,,,"https://doi.org/10.1289/ehp.1002775,https://doi.org/10.1186/1476-069X-12-102,https://doi.org/10.1016/j.envint.2015.05.009",,"Depends on the data used (birth, 2, 6, 12 years old). Main funders includes: Inserm, Institut national de Veille Sanitaire, Ministère du travail, Ministère de la recherche, Agence Nationale de la Recherche, Afsset/Anses, DRASS de Bretagne, EU commission, Programme 189 Post Grenelle, Fondation de ","We are grateful to the midwives, obstetricians, gynecologists, sonographers, pediatricians, and families who participated in the study.", BIB,,BIB,Born in Bradford,,Multicultural,https://borninbradford.nhs.uk/,BRI,,"Born in Bradford (BiB) started in 2007 as a response to the poor health outcomes for children in Bradford. Pregnant women were recruited when they attended the Bradford Royal Infirmary for their routine maternity care. Participants were asked to complete a questionnaire about different aspects of their lives including demographics information, physical and mental health and socio-economic information. Between 2007 and 2011, 12,453 pregnant women were recruited and 3,353 of their partners. Participants also gave permission for routine data linkage for themselves and their children. The BiB cohort is split between approximately 50% South Asian and 50% non-South Asian participants. @@ -24,15 +24,15 @@ From the findings of these studies, BiB have developed a range of additional res Information about the different data sets are available here: https://borninbradford.nhs.uk/research/documents-data/ -More information about how to access BiB data can be found here: https://borninbradford.nhs.uk/research/how-to-access-data/",,10.1093/ije/dys112,"10.14301/llcs.v4i2.221,10.1186/s12889-019-7222-2,10.1186/1471-2458-8-327",,"BiB receives core infrastructure funding from the Wellcome Trust (WT101597MA) and the National Institute for Health Research (NIHR) under its Applied Research Collaboration Yorkshire and Humber [NIHR200166]. Further support for genome-wide and multiple ‘omics measurements is from the UK Medical Research Council (G0600705), National Institute of Health Research (NF-SI-0611-10196), US National Institute of Health (R01 DK10324), and the European Research Council under the European Union’s Seventh Framework Programme (FP7/2007–2013) / ERC grant agreement no 669545. The recent follow-up of BiB participants was funded by a joint grant from the UK Medical Research Council and UK Economic and Social Science Research Council (MR/N024397/1) and a grant from the British Heart Foundation (CS/","Born in Bradford is only possible because of the enthusiasm and commitment of the children and parents in BiB. We are grateful to all the participants, health professionals, schools and researchers who have made Born in Bradford happen.", +More information about how to access BiB data can be found here: https://borninbradford.nhs.uk/research/how-to-access-data/",,https://doi.org/10.1093/ije/dys112,"https://doi.org/10.14301/llcs.v4i2.221,https://doi.org/10.1186/s12889-019-7222-2,https://doi.org/10.1186/1471-2458-8-327",,"BiB receives core infrastructure funding from the Wellcome Trust (WT101597MA) and the National Institute for Health Research (NIHR) under its Applied Research Collaboration Yorkshire and Humber [NIHR200166]. Further support for genome-wide and multiple ‘omics measurements is from the UK Medical Research Council (G0600705), National Institute of Health Research (NF-SI-0611-10196), US National Institute of Health (R01 DK10324), and the European Research Council under the European Union’s Seventh Framework Programme (FP7/2007–2013) / ERC grant agreement no 669545. The recent follow-up of BiB participants was funded by a joint grant from the UK Medical Research Council and UK Economic and Social Science Research Council (MR/N024397/1) and a grant from the British Heart Foundatio","Born in Bradford is only possible because of the enthusiasm and commitment of the children and parents in BiB. We are grateful to all the participants, health professionals, schools and researchers who have made Born in Bradford happen.", GenR,,GenR,The Generation R Study,,multi ethnic population-based prospective cohort study from fetal life onwards,https://generationr.nl/,EMC,,"The Generation R Study is a population-based prospective cohort study from fetal life until adulthood and is conducted in Rotterdam, the second largest city in the Netherlands. The study is designed to identify early environmental and genetic causes and causal pathways leading to normal and abnormal growth, development and health from fetal life, childhood and young adulthood. This multidisciplinary study focuses on several health outcomes including behaviour and cognition, body composition, eye development, growth, hearing, heart and vascular development, infectious disease and immunity, oral health and facial growth, respiratory health, allergy and skin disorders of children and their parents. Main exposures of interest include environmental, endocrine, genomic (genetic, epigenetic, microbiome), lifestyle related, nutritional and socio-demographic determinants. The Generation R study enrolled 9,153 mothers with a delivery date from April 2002 until January 2006, who participated with a total of 9,778 pregnancies in the study from which 9,901 children were expected to be born; the mothers gave birth to 9,749 live born children.",mariekewelten@gmail.com,"Population cohort,Birth cohort",,Longitudinal,"The design of Generation R has been described in detail previously, the following information was extracted from the most recent paper on design and cohort update 2017 (https://doi.org/10.1007/s10654-016-0224-9): The Generation R Study is a population-based prospective cohort study from fetal life until adulthood and is conducted in Rotterdam, the second largest city in the Netherlands. The study is designed to identify early environmental and genetic causes and causal pathways leading to normal and abnormal growth, development and health from fetal life, childhood and young adulthood. This multidisciplinary study focuses on several health outcomes including behaviour and cognition, body composition, eye development, growth, hearing, heart and vascular development, infectious disease and immunity, oral health and facial growth, respiratory health, allergy and skin disorders of children and their parents. Main exposures of interest include environmental, endocrine, genomic (genetic, epigenetic, microbiome), lifestyle related, nutritional and socio-demographic determinants. - Pregnant women with an expected delivery date between April 2002 and January 2006 living in Rotterdam were eligible for participation in the study. In total, 9778 mothers were enrolled in the study, who gave birth to 9749 live born children. Extensive assessments are performed in mothers, fathers and their children. Measurements were planned in early pregnancy (gestational age <18 weeks), mid pregnancy (gestational age 18–25 weeks) and late pregnancy (gestational age >25 weeks). The fathers were assessed once during the pregnancy of their partner. The children form a prenatally recruited birth cohort that will be followed at least until young adulthood. In the preschool period, which in the Netherlands refers to the period from birth until the age of 4 years, data collection was performed by a home-visit at the age of 3 months, and by repeated questionnaires and routine child health centers visits. Information from these routine visits obtained and used for the study. Additional detailed measurements of fetal and postnatal growth and development were conducted in a randomly selected subgroup of Dutch children and their parents at a gestational age of 32 weeks and postnatally at the ages of 1.5, 6, 14, 24, 36 and 48 months in a dedicated research center. + Pregnant women with an expected delivery date between April 2002 and January 2006 living in Rotterdam were eligible for participation in the study. In total, 9778 mothers were enrolled in the study, who gave birth to 9749 live born children. Extensive assessments are performed in mothers, fathers and their children. Measurements were planned in early pregnancy (gestational age <18 weeks), mid pregnancy (gestational age 18–25 weeks) and late pregnancy (gestational age >25 weeks). The fathers were assessed once during the pregnancy of their partner. The children form a prenatally recruited birth cohort that will be followed at least until young adulthood. In the preschool period, which in the Netherlands refers to the period from birth until the age of 4 years, data collection was performed by a home-visit at the age of 3 months, and by repeated questionnaires and routine child health centers visits. Information from these routine visitobtained and used for the study. Additional detailed measurements of fetal and postnatal growth and development were conducted in a randomly selected subgroup of Dutch children and their parents at a gestational age of 32 weeks and postnatally at the ages of 1.5, 6, 14, 24, 36 and 48 months in a dedicated research center. -Around the ages of 6 and 10 years all children and their parents were invited to visit our research center in the Erasmus MC-Sophia Children’s Hospital to participate in hands-on measurements, advanced imaging modalities, behavioural observations and biological sample collection. MRI scans of all participating children were made in order to image abdominal composition, brain, lungs, cardiovascular system, fat tissue, kidney, liver, and hip development. Furthermore, the parents received 6 questionnaires during this period. Children also received their own questionnaire around the age of 10. Information from municipal health services, schools and general practicionars has also been cleed. Response at baseline was 61%, and general follow-up rates until the age of 10 years were around 80%.",,"Retrospective,Prospective",,9901,,Netherlands (the),Rotterdam,"Prenatal,Infant (0-23 months),Newborn (0-1 months),Infants and toddlers (2-23 months),Child (2-12 years),Adolescent (13-17 years)",,,2001,,,,,Periodically,"Upon completion of follow-up, data will be cleaned for usage by the Generation R study group and then harmonized and uploaded for ECCN availability.",,,,"""not for profit, non commercial use only""","1. LifeCycle Project Research proposal has been discussed in the WP and distributed to all LifeCycle Project cohorts +Around the ages of 6 and 10 years all children and their parents were invited to visit our research center in the Erasmus MC-Sophia Children’s Hospital to participate in hands-on measurements, advanced imaging modalities, behavioural observations and biological sample collection. MRI scans of all participating children were made in order to image abdominal composition, brain, lungs, cardiovascular system, fat tissue, kidney, liver, and hip development. Furthermore, the parents received 6 questionnaires during this period. Children also received their own questionnaire around the age of 10. Information from municipal health services, schools and general practicionars has also been cleedResponse at baseline was 61%, and general follow-up rates until the age of 10 years were around 80%.",,"Retrospective,Prospective",,9901,,Netherlands (the),Rotterdam,"Prenatal,Infant (0-23 months),Newborn (0-1 months),Infants and toddlers (2-23 months),Child (2-12 years),Adolescent (13-17 years)",,,2001,,,,,Periodically,"Upon completion of follow-up, data will be cleaned for usage by the Generation R study group and then harmonized and uploaded for ECCN availability.",,,,"""not for profit, non commercial use only""","1. LifeCycle Project Research proposal has been discussed in the WP and distributed to all LifeCycle Project cohorts 2. The lead researcher sends a request to Generation R contact (primary contact for internal discussion: LifeCycle Project manager lifecycle@erasmusmc.nl). If it concerns another proposal than a LifeCycle proposal please state this. 3. The proposal is discussed in the Generation R Management Team by the involved Generation R PI 4. The involved PI notifies the lead researcher and the LifeCycle project manager of the MT outcome, with lifecycle@erasmusmc.nl in the cc. @@ -43,7 +43,7 @@ Around the ages of 6 and 10 years all children and their parents were invited to 9. The project manager finalizes the Data Access Process: - creates the Opal-views - emails the researcher the fully signed DUA, and the Opal-login details to access the data - - Informs Generation R contact that data access has been granted",,10.1007/s10654-006-9022-0,"https://doi.org/10.1007/s10654-010-9516-7,https://doi.org/10.1007/s10654-006-9022-0,https://doi.org/10.1111/j.1365-3016.2003.00521.x,https://doi.org/10.1007/s10654-007-9209-z,https://doi.org/10.1007/s10654-008-9309-4,https://doi.org/10.1007/s10654-012-9735-1,https://doi.org/10.1007/s10654-014-9980-6,https://doi.org/10.1007/s10654-016-0224-9,https://doi.org/10.1007/s10654-013-9768-0",,"The general design of the Generation R Study is made possible by financial support from the Erasmus + - Informs Generation R contact that data access has been granted",,https://doi.org/10.1007/s10654-006-9022-0,"https://doi.org/10.1007/s10654-010-9516-7,https://doi.org/10.1007/s10654-006-9022-0,https://doi.org/10.1111/j.1365-3016.2003.00521.x,https://doi.org/10.1007/s10654-007-9209-z,https://doi.org/10.1007/s10654-008-9309-4,https://doi.org/10.1007/s10654-012-9735-1,https://doi.org/10.1007/s10654-014-9980-6,https://doi.org/10.1007/s10654-016-0224-9,https://doi.org/10.1007/s10654-013-9768-0",,"The general design of the Generation R Study is made possible by financial support from the Erasmus MC, University Medical Center, Rotterdam, Erasmus University Rotterdam, Netherlands Organization for Health Research and Development (ZonMw), Netherlands Organisation for Scientific Research (NWO), Ministry of Health, Welfare and Sport and Ministry of Youth and @@ -64,74 +64,73 @@ Early childhood follow-up consisted of questionnaires at 1 month, 6 months and 1 - creates the Opal-views - emails the researcher the fully signed DUA, and the Opal-login details to access the data - Informs Generation R contact that data access has been granted",,,,,,, -Sepages,,SEPAGES,Suivi de l'Exposition à la Pollution Atmosphérique et Effet sur la s,Suivi de l'Exposition à la Pollution Atmosphérique et Effet sur la s,"Novel tools for integrating early-life environmental exposures and child health, biomarkers",https://cohorte-sepages.fr/en,INSERM,,"(1) characterize finely environmental exposures during pregnancy and early life ; (2) evaluate the impact of the environment on mother-child health, focusing on three main outcomes: growth, respiratory health and neurodevelopment; (3) explore possible underlying biological pathways, focusing on those mediated by epigenetic marks, immunologic and hormonal parameters and by the gut microbiota.",,Population cohort,,Longitudinal,,,Prospective,,484,480,France,Auvergne-Rhône-Alpes,"Prenatal,Infant (0-23 months),Newborn (0-1 months),Infants and toddlers (2-23 months),Child (2-12 years)",,,2014,,,,,Annually,,No,,,project specific restriction,Send a first request to Sarah Lyon-Caen (sarah.lyon-caen@univ-grenoble-alpes.fr),,10.3390/ijerph16203888,"10.1016/j.envpol.2021.117650,10.1016/j.envint.2020.105678,10.1016/j.envint.2021.106697,10.3390/ijerph16203888",,"The SEPAGES cohort was supported by the European Research Council (N°311765-E-DOHaD), the European Community’s Seventh Framework Programme (FP7/2007-206 - N°308333-892 HELIX), the European Union’s Horizon 2020 research and innovation programme (N° 874583 ATHLETE Project, N°825712 OBERON Project), the French Research Agency - ANR (PAPER project ANR-12-PDOC-0029-01, SHALCOH project ANR-14-CE21-0007, ANR-15-IDEX-02 and ANR-15-IDEX5, GUMME project ANR-18-CE36-005, ETAPE project ANR - EDeN project ANR -19-CE36-0003-01), the French Agency for Food, Environmental and Occupational Health & Safety - ANSES (CNAP project EST-2016-121, PENDORE project EST-2016-121, HyPAxE project EST-2019/1/039), the Plan Cancer (Canc’Air project), the French Cancer Research Foundation Association de Recherche sur le Cancer – ARC, the French Endowment Fund AGIR for chronic diseases – APMC (projects PRENAPAR and LCI-FOT), the French Endowment Fund for Respiratory Health, the French Fund – Fondation de France (CLIMATHES -3).","SEPAGES biospecimens are stored at Grenoble University Hospital(CHU-GA) biobank (bb-0033-00069); we would like to thank the whole CRB team, led by Pr. P. Mossuz and Mr. P. Lorimier, and in particular the technicians for the huge work of biospecimens processing and pooling: Mrs. W. Jayar and Mrs. L. Than, as well as Mr. G. Schummer. We thank the SEPAGES study group : E. Eyriey, A. Licinia, A. Vellement (Groupe Hospitalier Mutualiste, Grenoble), I. Pin, P. Hoffmann, E. Hullo, C. Llerena (Grenoble Alpes University Hospital, La Tronche), X. Morin (Clinique des Cèdres, Echirolles), A. Morlot (Clinique Belledonne, Saint-Martin d’Hères), J. Lepeule, S. Lyon-Caen, C. Philippat, I. Pin, J. Quentin, V. Siroux, R. Slama (Grenoble Alpes University, Inserm, CNRS, IAB). -We thank Mrs. A. Benlakhryfa, Mrs. L. Borges, Mr. Y. Gioria, clinical research assistants; Mrs. J. Giraud, Mrs. M. Marceau, Mrs. M-P. Martin, nurses; Mrs. E. Charvet, Mrs A. Putod, midwives; Mrs. M. Graca, Mrs. K.Gridel, Mrs. C. Pelini, Mrs Maïlys Barieldworkers; Mrs. A.Bossant, K. Guichardet, J-T Iltis A. Levanic, C.Martel, E. Quinteiro, S.Raffin neuropsychologists; the staff from Grenoble Center for Clinical Investigation (CIC): Prof. J.-L. Cracowski, Dr. C. Cracowski,Dr. E. Hodaj, Mrs. D. Abry, Mr. N. Gonnet and Mrs. A. Tournier. A warm thank you also to Dr. M. Althuser, Mr. S. Althuser, Dr. F. Camus-Chauvet, Mr. P. Dusonchet, Mrs. S. Dusonchet, Dr. L. Emery, Mrs. P.Fabbrizio, Prof. P. Hoffmann, Dr. D. Marchal André, Dr. X. Morin, Dr. E.Opoix, Dr. L. Pacteau, Dr. P. Rivoire, Mrs. A. Royannais, Dr. C.Tomasella, Dr. T. Tomasella, Dr. D. Tournadre, Mr. P. Viossat, Mrs. E.Volpi, Mrs. S. Rey, Dr. E. Warembourg and clinicians from Grenoble University Hospital for their support in the recruitment of the study volunteers. We also thank Mrs. A. Buchet, Mrs. SF. Caraby, Dr. J-N.Canonica, Mrs. J. Dujourdil, Dr. E. Eyriey, Prof. P. Hoffmann, Mrs. M.Jeannin, Mrs. A. Licina, Dr. X. Morin, Mrs. A. Nicolas, and all midwives from the four maternity wards of Grenole uran areas. SEPAGES data are stored thanks to Inserm RE-CO-NAI platform funded by Commissariat Général à l’Investissement, with the implication of Sophiede Visme (Inserm DSI). Many thanks to Dr. M.A. Charles, RE-CO-NAI coordinator, for her support. Finally, and importantly, we would like to express our sincere thanks to participants of the SEPA", -INMA,,INMA,INfancia y Medio Ambiente [Environment and Childhood] Project,,,https://www.proyectoinma.org,,,"INMA – INfancia y Medio Ambiente [Environment and Childhood] Project is a research network of several Spanish groups that created a project with the aim to study the paper of the more relevant environmental pollutants in the air, water and diet during the pregnancy and beginning of life, and their effects in the growth and developm",,Population cohort,,Longitudinal,,,Prospective,,2242,1600,Spain,"Gipuzkoa,Barcelona,Valencia","Prenatal,Newborn (0-1 months),Infants and toddlers (2-23 months),Child (2-12 years),Adolescent (13-17 years)",,,2003,2008,,,,Continuous,Released once data collected and cleaned,No,,,,,,10.1093/ije/dyr054,,,,, -EDEN,,EDEN,Study on the pre and early postnatal determinants of child health and development,,,http://eden.vjf.inserm.fr/en/,INSERM,,"The EDEN study is a French bicentric generalist cohort study (Heude et al. IJE 2016). Pregnant women were invited to participate during their prenatal visit, before the 24th week of amenorrhea, in the obstetrics and gynecology departments of the Poitiers and Nancy university hospitals. Recruitment took place from February 2003 to January 2006. The exclusion criteria were: twin pregnancy, diagnosis of insulin-dependent diabetes, inability to speak and read French, intention to move from the region within three years of inclusion . Among the women invited, 53% agreed to participate (n = 2,002). Detailed information on phenotypes and exposures were collected through questionnaires from pregnancy until 8 years and through clinical examinations of the mother (at 24 weeks of amenorrhoea, at delivery and 5–6 years after delivery) and the child (at birth and 1, 3 and 5–6 years, including cognitive assessments at 3 and 5–6 years). At 5–6 years, 1255 children were st",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,2002,,,,,,,2003,2006,,,,Continuous,,,,health or medical or biomedical research,,"- Submission of the project proposal to the cohort PI (Barbara Heude) +Sepages,,SEPAGES,Suivi de l'Exposition à la Pollution Atmosphérique et Effet sur la,Suivi de l'Exposition à la Pollution Atmosphérique et Effet sur la,"Novel tools for integrating early-life environmental exposures and child health, biomarkers",https://cohorte-sepages.fr/en,INSERM,,"(1) characterize finely environmental exposures during pregnancy and early life ; (2) evaluate the impact of the environment on mother-child health, focusing on three main outcomes: growth, respiratory health and neurodevelopment; (3) explore possible underlying biological pathways, focusing on those mediated by epigenetic marks, immunologic and hormonal parameters and by the gut microbiota.",,Population cohort,,Longitudinal,,,Prospective,,484,480,France,Auvergne-Rhône-Alpes,"Prenatal,Infant (0-23 months),Newborn (0-1 months),Infants and toddlers (2-23 months),Child (2-12 years)",,,2014,,,,,Annually,,No,,,project specific restriction,Send a first request to Sarah Lyon-Caen (sarah.lyon-caen@univ-grenoble-alpes.fr),,https://doi.org/10.3390/ijerph16203888,"https://doi.org/10.1016/j.envpol.2021.117650,https://doi.org/10.1016/j.envint.2020.105678,https://doi.org/10.1016/j.envint.2021.106697,https://doi.org/10.3390/ijerph16203888",,"The SEPAGES cohort was supported by the European Research Council (N°311765-E-DOHaD), the European Community’s Seventh Framework Programme (FP7/2007-206 - N°308333-892 HELIX), the European Union’s Horizon 2020 research and innovation programme (N° 874583 ATHLETE Project, N°825712 OBERON Project), the French Research Agency - ANR (PAPER project ANR-12-PDOC-0029-01, SHALCOH project ANR-14-CE21-0007, ANR-15-IDEX-02 and ANR-15-IDEX5, GUMME project ANR-18-CE36-005, ETAPE project ANR - EDeN project ANR -19-CE36-0003-01), the French Agency for Food, Environmental and Occupational Health & Safety - ANSES (CNAP project EST-2016-121, PENDORE project EST-2016-121, HyPAxE project EST-2019/1/039), the Plan Cancer (Canc’Air project), the French Cancer Research Foundation Association de Recherche sur le Cancer – ARC, the French Endowment Fund AGIR for chronic diseases – APMC (projects PRENAPAR and LCI-FOT), the French Endowment Fund for Respiratory Health, the French Fund – Fondation de France ","SEPAGES biospecimens are stored at Grenoble University Hospital(CHU-GA) biobank (bb-0033-00069); we would like to thank the whole CRB team, led by Pr. P. Mossuz and Mr. P. Lorimier, and in particular the technicians for the huge work of biospecimens processing and pooling: Mrs. W. Jayar and Mrs. L. Than, as well as Mr. G. Schummer. We thank the SEPAGES study group : E. Eyriey, A. Licinia, A. Vellement (Groupe Hospitalier Mutualiste, Grenoble), I. Pin, P. Hoffmann, E. Hullo, C. Llerena (Grenoble Alpes University Hospital, La Tronche), X. Morin (Clinique des Cèdres, Echirolles), A. Morlot (Clinique Belledonne, Saint-Martin d’Hères), J. Lepeule, S. Lyon-Caen, C. Philippat, I. Pin, J. Quentin, V. Siroux, R. Slama (Grenoble Alpes University, Inserm, CNRS, IAB). +We thank Mrs. A. Benlakhryfa, Mrs. L. Borges, Mr. Y. Gioria, clinical research assistants; Mrs. J. Giraud, Mrs. M. Marceau, Mrs. M-P. Martin, nurses; Mrs. E. Charvet, Mrs A. Putod, midwives; Mrs. M. Graca, Mrs. K.Gridel, Mrs. C. Pelini, Mrs Maïlys Barorkers; Mrs. A.Bossant, K. Guichardet, J-T Iltis A. Levanic, C.Martel, E. Quinteiro, S.Raffin neuropsychologists; the staff from Grenoble Center for Clinical Investigation (CIC): Prof. J.-L. Cracowski, Dr. C. Cracowski,Dr. E. Hodaj, Mrs. D. Abry, Mr. N. Gonnet and Mrs. A. Tournier. A warm thank you also to Dr. M. Althuser, Mr. S. Althuser, Dr. F. Camus-Chauvet, Mr. P. Dusonchet, Mrs. S. Dusonchet, Dr. L. Emery, Mrs. P.Fabbrizio, Prof. P. Hoffmann, Dr. D. Marchal André, Dr. X. Morin, Dr. E.Opoix, Dr. L. Pacteau, Dr. P. Rivoire, Mrs. A. Royannais, Dr. C.Tomasella, Dr. T. Tomasella, Dr. D. Tournadre, Mr. P. Viossat, Mrs. E.Volpi, Mrs. S. Rey, Dr. E. Warembourg and clinicians from Grenoble University Hospital for their support in the recruitment of the study volunteers. We also thank Mrs. A. Buchet, Mrs. SF. Caraby, Dr. J-N.Canonica, Mrs. J. Dujourdil, Dr. E. Eyriey, Prof. P. Hoffmann, Mrs. M.Jeannin, Mrs. A. Licina, Dr. X. Morin, Mrs. A. Nicolas, and all midwives from the four maternity wards of Grenole uran aras. SEPAGES data are stored thanks to Inserm RE-CO-NAI platform funded by Commissariat Général à l’Investissement, with the implication of Sophiede Visme (Inserm DSI). Many thanks to Dr. M.A. Charles, RE-CO-NAI coordinator, for her support. Finally, and importantly, we would like to express our sincere thanks to participants of the", +INMA,,INMA,INfancia y Medio Ambiente [Environment and Childhood] Project,,,https://www.proyectoinma.org,,,"INMA – INfancia y Medio Ambiente [Environment and Childhood] Project is a research network of several Spanish groups that created a project with the aim to study the paper of the more relevant environmental pollutants in the air, water and diet during the pregnancy and beginning of life, and their effects in the growth and develo",,Population cohort,,Longitudinal,,,Prospective,,2242,1600,Spain,"Gipuzkoa,Barcelona,Valencia","Prenatal,Newborn (0-1 months),Infants and toddlers (2-23 months),Child (2-12 years),Adolescent (13-17 years)",,,2003,2008,,,,Continuous,Released once data collected and cleaned,No,,,,,,https://doi.org/10.1093/ije/dyr054,,,,, +EDEN,,EDEN,Study on the pre and early postnatal determinants of child health and development,,,http://eden.vjf.inserm.fr/en/,INSERM,,"The EDEN study is a French bicentric generalist cohort study (Heude et al. IJE 2016). Pregnant women were invited to participate during their prenatal visit, before the 24th week of amenorrhea, in the obstetrics and gynecology departments of the Poitiers and Nancy university hospitals. Recruitment took place from February 2003 to January 2006. The exclusion criteria were: twin pregnancy, diagnosis of insulin-dependent diabetes, inability to speak and read French, intention to move from the region within three years of inclusion . Among the women invited, 53% agreed to participate (n = 2,002). Detailed information on phenotypes and exposures were collected through questionnaires from pregnancy until 8 years and through clinical examinations of the mother (at 24 weeks of amenorrhoea, at delivery and 5–6 years after delivery) and the child (at birth and 1, 3 and 5–6 years, including cognitive assessments at 3 and 5–6 years). At 5–6 years, 1255 children",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,2002,,,,,,,2003,2006,,,,Continuous,,,,health or medical or biomedical research,,"- Submission of the project proposal to the cohort PI (Barbara Heude) - Approval of the project by the EDEN steering committee (takes 1 month) ​- Signature of a data access agreement (DAA) -- Data made available for analysis by the data manager (credentials will be sent to the person requesting access to the d",,10.1093/ije/dyv151,,,"The study relied on many different sources of funding including Inserm, University Paris 11, the French Medical Research Foundation and the National Research Agency and also from national and European programs and project grants. A more comprehensive list of funders can be found on the EDEN website, along with other information","The authors are extremely grateful to all the families who took part in this study, the midwives and psychologists for recruiting and following +- Data made available for analysis by the data manager (credentials will be sent to the person requesting access to the",,https://doi.org/10.1093/ije/dyv151,,,"The study relied on many different sources of funding including Inserm, University Paris 11, the French Medical Research Foundation and the National Research Agency and also from national and European programs and project grants. A more comprehensive list of funders can be found on the EDEN website, along with other information","The authors are extremely grateful to all the families who took part in this study, the midwives and psychologists for recruiting and following them, and the whole EDEN team, including research scientists, engineers, technicians and managers.", -ELFE,,ELFE,"Etude Longitudinale Française depuis l'Enfan"" (French longitudinal study of children)",,National,https://www.elfe-france.fr/en/,INSERM,,"ELFE is the first French national birth cohort. Its objective is to study determinants of the development, health and socialization of children from birth to adulthood through a multidisciplinary approach. A total of 18 329 children were recruited at birth in a random sample of maternity units in metropolitan France during 25 selected days of 2011 spread over the year. Participation rates at the age 2-month, 1- and 2-year and 3.5-year parental interviews were 92%, 86%, 82% and 80%, respectively, of contacted participants. The ELFE has an open-data policy after an 18-month exclusivity period following each release of new data. The data-access policy, study protocols, questionnaires and data catalogue can be found online: [https://www.ELFE-france.fr/en/].",,Birth cohort,,Longitudinal,,,"Retrospective,Prospective",,18329,,France,,,,,2011,2011,,,,Continuous,,yes,,health or medical or biomedical research,,"- Submission of the project proposal to the cohort PI (Marie-Aline Charles) +ELFE,,ELFE,"Etude Longitudinale Française depuis l'Enfa"" (French longitudinal study of children)",,National,https://www.elfe-france.fr/en/,INSERM,,"ELFE is the first French national birth cohort. Its objective is to study determinants of the development, health and socialization of children from birth to adulthood through a multidisciplinary approach. A total of 18 329 children were recruited at birth in a random sample of maternity units in metropolitan France during 25 selected days of 2011 spread over the year. Participation rates at the age 2-month, 1- and 2-year and 3.5-year parental interviews were 92%, 86%, 82% and 80%, respectively, of contacted participants. The ELFE has an open-data policy after an 18-month exclusivity period following each release of new data. The data-access policy, study protocols, questionnaires and data catalogue can be found online: [https://www.ELFE-france.fr/en/].",,Birth cohort,,Longitudinal,,,"Retrospective,Prospective",,18329,,France,,,,,2011,2011,,,,Continuous,,yes,,health or medical or biomedical research,,"- Submission of the project proposal to the cohort PI (Marie-Aline Charles) - Approval of the project by Elfe data access committee (takes 2 weeks)​ - Signature of a data access agreement -- Data made available for analysis by Lucinda (credentials will be sent to the person requesting access to the d",,10.1093/ije/dyz227,,,"The Ministries of the Environment, Health and Research provided initial funding for the ELFE cohort. +- Data made available for analysis by Lucinda (credentials will be sent to the person requesting access to the",,https://doi.org/10.1093/ije/dyz227,,,"The Ministries of the Environment, Health and Research provided initial funding for the ELFE cohort. The funding for the first 5 years of childhood follow-up was obtained from the national ‘Investment for the Future’ -research funding program for a joint project bringing together the ELFE and Epipage 2 ",, +research funding program for a joint project bringing together the ELFE and Epipag",, NBC1967-1976,,,NBC1967-1976,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, NFBC1986,,NFBC1986,Northern Finland Birth Cohort 1986,,,https://www.oulu.fi/nfbc,UOULU,,"NFBC1986 is a longitudinal one-year birth cohort study from an unselected population. The cohort included all the mothers (N=9362) with children whose expected date of birth felt between July 1st 1985 - June 30th 1986 in the two northernmost provinces on Finland (Oulu and Lapland). A small percentage of the births occurred towards the end of June 1985 and begin of July 1986. -The number of deliveries in the cohort was 9362, which was 99% of all the deliveries taking place in the target period of the cohort. Altogether 9479 children were born into the cohort, 9432 of them live born. The original data have been supplemented by data collected with postal questionnaires at the ages of 7, 8 and 16 years and various hospital records and statistical register data. The latest data collection with clinical examinations and questionnaires was at the age of 33 years.",,Birth cohort,,Longitudinal,,,Prospective,,9479,,Finland,"Lapland,North Ostrobothnia","Prenatal,Infant (0-23 months),Newborn (0-1 months),Infants and toddlers (2-23 months),Child (2-12 years),Adolescent (13-17 years),Young adult (18-24 years),Adult (25-44 years)",,,1985,,,,,Closed dataset,,yes,,health or medical or biomedical research,,https://www.oulu.fi/nfbc/materialrequest,,http://urn.fi/urn:nbn:fi:att:f5c10eef-3d25-4bd0-beb8-f2d59df95b8e,"10.1111/j.1365-3016.1997.tb00007.x,10.1111/j.1471-0528.1993.tb12971.x",,"EU QLG1-CT-2000-01643 (EUROBLCS) Grant no. E51560, NorFA Grant no. 731, 20056, 30167, USA / NIH 2000 G DF682 Grant no. 50945",We thank all cohort members and researchers who have participated in the study. We also wish acknowledge the work of the NFBC project center., -PSYCONN,,PSYCONN,Psychosis Cohort Northern Netherlands,Psychose Cohort Noord Nederland,schizophrenia schizofrenie psychotic psychotisch psychose databank Nederland PHAMOUS EPO PROGRS PROGR-S RQMIS RQ-MIS,https://ipec-project.com,"GGZ Drenthe,GGZ Friesland,Lentis Research,UMCG","GGZ Drenthe,GGZ Friesland,Lentis Research,UMCG","PSYCONN (=Psychosis Cohort Northern Netherlands) is a combination of 3 datasets from the northern part of the Netherlands; PROGR-S (=Psychosis Recent Onset GRoningen Survey), PHAMOUS (=Pharmacotherapy Outcome and Monitoring Survey) and RQ-MIS (=RoQua Management Information System). Also, see https://umcgresearchdatacatalogue.nl/UMCG/ssr-catalogue/cohorts/MindLines. PROGR-S, initiated in 1997 without an age restriction, collects diagnostic measurements of patients with a suspected recent-onset psychotic episode or recurrent psychotic episode not diagnosed as such before. PHAMOUS, initiated in 2006, contains follow-up data of patients ≥18 years of age with a psychotic disorder who receive care from mental health care institutions in the northern part of the Netherlands (University Medical Center Groningen (UMCG)/University Center for Psychiatry (UCP), Lentis, GGZ Drenthe or GGZ Friesland). PROGR-S and PHAMOUS are both ongoing naturalistic prospective cohort studies. PHAMOUS participants will only be included they are also enrolled in PROGR-S. RQ-MIS contains health care consumption, socio-demographic, and diagnosis data. PSYCONN baseline data is collected from PROGR-S and RQ-MIS. PSYCONN outcome data is collected from PHAMOUS, RQ-MIS, and an additional medical records file search.",ipec@umcg.nl,"Clinical cohort,Registry",,Longitudinal,,,,,1000,,Netherlands (the),"Groningen,Friesland,Drenthe","Adult (18+ years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years),Aged (65-79 years),Aged (80+ years)",,,1998,2018,,,,Periodically,,,UMCG,,,"IPEC set up a virtual databank and implemented a technical infrastructure for remote federated analysis of individual-level data of psychosis patients. With this platform, individual patient data remains stored on local servers of participating institutions and individual patient data will not be transported to one central site. +The number of deliveries in the cohort was 9362, which was 99% of all the deliveries taking place in the target period of the cohort. Altogether 9479 children were born into the cohort, 9432 of them live born. The original data have been supplemented by data collected with postal questionnaires at the ages of 7, 8 and 16 years and various hospital records and statistical register data. The latest data collection with clinical examinations and questionnaires was at the age of 33 years.",,Birth cohort,,Longitudinal,,,Prospective,,9479,,Finland,"Lapland,North Ostrobothnia","Prenatal,Infant (0-23 months),Newborn (0-1 months),Infants and toddlers (2-23 months),Child (2-12 years),Adolescent (13-17 years),Young adult (18-24 years),Adult (25-44 years)",,,1985,,,,,Closed dataset,,yes,,health or medical or biomedical research,,https://www.oulu.fi/nfbc/materialrequest,,,"https://doi.org/10.1111/j.1365-3016.1997.tb00007.x,https://doi.org/10.1111/j.1471-0528.1993.tb12971.x",,"EU QLG1-CT-2000-01643 (EUROBLCS) Grant no. E51560, NorFA Grant no. 731, 20056, 30167, USA / NIH 2000 G DF682 Grant no. 50945",We thank all cohort members and researchers who have participated in the study. We also wish acknowledge the work of the NFBC project center., +PSYCONN,,PSYCONN,Psychosis Cohort Northern Netherlands,Psychose Cohort Noord Nederland,schizophrenia schizofrenie psychotic psychotisch psychose databank Nederland PHAMOUS EPO PROGRS PROGR-S RQMIS RQ-MIS,https://ipec-project.com,"GGZ Drenthe,GGZ Friesland,Lentis Research,UMCG","GGZ Drenthe,GGZ Friesland,Lentis Research,UMCG","PSYCONN (=Psychosis Cohort Northern Netherlands) is a combination of 3 datasets from the northern part of the Netherlands; PROGR-S (=Psychosis Recent Onset GRoningen Survey), PHAMOUS (=Pharmacotherapy Outcome and Monitoring Survey) and RQ-MIS (=RoQua Management Information System). Also, see https://umcgresearchdatacatalogue.nl/UMCG/ssr-catalogue/cohorts/MindLines. PROGR-S, initiated in 1997 without an age restriction, collects diagnostic measurements of patients with a suspected recent-onset psychotic episode or recurrent psychotic episode not diagnosed as such before. PHAMOUS, initiated in 2006, contains follow-up data of patients ≥18 years of age with a psychotic disorder who receive care from mental health care institutions in the northern part of the Netherlands (University Medical Center Groningen (UMCG)/University Center for Psychiatry (UCP), Lentis, GGZ Drenthe or GGZ Friesland). PROGR-S and PHAMOUS are both ongoing naturalistic prospective cohort studies. PHAMOUS participants will only be included hey are also enrolled in PROGR-S. RQ-MIS contains health care consumption, socio-demographic, and diagnosis data. PSYCONN baseline data is collected from PROGR-S and RQ-MIS. PSYCONN outcome data is collected from PHAMOUS, RQ-MIS, and an additional medical records file search.",ipec@umcg.nl,"Clinical cohort,Registry",,Longitudinal,,,,,1000,,Netherlands (the),"Groningen,Friesland,Drenthe","Adult (18+ years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years),Aged (65-79 years),Aged (80+ years)",,,1998,2018,,,,Periodically,,,UMCG,,,"IPEC set up a virtual databank and implemented a technical infrastructure for remote federated analysis of individual-level data of psychosis patients. With this platform, individual patient data remains stored on local servers of participating institutions and individual patient data will not be transported to one central site. Contact IPEC via ipec@umcg.nl to receive more information about access to the virtual databank.",,,,,"IPEC was supported by the 2020 SIRS Research Harmonization Group Award which included a grant in the amount of $5000 USD to Prof. Wim Veling and Prof. Craig Morgan; an internal MD-PhD research grant of the University Medical Center Groningen (grant number 18-41) to Vera Brink, MSc; a grant in the amount of 400,000 DKK of the Capital Regions Research council to Dr Nikolai Albert; and the participating institutions (Lentis Psychiatric Institute, GGZ Drenthe, GGZ Friesland, University Center Psychiatry of University Medical Center Groningen, Bispebjerg hospital, Psychiatric Hospital Risskov, and Psychiatric Centre Copenhagen).","IPEC wishes to acknowledge the services of all the study participants, staff, research groups, and institutions collaborating in IPEC, and of MOLGENIS and RoQua, who helped build and maintain the technical infrastructure.", CRC-Cohort,,CRC-Cohort,Colorectal Cancer Cohort,,,,BBMRI-ERIC,,"The colorectal cancer cohort (CRC-Cohort) was developed by BBMRI-ERIC, its National Nodes and BBMRI-ERIC partner biobanks within the EU-funded project ADOPT BBMRI-ERIC (H2020) as a use case for piloting access to European biobanks. It became a permanent asset of the BBMRI-ERIC infrastructure after the end of the ADOPT project. The CRC-Cohort collection is a joint long-term European endeavor, which enables existing, well-established biobanks to connect with BBMRI-ERIC and obtain increased recognition and visibility along with new users and data.",petr.holub@bbmri-eric.eu,Biobank,,Cross-sectional,,,Retrospective,,10500,,,,,,,,,,,,,,,,"general research use,health or medical or biomedical research,disease specific research",,,,,,,,, -RS3,,Rotterdam Study third cohort,"Rotterdam Study, third cohort",,Biomarkers · Cancer and related diseases · Cardiovascular diseases · Cohort study · Dermatological diseases · Endocrine diseases · Epidemiologic methods · Genetic and molecular epidemiology · Nutrition and lifestyle epidemiology · Liver diseases · Neurological diseases · Oncology · Ophthalmic diseases · Otolaryngological diseases · Pharmacoepidemiology · Population imaging · Renal dis,https://www.ergo-onderzoek.nl/,EMC,,"The Rotterdam Study is an ongoing prospective cohort study that started in 1990 in the city of Rotterdam, The Netherlands. The study aims to unravel etiology, preclinical course, natural history and potential targets for intervention for chronic diseases in mid-life and late-life.",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,6057,3932,Netherlands (the),Rotterdam,"Middle-aged (45-64 years),Aged (65+ years)",,,2006,,,,,Continuous,Once a follow-up is completed and data have been processed.,no,,health or medical or biomedical research,,,,10.1007/s10654-020-00640-5,"10.1007/s10654-009-9386-z,10.1007/BF00145007,10.1007/s10654-007-9199-x,10.1007/s10654-011-9610-5,10.1007/s10654-013-9866-z,10.1007/s10654-015-0082-x,10.1007/s10654-017-0321-4",,"The Rotterdam Study is supported by the Erasmus MC University Medical Center and Erasmus University Rotterdam; The Netherlands Organisation for Scientifc Research (NWO); The Netherlands Organisation for Health Research and Development (ZonMw); the Research Institute for Diseases in the Elderly (RIDE); The Netherlands Genomics Initiative (NGI); the Ministry of Education, Culture and Science; the Ministry of Health, Welfare and Sports; the European Commission (DG XII); and the Municipality of Rotterdam.","The contribution of inhabitants, general practitioners and pharmacists of the Ommoord district to the Rotterdam Study is gratefully acknowledged. The contributions to the Management Team specifcally and the Rotterdam Study at large of the following persons are pivotal to the daily operations of the study and highly appreciated: Jacobus Lubbe, Gabriëlle Bakker, Eric Neeleman, Jeannette Vergeer, Anneke Korving, Pauline van Wijngaarden, Jolande Verkroost – van Heemst, Silvan Licher, Isabel van de", +RS3,,Rotterdam Study third cohort,"Rotterdam Study, third cohort",,Biomarkers · Cancer and related diseases · Cardiovascular diseases · Cohort study · Dermatological diseases · Endocrine diseases · Epidemiologic methods · Genetic and molecular epidemiology · Nutrition and lifestyle epidemiology · Liver diseases · Neurological diseases · Oncology · Ophthalmic diseases · Otolaryngological diseases · Pharmacoepidemiology · Population imag,https://www.ergo-onderzoek.nl/,EMC,,"The Rotterdam Study is an ongoing prospective cohort study that started in 1990 in the city of Rotterdam, The Netherlands. The study aims to unravel etiology, preclinical course, natural history and potential targets for intervention for chronic diseases in mid-life and late-life.",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,6057,3932,Netherlands (the),Rotterdam,"Middle-aged (45-64 years),Aged (65+ years)",,,2006,,,,,Continuous,Once a follow-up is completed and data have been processed.,no,,health or medical or biomedical research,,,,https://doi.org/10.1007/s10654-020-00640-5,"https://doi.org/10.1007/s10654-009-9386-z,https://doi.org/10.1007/BF00145007,https://doi.org/10.1007/s10654-007-9199-x,https://doi.org/10.1007/s10654-011-9610-5,https://doi.org/10.1007/s10654-013-9866-z,https://doi.org/10.1007/s10654-015-0082-x,https://doi.org/10.1007/s10654-017-0321-4",,"The Rotterdam Study is supported by the Erasmus MC University Medical Center and Erasmus University Rotterdam; The Netherlands Organisation for Scientifc Research (NWO); The Netherlands Organisation for Health Research and Development (ZonMw); the Research Institute for Diseases in the Elderly (RIDE); The Netherlands Genomics Initiative (NGI); the Ministry of Education, Culture and Science; the Ministry of Health, Welfare and Sports; the European Commission (DG XII); and the Municipality of Rotterdam.","The contribution of inhabitants, general practitioners and pharmacists of the Ommoord district to the Rotterdam Study is gratefully acknowledged. The contributions to the Management Team specifcally and the Rotterdam Study at large of the following persons are pivotal to the daily operations of the study and highly appreciated: Jacobus Lubbe, Gabriëlle Bakker, Eric Neeleman, Jeannette Vergeer, Anneke Korving, Pauline van Wijngaarden, Jolande Verkroost – van Heemst, Silvan Licher, Isabel van", NFBC1966 G0,,NFBC1966 G0,Parents of NFBC1966,,,,UOULU,,,,Population cohort,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, FinnGeDi,,FinnGeDi,Finnish Gestational Diabetes Study,,,,UOULU,,,,Clinical cohort,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -MoBa,,MoBa,"The Norwegian Mother, Father and Child Cohort Study",,,https://www.fhi.no/en/studies/moba/,NIPH,,"MoBa is a Norwegian population-based longitudinal cohort study of around 114,500 children, 95,000 mothers and 75,000 fathers. Recruitment of pregnant women attending routine ultrasound examination took place between 1999 and 2009, and MoBa has since grown to become one of the largest population studies in the world. Baseline questionnaires were completed at around 15 weeks’ gestation, and two further prenatal questionnaires were administered at around 17-22 and 30 weeks’ gestation. In addition, fathers completed a questionnaire at around 15’ weeks gestation. Follow-up questionnaires after birth were completed at 6 months, 18 months, 36 months, 5 years, 7 years, and 8 years of age, and data are currently being collected through questionnaires for children aged 13 and 14 years. The purpose of MoBa is to improve our knowledge of diseases and health, by estimating exposure-outcome associations among children and their parents from preconception and pregnanc",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,114000,,Norway,,"Prenatal,Newborn (0-1 months),Child (2-12 years),Adolescent (13-17 years)",,,1999,2008,,,,Continuous,Continuously; the most recent release is version 12,Data can be linked to the medical birth registry.,,,,,,10.1093/ije/dyl170,https://doi.org/10.1093/ije/dyw029,,"The Norwegian Mother, Father and Child Cohort Study is supported by the Norwegian Ministry of Health and Care Services and the Ministry of Education and Research.",The authors are grateful to all the participating families in Norway who take part in this on-going cohort study., +MoBa,,MoBa,"The Norwegian Mother, Father and Child Cohort Study",,,https://www.fhi.no/en/studies/moba/,NIPH,,"MoBa is a Norwegian population-based longitudinal cohort study of around 114,500 children, 95,000 mothers and 75,000 fathers. Recruitment of pregnant women attending routine ultrasound examination took place between 1999 and 2009, and MoBa has since grown to become one of the largest population studies in the world. Baseline questionnaires were completed at around 15 weeks’ gestation, and two further prenatal questionnaires were administered at around 17-22 and 30 weeks’ gestation. In addition, fathers completed a questionnaire at around 15’ weeks gestation. Follow-up questionnaires after birth were completed at 6 months, 18 months, 36 months, 5 years, 7 years, and 8 years of age, and data are currently being collected through questionnaires for children aged 13 and 14 years. The purpose of MoBa is to improve our knowledge of diseases and health, by estimating exposure-outcome associations among children and their parents from preconception and pr",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,114000,,Norway,,"Prenatal,Newborn (0-1 months),Child (2-12 years),Adolescent (13-17 years)",,,1999,2008,,,,Continuous,Continuously; the most recent release is version 12,Data can be linked to the medical birth registry.,,,,,,https://doi.org/10.1093/ije/dyl170,,,"The Norwegian Mother, Father and Child Cohort Study is supported by the Norwegian Ministry of Health and Care Services and the Ministry of Education and Research.",The authors are grateful to all the participating families in Norway who take part in this on-going cohort study., GECKO,,GECKO,The Groningen Expert Center for Kids with Obesity Drenthe cohort,,,https://www.rug.nl/research/portal/datasets/gecko-drenthe-cohort(3e14f7ca-a55d-41d9-9803-19302aa4ea87).html,,,,,Clinical cohort,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, ALSPAC,,ALSPAC,Avon Longitudinal Study of Parents and Children,,,http://www.bristol.ac.uk/alspac/,UOB,,"Based at the University of Bristol, the Avon Longitudinal Study of Parents and Children (ALSPAC), also known as Children of the 90s, is a world-leading birth cohort study. Between April 1991 and December 1992 we recruited more than 14,000 pregnant women into the study and these women (some of whom had two pregnancies or multiple births during the recruitment period), the children arising from the pregnancy, and their partners have been followed up intensively over two decades. -We are the most detailed study of its kind in the world and we provide the international research community with a rich resource for the study of the environmental and genetic factors that affect a person’s health and development. Through our research we aim to inform policy and practices that will provide a better life for future genera",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,14541,,United Kingdom of Great Britain and Northern Ireland (the),,,,,1990,1992,,,,Closed dataset,Released once data collected and cleaned,,,,"""not for profit, non commercial use only""",,,10.1093/ije/dys064,10.1093/ije/dys066,,"The UK Medical Research Council and Wellcome (Grant ref: 217065/Z/19/Z) and the University of Bristol provide core support for ALSPAC. This publication is the work of the authors and will serve as guarantors for the contents of this paper.? +We are the most detailed study of its kind in the world and we provide the international research community with a rich resource for the study of the environmental and genetic factors that affect a person’s health and development. Through our research we aim to inform policy and practices that will provide a better life for future gene",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,14541,,United Kingdom of Great Britain and Northern Ireland (the),,,,,1990,1992,,,,Closed dataset,Released once data collected and cleaned,,,,"""not for profit, non commercial use only""",,,https://doi.org/10.1093/ije/dys064,https://doi.org/10.1093/ije/dys066,,"The UK Medical Research Council and Wellcome (Grant ref: 217065/Z/19/Z) and the University of Bristol provide core support for ALSPAC. This publication is the work of the authors and will serve as guarantors for the contents of this paper.? In addition, you are expected to acknowledge the grant(s) which supported the collection of the primary exposure(s) and outcome(s) used in your study and any other grants in the checklist, which are pertinent to your study. The following sentences should be included with the above section: ?A comprehensive list of grants funding (PDF, 330KB) is available on the ALSPAC website. This research was specifically funded by .","We are extremely grateful to all the families who took part in this study, the midwives for their help in recruiting them, and the whole ALSPAC team, which includes interviewers, computer and laboratory technicians, clerical workers, research scientists, volunteers, managers, receptionists and nurses.", CELSPAC-TNG,,CELSPAC-TNG,Central European Longitudinal Study of Parents and Children: The Next Generation,,"EXPOSOME, Central Europe",https://www.celspac.cz/tng,RECETOX,,This prospective birth cohort is designed to follow up 7000 children from their prenatal period to adolescence with the aim of assessing EXPOSOME factors potentially affecting children health.,,Birth cohort,,Longitudinal,,,"Retrospective,Prospective",,950,900,Czechia,,,,,2018,,,,,Continuous,,,,health or medical or biomedical research,,"If you are interested in using available data from the ELSPAC.CZ database, please follow these steps: 1) Send your request to info@celspac.cz. 2) You will be contacted by the project manager who will discuss and, if necessary, work up your project plan. 3) The ELSPAC Executive Council will assess the proposed project plans with regard to their quality and feasibility and will recommend the selected ones to be carried out. 4) If your request is approved, a contract will be concluded between the Faculty of Science at Masaryk University and your institution. 5) Data in the agreed form and extent will be provided based on this contract.",,,,,"The RECETOX Research Infrastructure (TNG is included) is supported by the Ministry of Education, Youth and Sports of the Czech Republic (LM2018121)",The authors are grateful to all participants of the TNG cohort., -RS2,,Rotterdam Study second cohort,"Rotterdam Study, second cohort",,Biomarkers · Cancer and related diseases · Cardiovascular diseases · Cohort study · Dermatological diseases · Endocrine diseases · Epidemiologic methods · Genetic and molecular epidemiology · Nutrition and lifestyle epidemiology · Liver diseases · Neurological diseases · Oncology · Ophthalmic diseases · Otolaryngological diseases · Pharmacoepidemiology · Population imaging · Renal dis,https://www.ergo-onderzoek.nl/,EMC,,"The Rotterdam Study is an ongoing prospective cohort study that started in 1990 in the city of Rotterdam, The Netherlands. The study aims to unravel etiology, preclinical course, natural history and potential targets for intervention for chronic diseases in mid-life and late-life.",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,4472,3011,Netherlands (the),Rotterdam,"Middle-aged (45-64 years),Aged (65+ years)",,,2000,,,,,Continuous,Once a follow-up is completed and data have been processed.,no,,health or medical or biomedical research,,,,10.1007/s10654-020-00640-5,"10.1007/s10654-009-9386-z,10.1007/BF00145007,10.1007/s10654-007-9199-x,10.1007/s10654-011-9610-5,10.1007/s10654-013-9866-z,10.1007/s10654-015-0082-x,10.1007/s10654-017-0321-4",,"The Rotterdam Study is supported by the Erasmus MC University Medical Center and Erasmus University Rotterdam; The Netherlands Organisation for Scientifc Research (NWO); The Netherlands Organisation for Health Research and Development (ZonMw); the Research Institute for Diseases in the Elderly (RIDE); The Netherlands Genomics Initiative (NGI); the Ministry of Education, Culture and Science; the Ministry of Health, Welfare and Sports; the European Commission (DG XII); and the Municipality of Rotterdam.","The contribution of inhabitants, general practitioners and pharmacists of the Ommoord district to the Rotterdam Study is gratefully acknowledged. The contributions to the Management Team specifcally and the Rotterdam Study at large of the following persons are pivotal to the daily operations of the study and highly appreciated: Jacobus Lubbe, Gabriëlle Bakker, Eric Neeleman, Jeannette Vergeer, Anneke Korving, Pauline van Wijngaarden, Jolande Verkroost – van Heemst, Silvan Licher, Isabel van de", -NOMA,,NOMA,Effect of fat quality on blood lipids and immune response,,"RCT, SFA, PUFA, LDL-cholesterol",https://www.med.uio.no/imb/english/research/projects/noma/,UiO,,"This randomized controlled trial aimed to investigate the health effects of replacing food items with a high content of saturated fatty acids with food items where a large portion of the saturated fat had been replaced with polyunsaturated fat. The main finding was that compared to intake of food items with saturated fat, intake of food items with polyunsaturated fat reduced total and low-density lipoprotein (LDL)-cholesterol by 9 and 11%, respectively. In this study, we collected data on standard clinical and biochemical variables, in addition to dietary intake as well as standard clinical and biochemical variables, in addition to dietary intake as well as metabolic and transcriptomic profiles.",,Clinical trial,,Cross-sectional,,,Prospective,,251,,,Oslo,"Adult (18+ years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years),Aged (80+ years)",,,2012,2014,,,,Other release type,Data collection is completed.,No,,health or medical or biomedical research,,Contact Stine Marie Ulven (smulven@medisin.uio.no) to use data from this study.,,10.1017/S0007114516003445,"10.1093/ajcn/nqy356,10.1016/j.numecd.2020.06.018",,"This study was funded by the University of Oslo, Norway, and the Throne-Holst Foundation for Nutrition Research, Oslo, Norway.","We thank the NoMa team at the Oslo and Akershus University College of Applied Sciences, Department of Health, Nutrition and Management and at the University of Oslo, Department for Nutrition, Oslo, Norway.", -ABCD,,ABCD,The Amsterdam Born Children and their Development Study,,"multicultural, psychosocial outcomes, child development, growth, cardiometabolic profile",http://www.abcd-studie.nl/,AMC,,The ABCD-study is a large longitudinal research study on the health of 8000 Amsterdam born children. We investigate which factors in early pregnancy and childhood influence the health of these children.,,Birth cohort,,Longitudinal,,,"Retrospective,Prospective",,8266,4300,Netherlands (the),Amsterdam,Prenatal,,,2003,2004,,,,Continuous,Once a follow-up is completed and data have been processed.,no,,,,reach out to contact person,,10.1093/ije/dyq128,,,The ABCD study has been supported by grants from The Netherlands Organisation for Health Research and Development (ZonMW) and The Netherlands Heart Foundation.,"We thank all participating families, and are grateful to all obstetric care providers, primary schools, students, and youth healthcare centers in Amsterdam for their contribution to the data collection of the ABCD-study", +RS2,,Rotterdam Study second cohort,"Rotterdam Study, second cohort",,Biomarkers · Cancer and related diseases · Cardiovascular diseases · Cohort study · Dermatological diseases · Endocrine diseases · Epidemiologic methods · Genetic and molecular epidemiology · Nutrition and lifestyle epidemiology · Liver diseases · Neurological diseases · Oncology · Ophthalmic diseases · Otolaryngological diseases · Pharmacoepidemiology · Population imag,https://www.ergo-onderzoek.nl/,EMC,,"The Rotterdam Study is an ongoing prospective cohort study that started in 1990 in the city of Rotterdam, The Netherlands. The study aims to unravel etiology, preclinical course, natural history and potential targets for intervention for chronic diseases in mid-life and late-life.",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,4472,3011,Netherlands (the),Rotterdam,"Middle-aged (45-64 years),Aged (65+ years)",,,2000,,,,,Continuous,Once a follow-up is completed and data have been processed.,no,,health or medical or biomedical research,,,,https://doi.org/10.1007/s10654-020-00640-5,"https://doi.org/10.1007/s10654-009-9386-z,https://doi.org/10.1007/BF00145007,https://doi.org/10.1007/s10654-007-9199-x,https://doi.org/10.1007/s10654-011-9610-5,https://doi.org/10.1007/s10654-013-9866-z,https://doi.org/10.1007/s10654-015-0082-x,https://doi.org/10.1007/s10654-017-0321-4",,"The Rotterdam Study is supported by the Erasmus MC University Medical Center and Erasmus University Rotterdam; The Netherlands Organisation for Scientifc Research (NWO); The Netherlands Organisation for Health Research and Development (ZonMw); the Research Institute for Diseases in the Elderly (RIDE); The Netherlands Genomics Initiative (NGI); the Ministry of Education, Culture and Science; the Ministry of Health, Welfare and Sports; the European Commission (DG XII); and the Municipality of Rotterdam.","The contribution of inhabitants, general practitioners and pharmacists of the Ommoord district to the Rotterdam Study is gratefully acknowledged. The contributions to the Management Team specifcally and the Rotterdam Study at large of the following persons are pivotal to the daily operations of the study and highly appreciated: Jacobus Lubbe, Gabriëlle Bakker, Eric Neeleman, Jeannette Vergeer, Anneke Korving, Pauline van Wijngaarden, Jolande Verkroost – van Heemst, Silvan Licher, Isabel van", +NOMA,,NOMA,Effect of fat quality on blood lipids and immune response,,"RCT, SFA, PUFA, LDL-cholesterol",https://www.med.uio.no/imb/english/research/projects/noma/,UiO,,"This randomized controlled trial aimed to investigate the health effects of replacing food items with a high content of saturated fatty acids with food items where a large portion of the saturated fat had been replaced with polyunsaturated fat. The main finding was that compared to intake of food items with saturated fat, intake of food items with polyunsaturated fat reduced total and low-density lipoprotein (LDL)-cholesterol by 9 and 11%, respectively. In this study, we collected data on standard clinical and biochemical variables, in addition to dietary intake as well as standard clinical and biochemical variables, in addition to dietary intake as well as metabolic and transcriptomic profiles.",,Clinical trial,,Cross-sectional,,,Prospective,,251,,,Oslo,"Adult (18+ years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years),Aged (80+ years)",,,2012,2014,,,,Other release type,Data collection is completed.,No,,health or medical or biomedical research,,Contact Stine Marie Ulven (smulven@medisin.uio.no) to use data from this study.,,https://doi.org/10.1017/S0007114516003445,"https://doi.org/10.1093/ajcn/nqy356,https://doi.org/10.1016/j.numecd.2020.06.018",,"This study was funded by the University of Oslo, Norway, and the Throne-Holst Foundation for Nutrition Research, Oslo, Norway.","We thank the NoMa team at the Oslo and Akershus University College of Applied Sciences, Department of Health, Nutrition and Management and at the University of Oslo, Department for Nutrition, Oslo, Norway.", +ABCD,,ABCD,The Amsterdam Born Children and their Development Study,,"multicultural, psychosocial outcomes, child development, growth, cardiometabolic profile",http://www.abcd-studie.nl/,AMC,,The ABCD-study is a large longitudinal research study on the health of 8000 Amsterdam born children. We investigate which factors in early pregnancy and childhood influence the health of these children.,,Birth cohort,,Longitudinal,,,"Retrospective,Prospective",,8266,4300,Netherlands (the),Amsterdam,Prenatal,,,2003,2004,,,,Continuous,Once a follow-up is completed and data have been processed.,no,,,,reach out to contact person,,https://doi.org/10.1093/ije/dyq128,,,The ABCD study has been supported by grants from The Netherlands Organisation for Health Research and Development (ZonMW) and The Netherlands Heart Foundation.,"We thank all participating families, and are grateful to all obstetric care providers, primary schools, students, and youth healthcare centers in Amsterdam for their contribution to the data collection of the ABCD-study", HELIX,,HELIX,The Human Early-Life Exposome,,Novel tools for integrating early-life environmental exposures and child health across Europe,http://www.projecthelix.eu,ISGLOBAL,,"Six prospective birth cohort studies are contributing to HELIX as the only realistic and feasible way to obtain the comprehensive, longitudinal, human data needed to build this early-life exposome. These cohorts have already collected large amounts of data as part of national and EU-funded projects. Results will be integrated with data from European cohorts (>300,000 subjects) and registers, to estimate health impacts at the large European scale.",,Birth cohort,,Longitudinal,,,Prospective,,32000,1300,"Norway,France,Spain,United Kingdom of Great Britain and Northern Ireland (the),Greece,Lithuania",,,,,2014,2016,,,,,,,,,,,,,,,https://cordis.europa.eu/project/id/308333,, -SWS,,SWS,Southampton Women’s ,,"pre-conception, pregnancy, childhood, growth, asthma, cardiovascular, mental health",https://www.mrc.soton.ac.uk/sws/,Soton,,"SWS is a population-based prospective birth cohort study of 12 583, initially non-pregnant, women aged 20–34 years, living in the city of Southampton, UK. Assessments of lifestyle, diet and anthropometry were done at study entry in 1998–2002. Women who subsequently became pregnant with singleton pregnancies were followed up during pregnancy; and their 3158 offspring have been studied in infa",src@mrc.soton.ac.uk,Birth cohort,,Longitudinal,,,Prospective,,12583,,United Kingdom of Great Britain and Northern Ireland (the),,"Young adult (18-24 years),Adult (25-44 years)",,,1998,2002,,,,Continuous,Upon completion of the procedures outlined above,No,,,,"Initial approach to the cohort contact with a short proposal. This will be sent to the SWS oversight group. If approved, data agreements need to be completed",,10.1093/ije/dyi202,,,"The SWS is supported by grants from the Medical Research Council, National Institute for Health Research Southampton Biomedical Research Centre, British Heart Foundation, University of Southampton and University Hospital Southampton National Health Service Foundation Trust, and the European Union’s Seventh Framework Programme (FP7/2007-2013), project EarlyNutrition (grant 289346) and from the European Union's Horizon 2020 research and innovation programme (LIFECYCLE, grant agreement No 733206). Study participants were drawn from a cohort study funded by the Medical Research Council and the Dunhill Medical ",The authors are grateful to the women of Southampton who gave their time to take part in the Southampton Women’s Survey and to the research nurses and other staff who collected and processed the, +SWS,,SWS,Southampton Women’,,"pre-conception, pregnancy, childhood, growth, asthma, cardiovascular, mental health",https://www.mrc.soton.ac.uk/sws/,Soton,,"SWS is a population-based prospective birth cohort study of 12 583, initially non-pregnant, women aged 20–34 years, living in the city of Southampton, UK. Assessments of lifestyle, diet and anthropometry were done at study entry in 1998–2002. Women who subsequently became pregnant with singleton pregnancies were followed up during pregnancy; and their 3158 offspring have been studied i",src@mrc.soton.ac.uk,Birth cohort,,Longitudinal,,,Prospective,,12583,,United Kingdom of Great Britain and Northern Ireland (the),,"Young adult (18-24 years),Adult (25-44 years)",,,1998,2002,,,,Continuous,Upon completion of the procedures outlined above,No,,,,"Initial approach to the cohort contact with a short proposal. This will be sent to the SWS oversight group. If approved, data agreements need to be completed",,https://doi.org/10.1093/ije/dyi202,,,"The SWS is supported by grants from the Medical Research Council, National Institute for Health Research Southampton Biomedical Research Centre, British Heart Foundation, University of Southampton and University Hospital Southampton National Health Service Foundation Trust, and the European Union’s Seventh Framework Programme (FP7/2007-2013), project EarlyNutrition (grant 289346) and from the European Union's Horizon 2020 research and innovation programme (LIFECYCLE, grant agreement No 733206). Study participants were drawn from a cohort study funded by the Medical Research Council and the Dunhill Medica",The authors are grateful to the women of Southampton who gave their time to take part in the Southampton Women’s Survey and to the research nurses and other staff who collected and processed t, Immune-Image_Task_6.2a,,Task 6.2a,Serial CD8 PET imaging in cancer patients before and during immune checkpoint inhibition,,,,UMCG,,"Upfront imaging of CD8 (Pandit-Taskar N. et al., J Nucl Med 2018, abstract) is feasible in patients, however data on serial imaging are lacking. When T-cells encounter tumour antigens they are activated and promote lymphocyte proliferation and recruitment to the tumour site. This inflammatory response increases blood flow and vascular permeability, which can be measured with dynamic contrast enhanced (DCE) MRI. Early after starting treatment, immune activation will also lead to T-cell tumour infiltration and increased tumour cellularity, which potentially can be quantified by diffusion weighted imaging (DWI) and radiomics features derived from computational image analysis. Preliminary data suggests that texture measures can capture the spatial heterogeneity of different sites of disease and predict treatment resistance and progression-free survival in the metastatic setting. Moreover, MRI measurements that integrate whole tumour volumetry, diffusion-weighted imaging (ADC) and radiomics (texture analysis) (Larkin T.J. et al.; 2014, Booth T.C. et al., 2017; Newman A.M. et al., 2015) may provide a therapy response assessment. Preliminary data suggests that texture measures can capture the spatial heterogeneity of metastases and predict treatment resistance and progression-free survival in patients with ovarian cancer. Aim Demonstrate and validate (1) PET with a 89Zrlabelled anti-CD8 minibody, to visualize serially the presence of CD8 expressing T-lymphocytes in cancer patients receiving CIT; (2) contribution of MRI measurements of intra- and inter-site tumour heterogeneity. Study design In patients with potential relevant indication for treatment with immune checkpoint inhibition treatment (CIT) (e.g. melanoma or NSCLC) and at least one accessible lesion for biopsy, [89Zr]Zr-anti-CD8 PET imaging (1.5 mg anti human CD8 minibody [89Zr]Zr-Df-IAB22M2C, 37 MBq) will be injected and imaging will be performed before the start and 7 days before the 3rd checkpoint inhibitor administration. Whole body [89Zr]Zr-anti-CD8 distribution will be determined: tumour and normal organ radioactive tracer uptake will be quantified as standardized uptake values (SUV). Patients will undergo multi-parametric MR before and after the 1st cycle and before the 3rd cycle (drugs administered on 2-3 weekly cycles). The third and final MRI will be performed within ~9 weeks after the start of treatment, or on disease progression, whichever is sooner. All metastases determined to be suitable for MR imaging will be evaluated to assess differential tumour response and pseudo-progression. Patients will also undergo contrast-enhanced CT to determine tumour response using iRECIST. Tumour biopsies will be extensively analysed. Statistical considerations: Initial feasibility will be shown in 12 patients. Depending on the tumour type chosen the number of patients required may be increased in order to have sufficient patients with a tumour response to immune checkpoint inhibitor treatment.",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -RS1,,Rotterdam Study first cohort,"Rotterdam Study, first cohort",,Biomarkers · Cancer and related diseases · Cardiovascular diseases · Cohort study · Dermatological diseases · Endocrine diseases · Epidemiologic methods · Genetic and molecular epidemiology · Nutrition and lifestyle epidemiology · Liver diseases · Neurological diseases · Oncology · Ophthalmic diseases · Otolaryngological diseases · Pharmacoepidemiology · Population imaging · Renal dis,https://www.ergo-onderzoek.nl/,EMC,,"The Rotterdam Study is an ongoing prospective cohort study that started in 1990 in the city of Rotterdam, The Netherlands. The study aims to unravel etiology, preclinical course, natural history and potential targets for intervention for chronic diseases in mid-life and late-life.",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,10215,7983,Netherlands (the),Rotterdam,"Middle-aged (45-64 years),Aged (65+ years)",,,1989,0,,,,Continuous,Once a follow-up is completed and data have been processed.,no,,health or medical or biomedical research,,,,10.1007/s10654-020-00640-5,"10.1007/s10654-009-9386-z,10.1007/BF00145007,10.1007/s10654-007-9199-x,10.1007/s10654-011-9610-5,10.1007/s10654-013-9866-z,10.1007/s10654-015-0082-x,10.1007/s10654-017-0321-4",,"The Rotterdam Study is supported by the Erasmus MC University Medical Center and Erasmus University Rotterdam; The Netherlands Organisation for Scientifc Research (NWO); The Netherlands Organisation for Health Research and Development (ZonMw); the Research Institute for Diseases in the Elderly (RIDE); The Netherlands Genomics Initiative (NGI); the Ministry of Education, Culture and Science; the Ministry of Health, Welfare and Sports; the European Commission (DG XII); and the Municipality of Rotterdam.","The contribution of inhabitants, general practitioners and pharmacists of the Ommoord district to the Rotterdam Study is gratefully acknowledged. The contributions to the Management Team specifcally and the Rotterdam Study at large of the following persons are pivotal to the daily operations of the study and highly appreciated: Jacobus Lubbe, Gabriëlle Bakker, Eric Neeleman, Jeannette Vergeer, Anneke Korving, Pauline van Wijngaarden, Jolande Verkroost – van Heemst, Silvan Licher, Isabel van de", +RS1,,Rotterdam Study first cohort,"Rotterdam Study, first cohort",,Biomarkers · Cancer and related diseases · Cardiovascular diseases · Cohort study · Dermatological diseases · Endocrine diseases · Epidemiologic methods · Genetic and molecular epidemiology · Nutrition and lifestyle epidemiology · Liver diseases · Neurological diseases · Oncology · Ophthalmic diseases · Otolaryngological diseases · Pharmacoepidemiology · Population imag,https://www.ergo-onderzoek.nl/,EMC,,"The Rotterdam Study is an ongoing prospective cohort study that started in 1990 in the city of Rotterdam, The Netherlands. The study aims to unravel etiology, preclinical course, natural history and potential targets for intervention for chronic diseases in mid-life and late-life.",,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,10215,7983,Netherlands (the),Rotterdam,"Middle-aged (45-64 years),Aged (65+ years)",,,1989,0,,,,Continuous,Once a follow-up is completed and data have been processed.,no,,health or medical or biomedical research,,,,https://doi.org/10.1007/s10654-020-00640-5,"https://doi.org/10.1007/s10654-009-9386-z,https://doi.org/10.1007/BF00145007,https://doi.org/10.1007/s10654-007-9199-x,https://doi.org/10.1007/s10654-011-9610-5,https://doi.org/10.1007/s10654-013-9866-z,https://doi.org/10.1007/s10654-015-0082-x,https://doi.org/10.1007/s10654-017-0321-4",,"The Rotterdam Study is supported by the Erasmus MC University Medical Center and Erasmus University Rotterdam; The Netherlands Organisation for Scientifc Research (NWO); The Netherlands Organisation for Health Research and Development (ZonMw); the Research Institute for Diseases in the Elderly (RIDE); The Netherlands Genomics Initiative (NGI); the Ministry of Education, Culture and Science; the Ministry of Health, Welfare and Sports; the European Commission (DG XII); and the Municipality of Rotterdam.","The contribution of inhabitants, general practitioners and pharmacists of the Ommoord district to the Rotterdam Study is gratefully acknowledged. The contributions to the Management Team specifcally and the Rotterdam Study at large of the following persons are pivotal to the daily operations of the study and highly appreciated: Jacobus Lubbe, Gabriëlle Bakker, Eric Neeleman, Jeannette Vergeer, Anneke Korving, Pauline van Wijngaarden, Jolande Verkroost – van Heemst, Silvan Licher, Isabel van", RAINE,,RAINE,The West Australian Pregnancy Cohort,,,https://rainestudy.org.au/,UWA,,"The Raine Study is one of the most comprehensive pregnancy birth cohort studies in the world. It is a rich resource for the study of environmental and genetic factors that affect health and development and can provide unique insights into the natural history of human diseases. Between May 1989 and November 1991, 2900 pregnant women were recruited into the study and their children have been assessed intensively over the past three decades. Prospective longitudinal data has been collected at multiple time-points over pregnancy, infancy, childhood, adolescence and young adulthood and there is broad multidisciplinary data on physical, mental and social aspects of development. -The main cohort (Gen2) are now almost 30 years of age and maintain a keen sense of commitment to the Raine Study. Their data has been genotyped and data linkage with other publicly held datasets (e.g. school results and hospital records) is available. There are stored biological samples and established collaborative research networks across a wide variety of disciplines.",,Population cohort,,Longitudinal,,,Prospective,,2868,,,Western Australia,,,,1989,2020,,,,Continuous,Upon completion of follow-up,,,health or medical or biomedical research,,1. Submit a project proposal 2. Approval of the project ​3. Sign a data access agreement (DAA) 4. Data access with credentials (via DATAS,,10.5694/mja12.10698,,,"The Raine Study has been funded by program and project grants from the Australian National Health and Medical Research Council, the Commonwealth Scientific and Industrial Research Organisation, Healthway and the Lions Eye Institute in Western Australia. The Raine Study Gen2-17 year follow-up was funded by the NHMRC Program Grant (Stanley et al, ID 353514) and the GWAS data from the Gen2-17 year follow-up was funded by the NHMRC (Huang et al, ID 1059711) grant. The Raine Study participation in LIFECYCLE was funded by a grant from the National Health and Medical Research Council, Australia (GNT114285). The University of Western Australia (UWA), Curtin University, the Raine Medical Research Foundation, the Telethon Kids Institute, the Women’s and Infant’s Research Foundation (KEMH), Murdoch University, The University of Notre Dame Australia and Edith Cowan University provide funding for the Core Management of the Raine Study. REF is a recipient of a National Health and Medical Research Council Early Career",, -DFBC,,DFBC,Dutch Famine Birth Cohort study,Hongerwinter Onderzoek,"famine, Dutch, older adults, ageing, pregnancy, early life, prenatal famine exposure",https://www.hongerwinter.nl/,Amsterdam UMC (AMC),,"The cohort was set up to investigate the effects of acute maternal undernutrition during specific stages of gestation on the offspring’s adult health. The main outcomes of interest of the DFBC are chronic cardiovascular and metabolic diseases, ageing and mental health. Differences in various outcomes have been found between participants exposed to famine and unexposed participants. Although statistically significant, these differences are not very large, therefore adjusting for exposure group enables researchers to use DFBC data in pooled ana",,Birth cohort,test,Longitudinal,,,"Retrospective,Prospective",,2414,,Netherlands (the),,,,,1994,2021,,,,Other release type,upon completion of fu,no,,health or medical or biomedical research,,1. Submit a DFBC project proposal (s.r.derooij@amsterdamumc.nl) 2. Approval of the project ​3. Sign a data access agreement (DAA) 4. Data access with credentials (via DATAS,,10.1136/bmjopen-2020-042078,10.1136/bmjopen-2013-003167,,"The Dutch famine birth cohort study has been funded by the Diabetes Fonds (The Netherlands, Grant Number NA), the Netherlands Heart Foundation ((NHS2001B087, NHS2007B083), The European Science Foundation (EUROSTRESS-DOME project), the European Commission (Brainage (Seventh Framework Programme Project 279281), Dynahealth (Horizon 2020 Project 633595), Longitools (Horizon 2020 Project 874739), Well-being (UK, Grant Number NA), the Medical Research Council (UK, Grant Number NA), the Dutch Research Council (NWO Aspasia Project 015014039) and the Academic Medical Centre (Amsterdam, The Netherlands, Grant number NA). We declare no conflict of interest.","We would like to thank the members of the Dutch famine birth cohort for their participation in our studies. We would also like to thank all researchers and other staff who have worked with us on study design, data collection, assessments, analyses and drafting of manuscripts.", +The main cohort (Gen2) are now almost 30 years of age and maintain a keen sense of commitment to the Raine Study. Their data has been genotyped and data linkage with other publicly held datasets (e.g. school results and hospital records) is available. There are stored biological samples and established collaborative research networks across a wide variety of disciplines.",,Population cohort,,Longitudinal,,,Prospective,,2868,,,Western Australia,,,,1989,2020,,,,Continuous,Upon completion of follow-up,,,health or medical or biomedical research,,1. Submit a project proposal 2. Approval of the project ​3. Sign a data access agreement (DAA) 4. Data access with credentials (via DAT,,https://doi.org/10.5694/mja12.10698,,,"The Raine Study has been funded by program and project grants from the Australian National Health and Medical Research Council, the Commonwealth Scientific and Industrial Research Organisation, Healthway and the Lions Eye Institute in Western Australia. The Raine Study Gen2-17 year follow-up was funded by the NHMRC Program Grant (Stanley et al, ID 353514) and the GWAS data from the Gen2-17 year follow-up was funded by the NHMRC (Huang et al, ID 1059711) grant. The Raine Study participation in LIFECYCLE was funded by a grant from the National Health and Medical Research Council, Australia (GNT114285). The University of Western Australia (UWA), Curtin University, the Raine Medical Research Foundation, the Telethon Kids Institute, the Women’s and Infant’s Research Foundation (KEMH), Murdoch University, The University of Notre Dame Australia and Edith Cowan University provide funding for the Core Management of the Raine Study. REF is a recipient of a National Health and Medical Research Council Early Ca",, +DFBC,,DFBC,Dutch Famine Birth Cohort study,Hongerwinter Onderzoek,"famine, Dutch, older adults, ageing, pregnancy, early life, prenatal famine exposure",https://www.hongerwinter.nl/,Amsterdam UMC (AMC),,"The cohort was set up to investigate the effects of acute maternal undernutrition during specific stages of gestation on the offspring’s adult health. The main outcomes of interest of the DFBC are chronic cardiovascular and metabolic diseases, ageing and mental health. Differences in various outcomes have been found between participants exposed to famine and unexposed participants. Although statistically significant, these differences are not very large, therefore adjusting for exposure group enables researchers to use DFBC data in pooled a",,Birth cohort,test,Longitudinal,,,"Retrospective,Prospective",,2414,,Netherlands (the),,,,,1994,2021,,,,Other release type,upon completion of fu,no,,health or medical or biomedical research,,1. Submit a DFBC project proposal (s.r.derooij@amsterdamumc.nl) 2. Approval of the project ​3. Sign a data access agreement (DAA) 4. Data access with credentials (via DAT,,https://doi.org/10.1136/bmjopen-2020-042078,https://doi.org/10.1136/bmjopen-2013-003167,,"The Dutch famine birth cohort study has been funded by the Diabetes Fonds (The Netherlands, Grant Number NA), the Netherlands Heart Foundation ((NHS2001B087, NHS2007B083), The European Science Foundation (EUROSTRESS-DOME project), the European Commission (Brainage (Seventh Framework Programme Project 279281), Dynahealth (Horizon 2020 Project 633595), Longitools (Horizon 2020 Project 874739), Well-being (UK, Grant Number NA), the Medical Research Council (UK, Grant Number NA), the Dutch Research Council (NWO Aspasia Project 015014039) and the Academic Medical Centre (Amsterdam, The Netherlands, Grant number NA). We declare no conflict of interest.","We would like to thank the members of the Dutch famine birth cohort for their participation in our studies. We would also like to thank all researchers and other staff who have worked with us on study design, data collection, assessments, analyses and drafting of manuscripts.", Immune-Image_Task_6.2b,,Task 6.2b,PD-1 therapy study,,,,UMCG,,An investigator initiated clinical treatment trial with PD-1 antibody provided by Janssen will include the patients who underwent CD8 imaging in task 6.2a. This study will be designed and executed during the course of Immune-Image according to cGCP standards. This study in contrast to the CD8 imaging part will extent till the end of Immune-Image in order to allow appropriate follow-up of the patients.,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, Immune-Image_Task_6.3,,Task 6.3,Optical PD-L1 imaging trial in locally advanced oesophageal cancer patients,,,,UMCG,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -PANIC,,PANIC,Physical Activity and Nutrition In Children,,"adipokines, adolescence, adolescents, aldosterone, arterial, imaging, arteriosclerosis, artery, wall, elasticity, artery, wall, stiffness, artery, wall, thickness, atherosclerosis, bioelectrical, impedance, bioimpedance, birth, birth, weight, body, fat, content, body, mineral, content, body, mineral, density, cardiovascular, disease, carotid, ultrasound, childhood, children, depression, development, diet, dietary, recording, dual-energy, x-ray, absorptiometry, dxa, eating, behavior, ergospirometry, exercise, exercise, test, fat, mass, fat, percentage, fat-free, mass, fetal, period, gene, genome-wide, association, studies, glucose, glucose, tolerance, test, growth, impedance, cardiography, insulin, intervention, intima-media, thickness, lean, body, mass, lean, mass, lifestyle, lipids, lipoproteins, liver, enzymes, maximal, oxygen, consumption, maximal, oxygen, uptake, mental, problems, metabolite, profiling, metabolomics, musculoskeletal, disease, myokines, nutrition, obesity, osteoporosis, overweight, oxidative, stress, physical, activity, pregnancy, prepubertal, psychic, problems, puberal, puberty, questionnaire, redox-regulation, respiratory, gas, analysis, respiratory, gases, school, age, screen, time, sedentary, behavior, sedentary, time, sex, hormones, sex, steroids, sitting, sleep, sleeping, stress, stress, hormones, thyroid, hormones, trial, type, 2, diabetes",https://www.panicstudy.fi/,UEF,,The Physical Activity and Nutrition in Children (PANIC) Study is an ongoing 8-year controlled physical activity and dietary intervention study in a large population sample of children who have been followed retrospectively since pregnancy and prospectively until adolescence and will finally be followed until early adulthood,,Clinical trial,,Longitudinal,,,"Prospective,Retrospective",,736,512,Finland,,"Child (2-12 years),Adolescent (13-17 years)",,,2007,,,,,Other release type,"Data from baseline, 2-year follow-up and 8-year follow-up have been processed",no,,,,reach out to contact person,,10.1007/s00125-020-05250-0,10.1016/j.numecd.2016.05.005,,"Ministry of Social Affairs and Health of Finland, Ministry of Education and Culture of Finland, Finnish Innovation Fund Sitra, Social Insurance Institution of Finland, Research Committee of the Kuopio University Hospital Catchment Area (State Research Funding), Kuopio University Hospital (EVO funding number 5031343), the city of Kuopio, Finnish Cultural Foundation, Juho Vainio Foundation, Foundation for Paediatric Research, Paavo Nurmi Foundation, Jenny and Antti Wihuri Foundation, Paulo Foundation, Diabetes Research Foundation, Finnish Foundation of Cardiovascular Research, OLVI-foundation, The Finnish Medical Foundation, Yrjö Jahnsson Foundation, Aarne and Aili Turunen Foundation, Päivikki and Sakari Sohlberg Foundation, Jalmari and Rauha Ahokas Foundation, Doctoral Programs in Public Health (DPPH), Finnish Doctoral Program in Oral Sciences (FINDOS), The Kuopio Naturalists' Society, Finnish Association for the Study of Pain, the Finnish Dental Society Apollonia and other Finnish Associations and Soci",, +PANIC,,PANIC,Physical Activity and Nutrition In Children,,"adipokines, adolescence, adolescents, aldosterone, arterial, imaging, arteriosclerosis, artery, wall, elasticity, artery, wall, stiffness, artery, wall, thickness, atherosclerosis, bioelectrical, impedance, bioimpedance, birth, birth, weight, body, fat, content, body, mineral, content, body, mineral, density, cardiovascular, disease, carotid, ultrasound, childhood, children, depression, development, diet, dietary, recording, dual-energy, x-ray, absorptiometry, dxa, eating, behavior, ergospirometry, exercise, exercise, test, fat, mass, fat, percentage, fat-free, mass, fetal, period, gene, genome-wide, association, studies, glucose, glucose, tolerance, test, growth, impedance, cardiography, insulin, intervention, intima-media, thickness, lean, body, mass, lean, mass, lifestyle, lipids, lipoproteins, liver, enzymes, maximal, oxygen, consumption, maximal, oxygen, uptake, mental, problems, metabolite, profiling, metabolomics, musculoskeletal, disease, myokines, nutrition, obesity, osteoporosis, overweight, oxidative, stress, physical, activity, pregnancy, prepubertal, psychic, problems, puberal, puberty, questionnaire, redox-regulation, respiratory, gas, analysis, respiratory, gases, school, age, screen, time, sedentary, behavior, sedentary, time, sex, hormones, sex, steroids, sitting, sleep, sleeping, stress, stress, hormones, thyroid, hormones, trial, type, 2, diabetes",https://www.panicstudy.fi/,UEF,,The Physical Activity and Nutrition in Children (PANIC) Study is an ongoing 8-year controlled physical activity and dietary intervention study in a large population sample of children who have been followed retrospectively since pregnancy and prospectively until adolescence and will finally be followed until early adulthood,,Clinical trial,,Longitudinal,,,"Prospective,Retrospective",,736,512,Finland,,"Child (2-12 years),Adolescent (13-17 years)",,,2007,,,,,Other release type,"Data from baseline, 2-year follow-up and 8-year follow-up have been processed",no,,,,reach out to contact person,,https://doi.org/10.1007/s00125-020-05250-0,https://doi.org/10.1016/j.numecd.2016.05.005,,"Ministry of Social Affairs and Health of Finland, Ministry of Education and Culture of Finland, Finnish Innovation Fund Sitra, Social Insurance Institution of Finland, Research Committee of the Kuopio University Hospital Catchment Area (State Research Funding), Kuopio University Hospital (EVO funding number 5031343), the city of Kuopio, Finnish Cultural Foundation, Juho Vainio Foundation, Foundation for Paediatric Research, Paavo Nurmi Foundation, Jenny and Antti Wihuri Foundation, Paulo Foundation, Diabetes Research Foundation, Finnish Foundation of Cardiovascular Research, OLVI-foundation, The Finnish Medical Foundation, Yrjö Jahnsson Foundation, Aarne and Aili Turunen Foundation, Päivikki and Sakari Sohlberg Foundation, Jalmari and Rauha Ahokas Foundation, Doctoral Programs in Public Health (DPPH), Finnish Doctoral Program in Oral Sciences (FINDOS), The Kuopio Naturalists' Society, Finnish Association for the Study of Pain, the Finnish Dental Society Apollonia and other Finnish Associations and So",, NFBC1966,,NFBC1966,NFBC1966,,,https://www.oulu.fi/nfbc,UOULU,,"The study was started by professor Paula Rantakallio in the two Northernmost provinces in Finland (Oulu and Lapland) already in the year 1965 when the mothers were pregnant. Data on the individuals born into this cohort was collected since the 24th gestational week as well as their mothers and, to a lesser extent, fathers. The cohort included 12055 mothers and they had 12068 deliveries (13 women delivered twice). Cases belonging to survey were determined by the calculated term. A small percentage of the births occurred towards the end of 1965 and early in 1967. The calculated term, as was customary at that time, was counted from the first day of the last menstrual period. Where this date was unknown the expected term was estimated from the date of commencement of foetal ovements and progress of the pregnancy. The study covered all live born and stillborn infants with birth weight of 600 grams or more. -According to the Finland's central Office of Statistics, births in the study area during 1966 totalled 12527, so study population comprised 96.3 per cent of all births during 1966 in that area. Altogether 12231 children were born into the cohort, 12058 of them live born. The original data have been supplemented by data collected with postal questionnaires at the ages of 1, 14 and 31 years and various hospital records and national register data.",,Birth cohort,,Longitudinal,,,Prospective,,12231,,Finland,"Lapland,North Ostrobothnia","Prenatal,Newborn (0-1 months),Infants and toddlers (2-23 months),Infant (0-23 months),Adolescent (13-17 years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years)",,,1965,,,,,Closed dataset,,yes,,health or medical or biomedical research,,https://www.oulu.fi/nfbc/materialrequest,,http://urn.fi/urn:nbn:fi:att:bc1e5408-980e-4a62-b899-43bec3755243,10.1093/ije/dyab109,,"For the 31yr follow-up: NFBC1966 received financial support from University of Oulu Grant no. 65354, Oulu University Hospital Grant no. 2/97, 8/97, Ministry of Health and Social Affairs Grant no. 23/251/97, 160/97, 190/97, National Institute for Health and Welfare, Helsinki Grant no. 54121, Regional Institute of Occupational Health, Oulu, Finland Grant no. 50621, 54231 +According to the Finland's central Office of Statistics, births in the study area during 1966 totalled 12527, so study population comprised 96.3 per cent of all births during 1966 in that area. Altogether 12231 children were born into the cohort, 12058 of them live born. The original data have been supplemented by data collected with postal questionnaires at the ages of 1, 14 and 31 years and various hospital records and national register data.",,Birth cohort,,Longitudinal,,,Prospective,,12231,,Finland,"Lapland,North Ostrobothnia","Prenatal,Newborn (0-1 months),Infants and toddlers (2-23 months),Infant (0-23 months),Adolescent (13-17 years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years)",,,1965,,,,,Closed dataset,,yes,,health or medical or biomedical research,,https://www.oulu.fi/nfbc/materialrequest,,,https://doi.org/10.1093/ije/dyab109,,"For the 31yr follow-up: NFBC1966 received financial support from University of Oulu Grant no. 65354, Oulu University Hospital Grant no. 2/97, 8/97, Ministry of Health and Social Affairs Grant no. 23/251/97, 160/97, 190/97, National Institute for Health and Welfare, Helsinki Grant no. 54121, Regional Institute of Occupational Health, Oulu, Finland Grant no. 50621, 54231 For the 46yr follow-up: NFBC1966 received financial support from University of Oulu Grant no. 24000692, Oulu University Hospital Grant no. 24301140, ERDF European Regional Development Fund Grant no. 539/2010 A31592",We thank all cohort members and researchers who have participated in the study. We also wish acknowledge the work of the NFBC project center., SAPALDIA,,,SAPALDIA,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, mtFIT study,,mtFIT study,Multitarget FIT study,,"colon, crc, colorectal, cancer, early detection, early detection of cancer, feces, mass screening, biomarkers, colonoscopy, routine diagnostic tests",https://www.avl.nl/multitargetFIT,NKI,,"The mulitarget FIT study is a prospective study that is running within the Dutch National CRC screening program. The aim of this study is to compare the performance of a new stool test (mtFIT) with the currently used FIT. In this study we collect the lab data (results mtFIT and FIT) as well as the clinical data associated; age and sex of participants, colonoscopy findings (when one or both tests are positive), pathology data (when a lesion is detected during colonoscopy).",b.carvalho@nki.nl,"Population cohort,Clinical trial,Biobank",,Cross-sectional,,,Prospective,,13300,,,,,,,,,,,,,,,,"health or medical or biomedical research,disease specific research",,,,,,,,, -SYSDIMET,,SYSDIMET,Health Grain intervention,,"RCT, healthy diet, cereals, berries, n-3 fatty acids, dietary fibre, inflammation, endothelial dysfunction",,UEF,,,,Clinical trial,,Longitudinal,,,Prospective,,131,104,Finland,,"Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years)",,,2007,2007,,,,Closed dataset,Contact Marjukka Kolehmainen to use data from this study,no,,health or medical or biomedical research,,Contact Marjukka Kolehmainen to use data from this study,,10.1007/s00125-011-2285-3,"10.1371/journal.pone.0090352,10.1371/journal.pone.0022646",,"This study was supported by: the Academy of Finland (117844 and 118590 [to M. Uusitupa]; 131460 [to K. Poutanen]; 130469 [to H. Mykkänen] and 131593 [to V. D. F. de Mello]); the Kuopio University Hospital (5106, 5168, 5254 [to M. Uusitupa]); the Finnish Diabetes Research Foundation; the Sigrid Juselius Foundation; the Nordic Centre of Excellence on ‘Systems biology in controlled dietary interventions and cohort studies’ -(SYSDIET; 070014); and the European Commission in the Communities 6th Framework Programme, Project HEALTHGRAIN (FOODC","We thank the laboratory technologists T. Onnukka and K. Kettunen from the Department of Clinical Nutrition, University of Eastern Finland, Kuopio, Finland, and G. Trischler from the Department of Internal Medicine II-Cardiology, University of Ulm Medical Center, Ulm, Germany for their skilful work.", +SYSDIMET,,SYSDIMET,Health Grain intervention,,"RCT, healthy diet, cereals, berries, n-3 fatty acids, dietary fibre, inflammation, endothelial dysfunction",,UEF,,,,Clinical trial,,Longitudinal,,,Prospective,,131,104,Finland,,"Adult (25-44 years),Middle-aged (45-64 years),Aged (65+ years)",,,2007,2007,,,,Closed dataset,Contact Marjukka Kolehmainen to use data from this study,no,,health or medical or biomedical research,,Contact Marjukka Kolehmainen to use data from this study,,https://doi.org/10.1007/s00125-011-2285-3,"https://doi.org/10.1371/journal.pone.0090352,https://doi.org/10.1371/journal.pone.0022646",,"This study was supported by: the Academy of Finland (117844 and 118590 [to M. Uusitupa]; 131460 [to K. Poutanen]; 130469 [to H. Mykkänen] and 131593 [to V. D. F. de Mello]); the Kuopio University Hospital (5106, 5168, 5254 [to M. Uusitupa]); the Finnish Diabetes Research Foundation; the Sigrid Juselius Foundation; the Nordic Centre of Excellence on ‘Systems biology in controlled dietary interventions and cohort studies’ +(SYSDIET; 070014); and the European Commission in the Communities 6th Framework Programme, Project HEALTHGRAIN (","We thank the laboratory technologists T. Onnukka and K. Kettunen from the Department of Clinical Nutrition, University of Eastern Finland, Kuopio, Finland, and G. Trischler from the Department of Internal Medicine II-Cardiology, University of Ulm Medical Center, Ulm, Germany for their skilful work.", DNBC,,DNBC,The Danish National Birth Cohort,,,https://www.dnbc.dk,UCPH,,"The Danish National Birth Cohort (DNBC) was established to investigate the causal link between exposures in early life and disease later on and the possibilites for disease prevention. Initial data collection information was collected by computer-assisted telephone interviews with the women twice during pregnancy and when their children were six and 18 months old. Further data collections were conducted when the children were 7, 11 and 18 years old. For more details, please visit the DNBC homepage.",,Birth cohort,,Longitudinal,,,Prospective,,100410,96825,Denmark,,,,,1996,2003,,,,Other release type,,The non-harmonized DNBC data can be linked to Danish registries,,health or medical or biomedical research,,"1. Once a proposal has been circulated, the DNBC LifeCycle team at UCPH discuss the proposal to determine whether DNBC can participate and assign a lead DNBC researcher to the proposal. This DNBC researcher then contacts the study PI to confirm our participation and request that the study PI fill in a DNBC application form. 2. The DNBC application form is sent to the DNBC managerial team via UCPH. 3. After approval of the application, the researcher/study PI will receive a letter (by e-mail) of approval from the DNBC and asked to sign a consent form for correct data use. -4. Once this consent form has been signed and returned, the researcher can contact the DNBC data manager who will provide access to the requested variables.",,10.1177/14034948010290040201,,,"The Danish National Birth Cohort was established with a significant grant from the Danish National Research Foundation. Additional support was obtained from the Danish Regional Committees, the Pharmacy Foundation, the Egmont Foundation, the March of Dimes Birth Defects Foundation, the Health Foundation and other minor grants. The DNBC Biobank has been supported by the Novo Nordisk Foundation and the Lundbeck Foundation. Follow-up of mothers and children have been supported by the Danish Medical Research Council (SSVF 0646, 271-08-0839/06-066023, O602-01042B, 0602-02738B), the Lundbeck Foundation (195/04, R100-A9193), The Innovation Fund Denmark 0603-00294B (09-067124), the Nordea Foundation (02-2013-2014), Aarhus Ideas (AU R9-A959-13-S804), University of Copenhagen Strategic Grant (IFSV 2012), and the Danish Council for Independent Research (DFF – 4183-00594 and DFF - 4183-001","The authors would like to thank the participants, the first Principal Investigator of DNBC Prof. Jørn Olsen, the scientific managerial team, and DNBC secretariat for being, establishing, developing and consolidating the Danish National Birth Cohor", +4. Once this consent form has been signed and returned, the researcher can contact the DNBC data manager who will provide access to the requested variables.",,https://doi.org/10.1177/14034948010290040201,,,"The Danish National Birth Cohort was established with a significant grant from the Danish National Research Foundation. Additional support was obtained from the Danish Regional Committees, the Pharmacy Foundation, the Egmont Foundation, the March of Dimes Birth Defects Foundation, the Health Foundation and other minor grants. The DNBC Biobank has been supported by the Novo Nordisk Foundation and the Lundbeck Foundation. Follow-up of mothers and children have been supported by the Danish Medical Research Council (SSVF 0646, 271-08-0839/06-066023, O602-01042B, 0602-02738B), the Lundbeck Foundation (195/04, R100-A9193), The Innovation Fund Denmark 0603-00294B (09-067124), the Nordea Foundation (02-2013-2014), Aarhus Ideas (AU R9-A959-13-S804), University of Copenhagen Strategic Grant (IFSV 2012), and the Danish Council for Independent Research (DFF – 4183-00594 and DFF - 4183-0","The authors would like to thank the participants, the first Principal Investigator of DNBC Prof. Jørn Olsen, the scientific managerial team, and DNBC secretariat for being, establishing, developing and consolidating the Danish National Birth Coho", ELSPAC,,ELSPAC,ELSPAC,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -FEF,,FEF,Fibrefects - Grain fibre modification for gut-mediated health effects,,"RCT, Postprandial, Glucose metabolism, Short chain fatty acids, Rye, Wheat, Wholegrain, Bran, Bioprocessing, Gastrointestinal symptoms",,UEF,,,,Clinical trial,,Longitudinal,,,Prospective,,25,25,Finland,,"Adult (25-44 years),Middle-aged (45-64 years)",,,2011,2011,,,,Closed dataset,,no,,health or medical or biomedical research,,,,10.1186/1475-2891-13-104,10.1093/ajcn/nqy394,,"This study was carried out as a part of the FIBREFECTS (Grain fiber modification for gut mediated health effects) funded mainly by TEKES – The Finnish Funding Agency for Technology and Innovation – and to a smaller extent by Oy Karl Fazer Ab, Vaasan Oy, and Ravintoraisio Oy. The study was supported by a grant from Raisio Plc Research Foundation to J. Lappi, Academy of Finland to K. Poutanen, the Nordic Centre of Excellence on ‘Systems biology in controlled dietary interventions and cohort studies’ (SYSDIET; 070014) to M. Kolehmainen, the Nordic Centre of Excellence on ‘Nordic health – whole grain food (HELGA; 070015) to H. Mykkänen, and the Danish Strategic Research Council, ‘Concepts for enhanced butyrate production to improve colonic health and insulin sensi","Authors thank Erja Kinnunen, Eeva Lajunen and Lilli Lotvonen for their excellent technical assistance during the intervention.", +FEF,,FEF,Fibrefects - Grain fibre modification for gut-mediated health effects,,"RCT, Postprandial, Glucose metabolism, Short chain fatty acids, Rye, Wheat, Wholegrain, Bran, Bioprocessing, Gastrointestinal symptoms",,UEF,,,,Clinical trial,,Longitudinal,,,Prospective,,25,25,Finland,,"Adult (25-44 years),Middle-aged (45-64 years)",,,2011,2011,,,,Closed dataset,,no,,health or medical or biomedical research,,,,https://doi.org/10.1186/1475-2891-13-104,https://doi.org/10.1093/ajcn/nqy394,,"This study was carried out as a part of the FIBREFECTS (Grain fiber modification for gut mediated health effects) funded mainly by TEKES – The Finnish Funding Agency for Technology and Innovation – and to a smaller extent by Oy Karl Fazer Ab, Vaasan Oy, and Ravintoraisio Oy. The study was supported by a grant from Raisio Plc Research Foundation to J. Lappi, Academy of Finland to K. Poutanen, the Nordic Centre of Excellence on ‘Systems biology in controlled dietary interventions and cohort studies’ (SYSDIET; 070014) to M. Kolehmainen, the Nordic Centre of Excellence on ‘Nordic health – whole grain food (HELGA; 070015) to H. Mykkänen, and the Danish Strategic Research Council, ‘Concepts for enhanced butyrate production to improve colonic health an","Authors thank Erja Kinnunen, Eeva Lajunen and Lilli Lotvonen for their excellent technical assistance during the intervention.", KANC,,KANC,Kaunas cohort,,,,VMU,,,,Population cohort,,Longitudinal,,,"Retrospective,Prospective",,4000,4000,Lithuania,,"Prenatal,Child (2-12 years),Adolescent (13-17 years)",,,2007,,,,,,,,,health or medical or biomedical research,,,,,,,,, -ENVIRONAGE,,ENVIRONAGE,ENVIRonmental influence ON early AGEing,,"air pollution, child, aging, newborn, exposome, telomere length, placenta, omics, cardiovascular, system cognition",https://www.uhasselt.be/Algemene-Informatie-Limburgs-Geboortecohort,UHASSELT,,Within the ongoing population-based prospective birth cohort study ENVIRONAGE we explore new dimensions in the current understanding of human ageing and its interaction with the environment,,Birth cohort,,Longitudinal,,,"Retrospective,Prospective",,2000,2000,Belgium,,,,,2010,,,,,Annually,"Due to ongoing recruitment and follow-up, annual updates are provided of the datasets",No,,health or medical or biomedical research,,Contact Tim Nawrot (tim.nawrot@uhasselt.be) or Michelle Plusquin (Michelle.plusquin@uhasselt.Be),,10.1093/ije/dyw269,,,"The ENVIRONAGE birth cohort is supported by the European Research Council [ERC-2012-StG.310898], and by funds of the Flemish Scientific Research council [FWO, G.0.733.15.N, G.0.592.14].","The authors are extremely grateful to the participating women and neonates, as well as the staff of the maternity ward, midwives and the staff of the clinical laboratory of East-Limburg Hospital in Genk.", \ No newline at end of file +ENVIRONAGE,,ENVIRONAGE,ENVIRonmental influence ON early AGEing,,"air pollution, child, aging, newborn, exposome, telomere length, placenta, omics, cardiovascular, system cognition",https://www.uhasselt.be/Algemene-Informatie-Limburgs-Geboortecohort,UHASSELT,,Within the ongoing population-based prospective birth cohort study ENVIRONAGE we explore new dimensions in the current understanding of human ageing and its interaction with the environment,,Birth cohort,,Longitudinal,,,"Retrospective,Prospective",,2000,2000,Belgium,,,,,2010,,,,,Annually,"Due to ongoing recruitment and follow-up, annual updates are provided of the datasets",No,,health or medical or biomedical research,,Contact Tim Nawrot (tim.nawrot@uhasselt.be) or Michelle Plusquin (Michelle.plusquin@uhasselt.Be),,https://doi.org/10.1093/ije/dyw269,,,"The ENVIRONAGE birth cohort is supported by the European Research Council [ERC-2012-StG.310898], and by funds of the Flemish Scientific Research council [FWO, G.0.733.15.N, G.0.592.14].","The authors are extremely grateful to the participating women and neonates, as well as the staff of the maternity ward, midwives and the staff of the clinical laboratory of East-Limburg Hospital in Genk.", diff --git a/data/_demodata/applications/datacatalogue/Databanks.csv b/data/_demodata/applications/datacatalogue/Databanks.csv index 724da1eb88..5b4c17899b 100644 --- a/data/_demodata/applications/datacatalogue/Databanks.csv +++ b/data/_demodata/applications/datacatalogue/Databanks.csv @@ -27,7 +27,7 @@ EXE,,EXE,EXE,,,,,,,,Exemptions from copayment,,,,,,,,,It is collected for all re It may not be accurate for certain conditions, such as diabetes, which is an over-used and misused exemption code. It can be used as a proxy of some diseases which exempt NHS beneficiaries from co-payment, but it is not always requested by patients. -For example, a person with movement disorders may not request a co-pay exemption because they may not need it and it���������s an administrative burden, or are afraid of stigma +For example, a person with movement disorders may not request a co-pay exemption because they may not need it and it's an administrative burden, or are afraid of stigma Another point to bear in mind is that some co-payment exemptions trump others; for example, a patient may have a rare disease and a poor socioeconomic status, but since the exemption for socioeconomic status is more economically advantageous, the patient may opt for the latter. Data on rare disease exemption would the","The lag time between collection of new data and availability of new data in house is usually 1 year at most. For exception technical reasons (change in data collection platform), we have not yet updated data to 2019. We expect to do this in the next 3-6 months.",,,,,,,,,,,,,,,,,, SPECIALISTICA,,SPECIALISTICA,SPECIALISTICA,,,,,,,,Specialist visits and diagnostic tests,,,,,,,,,"It is collected for all NHS beneficiaries in the catchment area, provided that the specialist visits and diagnostic tests are conducted within an NHS setting.",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,The claim for a specialist practitioner visit or for a diagnostic test.,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"This is usually considered to have an acceptable level of quality. @@ -45,7 +45,7 @@ DIAGNOSTIC,,DIAGNOSTIC,DIAGNOSTIC,,Other,"table containing: - chronic diseases, occupational diseases, work accident and disability status, with codes and dates - annual comorbidities indicators Y/N (n=56: categories of cancer, cardiovascular diseases, diabetes...) according to CNAM algorithm based on diagnoses, chronic diseases and disability status, specific drugs and medical procedures, with dates - pregnancy outcomes (stillbirth, live birth, termination...), with dates",,,UB,,"Table with all inpatient stay diagnoses (ICD10 code + dates), long term disease / occupational diseases / work accident registration (ICD10 code + dates), pregnancy outcomes (specific BPE code + dates start/end of pregnancy) or 56 specific comorbidities annual indicators (specific BPE code + dates)",,,,,"Outpatient healthcare data (DCIR) are loaded monthly in SNDS. Data of month M are loaded in SNDS on Month M+1 with 90% of completion and need 6 months up to 1 year to be 99% complete -Inpatient data (PMSI) are loaded monthly but made available once a year in SNDS for extractions. Data of year N are loaded in SNDS on Q3-Q4 N+1",,67000000,,All,France,,"All ages,Newborn (0-1 months),Infants and toddlers (2-23 months),Child (2-12 years),Adolescent (13-17 years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65-79 years),Aged (80+ years)",Insurance coverage start,"Any person who has French nationality and has resided in France in a stable and regular manner, i.e. for more than 3 months, is obliged to be affiliated to the S��curit�� Sociale (French national healthcare insurance) whatever their situation (e.g. employees, civil servants, retired people, self-employed people, trainees, students, unemployed people, children and infants) +Inpatient data (PMSI) are loaded monthly but made available once a year in SNDS for extractions. Data of year N are loaded in SNDS on Q3-Q4 N+1",,67000000,,All,France,,"All ages,Newborn (0-1 months),Infants and toddlers (2-23 months),Child (2-12 years),Adolescent (13-17 years),Young adult (18-24 years),Adult (25-44 years),Middle-aged (45-64 years),Aged (65-79 years),Aged (80+ years)",Insurance coverage start,"Any person who has French nationality and has resided in France in a stable and regular manner, i.e. for more than 3 months, is obliged to be affiliated to the S curit Sociale (French national healthcare insurance) whatever their situation (e.g. employees, civil servants, retired people, self-employed people, trainees, students, unemployed people, children and infants) Citizens of the European Union or the European Economic Area (EEA) benefit from the same status as the French. Citizens from outside of the European Union and the EEA, must be able to prove their stay in the country in order to benefit from the national healthcare coverage (e.g. valid residence permit) except if they are victime of an accident at work or an occupational","Death,Emigration,Insurance coverage end",,,,,,,"Diagnostic codes included,Rare diseases,""Primary care - GP, community pharmacist level"",Population pregnancy and neonates",,,,,Healthcare claims database,,,,,,,true,,,,French,"Diagnoses from hospital stay, long term disease coverage, occupational diseases, work accident or 56 specific comorbidities indicators* collected and created by the CNAM. @@ -64,7 +64,7 @@ MBRN,,MBRN,MBRN,,Birth registry,,,,UiO,,Medical Birth Registry,,,,,,,,,"Any preg SALM,,SALM,SALM,,,,,,,,Mental health services,,,,,,,,,Yes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,When a patient is admitted to a treatment pathway in one of the regional Mental Health services.,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"Much of mental care is purchased by patients outside of the regional Mental Health services, for instance diagnoses of depression are almost never found. There are exceptions, for instance autism is expected to be fairly complete, and in general infant mental healthcare is well recorded because Tuscany has a national-level specialist center in infant mental health.","It was created in 2010. The table is updated every 3-4 months and is complete up to around 6 months before that date. At April of every year the final table is closed, except for record generated outside of the region which are obtained in September.",,,,,,,,,,,,,,,,,, SEA,,SEA,SEA,,Exemptions from co-payment,,,,"ARS.TOSCANA,CNR-IFC",,Exemptions from copayment,,,,,,,,,All the population in the database,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"Citizens are required to share in the cost of health care by paying a share of the value of certain services. Exemption from this duty is available for various reasons. Relevant to our purposes are income exemptions, which can provide an indication of socio-economic status, and pathology exemptions. The exemption database is coded using a system that, in the case of pathologies, can be mapped to the ICD-9-CM coding. Citizens under the age of six or over sixty-five are exempt, unless they are in a favourable income situation.",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -SPA,,SPA,SPA,,,,,,,,outpatient activities,,,,,,,,,All the population.,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"Access to an ambulatory procedure belonging to the list of reimbursed procedures (Nomenclatore), provided it is prescribed by the primary care physician of the patient, or of a specialist employed by the National healthcare System. Nomeclatore is updated every few years, and contains laboratory and instrumental diagnostic tests, diagnostic and rehabilitation procedures, specialist encounters.",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"Many ambulatory services are purchased by patients to avoid waiting lists ��������� for instance around half of specialist visits, according to estimates of the National","The table is updated every 3-4 months and is complete up to around 6 months before that date. At April of every year the final table is closed, except for record generated outside of the region which are obtained in September.",,,,,,,,,,,,,,,,,, +SPA,,SPA,SPA,,,,,,,,outpatient activities,,,,,,,,,All the population.,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"Access to an ambulatory procedure belonging to the list of reimbursed procedures (Nomenclatore), provided it is prescribed by the primary care physician of the patient, or of a specialist employed by the National healthcare System. Nomeclatore is updated every few years, and contains laboratory and instrumental diagnostic tests, diagnostic and rehabilitation procedures, specialist encounters.",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"Many ambulatory services are purchased by patients to avoid waiting lists for instance around half of specialist visits, according to estimates of the National","The table is updated every 3-4 months and is complete up to around 6 months before that date. At April of every year the final table is closed, except for record generated outside of the region which are obtained in September.",,,,,,,,,,,,,,,,,, AVOHILMO,,AVOHILMO,Register of Primary Health Care Visits (AVOHILMO),,Primary care medical records,,,,DHL.I,,Primary care administrative data,,,,,,,,,National data.,,,,"Birth,Immigration",,,,,,,,,"""Primary care - GP, community pharmacist level""",,,,,,,,,,,,,,,,,"Real time data that comes every night, as soon as a person visits their primary care physician. You can access only primary care centers of your own region, you are appointed to a specific primary health center. Primary care makes referral to secondary care (also doctors working at private sector i.e. private health care and occupational care can make referrals).",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Since 2011 the data collection has covered outpatient primary health care delivered in Finland with more complete data 2013 onwards. Data collection is continuous and statistics are published on a monthly basis.,,,,,,,,,,"ICPC,ICD10",,,,,,,, SPF,,SPF,SPF,,,,,,"ARS.TOSCANA,CNR-IFC",,Dispensation of medicines in community pharmacies,,,,,,,,,For all the dispensations taking place in Tuscany or to Tuscan inhabitants in other Italian regions are recorded.,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"A dispensation by a community pharmacy of a medicine that belongs to the list of reimbursed drugs maintained by the Italian Regulatory Agency (AIFA), upon prescription by the primary care physician of the patient, or of a specialist employed by the National healthcare System. AIFA has indeed two roles: admission of drugs to the Italia market (mostly after European procedure) and admission to reimbursement (which may take place many months after admission to market). Records are transmitted to the Region by pharmacies to obtain reimbursement (medicines are free at the point of dispensation, except for a copayment). The table is compulsory upon national law and is transmitted yearly to the Ministry of Finance.",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"Cheap medicines, even if potentially reimbursed, may be bought out-of-the-counter to avoid queing to renew a prescription. diff --git a/data/_demodata/applications/datacatalogue/Publications.csv b/data/_demodata/applications/datacatalogue/Publications.csv index 259f836ed2..a31b23dd33 100644 --- a/data/_demodata/applications/datacatalogue/Publications.csv +++ b/data/_demodata/applications/datacatalogue/Publications.csv @@ -1,30 +1,30 @@ doi,title,authors,year,journal,volume,number,pagination,publisher,school,abstract -https://doi.org/10.1371/journal.pone.0209003,Cervical cancer screening in Sweden 2014-2016.,"""Hortlund, M."",""Elfström, K.M"",""Sparén, P."",""Almstedt, P."",""Strander, B."",""Dillner, J.""",2018,PLoS ONE,,13,,,, -10.1002/ijc.32791,Effectiveness of AS04-adjuvanted human papillomavirus (HPV) type 16/18 vaccine in reducing oropharyngeal HPV infections in young females – results from a community-randomized trial.,"Lehtinen M,Apter D,Eriksson T,Harjula K,Hokkanen M,Lehtinen T,Natunen K,Damaso S,Soila M,Bi D,Struyf F,"",""",2020,Int J Cancer,,,,,, -10.1016/S1473-3099(20)30873-2,"Sustainability of neutralising antibodies induced by bivalent or quadrivalent HPV vaccines and correlation with efficacy: a combined follow-up analysis of data from two randomised, double-blind, multicentre, phase 3 trials.","Filipe Colaço Mariz,Penelope Gray,Noemi Bender,Tiina Eriksson,Hanna Kann,Dan Apter,Jorma Paavonen,Emma Pajunen,Kristina M Prager,Peter Sehr,Heljä-Marja Surcel,Tim Waterboer,Martin Müller,Michael Pawlita,Matti Lehtinen",2021,Lancet Inf. Disease,,,,,, -10.1093/infdis/jiaa099,Vaccination With Moderate Coverage Eradicates Oncogenic Human Papillomaviruses If a Gender-Neutral Strategy Is Applied.,"Vänskä S,Luostarinen T,Baussano I,Apter D,Eriksson T,Natunen K,Nieminen P,Paavonen J,Pimenoff VN,Pukkala E,Söderlund-Strand A,Dubin G,Garnett G,Dillner J,Lehtinen M,"",""",2020,JID,,,,,, -10.1002/ijc.27586,Occurrence of vaccine and non-vaccine human papillomavirus types in adolescent Finnish females 4 years post-vaccination,"Palmroth J,Merikukka M,Paavonen J,Apter D,Eriksson T,Natunen K,Dubin G,Lehtinen M",2012,Int J Cancer,,,,,, -10.1080/21645515.2016.1183847,Safety of the human papillomavirus (HPV)-16/18 AS04-adjuvanted vaccine in adolescents aged 12-15 years: Interim analysis of a large community-randomized controlled trial,"Lehtinen M,Eriksson T,Apter D,Hokkanen M,Natunen K,Paavonen J,Pukkala E,Angelo MG,Zima J,David MP,Datta S,Bi D,Struyf F,Dubin G",2016,Hum Vaccin Immunother,,,,,, -10.1136/bmjopen-2017-015867,Ten-year follow-up of human papillomavirus vaccine efficacy against the most stringent cervical neoplasia end-point-registry-based follow-up of three cohorts from randomized trials,"Lehtinen M,Lagheden C,Luostarinen T,Eriksson T,Apter D,Harjula K,Kuortti M,Natunen K,Palmroth J,Petäjä T,Pukkala E,Siitari-Mattila M,Struyf F,Nieminen P,Paavonen J,Dubin G,Dillner J",2017,BMJ Open,,,,,, -10.1002/ijc.31618,Gender-neutral vaccination provides improved control of human papillomavirus types 18/31/33/35 through herd immunity: Results of a community randomized trial (III),"Lehtinen M,Luostarinen T,Vänskä S,Söderlund-Strand A,Eriksson T,Natunen K,Apter D,Baussano I,Harjula K,Hokkanen M,Kuortti M,Palmroth J,Petäjä T,Pukkala E,Rekonen S,Siitari-Mattila M,Surcel HM,Tuomivaara L,Paavonen J,Nieminen P,Dillner J",2018,Int J Cancer,,,,,, -10.1002/ijc.32802,Baseline findings and safety of infrequent vs. frequent screening of human papillomavirus vaccinated women,"Louvanto K,Eriksson T,Gray P,Apter D,Baussano I,Bly A,Harjula K,Heikkilä K,Hokkanen M,Huhtinen L,Ikonen M,Karttunen H,Nummela M,Söderlund-Strand A,Veivo U,Dillner J,Elfstöm M,Nieminen P,Lehtinen M",2020,Int J Cancer,,,,,, -10.1002/ijc.33169,Long-term follow-up of human papillomavirus type replacement among young pregnant Finnish females before and after a community-randomised HPV vaccination trial with moderate coverage,"Gray P,Kann H,Pimenoff VN,Adhikari I,Eriksson T,Surcel HM,Vänskä S,Dillner J,Faust H,Lehtinen M",2020,Int J Cancer,,,,,, -10.1258/095646206776253453,Population based enrolment of adolescent girls into long-term human papillomavirus vaccination trial.,"Lehtinen M,Idänpään-Heikkilä I,Lunnas T,Palmroth J,Barr E,Kuortti M,Cacciatore R,Isaksson R,Kekki M,Koskela P,Kosunen E,Lahti L,Liljamo T,Luostarinen T,Apter D,Pukkala E,Paavonen J",2006,Int J STD & AIDS,,,,,, -10.1002/ijc.31281,Evaluation of HPV type-replacement in unvaccinated and vaccinated adolescent females-Post-hoc analysis of a community-randomized clinical trial (II).,"Gray P,Palmroth J,Luostarinen T,Apter D,Dubin G,Garnett G,Eriksson T,Natunen K,Merikukka M,Pimenoff V,Söderlund-Strand A,Vänskä S,Paavonen J,Pukkala E,Dillner J,Lehtinen M.",2018,Int J Cancer,,,,,, -10.1371/journal.pmed.1003588,Human papillomavirus seroprevalence in pregnant women following gender-neutral and girls-only vaccination programs in Finland: A cross-sectional cohort analysis following a cluster randomized trial,"Gray P,Kann H,Pimenoff VN,Eriksson T,Luostarinen T,Vänskä S,Surcel HM,Faust H,Dillner J,Lehtinen M,"",""",2021,PLoS Med,,,,,, -10.1371/journal.pone.0072088,Impact of vaccination on 14 high-risk HPV type infections: a mathematical modelling approach,"Vänskä S,Auranen K,Leino T,Salo H,Nieminen P,Kilpi T,Tiihonen P,Apter D,Lehtinen M",2013,Plos One,,,,,, -10.1016/j.vaccine.2014.12.019,Characteristics of a cluster-randomized phase IV human papillomavirus vaccination effectiveness trial,"Lehtinen M,Apter D,Baussano I,Eriksson T,Natunen K,Paavonen J,Vänskä S,Bi D,David MP,Datta S,Struyf F,Jenkins D,Pukkala E,Garnett G,Dubin G",2015,Vaccine,,,,,, -10.1002/ijc.32189,Occurrence of human papillomavirus (HPV) type replacement by sexual risk-taking behaviour group: Post-hoc analysis of a community randomized clinical trial up to 9 years after vaccination (IV),"Gray P,Luostarinen T,Vänskä S,Eriksson T,Lagheden C,Man I,Palmroth J,Pimenoff VN,Söderlund-Strand A,Dillner J,Lehtinen M",2019,Int J Cancer,,,,,, -10.1258/095646206778145550,"Enrolment of 22,000 adolescents to a population based HPV","""Lehtinen M,""",2006,Int J STD & AIDS,,,,,, -10.1002/ijc.31119,Impact of gender-neutral or girls-only vaccination against HPV. Results of a community randomized trial (I),"""Lehtinen M,""",2018,Int J Cancer,,,,,, -10.1002/cam4.1222,Cancer Registry follow-up for 17 million person-years of a nationwide maternity cohort.,"Lehtinen M,Surcel HM,Natunen K,Pukkala E,Dillner J",2017,Cancer Med.,,,,,, -10.1097/EE9.0000000000000182,Human exposome assessment platform,"Trier Møller F,Clements MS,Kozlakidis Z,Pimenoff VN,Wilkowski B,Boeckhout M,Öhman H,Chong S,Holzinger A,Lehtinen M,van Veen EB,Bała P,Widschwendter M,Snyder MP,Dillner J,Merino Martinez R,Müller H,Negru S,Ormenisan A,Arroyo Mühr LS,Zhang X",2021,Environmental Epidemiology,,,,,, -10.1093/ije/dyv151,EDEN mother-child cohort study group. Cohort Profile: The EDEN mother-child cohort on the prenatal and early postnatal determinants of child health and development.,,,,,,,,, -10.1093/ije/dys064,Cohort Profile: the 'children of the 90s'--the index offspring of the Avon Longitudinal Study of Parents and Children.,"Boyd A,Golding J,Macleod J,Lawlor DA,Fraser A,Henderson J,Molloy L,Ness A,Ring S,Davey Smith G.",2013,Int J Epidemiol.,42,1,111-27,,,"The Avon Longitudinal Study of Parents and Children (ALSPAC) is a transgenerational prospective observational study investigating influences on health and development across the life course. It considers multiple genetic, epigenetic, biological, psychological, social and other environmental exposures in relation to a similarly diverse range of health, social and developmental outcomes. Recruitment sought to enroll pregnant women in the Bristol area of the UK during 1990-92; this was extended to include additional children eligible using the original enrollment definition up to the age of 18 years. The children from 14541 pregnancies were recruited in 1990-92, increasing to 15247 pregnancies by the age of 18 years. This cohort profile describes the index children of these pregnancies. Follow-up includes 59 questionnaires (4 weeks-18 years of age) and 9 clinical assessment visits (7-17 years of age). The resource comprises a wide range of phenotypic and environmental measures in addition to biological samples, genetic (DNA on 11343 children, genome-wide data on 8365 children, complete genome sequencing on 2000 children) and epigenetic (methylation sampling on 1000 children) information and linkage to health and administrative records. Data access is described in this article and is currently set up as a supported access resource. To date, over 700 peer-reviewed articles have been published using ALSPAC data." +https://doi.org/10.1371/journal.pone.0209003,Cervical cancer screening in Sweden 2014-2016.,"""Hortlund, M."",""Elfström, K."",""Sparén, P"",""Almstedt, P."",""Strander, B."",""Dillner, J.""",2018,PLoS ONE,,13,,,, +https://doi.org/10.1002/ijc.32791,Effectiveness of AS04-adjuvanted human papillomavirus (HPV) type 16/18 vaccine in reducing oropharyngeal HPV infections in young females – results from a community-randomized tria,"Lehtinen M,Apter D,Eriksson T,Harjula K,Hokkanen M,Lehtinen T,Natunen K,Damaso S,Soila M,Bi D,Struyf F,"",""",2020,Int J Cancer,,,,,, +https://doi.org/10.1016/S1473-3099(20)30873-2,"Sustainability of neutralising antibodies induced by bivalent or quadrivalent HPV vaccines and correlation with efficacy: a combined follow-up analysis of data from two randomised, double-blind, multicentre, phase 3 trials.","Filipe Colaço Mariz,Penelope Gray,Noemi Bender,Tiina Eriksson,Hanna Kann,Dan Apter,Jorma Paavonen,Emma Pajunen,Kristina M Prager,Peter Sehr,Heljä-Marja Surcel,Tim Waterboer,Martin Müller,Michael Pawlita,Matti Lehti",2021,Lancet Inf. Disease,,,,,, +https://doi.org/10.1093/infdis/jiaa099,Vaccination With Moderate Coverage Eradicates Oncogenic Human Papillomaviruses If a Gender-Neutral Strategy Is Applied.,"Vänskä S,Luostarinen T,Baussano I,Apter D,Eriksson T,Natunen K,Nieminen P,Paavonen J,Pimenoff VN,Pukkala E,Söderlund-Strand A,Dubin G,Garnett G,Dillner J,Lehtinen"",""",2020,JID,,,,,, +https://doi.org/10.1002/ijc.27586,Occurrence of vaccine and non-vaccine human papillomavirus types in adolescent Finnish females 4 years post-vaccination,"Palmroth J,Merikukka M,Paavonen J,Apter D,Eriksson T,Natunen K,Dubin G,Lehtinen M",2012,Int J Cancer,,,,,, +https://doi.org/10.1080/21645515.2016.1183847,Safety of the human papillomavirus (HPV)-16/18 AS04-adjuvanted vaccine in adolescents aged 12-15 years: Interim analysis of a large community-randomized controlled tria,"Lehtinen M,Eriksson T,Apter D,Hokkanen M,Natunen K,Paavonen J,Pukkala E,Angelo MG,Zima J,David MP,Datta S,Bi D,Struyf F,Dubin G",2016,Hum Vaccin Immunother,,,,,, +https://doi.org/10.1136/bmjopen-2017-015867,Ten-year follow-up of human papillomavirus vaccine efficacy against the most stringent cervical neoplasia end-point-registry-based follow-up of three cohorts from randomized trial,"Lehtinen M,Lagheden C,Luostarinen T,Eriksson T,Apter D,Harjula K,Kuortti M,Natunen K,Palmroth J,Petäjä T,Pukkala E,Siitari-Mattila M,Struyf F,Nieminen P,Paavonen J,Dubin G,Dillner",2017,BMJ Open,,,,,, +https://doi.org/10.1002/ijc.31618,Gender-neutral vaccination provides improved control of human papillomavirus types 18/31/33/35 through herd immunity: Results of a community randomized trial (III),"Lehtinen M,Luostarinen T,Vänskä S,Söderlund-Strand A,Eriksson T,Natunen K,Apter D,Baussano I,Harjula K,Hokkanen M,Kuortti M,Palmroth J,Petäjä T,Pukkala E,Rekonen S,Siitari-Mattila M,Surcel HM,Tuomivaara L,Paavonen J,Nieminen P,Dill",2018,Int J Cancer,,,,,, +https://doi.org/10.1002/ijc.32802,Baseline findings and safety of infrequent vs. frequent screening of human papillomavirus vaccinated women,"Louvanto K,Eriksson T,Gray P,Apter D,Baussano I,Bly A,Harjula K,Heikkilä K,Hokkanen M,Huhtinen L,Ikonen M,Karttunen H,Nummela M,Söderlund-Strand A,Veivo U,Dillner J,Elfstöm M,Nieminen P,Lehtine",2020,Int J Cancer,,,,,, +https://doi.org/10.1002/ijc.33169,Long-term follow-up of human papillomavirus type replacement among young pregnant Finnish females before and after a community-randomised HPV vaccination trial with moderate coverage,"Gray P,Kann H,Pimenoff VN,Adhikari I,Eriksson T,Surcel HM,Vänskä S,Dillner J,Faust H,Lehtinen",2020,Int J Cancer,,,,,, +https://doi.org/10.1258/095646206776253453,Population based enrolment of adolescent girls into long-term human papillomavirus vaccination trial.,"Lehtinen M,Idänpään-Heikkilä I,Lunnas T,Palmroth J,Barr E,Kuortti M,Cacciatore R,Isaksson R,Kekki M,Koskela P,Kosunen E,Lahti L,Liljamo T,Luostarinen T,Apter D,Pukkala E,Paavon",2006,Int J STD & AIDS,,,,,, +https://doi.org/10.1002/ijc.31281,Evaluation of HPV type-replacement in unvaccinated and vaccinated adolescent females-Post-hoc analysis of a community-randomized clinical trial (II).,"Gray P,Palmroth J,Luostarinen T,Apter D,Dubin G,Garnett G,Eriksson T,Natunen K,Merikukka M,Pimenoff V,Söderlund-Strand A,Vänskä S,Paavonen J,Pukkala E,Dillner J,Lehtinen",2018,Int J Cancer,,,,,, +https://doi.org/10.1371/journal.pmed.1003588,Human papillomavirus seroprevalence in pregnant women following gender-neutral and girls-only vaccination programs in Finland: A cross-sectional cohort analysis following a cluster randomized trial,"Gray P,Kann H,Pimenoff VN,Eriksson T,Luostarinen T,Vänskä S,Surcel HM,Faust H,Dillner J,Lehtinen "",""",2021,PLoS Med,,,,,, +https://doi.org/10.1371/journal.pone.0072088,Impact of vaccination on 14 high-risk HPV type infections: a mathematical modelling approach,"Vänskä S,Auranen K,Leino T,Salo H,Nieminen P,Kilpi T,Tiihonen P,Apter D,Lehtinen",2013,Plos One,,,,,, +https://doi.org/10.1016/j.vaccine.2014.12.019,Characteristics of a cluster-randomized phase IV human papillomavirus vaccination effectiveness trial,"Lehtinen M,Apter D,Baussano I,Eriksson T,Natunen K,Paavonen J,Vänskä S,Bi D,David MP,Datta S,Struyf F,Jenkins D,Pukkala E,Garnett G,Dubin",2015,Vaccine,,,,,, +https://doi.org/10.1002/ijc.32189,Occurrence of human papillomavirus (HPV) type replacement by sexual risk-taking behaviour group: Post-hoc analysis of a community randomized clinical trial up to 9 years after vaccination (IV),"Gray P,Luostarinen T,Vänskä S,Eriksson T,Lagheden C,Man I,Palmroth J,Pimenoff VN,Söderlund-Strand A,Dillner J,Lehtine",2019,Int J Cancer,,,,,, +https://doi.org/10.1258/095646206778145550,"Enrolment of 22,000 adolescents to a population based HPV","""Lehtinen M,""",2006,Int J STD & AIDS,,,,,, +https://doi.org/10.1002/ijc.31119,Impact of gender-neutral or girls-only vaccination against HPV. Results of a community randomized trial (I),"""Lehtinen M,""",2018,Int J Cancer,,,,,, +https://doi.org/10.1002/cam4.1222,Cancer Registry follow-up for 17 million person-years of a nationwide maternity cohort,"Lehtinen M,Surcel HM,Natunen K,Pukkala E,Dillner J",2017,Cancer Med.,,,,,, +https://doi.org/10.1097/EE9.0000000000000182,Human exposome assessment platform,"Trier Møller F,Clements MS,Kozlakidis Z,Pimenoff VN,Wilkowski B,Boeckhout M,Öhman H,Chong S,Holzinger A,Lehtinen M,van Veen EB,Bała P,Widschwendter M,Snyder MP,Dillner J,Merino Martinez R,Müller H,Negru S,Ormenisan A,Arroyo Mühr LS,Zh",2021,Environmental Epidemiology,,,,,, +https://doi.org/10.1093/ije/dyv151,EDEN mother-child cohort study group. Cohort Profile: The EDEN mother-child cohort on the prenatal and early postnatal determinants of child health and development.,,,,,,,,, +https://doi.org/10.1093/ije/dys064,Cohort Profile: the 'children of the 90s'--the index offspring of the Avon Longitudinal Study of Parents and Children.,"Boyd A,Golding J,Macleod J,Lawlor DA,Fraser A,Henderson J,Molloy L,Ness A,Ring S,Davey Smith G.",2013,Int J Epidemiol.,42,1,111-27,,,"The Avon Longitudinal Study of Parents and Children (ALSPAC) is a transgenerational prospective observational study investigating influences on health and development across the life course. It considers multiple genetic, epigenetic, biological, psychological, social and other environmental exposures in relation to a similarly diverse range of health, social and developmental outcomes. Recruitment sought to enroll pregnant women in the Bristol area of the UK during 1990-92; this was extended to include additional children eligible using the original enrollment definition up to the age of 18 years. The children from 14541 pregnancies were recruited in 1990-92, increasing to 15247 pregnancies by the age of 18 years. This cohort profile describes the index children of these pregnancies. Follow-up includes 59 questionnaires (4 weeks-18 years of age) and 9 clinical assessment visits (7-17 years of age). The resource comprises a wide range of phenotypic and environmental measures in addition to biological samples, genetic (DNA on 11343 children, genome-wide data on 8365 children, complete genome sequencing on 2000 children) and epigenetic (methylation sampling on 1000 children) information and linkage to health and administrative records. Data access is described in this article and is currently set up as a supported access resource. To date, over 700 peer-reviewed articles have been published using ALSPAC data." https://doi.org/10.1093/ije/dyw029,Cohort profile update: the Norwegian mother and child cohort study (MoBa).,"Magnus P,Birke C,Vejrup K,Haugan A,Alsaker E,Daltveit AK,Handal M,Haugen M,H iseth G,Knudsen GP,Paltiel L,Schreuder P,Tambs K,Vold",2016,Int J Epidemiol,45,2,"382,388",,, -10.1093/ajcn/nqy356,Using metabolic profiling and gene expression analyses to explore molecular effects of replacing saturated fat with polyunsaturated fat-a randomized controlled dietary intervention study.,"""Ulven, Stine M et al""",2019,The American journal of clinical nutrition,109,5,"12,391,250",,, -10.1016/j.numecd.2020.06.018,Associations between dietary patterns and gene expression pattern in peripheral blood mononuclear cells: A cross-sectional study.,"""Christensen, Jacob J et al.""",2020,"Nutrition, metabolism, and cardiovascular diseases : NMCD",30,11,"21,112,122",,, -10.2196/resprot.3873,Internet-Based Birth-Cohort Studies: Is This the Future for Epidemiology?,"Firestone R,Cheng S,Pearce N,Douwes J,Merletti F,Pizzi C,Pivetta E,Rusconi F,Richiardi L",2015,JMIR Research Protocols,,,,,,"Background: International collaborative cohorts the NINFEA and the ELF studies are mother-child cohorts that use the internet for recruitment and follow-up of their members. The cohorts investigated the association of early life exposures and a wide range of non-communicable diseases. +https://doi.org/10.1093/ajcn/nqy356,Using metabolic profiling and gene expression analyses to explore molecular effects of replacing saturated fat with polyunsaturated fat-a randomized controlled dietary intervention study.,"""Ulven, Stine M et al""",2019,The American journal of clinical nutrition,109,5,"12,391,250",,, +https://doi.org/10.1016/j.numecd.2020.06.018,Associations between dietary patterns and gene expression pattern in peripheral blood mononuclear cells: A cross-sectional study.,"""Christensen, Jacob J et al.""",2020,"Nutrition, metabolism, and cardiovascular diseases : NMCD",30,11,"21,112,122",,, +https://doi.org/10.2196/resprot.3873,Internet-Based Birth-Cohort Studies: Is This the Future for Epidemiology?,"Firestone R,Cheng S,Pearce N,Douwes J,Merletti F,Pizzi C,Pivetta E,Rusconi F,Richiardi L",2015,JMIR Research Protocols,,,,,,"Background: International collaborative cohorts the NINFEA and the ELF studies are mother-child cohorts that use the internet for recruitment and follow-up of their members. The cohorts investigated the association of early life exposures and a wide range of non-communicable diseases. Objective: The objective is to report the research methodology, with emphasis on the advantages and limitations offered by an Internet-based design. These studies were conducted in Turin, Italy and Wellington, New Zealand. @@ -33,7 +33,7 @@ Methods: The cohorts utilized various online/offline methods to recruit particip Results: The NINFEA study has recruited 7003 pregnant women, while the ELF study has recruited 2197 women. The cohorts targeted the whole country, utilizing a range of support processes to reduce the attrition rate of the participants. For the NINFEA and ELF cohorts, online participants were predominantly older (35% and 28.9%, respectively), highly educated (55.6% and 84.9%, respectively), and were in their final trimester of pregnancy (48.5% and 53.6%, respectively). Conclusions: Internet-based cohort epidemiological studies are feasible, however, it is clear that participants are self-selective samples, as is the case for many birth cohorts. Internet-based cohort studies are potentially cost-effective and novel methodology for conducting long-term epidemiology research. However, from our experience, participants tend to be self-selective. In marked time, if the cohorts are to form part of a larger research program they require further use and exploration to address biases and overcome limitations." -10.1371/journal.pone.0090352,"Effects of whole grain, fish and bilberries on serum metabolic profile and lipid transfer protein activities: a randomized trial (Sysdimet)","Maria Lankinen,Marjukka Kolehmainen,Tiina J skel inen,Jussi Paananen,Laura Joukamo,Antti J Kangas,Pasi Soininen,Kaisa Poutanen,Hannu Mykk nen,Helena Gylling,Matej Ore ",2014,pLoS One,,,,,,"Objective: We studied the combined effects of wholegrain, fish and bilberries on serum metabolic profile and lipid transfer protein activities in subjects with the metabolic syndrome. +https://doi.org/10.1371/journal.pone.0090352,"Effects of whole grain, fish and bilberries on serum metabolic profile and lipid transfer protein activities: a randomized trial (Sysdimet)","Maria Lankinen,Marjukka Kolehmainen,Tiina J skel inen,Jussi Paananen,Laura Joukamo,Antti J Kangas,Pasi Soininen,Kaisa Poutanen,Hannu Mykk nen,Helena Gylling,Matej Ore ",2014,pLoS One,,,,,,"Objective: We studied the combined effects of wholegrain, fish and bilberries on serum metabolic profile and lipid transfer protein activities in subjects with the metabolic syndrome. Methods: Altogether 131 subjects (40-70 y, BMI 26-39 kg/m(2)) with impaired glucose metabolism and features of the metabolic syndrome were randomized into three groups with 12-week periods according to a parallel study design. They consumed either: a) wholegrain and low postprandial insulin response grain products, fatty fish 3 times a week, and bilberries 3 portions per day (HealthyDiet), b) wholegrain and low postprandial insulin response grain products (WGED), or c) refined wheat breads as cereal products (Control). Altogether 106 subjects completed the study. Serum metabolic profile was studied using an NMR-based platform providing information on lipoprotein subclasses and lipids as well as low-molecular-weight metabolites. @@ -42,8 +42,8 @@ Results: There were no significant differences in clinical characteristics betwe Conclusions: The results suggest that consumption of diet rich in whole grain, bilberries and especially fatty fish causes changes in HDL particles shifting their subclass distribution toward larger particles. These changes may be related to known protective functions of HDL such as reverse cholesterol transport and could partly explain the known protective effects of fish consumption against atherosclerosis. Trial registration: The study was registered at ClinicalTrials.gov NCT00573781." -10.5694/mja12.10698,Birth of a cohort--the first 20 years of the Raine study.,,,,,,,,, -10.3945/ajcn.113.064071,Lower protein content in infant formula reduces BMI and obesity risk at school age: follow-up of a randomized trial,"Martina Weber,Veit Grote,Ricardo Closa-Monasterolo,Joaqu n Escribano,Jean-Paul Langhendries,Elena Dain,Marcello Giovannini,Elvira Verduci,Dariusz Gruszfeld,Piotr Socha,B",2014,The American Journal of Clinical Nutrition,99,5,1041-1051,,,"Background: Early nutrition is recognized as a target for the effective prevention of childhood obesity. Protein intake was associated with more rapid weight gain during infancy a known risk factor for later obesity. +https://doi.org/10.5694/mja12.10698,Birth of a cohort--the first 20 years of the Raine study.,,,,,,,,, +https://doi.org/10.3945/ajcn.113.064071,Lower protein content in infant formula reduces BMI and obesity risk at school age: follow-up of a randomized trial,"Martina Weber,Veit Grote,Ricardo Closa-Monasterolo,Joaqu n Escribano,Jean-Paul Langhendries,Elena Dain,Marcello Giovannini,Elvira Verduci,Dariusz Gruszfeld,Piotr Socha,B",2014,The American Journal of Clinical Nutrition,99,5,1041-1051,,,"Background: Early nutrition is recognized as a target for the effective prevention of childhood obesity. Protein intake was associated with more rapid weight gain during infancy a known risk factor for later obesity. Objective: We tested whether the reduction of protein in infant formula reduces body mass index (BMI; in kg/m2) and the prevalence of obesity at 6 y of age. @@ -52,8 +52,8 @@ Design: The Childhood Obesity Project was conducted as a European multicenter, d Results:; P = 0.009) at 6 y of age. The risk of becoming obese in the HP group was 2.43 (95% CI: 1.12, 5.27; P = 0.024) times that in the LP group. There was a tendency for a higher weight in HP children (0.67 kg; 95% CI: -0.04, 1.39 kg; P = 0.064) but no difference in height between the intervention groups. Anthropometric measurements were similar in the LP and breastfed groups. Conclusions: Infant formula with a lower protein content reduces BMI and obesity risk at school age. Avoidance of infant foods that provide excessive protein intakes could contribute to a reduction in childhood obesity. This trial was registered at clinicaltrials.gov as NCT00338689." -10.1016/j.envpol.2021.117650,Pre- and early post-natal exposure to phthalates and DINCH in a new type of mother-child cohort relying on the collection of repeated urine samples,Philippat et el.,2021,,,,,,, -doi.org/10.1002/oby.22203,Effect of Lower Versus Higher Protein Content in Infant Formula Through the First Year on Body Composition from 1 to 6 Years: Follow-Up of a Randomized Clinical Trial,"Martina Totzauer,Veronica Luque,Joaquin Escribano,Ricardo Closa-Monasterolo,Elvira Verduci,Alice ReDionigi,Joana Hoyos,Jean-Paul Langhendries,Dariusz Gruszfeld,Piotr Socha,Berthold Koletzko,Veit Grote",2018,Obesity (Silver Spring),,,1203-1210,,,"Objective: The objective of this study was to investigate the effect of lower protein (LP) versus higher protein (HP) content in infant formula on body composition from 3 months to 6 years. +https://doi.org/10.1016/j.envpol.2021.117650,Pre- and early post-natal exposure to phthalates and DINCH in a new type of mother-child cohort relying on the collection of repeated urine samples,Philippat et el.,2021,,,,,,, +https://doi.org/10.1002/oby.22203,Effect of Lower Versus Higher Protein Content in Infant Formula Through the First Year on Body Composition from 1 to 6 Years: Follow-Up of a Randomized Clinical Trial,"Martina Totzauer,Veronica Luque,Joaquin Escribano,Ricardo Closa-Monasterolo,Elvira Verduci,Alice ReDionigi,Joana Hoyos,Jean-Paul Langhendries,Dariusz Gruszfeld,Piotr Socha,Berthold Koletzko,Veit Grote",2018,Obesity (Silver Spring),,,1203-1210,,,"Objective: The objective of this study was to investigate the effect of lower protein (LP) versus higher protein (HP) content in infant formula on body composition from 3 months to 6 years. Methods: In a multicenter, double-blind European trial, healthy infants (N = 1,090) were randomly assigned to different protein content formulas (upper [HP] and lower [LP] limits of the European Union regulations in 2001) during the first year; breastfed infants (N = 588) were recruited for reference values. Weight, height, and triceps and subscapular skinfold (SF) thickness were measured repeatedly (N = 650 at 6 years), and body composition was estimated (Slaughter). The 99th percentile of fat mass index reference data were used to assess excess body fat at 6 years. @@ -62,46 +62,46 @@ Results: At 2 and 6 years, the study observed greater sum of SFs ( 2 years Conclusions: Infant formula with HP levels induced greater fat mass in children from 2 to 6 years. Lowering the protein content of infant formula may result in a healthier body composition in early childhood. Tri" -10.1016/j.envint.2020.105678,Exposure to phenols during pregnancy and the first year of life in a new type of couple-child cohort relying on repeated urine biospecimens,Rolland et al.,2020,,,,,,, -10.14301/llcs.v4i2.221,"Design and characteristics of a new birth cohort, to study the early origins and ethnic variation of childhood obesity: the BiB1000 study","Bryant M,Santorelli G,Fairley L,West J,Lawlor DA,Bhopal R,Petherick E,Sahota P,Hill A,Cameron N,Small N,Wright J",2013,Longitudinal and Life Course Studies,4,2,119-135,Born in Bradford,,"Epidemiological evidence indicates that early life factors are important for obesity development but there are gaps in knowledge regarding the impact of exposures during pregnancy and early life, especially in South Asian children. There is a corresponding lack of evidence to guide development of culturally-appropriate, obesity prevention programmes. This paper describes the methodology and characteristics of participants in Born in Bradford 1000 (BiB1000), a nested cohort of the Born in Bradford prospective birth cohort. BiB1000 aims to enable a deep and extensive understanding of the predictors and influences of health-related behaviours to develop a culturally-specific obesity prevention intervention. 1,735 mothers agreed to take part in detailed assessments focused on risk factors of obesity. Of these, 1,707 had singleton births. Data were collected from the families during pregnancy, at birth and when the infant was aged 6, 12, 18, 24 and 36 months. Approximately half of the mothers (n=933) are of South Asian ethnicity; of which, just under half were born in the UK. Prevalence of obesity in BiB1000 is similar to the full BiB cohort and to UK national averages. In addition to pre-specified hypothesised targets for obesity prevention, (e.g. parental feeding styles, diet and activity), BiB1000 is exploring qualitative determinants of behaviours andother exposures with a lesser evidence base (e.g. food environments, sleep, parenting practices). These data will enable a rich understanding of the behaviours and their determinants in order to inform the development of a culturally-relevant, childhood obesity prevention intervention." -10.1007/s10654-015-0082-x,The Rotterdam Study: 2016 objectives and design update,"Albert Hofman,Guy G O Brusselle,Sarwa Darwish Murad,Cornelia M van Duijn,Oscar H Franco,Andr Goedegebure,M Arfan Ikram,Caroline C W Klaver,Tamar E C Nijsten,Robin P Peeters,Bruno H Ch Stricker,Henning W Tiemeier,Andr ",2015,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in The Netherlands. The study targets cardiovascular, endocrine, hepatic, neurological, ophthalmic, psychiatric, dermatological, otolaryngological, locomotor, and respiratory diseases. As of 2008, 14,926 subjects aged 45 years or over comprise the Rotterdam Study cohort. The findings of the Rotterdam Study have been presented in over 1200 research articles and reports (see www.erasmus-epidemiology.nl/rotterdamstudy ). This article gives the rationale of the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." -10.1186/1471-2458-8-327,"Born in Bradford, a cohort study of babies born in Bradford, and their parents: protocol for the recruitment phase","Pauline Raynor,Born in Bradford Collaborative Group",2008,BMC Public Health,,,,,,"Background: Bradford, one of the most deprived cities in the United Kingdom, has a wide range of public health problems associated with socioeconomic deprivation, including an infant mortality rate almost double that for England and Wales. Infant mortality is highest for babies of Pakistani origin, who comprise almost half the babies born in Bradford. The Born in Bradford cohort study aims to examine environmental, psychological and genetic factors that impact on health and development perinatally, during childhood and subsequent adult life, and those that influence their parents' health and wellbeing. This protocol outlines methods for the recruitment phase of the study. +https://doi.org/10.1016/j.envint.2020.105678,Exposure to phenols during pregnancy and the first year of life in a new type of couple-child cohort relying on repeated urine biospecimens,Rolland et al.,2020,,,,,,, +https://doi.org/10.14301/llcs.v4i2.221,"Design and characteristics of a new birth cohort, to study the early origins and ethnic variation of childhood obesity: the BiB1000 study","Bryant M,Santorelli G,Fairley L,West J,Lawlor DA,Bhopal R,Petherick E,Sahota P,Hill A,Cameron N,Small N,Wright J",2013,Longitudinal and Life Course Studies,4,2,119-135,Born in Bradford,,"Epidemiological evidence indicates that early life factors are important for obesity development but there are gaps in knowledge regarding the impact of exposures during pregnancy and early life, especially in South Asian children. There is a corresponding lack of evidence to guide development of culturally-appropriate, obesity prevention programmes. This paper describes the methodology and characteristics of participants in Born in Bradford 1000 (BiB1000), a nested cohort of the Born in Bradford prospective birth cohort. BiB1000 aims to enable a deep and extensive understanding of the predictors and influences of health-related behaviours to develop a culturally-specific obesity prevention intervention. 1,735 mothers agreed to take part in detailed assessments focused on risk factors of obesity. Of these, 1,707 had singleton births. Data were collected from the families during pregnancy, at birth and when the infant was aged 6, 12, 18, 24 and 36 months. Approximately half of the mothers (n=933) are of South Asian ethnicity; of which, just under half were born in the UK. Prevalence of obesity in BiB1000 is similar to the full BiB cohort and to UK national averages. In addition to pre-specified hypothesised targets for obesity prevention, (e.g. parental feeding styles, diet and activity), BiB1000 is exploring qualitative determinants of behaviours andother exposures with a lesser evidence base (e.g. food environments, sleep, parenting practices). These data will enable a rich understanding of the behaviours and their determinants in order to inform the development of a culturally-relevant, childhood obesity prevention intervention." +https://doi.org/10.1007/s10654-015-0082-x,The Rotterdam Study: 2016 objectives and design update,"Albert Hofman,Guy G O Brusselle,Sarwa Darwish Murad,Cornelia M van Duijn,Oscar H Franco,Andr Goedegebure,M Arfan Ikram,Caroline C W Klaver,Tamar E C Nijsten,Robin P Peeters,Bruno H Ch Stricker,Henning W Tiemeier,Andr ",2015,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in The Netherlands. The study targets cardiovascular, endocrine, hepatic, neurological, ophthalmic, psychiatric, dermatological, otolaryngological, locomotor, and respiratory diseases. As of 2008, 14,926 subjects aged 45 years or over comprise the Rotterdam Study cohort. The findings of the Rotterdam Study have been presented in over 1200 research articles and reports (see www.erasmus-epidemiology.nl/rotterdamstudy ). This article gives the rationale of the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." +https://doi.org/10.1186/1471-2458-8-327,"Born in Bradford, a cohort study of babies born in Bradford, and their parents: protocol for the recruitment phase","Pauline Raynor,Born in Bradford Collaborative Group",2008,BMC Public Health,,,,,,"Background: Bradford, one of the most deprived cities in the United Kingdom, has a wide range of public health problems associated with socioeconomic deprivation, including an infant mortality rate almost double that for England and Wales. Infant mortality is highest for babies of Pakistani origin, who comprise almost half the babies born in Bradford. The Born in Bradford cohort study aims to examine environmental, psychological and genetic factors that impact on health and development perinatally, during childhood and subsequent adult life, and those that influence their parents' health and wellbeing. This protocol outlines methods for the recruitment phase of the study. Methods: Most Bradford women attend for antenatal care and give birth at the Bradford Royal Infirmary, which has approximately 5,800 births per year. Women are eligible for recruitment if they plan to give birth here. Babies born from March 2007 are eligible to participate, recruitment is planned to continue until 2010. Fathers of babies recruited are invited to participate. Women are usually recruited when they attend for a routine oral glucose tolerance test at 26-28 weeks gestation. Recruitment of babies is at birth. Fathers are recruited whenever possible during the antenatal period, or soon after the birth. The aim is to recruit 10,000 women, their babies, and the babies' fathers. At recruitment women have blood samples taken, are interviewed to complete a semi-structured questionnaire, weighed, and have height, arm circumference and triceps skinfold measured. Umbilical cord blood is collected at birth. Within two weeks of birth babies have their head, arm and abdominal circumference measured, along with subscapular and triceps skinfold thickness. Fathers self-complete a questionnaire at recruitment, have height and weight measured, and provide a saliva sample. Participants are allocated a unique study number. NHS numbers will be used to facilitate record linkage and access to routine data. A wide range of hospital and community sources is being accessed to provide data for the women and children. Data are checked for accuracy and consistency. Conclusion: Born in Bradford will increase understanding of the factors that contribute to health and wellbeing, and identify factors that influence differences in them between people of Pakistani and European origin." -10.1093/ije/dyr054,INMA Project. Cohort Profile: the INMA--INfancia y Medio Ambiente--(Environment and Childhood) Project.,,,,,,,,, -10.1136/bmjopen-2013-003167,The developmental origins of ageing: study protocol for the Dutch famine birth cohort study on ageing,"Susanne R de Rooij,Tessa J Roseboom",2013,BMJ Open,,,,,,"Introduction Evidence from animal studies suggest that the rate of ageing may be influenced not only by genetic and lifestyle factors, but also by the prenatal environment. We have previously shown that people who were exposed to famine during early gestation performed worse on a selective attention task, which may be a first sign of cognitive decline, and were on average 3 years younger at the time of coronary artery disease diagnosis. Women in this group seem to die at a younger age. We hypothesise that an accelerated ageing process, set in motion by the poor prenatal environment, underlies these findings. +https://doi.org/10.1093/ije/dyr054,INMA Project. Cohort Profile: the INMA--INfancia y Medio Ambiente--(Environment and Childhood) Project.,,,,,,,,, +https://doi.org/10.1136/bmjopen-2013-003167,The developmental origins of ageing: study protocol for the Dutch famine birth cohort study on ageing,"Susanne R de Rooij,Tessa J Roseboom",2013,BMJ Open,,,,,,"Introduction Evidence from animal studies suggest that the rate of ageing may be influenced not only by genetic and lifestyle factors, but also by the prenatal environment. We have previously shown that people who were exposed to famine during early gestation performed worse on a selective attention task, which may be a first sign of cognitive decline, and were on average 3 years younger at the time of coronary artery disease diagnosis. Women in this group seem to die at a younger age. We hypothesise that an accelerated ageing process, set in motion by the poor prenatal environment, underlies these findings. Methods and analysis The Dutch Famine Birth Cohort consists of 2414 men and women born in Amsterdam as term singletons around the time of the Dutch famine. In a subsample of 150 cohort members, who now are about 68 years of age, we are currently measuring cognitive decline and the incidence of white matter hyperintensities and cerebral microbleeds (throand physical performance, visual acuity and incidence of cataract operations. In this same subgroup, we will assess telomere length, oxidative stress and inflammatory status as potential underlying mechanisms. Furthermore, in the entire cohort, we will assess mortality as well as hospital admissions for age-related diseases up to the age of 68 years. Ethics and dissemination The study was approved by the local medical ethics committee (Academic Medical Centre, University of Amsterdam) and is being carried out in agreement with the Declaration of Helsinki. All participants give written informed consent. Study findings will be widely disseminated to the scientific public as well as to the medical society and general public." -10.1007/s10654-009-9386-z,The Rotterdam Study: 2010 objectives and design update,"Albert Hofman,Monique M B Breteler,Cornelia M van Duijn,Harry L A Janssen,Gabriel P Krestin,Ernst J Kuipers,Bruno H Ch Stricker,Henning Tiemeier,Andr G Uitterlinden,Johannes R Vingerling,Jacquel",2009,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in The Netherlands. The study targets cardiovascular, endocrine, hepatic, neurological, ophthalmic, psychiatric and respiratory diseases. As of 2008, 14,926 subjects aged 45 years or over comprise the Rotterdam Study cohort. The findings of the Rotterdam Study have been presented in close to a 1,000 research articles and reports (see www.epib.nl/rotterdamstudy). This article gives the rationale of the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." -10.1016/j.envint.2021.106697,Associations between a mixture of phenols and phthalates and child behaviour in a French mother-child cohort with repeated assessment of exposure,Guilbert et al.,2021,,,,,,, -10.1289/ehp.1002775,Urinary Biomarkers of Prenatal Atrazine Exposure and Adverse Birth Outcomes in the PELAGIE Birth Cohort,Chevrier C et al.,2011,EHP,,,,,, -10.1093/ije/dys066,Cohort Profile: the Avon Longitudinal Study of Parents and Children: ALSPAC mothers cohort,"Abigail Fraser,Corrie Macdonald-Wallis,Kate Tilling,Andy Boyd,Jean Golding,George Davey Smith,John Henderson,John Macleod,Lynn Molloy,Andy Ness,Susan Ring,Scott M Nelson,Debbie A Lawlor",2013,International journal of epidemiology,,,,,,"Summary The Avon Longitudinal Study of Children and Parents (ALSPAC) was established to understand how genetic and environmental characteristics influence health and development in parents and children. All pregnant women resident in a defined area in the South West of England, with an expected date of delivery between 1st April 1991 and 31st December 1992, were eligible and 13761 women (contributing 13867 pregnancies) were recruited. These women have been followed over the last 19-22 years and have completed up to 20 questionnaires, have had detailed data abstracted from their medical records and have information on any cancer diagnoses and deaths through record linkage. A follow-up assessment was completed 17-18 years postnatal at which anthropometry, blood pressure, fat, lean and bone mass and carotid intima media thickness were assessed, and a fasting blood sample taken. The second follow-up clinic, which additionally measures cognitive function, physical capability, physical activity (with accelerometer) and wrist bone architecture, is underway and two further assessments with similar measurements will take place over the next 5 years. There is a detailed biobank that includes DNA, with genome-wide data available on >10000, stored serum and plasma taken repeatedly since pregnancy and other samples; a wide range of data on completed biospecimen assays are available. Details of how to access these data are provided in this cohort profile." -10.1007/s10654-006-9022-0,The Generation R Study: Design and cohort profile,,,,,,,,, -10.1007/s00394-020-02438-3,Post-weight loss changes in fasting appetite- and energy balance-related hormone concentrations and the effect of the macronutrient content of a weight maintenance diet: a randomised controlled trial,"Mari Näätänen,Marjukka Kolehmainen,David E. Laaksonen,Karl-Heinz Herzig,Kaisa Poutanen",2021,European Journal of Nutrition,,,,,,"Purpose +https://doi.org/10.1007/s10654-009-9386-z,The Rotterdam Study: 2010 objectives and design update,"Albert Hofman,Monique M B Breteler,Cornelia M van Duijn,Harry L A Janssen,Gabriel P Krestin,Ernst J Kuipers,Bruno H Ch Stricker,Henning Tiemeier,Andr G Uitterlinden,Johannes R Vingerling,Jacquel",2009,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in The Netherlands. The study targets cardiovascular, endocrine, hepatic, neurological, ophthalmic, psychiatric and respiratory diseases. As of 2008, 14,926 subjects aged 45 years or over comprise the Rotterdam Study cohort. The findings of the Rotterdam Study have been presented in close to a 1,000 research articles and reports (see www.epib.nl/rotterdamstudy). This article gives the rationale of the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." +https://doi.org/10.1016/j.envint.2021.106697,Associations between a mixture of phenols and phthalates and child behaviour in a French mother-child cohort with repeated assessment of exposure,Guilbert et al.,2021,,,,,,, +https://doi.org/10.1289/ehp.1002775,Urinary Biomarkers of Prenatal Atrazine Exposure and Adverse Birth Outcomes in the PELAGIE Birth Cohort,Chevrier C et al.,2011,EHP,,,,,, +https://doi.org/10.1093/ije/dys066,Cohort Profile: the Avon Longitudinal Study of Parents and Children: ALSPAC mothers cohort,"Abigail Fraser,Corrie Macdonald-Wallis,Kate Tilling,Andy Boyd,Jean Golding,George Davey Smith,John Henderson,John Macleod,Lynn Molloy,Andy Ness,Susan Ring,Scott M Nelson,Debbie A Lawlor",2013,International journal of epidemiology,,,,,,"Summary The Avon Longitudinal Study of Children and Parents (ALSPAC) was established to understand how genetic and environmental characteristics influence health and development in parents and children. All pregnant women resident in a defined area in the South West of England, with an expected date of delivery between 1st April 1991 and 31st December 1992, were eligible and 13761 women (contributing 13867 pregnancies) were recruited. These women have been followed over the last 19-22 years and have completed up to 20 questionnaires, have had detailed data abstracted from their medical records and have information on any cancer diagnoses and deaths through record linkage. A follow-up assessment was completed 17-18 years postnatal at which anthropometry, blood pressure, fat, lean and bone mass and carotid intima media thickness were assessed, and a fasting blood sample taken. The second follow-up clinic, which additionally measures cognitive function, physical capability, physical activity (with accelerometer) and wrist bone architecture, is underway and two further assessments with similar measurements will take place over the next 5 years. There is a detailed biobank that includes DNA, with genome-wide data available on >10000, stored serum and plasma taken repeatedly since pregnancy and other samples; a wide range of data on completed biospecimen assays are available. Details of how to access these data are provided in this cohort profile." +https://doi.org/10.1007/s10654-006-9022-0,The Generation R Study: Design and cohort profile,,,,,,,,, +https://doi.org/10.1007/s00394-020-02438-3,Post-weight loss changes in fasting appetite- and energy balance-related hormone concentrations and the effect of the macronutrient content of a weight maintenance diet: a randomised controlled trial,"Mari Näätänen,Marjukka Kolehmainen,David E. Laaksonen,Karl-Heinz Herzig,Kaisa Pouta",2021,European Journal of Nutrition,,,,,,"Purpose We investigated the effects of the macronutrient composition of diets with differing satiety values on fasting appetite-related hormone concentrations after weight loss and examined whether the hormone secretion adapted to changes in body fat mass (FM) and fat-free mass (FFM) during the weight maintenance period (WM)." -10.1371/journal.pone.0022646,"Whole grain products, fish and bilberries alter glucose and lipid metabolism in a randomized, controlled trial: the Sysdimet study","Maria Lankinen,Ursula Schwab,Marjukka Kolehmainen,Jussi Paananen,Kaisa Poutanen,Hannu Mykk nen,Tuulikki Sepp nen-Laakso,Helena Gylli",2011,,,,,,,"Background: Due to the growing prevalence of type 2 diabetes, new dietary solutions are needed to help improve glucose and lipid metabolism in persons at high risk of developing the disease. Herein we investigated the effects of low-insulin-response grain products, fatty fish, and berries on glucose metabolism and plasma lipidomic profiles in persons with impaired glucose metabolism. +https://doi.org/10.1371/journal.pone.0022646,"Whole grain products, fish and bilberries alter glucose and lipid metabolism in a randomized, controlled trial: the Sysdimet study","Maria Lankinen,Ursula Schwab,Marjukka Kolehmainen,Jussi Paananen,Kaisa Poutanen,Hannu Mykk nen,Tuulikki Sepp nen-Laakso,Helena Gylli",2011,,,,,,,"Background: Due to the growing prevalence of type 2 diabetes, new dietary solutions are needed to help improve glucose and lipid metabolism in persons at high risk of developing the disease. Herein we investigated the effects of low-insulin-response grain products, fatty fish, and berries on glucose metabolism and plasma lipidomic profiles in persons with impaired glucose metabolism. Methodology/principal findings: Altogether 106 men and women with impaired glucose metabolism and with at least two other features of the metabolic syndrome were included in a 12-week parallel dietary intervention. The participants were randomized into three diet intervention groups: (1) whole grain and low postprandial insulin response grain products, fatty fish three times a week, and bilberries three portions per day (HealthyDiet group), (2) Whole grain enriched diet (WGED) group, which includes principally the same grain products as group (1), but with no change in fish or berry consumption, and (3) refined wheat breads (Control). Oral glucose tolerance, plasma fatty acids and lipidomic profiles were measured before and after the intervention. Self-reported compliance with the diets was good and the body weight remained constant. Within the HealthyDiet group two hour glucose concentration and area-under-the-curve for glucose decreased and plasma proportion of (n-3) long-chain PUFAs increased (False Discovery Rate p-values <0.05). Increases in eicosapentaenoic acid and docosahexaenoic acid associated curvilinearly with the improved insulin secretion and glucose disposal. Among the 364 characterized lipids, 25 changed significantly in the HealthyDiet group, including multiple triglycerides incorporating the long chain (n-3) PUFA. Conclusions/significance: The results suggest that the diet rich in whole grain and low insulin response grain products, bilberries, and fatty fish improve glucose metabolism and alter the lipidomic profile. Therefore, such a diet may have a beneficial effect in the efforts to prevent type 2 diabetes in high risk persons. Trial registration: ClinicalTrials.gov NCT00573781." -10.1007/BF00145007,Determinants of disease and disability in the elderly: the Rotterdam Elderly Study,"A Hofman,D E Grobbee,P T de Jong,F A van den Ouweland",1991,European Journal of epidemiology,,,,,,"In this paper the Rotterdam Elderly Study is presented. The aim of the study is to investigate determinants of disease occurrence and progression in the elderly. In addition to contributing to our understanding of the etiology of geriatric illnesses, the study is expected to lead to specific recommendations for intervention. The study focuses on causally related determinants of major diseases in the elderly. Fields of interest for the Rotterdam Elderly Study are conditions which interfere the most with the quality of life for the elderly. The aims of the Rotterdam Elderly Study are: (1) To investigate, by means of epidemiologic, clinical and basic research, the determinants of diseases in order to assess their etiologic significance. (2) To investigate potentially modifiable determinants in order to be able to develop preventive strategies by providing specific recommendations for intervention studies. The Rotterdam Elderly Study focuses on four primary areas of research: neurogeriatric diseases, cardiovascular diseases, locomotor diseases and ophthalmologic diseases. It is a prospective follow-up study, in which determinants of disease and determinants of progression of disease will be investigated in the total population of 55 years or over of the district of Ommoord in Rotterdam. It is anticipated that about 10,000 people will participate in the study and they will be examined in the period of 1991 to 1995." +https://doi.org/10.1007/BF00145007,Determinants of disease and disability in the elderly: the Rotterdam Elderly Study,"A Hofman,D E Grobbee,P T de Jong,F A van den Ouweland",1991,European Journal of epidemiology,,,,,,"In this paper the Rotterdam Elderly Study is presented. The aim of the study is to investigate determinants of disease occurrence and progression in the elderly. In addition to contributing to our understanding of the etiology of geriatric illnesses, the study is expected to lead to specific recommendations for intervention. The study focuses on causally related determinants of major diseases in the elderly. Fields of interest for the Rotterdam Elderly Study are conditions which interfere the most with the quality of life for the elderly. The aims of the Rotterdam Elderly Study are: (1) To investigate, by means of epidemiologic, clinical and basic research, the determinants of diseases in order to assess their etiologic significance. (2) To investigate potentially modifiable determinants in order to be able to develop preventive strategies by providing specific recommendations for intervention studies. The Rotterdam Elderly Study focuses on four primary areas of research: neurogeriatric diseases, cardiovascular diseases, locomotor diseases and ophthalmologic diseases. It is a prospective follow-up study, in which determinants of disease and determinants of progression of disease will be investigated in the total population of 55 years or over of the district of Ommoord in Rotterdam. It is anticipated that about 10,000 people will participate in the study and they will be examined in the period of 1991 to 1995." https://doi.org/10.1007/s10654-008-9309-4,The Generation R Study: design and cohort update until the age of 4 years,"""Vincent W. V. Jaddoe, Cornelia M. van Duijn, Albert J. van der Heijden, Johan P. Mackenbach, Henri tte A. Moll, Eric A. P. Steegers, Henning Tiemeier, Andre G. Uitterlinden, Frank C. Verhulst""",2008,European Journal of Epidemiology volume,23,,,,, -10.1007/s10654-007-9199-x,The Rotterdam Study: objectives and design update,"Albert Hofman,Monique M B Breteler,Cornelia M van Duijn,Gabriel P Krestin,Huibert A Pols,Bruno H Ch Stricker,Henning Tiemeier,Andr G Uitterlinden,Johannes R Vingerling,Jacquel",,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in the Netherlands. The study targets cardiovascular, neurological, ophthalmological and endocrine diseases. As of 2008 about 15,000 subjects aged 45 years or over comprise the Rotterdam Study cohort. The findings of the Rotterdam Study have been presented in some 600 research articles and reports (see http://www.epib.nl/rotterdamstudy). This article gives the reasons for the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." -10.1007/s10654-011-9610-5,The Rotterdam Study: 2012 objectives and design update,"Albert Hofman,Cornelia M van Duijn,Oscar H Franco,M Arfan Ikram,Harry L A Janssen,Caroline C W Klaver,Ernst J Kuipers,Tamar E C Nijsten,Bruno H Ch Stricker,Henning Tiemeier,Andr G Uitterlinden,Meike W Vernooij,Jacquel",2011,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in The Netherlands. The study targets cardiovascular, endocrine, hepatic, neurological, ophthalmic, psychiatric, dermatological, oncological, and respiratory diseases. As of 2008, 14,926 subjects aged 45 years or over comprise the Rotterdam Study cohort. The findings of the Rotterdam Study have been presented in over a 1,000 research articles and reports (see www.erasmus-epidemiology.nl/rotterdamstudy ). This article gives the rationale of the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." -10.1007/s10654-013-9866-z,The Rotterdam Study: 2014 objectives and design update,"Albert Hofman,Sarwa Darwish Murad,Cornelia M van Duijn,Oscar H Franco,Andr Goedegebure,M Arfan Ikram,Caroline C W Klaver,Tamar E C Nijsten,Robin P Peeters,Bruno H Ch Stricker,Henning W Tiemeier,Andr ",2013,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in The Netherlands. The study targets cardiovascular, endocrine, hepatic, neurological, ophthalmic, psychiatric, dermatological, oncological, and respiratory diseases. As of 2008, 14,926 subjects aged 45 years or over comprise the Rotterdam Study cohort. The findings of the Rotterdam Study have been presented in over a 1,000 research articles and reports (see www.erasmus-epidemiology.nl/rotterdamstudy ). This article gives the rationale of the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." -10.1007/s00125-011-2285-3,"A diet high in fatty fish, bilberries and wholegrain products improves markers of endothelial function and inflammation in individuals with impaired glucose metabolism in a randomised controlled trial: The Sysdimet study.",,,,,,,,, -10.1007/s10654-017-0321-4,"The Rotterdam Study: 2018 update on objectives, design and main results","M Arfan Ikram,Guy G O Brusselle,Sarwa Darwish Murad,Cornelia M van Duijn,Oscar H Franco,Andr Goedegebure,Caroline C W Klaver,Tamar E C Nijsten,Robin P Peeters,Bruno H Stricker,Henning Tiemeier,Andr G Uitterlinde",2017,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in The Netherlands. The study targets cardiovascular, endocrine, hepatic, neurological, ophthalmic, psychiatric, dermatological, otolaryngological, locomotor, and respiratory diseases. As of 2008, 14,926 subjects aged 45 years or over comprise the Rotterdam Study cohort. Since 2016, the cohort is being expanded by persons aged 40 years and over. The findings of the Rotterdam Study have been presented in over 1500 research articles and reports (see www.erasmus-epidemiology.nl/rotterdamstudy ). This article gives the rationale of the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." -10.1186/1476-069X-12-102,"Maternal fish and shellfish consumption and wheeze, eczema and food allergy at age two: a prospective cohort study in Brittany, France",Pelé F e,2013,Environ Health,,,,,, -10.1016/j.envint.2015.05.009,Pyrethroid insecticide exposure and cognitive developmental disabilities in children: The PELAGIE mother ,Viel et al.,2015,Environ Int,,,,,, +https://doi.org/10.1007/s10654-007-9199-x,The Rotterdam Study: objectives and design update,"Albert Hofman,Monique M B Breteler,Cornelia M van Duijn,Gabriel P Krestin,Huibert A Pols,Bruno H Ch Stricker,Henning Tiemeier,Andr G Uitterlinden,Johannes R Vingerling,Jacquel",,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in the Netherlands. The study targets cardiovascular, neurological, ophthalmological and endocrine diseases. As of 2008 about 15,000 subjects aged 45 years or over comprise the Rotterdam Study cohort. The findings of the Rotterdam Study have been presented in some 600 research articles and reports (see http://www.epib.nl/rotterdamstudy). This article gives the reasons for the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." +https://doi.org/10.1007/s10654-011-9610-5,The Rotterdam Study: 2012 objectives and design update,"Albert Hofman,Cornelia M van Duijn,Oscar H Franco,M Arfan Ikram,Harry L A Janssen,Caroline C W Klaver,Ernst J Kuipers,Tamar E C Nijsten,Bruno H Ch Stricker,Henning Tiemeier,Andr G Uitterlinden,Meike W Vernooij,Jacquel",2011,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in The Netherlands. The study targets cardiovascular, endocrine, hepatic, neurological, ophthalmic, psychiatric, dermatological, oncological, and respiratory diseases. As of 2008, 14,926 subjects aged 45 years or over comprise the Rotterdam Study cohort. The findings of the Rotterdam Study have been presented in over a 1,000 research articles and reports (see www.erasmus-epidemiology.nl/rotterdamstudy ). This article gives the rationale of the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." +https://doi.org/10.1007/s10654-013-9866-z,The Rotterdam Study: 2014 objectives and design update,"Albert Hofman,Sarwa Darwish Murad,Cornelia M van Duijn,Oscar H Franco,Andr Goedegebure,M Arfan Ikram,Caroline C W Klaver,Tamar E C Nijsten,Robin P Peeters,Bruno H Ch Stricker,Henning W Tiemeier,Andr ",2013,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in The Netherlands. The study targets cardiovascular, endocrine, hepatic, neurological, ophthalmic, psychiatric, dermatological, oncological, and respiratory diseases. As of 2008, 14,926 subjects aged 45 years or over comprise the Rotterdam Study cohort. The findings of the Rotterdam Study have been presented in over a 1,000 research articles and reports (see www.erasmus-epidemiology.nl/rotterdamstudy ). This article gives the rationale of the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." +https://doi.org/10.1007/s00125-011-2285-3,"A diet high in fatty fish, bilberries and wholegrain products improves markers of endothelial function and inflammation in individuals with impaired glucose metabolism in a randomised controlled trial: The Sysdimet study.",,,,,,,,, +https://doi.org/10.1007/s10654-017-0321-4,"The Rotterdam Study: 2018 update on objectives, design and main results","M Arfan Ikram,Guy G O Brusselle,Sarwa Darwish Murad,Cornelia M van Duijn,Oscar H Franco,Andr Goedegebure,Caroline C W Klaver,Tamar E C Nijsten,Robin P Peeters,Bruno H Stricker,Henning Tiemeier,Andr G Uitterlinde",2017,European Journal of epidemiology,,,,,,"The Rotterdam Study is a prospective cohort study ongoing since 1990 in the city of Rotterdam in The Netherlands. The study targets cardiovascular, endocrine, hepatic, neurological, ophthalmic, psychiatric, dermatological, otolaryngological, locomotor, and respiratory diseases. As of 2008, 14,926 subjects aged 45 years or over comprise the Rotterdam Study cohort. Since 2016, the cohort is being expanded by persons aged 40 years and over. The findings of the Rotterdam Study have been presented in over 1500 research articles and reports (see www.erasmus-epidemiology.nl/rotterdamstudy ). This article gives the rationale of the study and its design. It also presents a summary of the major findings and an update of the objectives and methods." +https://doi.org/10.1186/1476-069X-12-102,"Maternal fish and shellfish consumption and wheeze, eczema and food allergy at age two: a prospective cohort study in Brittany, France",Pelé F ,2013,Environ Health,,,,,, +https://doi.org/10.1016/j.envint.2015.05.009,Pyrethroid insecticide exposure and cognitive developmental disabilities in children: The PELAGIE mother ,Viel et al.,2015,Environ Int,,,,,, https://doi.org/10.1007/s10654-012-9735-1,The Generation R Study: design and cohort update 2012,"""Vincent W. V. Jaddoe, Cornelia M. van Duijn, Oscar H. Franco, Albert J. van der Heijden, Marinus H. van IIzendoorn, Johan C. de Jongste, Aad van der Lugt, Johan P. Mackenbach, Henri tte A. Moll, Hein Raat, Fernando Rivadeneira, Eric A. P.""",2012,European Journal of Epidemiology,27,,739-756,,, -10.1111/j.1365-3016.1997.tb00007.x,Ecological and individual predictors of birthweight in a northern Finland birth cohort 1986,"M R Järvelin,P Elliott,I Kleinschmidt,M Martuzzi,C Grundy,A L Hartikainen,P Rantak",1997,Paediatric and perinatal epidemiology,,,,,,"This multilevel study of spatial variability in, and determinants of, birthweight was conducted using individual and ecological data in a geographically defined prospective birth cohort for 1986 in northern Finland. The study area comprises three large areas defined by latitude: Northern Lapland (NL), Southern Lapland (SL) and Oulu province (OP), comprising 74 localities with a total study population of 9216 singleton births. The mean birthweight was 3482 g for NL, 3537 g for SL and 3587 g for OP (NL vs. OP and SL vs. OP: P < 0.05). The crude rate for stillbirths was highest in NL. The women in the northernmost area were socially less privileged and the localities less prosperous compared with those in the southernmost area. Significant spatial clustering of mean birthweights was found (P = 0.0016), with highest birthweight in the south-western part of the study area. A variable expressing the wealth of each locality, the financial capacity category (FCC), had its lowest mean value in NL, with a range of one to six for the localities studied here. A multilevel multiple regression model showed that, after allowing for sex, gestational age, mother's age, height and hypertensive disorders, parity, body mass index, previous low birthweight child and smoking as individual determinants of birthweight, part of the residual variation could be explained by the locality wealth parameter. Using the multilevel model, the differences in mean birthweight across the three latitude areas persisted but were reduced (difference OP vs. NL reduced from 105 g to 86.5 g). The relationship between birthweight and FCC was inverse U-shaped with the highest mean birthweight estimated for localities occurring in the middle of the range (FCC = 3). The wealthiest urban localities (FCC = 6) and the most deprived localities (FCC = 1) both had a predicted birthweight about 60 g below the maximum at FCC = 3, if all other factors were held constant. This result, taken together with the spatial clustering of birthweights, suggests that there may be important social and environmental determinants of birthweight that have yet to be identified." -10.1111/j.1471-0528.1993.tb12971.x,Labour induction policy in hospitals of different levels of specialisation,"M R Järvelin,A L Hartikainen-Sorri,P Rantak",1993,British journal of obstetrics and gynaecology,,,,,,"Objective: To examine indications for the induction of labour and variations in the current policy of induction at different levels of obstetric specialisation and to compare the outcome of induced and spontaneous labour. +https://doi.org/10.1111/j.1365-3016.1997.tb00007.x,Ecological and individual predictors of birthweight in a northern Finland birth cohort 1986,"M R Järvelin,P Elliott,I Kleinschmidt,M Martuzzi,C Grundy,A L Hartikainen,P Ranta",1997,Paediatric and perinatal epidemiology,,,,,,"This multilevel study of spatial variability in, and determinants of, birthweight was conducted using individual and ecological data in a geographically defined prospective birth cohort for 1986 in northern Finland. The study area comprises three large areas defined by latitude: Northern Lapland (NL), Southern Lapland (SL) and Oulu province (OP), comprising 74 localities with a total study population of 9216 singleton births. The mean birthweight was 3482 g for NL, 3537 g for SL and 3587 g for OP (NL vs. OP and SL vs. OP: P < 0.05). The crude rate for stillbirths was highest in NL. The women in the northernmost area were socially less privileged and the localities less prosperous compared with those in the southernmost area. Significant spatial clustering of mean birthweights was found (P = 0.0016), with highest birthweight in the south-western part of the study area. A variable expressing the wealth of each locality, the financial capacity category (FCC), had its lowest mean value in NL, with a range of one to six for the localities studied here. A multilevel multiple regression model showed that, after allowing for sex, gestational age, mother's age, height and hypertensive disorders, parity, body mass index, previous low birthweight child and smoking as individual determinants of birthweight, part of the residual variation could be explained by the locality wealth parameter. Using the multilevel model, the differences in mean birthweight across the three latitude areas persisted but were reduced (difference OP vs. NL reduced from 105 g to 86.5 g). The relationship between birthweight and FCC was inverse U-shaped with the highest mean birthweight estimated for localities occurring in the middle of the range (FCC = 3). The wealthiest urban localities (FCC = 6) and the most deprived localities (FCC = 1) both had a predicted birthweight about 60 g below the maximum at FCC = 3, if all other factors were held constant. This result, taken together with the spatial clustering of birthweights, suggests that there may be important social and environmental determinants of birthweight that have yet to be identified." +https://doi.org/10.1111/j.1471-0528.1993.tb12971.x,Labour induction policy in hospitals of different levels of specialisation,"M R Järvelin,A L Hartikainen-Sorri,P Ranta",1993,British journal of obstetrics and gynaecology,,,,,,"Objective: To examine indications for the induction of labour and variations in the current policy of induction at different levels of obstetric specialisation and to compare the outcome of induced and spontaneous labour. Design: A prospective 1 year birth cohort. @@ -115,17 +115,17 @@ Results: Labour was induced significantly more often at units of the lowest leve Conclusion: The practice of induction of labour are not consistent in different hospitals. The opinions of individual practitioners and staff routines influence the induction policy nearly as much as do medical reasons. Despite the safety of induction, a liberal induction policy leads to an increase in operative deliveries creating potential risks for the mother and child and greater expense." https://doi.org/10.1007/s10654-014-9980-6,The Generation R Study: Biobank update 2015,"""Claudia J. Kruithof, Marjolein N. Kooijman, Cornelia M. van Duijn, Oscar H. Franco, Johan C. de Jongste, Caroline C. W. Klaver, Johan P. Mackenbach, Henri tte A. Moll, Hein Raat, Edmond H. H. M. Rings, Fernando Rivadeneira, Eric A. P. Stee""",2014,European Journal of Epidemiology,29,,911-927,,, -10.1016/j.numecd.2016.05.005,Dietary quality indices in relation to cardiometabolic risk among Finnish children aged 6-8 years - The PANIC study,"A M Eloranta,U Schwab,T Ven l inen,S Kiiskinen,H M Lakka",2016,"Nutrition, metabolism, and cardiovascular diseases : NMCD",,,,,,"Background and aims: There are no studies on the relationships of dietary quality indices to the clustering of cardiometabolic risk factors in children. We therefore investigated the associations of four dietary quality indices with cardiometabolic risk score and cardiometabolic risk factors in Finnish children. +https://doi.org/10.1016/j.numecd.2016.05.005,Dietary quality indices in relation to cardiometabolic risk among Finnish children aged 6-8 years - The PANIC study,"A M Eloranta,U Schwab,T Ven l inen,S Kiiskinen,H M Lakka",2016,"Nutrition, metabolism, and cardiovascular diseases : NMCD",,,,,,"Background and aims: There are no studies on the relationships of dietary quality indices to the clustering of cardiometabolic risk factors in children. We therefore investigated the associations of four dietary quality indices with cardiometabolic risk score and cardiometabolic risk factors in Finnish children. Methods and results: Subjects were a population sample of 204 boys and 198 girls aged 6-8 years. We assessed diet by 4-day food records and calculated Dietary Approaches to Stop Hypertension (DASH) Score, Baltic Sea Diet Score (BSDS), Mediterranean Diet Score (MDS), and Finnish Children Healthy Eating Index (FCHEI). We calculated the age- and sex-adjusted cardiometabolic risk score summing up Z-scores for waist circumference, mean of systolic and diastolic blood pressure and concentrations of fasting serum insulin and fasting plasma glucose, triglycerides and HDL cholesterol, the last multiplying by -1. Higher FCHEI was associated with lower cardiometabolic risk score among boys (standardised regression coefficient = -0.14, P = 0.044) adjusted for age, physical activity, electronic media time and household income. Higher DASH Score was related to a lower serum insulin in boys ( = -0.15, P = 0.028). Higher DASH Score ( = -0.16, P = 0.023) and FCHEI ( = -0.17, P = 0.014) were related to lower triglyceride concentration in boys. Higher FCHEI was associated with lower triglyceride concentration in girls ( = -0.16, P = 0.033). Higher DASH Score ( = -0.19, P = 0.011) and BSDS ( = -0.23, P = 0.001) were associated with lower plasma HDL cholesterol concentration in girls. Conclusion: Higher FCHEI was associated with lower cardiometabolic risk among boys, whereas DASH Score, BSDS or MDS were not associate" -10.3390/ijerph16203888,Deciphering the Impact of Early-Life Exposures to Highly Variable Environmental Factors on Foetal and Child Health: Design of SEPAGES Couple-Child Cohort,"""Lyon-Caen S, Siroux V, Lepeule J, et al.""",2019,International Journal of Environmental Research and Public Health,16,,,,, -10.1136/bmjopen-2020-042078,Cohort profile: the Dutch famine birth cohort (DFBC)—a prospective birth cohort study in the Ne,,,,,,,,, +https://doi.org/10.3390/ijerph16203888,Deciphering the Impact of Early-Life Exposures to Highly Variable Environmental Factors on Foetal and Child Health: Design of SEPAGES Couple-Child Cohort,"""Lyon-Caen S, Siroux V, Lepeule J, et al.""",2019,International Journal of Environmental Research and Public Health,16,,,,, +https://doi.org/10.1136/bmjopen-2020-042078,Cohort profile: the Dutch famine birth cohort (DFBC)—a prospective birth cohort study in the ,,,,,,,,, https://doi.org/10.1007/s10654-010-9516-7,The Generation R Study: design and cohort update 2010,,2010,European Journal of Epidemiology,25,,,,, -10.1007/s10654-015-0096-4,"The French CONSTANCES population-based cohort: design, inclusion and follow-up.",,,,,,,,, -10.1093/ije/dyl170,Cohort profile: the Norwegian Mother and Child Cohort Study (MoBa).,,,,,,,,, -10.1093/ajcn/nqy394,Decreased plasma serotonin and other metabolite changes in healthy adults after consumption of wholegrain rye: an untargeted metabolomics study,"Pekka Keski-Rahkonen,Marjukka Kolehmainen,Jenni Lappi,Valerie Micard,Jenna Jokkala,Natalia Rosa-Sibakov,Jussi Pihlajam ki,Pirkka V Kirjavainen,Hannu Mykk nen,Kaisa Poutanen,""Marc J Gunter,""",2019,The American journal of clinical nutrition,6,,1630-1639,,,"Background: Wholegrain consumption has been associated with beneficial health effects including reduction of diabetes and cancer risk; however, the underlying mechanisms are not fully understood. +https://doi.org/10.1007/s10654-015-0096-4,"The French CONSTANCES population-based cohort: design, inclusion and follow-up.",,,,,,,,, +https://doi.org/10.1093/ije/dyl170,Cohort profile: the Norwegian Mother and Child Cohort Study (MoBa).,,,,,,,,, +https://doi.org/10.1093/ajcn/nqy394,Decreased plasma serotonin and other metabolite changes in healthy adults after consumption of wholegrain rye: an untargeted metabolomics study,"Pekka Keski-Rahkonen,Marjukka Kolehmainen,Jenni Lappi,Valerie Micard,Jenna Jokkala,Natalia Rosa-Sibakov,Jussi Pihlajam ki,Pirkka V Kirjavainen,Hannu Mykk nen,Kaisa Poutanen,""Marc J Gunter,""",2019,The American journal of clinical nutrition,6,,1630-1639,,,"Background: Wholegrain consumption has been associated with beneficial health effects including reduction of diabetes and cancer risk; however, the underlying mechanisms are not fully understood. Objective: The aim of this study was to characterize the effects of wholegrain rye intake on circulating metabolites in a human intervention study using untargeted metabolomics. @@ -136,23 +136,22 @@ Results: Five endogenous metabolites and 15 rye phytochemicals associated with W Conclusions: Wholegrain rye intake decreases plasma serotonin in healthy adults when compared with refined wheat. Intake of rye bran and wheat aleurone decreases colonic serotonin in mice. These results suggest that peripheral serotonin could be a potential link between wholegrain consumption and its associated health effects.Data used in the study were derived from a trial registered at www.clinicaltrials.gov as NCT03550365. Keywords: 5-HT; alkylresorcinol; dietary intervention; glycerophosphocholine; metabolomics; plasmalogen; rye; serotonin; taurine; wholegrain." -https://doi.org/10.1007/s10654-006-9022-0,The Generation R Study: Design and cohort profile,"""Vincent W. V. Jaddoe, Johan P. Mackenbach, Henri tte A. Moll, Eric A. P. Steegers, Henning Tiemeier, Frank C. Verhulst, Jacqueline C. M. Witteman""",2006,European Journal of Epidemiology volume,21,475,,,, https://doi.org/10.1111/j.1365-3016.2003.00521.x,"Growth, development and health from early fetal life until young adulthood: the Generation R Study","""Albert Hofman, Vincent W. V. Jaddoe, Johan P. Mackenbach, Henriette A. Moll, Rosalinde F. M. Snijders, Eric A. P. Steegers, Frank C. Verhulst, Jacqueline C. M. Witteman, Hans A. B """,2004,Pediatric and Perinatal Epidemiology,18,1,61-72,,, https://doi.org/10.1007/s10654-016-0224-9,The Generation R Study: design and cohort update 2017,"""Marjolein N. Kooijman, Claudia J. Kruithof, Cornelia M. van Duijn, Liesbeth Duijts, Oscar H. Franco, Marinus H. van IJzendoorn, Johan C. de Jongste, Caroline C. W. Klaver, Aad van der Lugt, Johan P. Mackenbach, Henri tte A. Moll, Robin P.""",2016,European Journal of Epidemiology,31,,1243-1264,,, -10.1177/14034948010290040201,"The Danish National Birth Cohort - its background, structure and aim. Scandinavian Journal of Public Health.",,,,,,,,, -10.3945/ajcn.2008.27091,Lower protein in infant formula is associated with lower weight up to age 2 y: a randomized clinical trial.,,,,,,,,, -10.1093/ije/dys112,Cohort Profile: the Born in Bradford multi-ethnic family cohort study.,,,,,,,,, -10.1017/S0007114516003445,"Exchanging a few commercial, regularly consumed food items with improved fat quality reduces total cholesterol and LDL-cholesterol: a double-blind, randomised controlled trial.",,,,,,,,, -10.1007/s10654-007-9194-2,Feasibility of recruiting a birth cohort through the Internet: the experience of the NINFEA cohort.,,,,,,,,, -10.1093/ije/dyw269,Cohort Profile: The ENVIRonmental influence ON early AGEing (ENVIRONAGE): a birth cohort study.,,,,,,,,, -10.1186/1475-2891-13-104,Postprandial glucose metabolism and SCFA after consuming wholegrain rye bread and wheat bread enriched with bioprocessed rye bran in individuals with mild gastrointestinal symptoms.,,,,,,,,, -10.1093/ije/dyi202,Cohort profile: the Southampton women's survey.,,,,,,,,, -10.1016/j.ejso.2017.08.018,Epidemiology of rare cancers and inequalities in oncologic outcomes,"""Gatta G, Trama A, Capocaccia R; RARECARENet Working Group""",2019,Eur J Surg Oncol.,45,1,3-11,,, -10.1007/s10654-020-00640-5,"Objectives, design and main findings until 2020 from the Rotterdam Study.",,,,,,,,, -10.1016/j.ejso.2018.02.006,Testicular germ-cell tumours and penile squamous cell carcinoma: Appropriate management makes the difference,"""Nicolai N, Biasoni D, Catanzaro MA, Colecchia M, Trama A; RARECAREnet Working Group.""",2019,Eur J Surg Oncol.,45,1,60-66,,, -0.1016/S0140-6736(17)33326-3,Global surveillance of trends in cancer survival 2000-14 (CONCORD-3): analysis of individual records for 37 513 025 patients diagnosed with one of 18 cancers from 322 population-based registries in 71 countries,"""Allemani C, Matsuda T, Di Carlo V, Harewood R, Matz M, Nikšić M, Bonaventure A, Valkov M, Johnson CJ, Estève J, Ogunbiyi OJ,"",""Azevedo E Silva G, Chen WQ, Eser S, Engholm G, Stiller CA, Monnereau A, Woods RR, Visser O, Lim GH, Aitken J, Weir HK, Coleman MP; CONCORD Working Group.""",2018,lancet,391,10125,1023-1075,,, +https://doi.org/10.1177/14034948010290040201,"The Danish National Birth Cohort - its background, structure and aim. Scandinavian Journal of Public Health.",,,,,,,,, +https://doi.org/10.3945/ajcn.2008.27091,Lower protein in infant formula is associated with lower weight up to age 2 y: a randomized clinical trial.,,,,,,,,, +https://doi.org/10.1093/ije/dys112,Cohort Profile: the Born in Bradford multi-ethnic family cohort study.,,,,,,,,, +https://doi.org/10.1017/S0007114516003445,"Exchanging a few commercial, regularly consumed food items with improved fat quality reduces total cholesterol and LDL-cholesterol: a double-blind, randomised controlled trial.",,,,,,,,, +https://doi.org/10.1007/s10654-007-9194-2,Feasibility of recruiting a birth cohort through the Internet: the experience of the NINFEA cohort.,,,,,,,,, +https://doi.org/10.1093/ije/dyw269,Cohort Profile: The ENVIRonmental influence ON early AGEing (ENVIRONAGE): a birth cohort study.,,,,,,,,, +https://doi.org/10.1186/1475-2891-13-104,Postprandial glucose metabolism and SCFA after consuming wholegrain rye bread and wheat bread enriched with bioprocessed rye bran in individuals with mild gastrointestinal symptoms.,,,,,,,,, +https://doi.org/10.1093/ije/dyi202,Cohort profile: the Southampton women's survey.,,,,,,,,, +https://doi.org/10.1016/j.ejso.2017.08.018,Epidemiology of rare cancers and inequalities in oncologic outcomes,"""Gatta G, Trama A, Capocaccia R; RARECARENet Working Group""",2019,Eur J Surg Oncol.,45,1,3-11,,, +https://doi.org/10.1007/s10654-020-00640-5,"Objectives, design and main findings until 2020 from the Rotterdam Study.",,,,,,,,, +https://doi.org/10.1016/j.ejso.2018.02.006,Testicular germ-cell tumours and penile squamous cell carcinoma: Appropriate management makes the difference,"""Nicolai N, Biasoni D, Catanzaro MA, Colecchia M, Trama A; RARECAREnet Working Group.""",2019,Eur J Surg Oncol.,45,1,60-66,,, +https://doi.org/0.1016/S0140-6736(17)33326-3,Global surveillance of trends in cancer survival 2000-14 (CONCORD-3): analysis of individual records for 37 513 025 patients diagnosed with one of 18 cancers from 322 population-based registries in 71 count,"""Allemani C, Matsuda T, Di Carlo V, Harewood R, Matz M, Nikšić M, Bonaventure A, Valkov M, Johnson CJ, Estève J, Ogunbiyi "",""Azevedo E Silva G, Chen WQ, Eser S, Engholm G, Stiller CA, Monnereau A, Woods RR, Visser O, Lim GH, Aitken J, Weir HK, Coleman MP; CONCORD Working Group.""",2018,lancet,391,10125,1023-1075,,, https://doi.org/10.1007/s10654-007-9209-z,The Generation R Study Biobank: a resource for epidemiological studies in children and their parents,"""Vincent W. V. Jaddoe, Rachel Bakker, Cock M. van Duijn, Albert J. van der Heijden, Jan Lindemans, Johan P. Mackenbach, Henri tte A. Moll, Eric A. P. Steegers, Henning Tiemeier, Andre G. Uitterlinden, Frank C. Verhulst""",2007,European Journal of Epidemiology,22,,917-923,,, -10.1186/s12889-019-7222-2,Growing up in Bradford: protocol for the age 7 11 follow up of the Born,"Philippa K Bird,Rosemary R. C. McEachan,Mark Mon-Williams,Neil Small,Jane West,Peter Whincup,John Wright,Elizabeth Andrews,Sally E Barber,Liam J B Hill,Laura Lennon,Dan Mason,Katy A Shire,Dagmar Waiblinger,Amanda H. Waterman,Deborah A. Lawlor,Kate E. Pickett",2019,BMC Public Health,19,,,Born in Bradford,,"Background +https://doi.org/10.1186/s12889-019-7222-2,Growing up in Bradford: protocol for the age 7 11 follow up of the Born,"Philippa K Bird,Rosemary R. C. McEachan,Mark Mon-Williams,Neil Small,Jane West,Peter Whincup,John Wright,Elizabeth Andrews,Sally E Barber,Liam J B Hill,Laura Lennon,Dan Mason,Katy A Shire,Dagmar Waiblinger,Amanda H. Waterman,Deborah A. Lawlor,Kate E. Pickett",2019,BMC Public Health,19,,,Born in Bradford,,"Background Born in Bradford (BiB) is a prospective multi-ethnic pregnancy and birth cohort study that was established to examine determinants of health and development during childhood and, subsequently, adult life in a deprived multi-ethnic population in the north of England. Between 2007 and 2010, the BiB cohort recruited 12,453 women who experienced 13,776 pregnancies and 13,858 births, along with 3353 of their partners. Forty five percent of the cohort are of Pakistani origin. Now that children are at primary school, the first full follow-up of the cohort is taking place. The aims of the follow-up are to investigate the determinants of children s pre-pubertal health and development, including through understanding parents health and wellbeing, and to obtain data on exposures in childhood that might influence future health. Methods @@ -160,12 +159,12 @@ We are employing a multi-method approach across three data collection arms (comm Discussion Our multi-method approach to recruitment and assessment provides an efficient method of collecting rich data on all family members. Data collected will enhance BiB as a resource for the international research community to study the interplay between ethnicity, socioeconomic circumstances and biology in relation to cardiometabolic healorimotor development and wellbeing." -10.1093/ije/dyab109,Cohort Profile: 46 years of follow-up of the Northern Finland Birth Cohort 1966 (NFBC1966),"Tanja Nordstr m,Jouko Miettunen,Juha Auvinen,Leena Ala-Mursula,Sirkka Kein nen-Kiukaanniemi,Juha Veijola,Marjo-Riitta J ",2021,International Journal of Epidemiology,,,,,, +https://doi.org/10.1093/ije/dyab109,Cohort Profile: 46 years of follow-up of the Northern Finland Birth Cohort 1966 (NFBC1966),"Tanja Nordstr m,Jouko Miettunen,Juha Auvinen,Leena Ala-Mursula,Sirkka Kein nen-Kiukaanniemi,Juha Veijola,Marjo-Riitta J ",2021,International Journal of Epidemiology,,,,,, https://doi.org/10.1007/s10654-013-9768-0,Pediatric population-based neuroimaging and the Generation R Study: the intersection of developmental neuroscience and epidemiology,"""Tonya White, Hanan El Marroun, Ilse Nijs, Marcus Schmidt, Aad van der Lugt, Piotr A. Wielopolki, Vincent W. V. Jaddoe, Albert Hofman, Gabriel P. Krestin, Henning Tiemeier & Frank C. Verhulst""",2013,European Journal of Epidemiology volume,28,,99-111,,, -10.1093/ije/dyz227,Cohort Profile: The French national cohort of children (ELFE): birth to 5 years.,,,,,,,,, -10.1155/2012/274068,Psychobehavioural Factors Are More Strongly Associated with Successful Weight Management Than Predetermined SatietyEffect or Other Characteristics of Diet.,,,,,,,,, +https://doi.org/10.1093/ije/dyz227,Cohort Profile: The French national cohort of children (ELFE): birth to 5 years.,,,,,,,,, +https://doi.org/10.1155/2012/274068,Psychobehavioural Factors Are More Strongly Associated with Successful Weight Management Than Predetermined SatietyEffect or Other Characteristics of Diet.,,,,,,,,, http://urn.fi/urn:nbn:fi:att:bc1e5408-980e-4a62-b899-43bec3755243,Northern Finland Birth Cohort 1966,,,,,,,,, -10.1007/s00125-020-05250-0,A 2 year physical activity and dietary intervention attenuates the increase in insulin resistance in a general population of children: the PANIC study.,,,,,,,,, -10.1093/ije/dyq128,Cohort profile: the Amsterdam born children and their development (ABCD) study.,,,,,,,,, +https://doi.org/10.1007/s00125-020-05250-0,A 2 year physical activity and dietary intervention attenuates the increase in insulin resistance in a general population of children: the PANIC study.,,,,,,,,, +https://doi.org/10.1093/ije/dyq128,Cohort profile: the Amsterdam born children and their development (ABCD) study.,,,,,,,,, http://urn.fi/urn:nbn:fi:att:f5c10eef-3d25-4bd0-beb8-f2d59df95b8e,Northern Finland Birth Cohort 1986,,,,,,,,, -10.1093/ije/dyx084,"Cohort Profile: The Mother-Child Cohort in Crete, Greece (Rhea Study).",,,,,,,,, +https://doi.org/10.1093/ije/dyx084,"Cohort Profile: The Mother-Child Cohort in Crete, Greece (Rhea Study).",,,,,,,,, diff --git a/data/_models/shared/DataCatalogue-TODO.csv b/data/_models/shared/DataCatalogue-TODO.csv index 3694b473f7..5fa4e1f3a3 100644 --- a/data/_models/shared/DataCatalogue-TODO.csv +++ b/data/_models/shared/DataCatalogue-TODO.csv @@ -1,5 +1,5 @@ tableName,tableExtends,columnName,columnType,key,required,refSchema,refTable,refLink,refBack,validation,semantics,description,profiles -Version,,,,,,,,,,,,3.10,"DataCatalogue,EMA,SharedStaging,CohortStaging" +Version,,,,,,,,,,,,3.11,"DataCatalogue,EMA,SharedStaging,CohortStaging" Catalogues,,,,,,,,,,,,A collection of resources within a network or consortium or about a common topic,DataCatalogue Catalogues,,network,ref,1,TRUE,,Networks,,,,,Network or consortium that leads this catalogue,DataCatalogue Catalogues,,type,ontology,,TRUE,CatalogueOntologies,CatalogueTypes,,,,,Type of catalogue,DataCatalogue @@ -194,7 +194,7 @@ Studies,,networks other,text,,,,,,,,,List the names of any other networks that a RWE resources,,networks,refback,,,,Networks,,data sources,,,List of networks that this datasource is associated with,"DataCatalogue,EMA" RWE resources,,studies,refback,,,,Studies,,data sources,,,List of studies that this datasource is associated with,"DataCatalogue,EMA" Publications,,,,,,,,,,,,Publications following bibtex format,"DataCatalogue,CohortStaging" -Publications,,doi,,1,TRUE,,,,,,,Digital object identifier,"DataCatalogue,EMA,CohortStaging" +Publications,,doi,hyperlink,1,TRUE,,,,,,,Digital object identifier,"DataCatalogue,EMA,CohortStaging" Publications,,title,text,,,,,,,,,Publication title,"DataCatalogue,EMA,CohortStaging" Publications,,authors,string_array,,,,,,,,,"List of authors, one entry per author",DataCatalogue Publications,,year,int,,,,,,,,,"Year of publication (or, if unpublished, year of creation)",DataCatalogue diff --git a/data/datacatalogue/stagingCohorts/molgenis.csv b/data/datacatalogue/stagingCohorts/molgenis.csv index e7d408ebb6..dd55757e85 100644 --- a/data/datacatalogue/stagingCohorts/molgenis.csv +++ b/data/datacatalogue/stagingCohorts/molgenis.csv @@ -1,5 +1,5 @@ tableName,tableExtends,columnName,columnType,key,required,refSchema,refTable,refLink,refBack,validation,semantics,description -Version,,,,,,,,,,,,3.10 +Version,,,,,,,,,,,,3.11 Resources,,,,,,,,,,,,"Generic listing of all resources. Should not be used directly, instead use specific types such as Databanks and Studies" Extended resources,Resources,,,,,,,,,,, Data resources,Extended resources,,,,,,,,,,,Resources for data @@ -62,7 +62,7 @@ Data resources,,collaborations,heading,,,,,,,,,List of relevant collaborations Cohorts,,studies,ref_array,,,catalogue,Studies,,cohorts,,,Listing of studies that used this cohort Cohorts,,networks,ref_array,,,catalogue,Networks,,cohorts,,,The consortia or networks that this cohort is involved in Publications,,,,,,,,,,,,Publications following bibtex format,,, -Publications,,doi,,1,TRUE,,,,,,,Digital object identifier +Publications,,doi,hyperlink,1,TRUE,,,,,,,Digital object identifier. Enter a valid hyperlink, i.e., starting with 'https://doi.org/' Publications,,title,text,,,,,,,,,Publication title Publications,,authors,string_array,,,,,,,,,"List of authors, one entry per author" Publications,,year,int,,,,,,,,,"Year of publication (or, if unpublished, year of creation)" diff --git a/data/datacatalogue/stagingCohortsUMCG/molgenis.csv b/data/datacatalogue/stagingCohortsUMCG/molgenis.csv index d18e670136..f92964f16e 100644 --- a/data/datacatalogue/stagingCohortsUMCG/molgenis.csv +++ b/data/datacatalogue/stagingCohortsUMCG/molgenis.csv @@ -1,5 +1,5 @@ tableName,tableExtends,columnName,columnType,key,required,refSchema,refTable,refLink,refBack,validation,semantics,description -Version,,,,,,,,,,,,3.10 +Version,,,,,,,,,,,,3.11 Resources,,,,,,,,,,,,"Generic listing of all resources. Should not be used directly, instead use specific types such as Databanks and Studies" Extended resources,Resources,,,,,,,,,,, Data resources,Extended resources,,,,,,,,,,, @@ -57,7 +57,7 @@ Extended resources,,documentation,refback,,,,Documentation,,resource,,,List of d Extended resources,,funding statement,text,,,,,,,,,Statement listing funding that was obtained for this resource Extended resources,,acknowledgements,text,,,,,,,,,Acknowledgement statement and citation regulation for this resource Publications,,,,,,,,,,,,Publications following bibtex format -Publications,,doi,,1,TRUE,,,,,,,Digital object identifier +Publications,,doi,hyperlink,1,TRUE,,,,,,,Digital object identifier. Enter a valid hyperlink, i.e., starting with 'https://doi.org/' Publications,,title,,,TRUE,,,,,,,Publication title Contacts,,resource,ref,1,TRUE,,Resources,,,,,Resource the contact is affiliated with Contacts,,role,ontology_array,,,CatalogueOntologies,Contribution types,,,,,Type(s) of contribution or role in the resource diff --git a/data/datacatalogue/stagingNetworks/molgenis.csv b/data/datacatalogue/stagingNetworks/molgenis.csv index 78b958f3fd..c16e380009 100644 --- a/data/datacatalogue/stagingNetworks/molgenis.csv +++ b/data/datacatalogue/stagingNetworks/molgenis.csv @@ -1,5 +1,5 @@ tableName,tableExtends,columnName,columnType,key,required,refSchema,refTable,refLink,refBack,validation,semantics,description -Version,,,,,,,,,,,,3.10 +Version,,,,,,,,,,,,3.11 Resources,,,,,,,,,,,,"Generic listing of all resources. Should not be used directly, instead use specific types such as Databanks and Studies" Extended resources,Resources,,,,,,,,,,, Models,Extended resources,,,,,,,,,,,Data models @@ -27,7 +27,7 @@ Extended resources,,funding statement,text,,,,,,,,,Statement listing funding tha Extended resources,,acknowledgements,text,,,,,,,,,Acknowledgement statement and citation regulation for this resource Extended resources,,documentation,refback,,,,Documentation,,resource,,,"Descriptive document(s) available for this resource, e.g. informed consent" Publications,,,,,,,,,,,,Publications following bibtex format -Publications,,doi,,1,TRUE,,,,,,,Digital object identifier +Publications,,doi,hyperlink,1,TRUE,,,,,,,Digital object identifier Publications,,title,text,,,,,,,,,Publication title Publications,,authors,string_array,,,,,,,,,"List of authors, one entry per author" Publications,,year,int,,,,,,,,,"Year of publication (or, if unpublished, year of creation)" diff --git a/data/datacatalogue/stagingRWE/molgenis.csv b/data/datacatalogue/stagingRWE/molgenis.csv index cff2a02a0c..b0c7c44dbd 100644 --- a/data/datacatalogue/stagingRWE/molgenis.csv +++ b/data/datacatalogue/stagingRWE/molgenis.csv @@ -1,5 +1,5 @@ tableName,tableExtends,columnName,columnType,key,required,refSchema,refTable,refLink,refBack,validation,semantics,description,profiles -Version,,,,,,,,,,,,3.10,EMA +Version,,,,,,,,,,,,3.11,EMA Resources,,,,,,,,,,,,"Generic listing of all resources. Should not be used directly, instead use specific types such as Databanks and Studies",EMA Extended resources,Resources,,,,,,,,,,,,EMA Data resources,Extended resources,,,,,,,,,,,Resources for data,EMA @@ -127,7 +127,7 @@ Extended resources,,documentation,refback,,,,Documentation,,resource,,,"Descript Data resources,,supplementary information,text,,,,,,,,,Any other information that needs to be disclosed for this resource, Data resources,,collaborations,heading,,,,,,,,,List of relevant collaborations, Publications,,,,,,,,,,,,Publications following bibtex format, -Publications,,doi,,1,TRUE,,,,,,,Digital object identifier, +Publications,,doi,hyperlink,1,TRUE,,,,,,,Digital object identifier, Publications,,title,text,,,,,,,,,Publication title, Publications,,authors,string_array,,,,,,,,,"List of authors, one entry per author", Publications,,year,int,,,,,,,,,"Year of publication (or, if unpublished, year of creation)", diff --git a/data/scripts/molgenis-model-update/run.py b/data/scripts/molgenis-model-update/run.py new file mode 100644 index 0000000000..f56d36db7d --- /dev/null +++ b/data/scripts/molgenis-model-update/run.py @@ -0,0 +1,238 @@ +import setuptools.discovery +from decouple import config +from util.client import Session +from update.update_3_11 import Transform +from util.zip_handling import Zip +import os + +if not os.path.isdir('./files'): + os.mkdir('./files') + +os.chdir('./files') + +# Data model details +DATA_MODEL_VERSION = config('MG_DATA_MODEL_VERSION') + +# Server details +SERVER_URL = config('MG_SERVER_URL') +SERVER_USERNAME = config('MG_SERVER_USERNAME') +SERVER_PASSWORD = config('MG_SERVER_PASSWORD') +CATALOGUE_SCHEMA_NAME = config('MG_CATALOGUE_SCHEMA_NAME') +ONTOLOGIES_SCHEMA_NAME = config('MG_ONTOLOGIES_SCHEMA_NAME') +SHARED_STAGING_NAME = config('MG_SHARED_STAGING_NAME') + +COHORTS = config('MG_COHORTS', cast=lambda v: [s.strip() for s in v.split(',')]) +DATA_SOURCES = config('MG_DATA_SOURCES', cast=lambda v: [s.strip() for s in v.split(',')]) +NETWORKS = config('MG_NETWORKS', cast=lambda v: [s.strip() for s in v.split(',')]) + +print('----- Config variables loaded ----') + +print('SERVER_URL: ' + SERVER_URL) +print('SERVER_USERNAME: ' + SERVER_USERNAME) +print('SERVER_PASSWORD: ******') +print('CATALOGUE_SCHEMA_NAME: ' + CATALOGUE_SCHEMA_NAME) +print('ONTOLOGIES_SCHEMA_NAME: ' + ONTOLOGIES_SCHEMA_NAME) +print('SHARED_STAGING_NAME: ' + SHARED_STAGING_NAME) + +print('----- ----') + +print('Updating catalogue data model to version ' + DATA_MODEL_VERSION) + + +# sign in to server +print('Sign in to server: ' + SERVER_URL) +session = Session( + url=SERVER_URL, + email=SERVER_USERNAME, + password=SERVER_PASSWORD +) + +# -------------------------------------------------------------- + +# extract data from catalogue schema +print('Extract data from ' + CATALOGUE_SCHEMA_NAME + ': ' + CATALOGUE_SCHEMA_NAME + '_data.zip') +session.download_zip(database_name=CATALOGUE_SCHEMA_NAME) + +# transform data from catalogue schema +print('Transform data from ' + CATALOGUE_SCHEMA_NAME) +# get instances of classes +zip_handling = Zip(CATALOGUE_SCHEMA_NAME) +update = Transform(CATALOGUE_SCHEMA_NAME, 'catalogue') + +# run zip and transform functions +zip_handling.unzip_data() +update.delete_data_model_file() # delete molgenis.csv from data folder +update.update_data_model_file() +update.transform_data() +zip_handling.zip_data() + +# -------------------------------------------------------------- + +# Cohorts update +print('-----------------------') +print('Cohort data update to data model ' + DATA_MODEL_VERSION) + +for cohort in COHORTS: + # sign in to server + print('Sign in to server: ' + SERVER_URL) + session = Session( + url=SERVER_URL, + email=SERVER_USERNAME, + password=SERVER_PASSWORD + ) + # extract data + print('Extract data for ' + cohort + ': ' + cohort + '_data.zip') + session.download_zip(database_name=cohort) + + # transform data from cohorts + print('Transform data from ' + cohort) + zip_handling = Zip(cohort) + update = Transform(cohort, 'cohort_UMCG') + + zip_handling.remove_unzipped_data() + zip_handling.unzip_data() + update.delete_data_model_file() + update.transform_data() + update.update_data_model_file() + zip_handling.zip_data() + zip_handling.remove_unzipped_data() + # delete and create new cohort schema + schema_description = session.get_database_description(database_name=cohort) + session.drop_database(database_name=cohort) + session.create_database(database_name=cohort, database_description=schema_description) + +# -------------------------------------------------------------- + +# Data sources update +print('-----------------------') +print('Data source update to data model ' + DATA_MODEL_VERSION) + +for data_source in DATA_SOURCES: + # sign in to server + print('Sign in to server: ' + SERVER_URL) + session = Session( + url=SERVER_URL, + email=SERVER_USERNAME, + password=SERVER_PASSWORD + ) + # extract data + print('Extract data for ' + data_source + ': ' + data_source + '_data.zip') + session.download_zip(database_name=data_source) + + # transform data from cohorts + print('Transform data from ' + data_source) + zip_handling = Zip(data_source) + update = Transform(data_source, 'data_source') + + zip_handling.remove_unzipped_data() + zip_handling.unzip_data() + update.delete_data_model_file() + update.transform_data() + update.update_data_model_file() + zip_handling.zip_data() + zip_handling.remove_unzipped_data() + # delete and create new cohort schema + schema_description = session.get_database_description(database_name=data_source) + session.drop_database(database_name=data_source) + session.create_database(database_name=data_source, database_description=schema_description) + +# Networks update +print('-----------------------') +print('Networks update to data model ' + DATA_MODEL_VERSION) + +for network in NETWORKS: + # sign in to server + print('Sign in to server: ' + SERVER_URL) + session = Session( + url=SERVER_URL, + email=SERVER_USERNAME, + password=SERVER_PASSWORD + ) + # extract data + print('Extract data for ' + network + ': ' + network + '_data.zip') + session.download_zip(database_name=network) + + # transform data from cohorts + print('Transform data from ' + network) + zip_handling = Zip(network) + update = Transform(network, 'network') + + zip_handling.remove_unzipped_data() + zip_handling.unzip_data() + update.delete_data_model_file() + update.transform_data() + update.update_data_model_file() + zip_handling.zip_data() + zip_handling.remove_unzipped_data() + # delete and create new cohort schema + schema_description = session.get_database_description(database_name=network) + session.drop_database(database_name=network) + session.create_database(database_name=network, database_description=schema_description) + +# --------------------------------------------------------------- + +# delete and create schemas +print('------------------------') +print('Updating catalogue schema') +# delete and create new UMCG schema +schema_description = session.get_database_description(database_name=CATALOGUE_SCHEMA_NAME) +session.drop_database(database_name=CATALOGUE_SCHEMA_NAME) +session.create_database(database_name=CATALOGUE_SCHEMA_NAME, database_description=schema_description) + +# upload molgenis.csv to catalogue schema +update_general = Transform(CATALOGUE_SCHEMA_NAME, 'catalogue') +data_model_file = update_general.update_data_model_file() +session.upload_zip(database_name=CATALOGUE_SCHEMA_NAME, data_to_upload='catalogue_data_model') + +# upload transformed catalogue data to catalogue schema +session.upload_zip(database_name=CATALOGUE_SCHEMA_NAME, data_to_upload=CATALOGUE_SCHEMA_NAME) + +# ---------------------------------------------------------------------- + +# Cohorts upload data +print('-----------------------') + +print('Updating data for cohorts') + +for cohort in COHORTS: + # sign in to server + print('Sign in to server: ' + SERVER_URL) + session = Session( + url=SERVER_URL, + email=SERVER_USERNAME, + password=SERVER_PASSWORD + ) + print('Upload transformed data for: ' + cohort) + session.upload_zip(database_name=cohort, data_to_upload=cohort) + +# Data sources upload data +print('-----------------------') + +print('Updating data for data sources') + +for data_source in DATA_SOURCES: + # sign in to server + print('Sign in to server: ' + SERVER_URL) + session = Session( + url=SERVER_URL, + email=SERVER_USERNAME, + password=SERVER_PASSWORD + ) + print('Upload transformed data for: ' + data_source) + session.upload_zip(database_name=data_source, data_to_upload=data_source) + +# Networks upload data +print('-----------------------') + +print('Updating data for networks') + +for network in NETWORKS: + # sign in to server + print('Sign in to server: ' + SERVER_URL) + session = Session( + url=SERVER_URL, + email=SERVER_USERNAME, + password=SERVER_PASSWORD + ) + print('Upload transformed data for: ' + network) + session.upload_zip(database_name=network, data_to_upload=network) diff --git a/data/scripts/molgenis-model-update/update/update_3_11.py b/data/scripts/molgenis-model-update/update/update_3_11.py new file mode 100644 index 0000000000..748c4f018e --- /dev/null +++ b/data/scripts/molgenis-model-update/update/update_3_11.py @@ -0,0 +1,129 @@ +import shutil +import os +import pandas as pd + + +def float_to_int(df): + """ + Cast float64 Series to Int64. + """ + for column in df.columns: + if df[column].dtype == 'float64': + df.loc[:, column] = df[column].astype('Int64') + + return df + + +def get_hyperlink(x): + """ + Return hyperlink for websites if not filled out correctly. + """ + x_string = '' + + if not pd.isna(x): + x_list = x.split(',') + for item in x_list: + if not item.startswith('http'): + item = 'https://doi.org/' + item + if item == 'https://doi.org/doi: 10.1002/nau.24996': + item = 'https://doi.org/10.1002/nau.24996' + x_string += item + ',' + x_string = x_string[:-1] + + return x_string + + +class Transform: + """General functions to update catalogue data model. + """ + + def __init__(self, database_name, database_type): + self.database_name = database_name + self.database_type = database_type + self.path = self.database_name + '_data/' + + def delete_data_model_file(self): + """Delete molgenis.csv + """ + os.remove(self.path + 'molgenis.csv') + + def update_data_model_file(self): + """Get path to data model file and copy molgenis.csv to appropriate folder if it does not exist + """ + # get molgenis.csv location + if self.database_type == 'catalogue': + data_model = os.path.abspath('../../../datacatalogue/molgenis.csv') + elif self.database_type == 'network': + data_model = os.path.abspath('../../../datacatalogue/stagingNetworks/molgenis.csv') + elif self.database_type == 'cohort': + data_model = os.path.abspath('../../../datacatalogue/stagingCohorts/molgenis.csv') + elif self.database_type == 'data_source': + data_model = os.path.abspath('../../../datacatalogue/stagingRWE/molgenis.csv') + elif self.database_type == 'cohort_UMCG': + data_model = os.path.abspath('../../../datacatalogue/stagingCohortsUMCG/molgenis.csv') + elif self.database_type == 'shared': + data_model = os.path.abspath('../../../datacatalogue/stagingShared/molgenis.csv') + + # copy molgenis.csv to appropriate folder + if self.database_type == 'catalogue': + path = './catalogue_data_model' + if not os.path.isdir(path): + os.mkdir(path) + shutil.copyfile(data_model, os.path.abspath(os.path.join(path, 'molgenis.csv'))) + shutil.make_archive('./catalogue_data_model_upload', 'zip', path) + else: + shutil.copyfile(data_model, os.path.abspath(os.path.join(self.path, 'molgenis.csv'))) + + def transform_data(self): + """Make changes per table + """ + # transformations for catalogue and cohorts + if self.database_type == 'catalogue': + self.cohorts() + self.data_sources() + self.databanks() + self.publications() + if self.database_type in ['cohort', 'cohort_UMCG']: + self.cohorts() + self.publications() + if self.database_type == 'data_sources': + self.data_sources() + self.databanks() + self.publications() + + def cohorts(self): + """Transform columns in cohorts + """ + df_cohorts = pd.read_csv(self.path + 'Cohorts.csv') + df_cohorts['design paper'] = df_cohorts['design paper'].apply(get_hyperlink) + if self.database_type == 'cohort': + df_cohorts['publications'] = df_cohorts['publications'].apply(get_hyperlink) + df_cohorts = float_to_int(df_cohorts) # convert float back to integer + df_cohorts.to_csv(self.path + 'Cohorts.csv', index=False) + + def data_sources(self): + """Transform columns in data sources + """ + df_data_sources = pd.read_csv(self.path + 'Data sources.csv') + df_data_sources['design paper'] = df_data_sources['design paper'].apply(get_hyperlink) + df_data_sources['publications'] = df_data_sources['publications'].apply(get_hyperlink) + df_data_sources = float_to_int(df_data_sources) # convert float back to integer + df_data_sources.to_csv(self.path + 'Data sources.csv', index=False) + + def databanks(self): + """Transform columns in databanks + """ + df_databanks = pd.read_csv(self.path + 'Databanks.csv') + df_databanks['design paper'] = df_databanks['design paper'].apply(get_hyperlink) + df_databanks['publications'] = df_databanks['publications'].apply(get_hyperlink) + df_databanks = float_to_int(df_databanks) # convert float back to integer + df_databanks.to_csv(self.path + 'Databanks.csv', index=False) + + def publications(self): + """Transform columns in publications + """ + df_publications = pd.read_csv(self.path + 'Publications.csv') + df_publications['doi'] = df_publications['doi'].apply(get_hyperlink) + df_publications = float_to_int(df_publications) # convert float back to integer + df_publications = df_publications.drop_duplicates(subset='doi') + df_publications.to_csv(self.path + 'Publications.csv', index=False) diff --git a/data/scripts/molgenis-model-update/util/client.py b/data/scripts/molgenis-model-update/util/client.py index 1ef85b2a2e..52bc421a4e 100644 --- a/data/scripts/molgenis-model-update/util/client.py +++ b/data/scripts/molgenis-model-update/util/client.py @@ -96,14 +96,14 @@ def upload_zip(self, database_name, data_to_upload): except: errors = responseJson['errors'][0] print(f'Upload failed: {errors}') - finally: - try: - if database_name not in ['UMCG', 'catalogue', 'DataCatalogue']: # otherwise data to upload >> - # << deleted when schema is uploaded - if os.path.exists(database_name + '_upload.zip'): - os.remove(database_name + '_upload.zip') - except PermissionError: - sys.exit('Error deleting upload.zip') + # finally: + # try: + # if database_name not in ['UMCG', 'catalogue', 'DataCatalogue']: # otherwise data to upload >> + # # << deleted when schema is uploaded + # if os.path.exists(database_name + '_upload.zip'): + # os.remove(database_name + '_upload.zip') + # except PermissionError: + # sys.exit('Error deleting upload.zip') def get_database_description(self, database_name) -> str: """ Get description of database diff --git a/data/scripts/molgenis-model-update/util/zip_handling.py b/data/scripts/molgenis-model-update/util/zip_handling.py index 9aeb6a07a9..0c275592b5 100644 --- a/data/scripts/molgenis-model-update/util/zip_handling.py +++ b/data/scripts/molgenis-model-update/util/zip_handling.py @@ -26,9 +26,9 @@ def remove_unzipped_data(self): def unzip_data(self): """Extract data.zip """ - data = ZipFile(self.path + '.zip') + data = ZipFile(self.path + '_data.zip') try: - data.extractall(self.path) + data.extractall(self.path + '_data') except FileNotFoundError: self.logger.error('unzip failed') exit() @@ -36,13 +36,13 @@ def unzip_data(self): self.logger.error('Error: unzip failed, permission denied') exit() try: - if os.path.exists(self.database + '.zip'): - os.remove(self.database + '.zip') + if os.path.exists(self.database + '_data.zip'): + os.remove(self.database + '_data.zip') except PermissionError: # remove fails on windows, is not needed on Windows, pass - self.logger.warning('Warning: Error deleting ' + self.database + '_data.zip') + self.logger.warning('Warning: Error deleting ' + self.database + '.zip') def zip_data(self): """Zip transformed data to upload.zip """ - shutil.make_archive(self.database + '_upload', 'zip', self.path) + shutil.make_archive(self.database + '_upload', 'zip', self.path + '_data') From 86ccb875f18ead7648db9a0d2f9a00fc0810cb28 Mon Sep 17 00:00:00 2001 From: Joris de Keijser Date: Wed, 7 Feb 2024 13:04:58 +0100 Subject: [PATCH 39/87] fix: required computeds (#3351) Prevents users from creating required computed fields in the schema editor; Fixes some issues with creating auto id columns; Fixes an issue where the user couldn't create columns after creating a new table. --- apps/dev-proxy.config.js | 2 +- apps/schema/package.json | 1 + .../schema/src/components/ColumnEditModal.vue | 158 ++++++++++-------- apps/schema/src/components/SchemaToc.vue | 3 + apps/schema/tsconfig.json | 11 ++ 5 files changed, 108 insertions(+), 67 deletions(-) create mode 100644 apps/schema/tsconfig.json diff --git a/apps/dev-proxy.config.js b/apps/dev-proxy.config.js index 58c8c468fe..d4a51ac3be 100644 --- a/apps/dev-proxy.config.js +++ b/apps/dev-proxy.config.js @@ -1,5 +1,5 @@ const HOST = process.env.MOLGENIS_APPS_HOST || "https://emx2.dev.molgenis.org"; -const SCHEMA = process.env.MOLGENIS_APPS_SCHEMA || "FAIR%20data%20hub"; +const SCHEMA = process.env.MOLGENIS_APPS_SCHEMA || "pet store"; const opts = { changeOrigin: true, secure: false, logLevel: "debug" }; diff --git a/apps/schema/package.json b/apps/schema/package.json index 13a9d9ff44..7c5028bbdd 100644 --- a/apps/schema/package.json +++ b/apps/schema/package.json @@ -13,6 +13,7 @@ "core-js": "3.34.0", "graphql-request": "5.2.0", "graphql-tag": "2.12.6", + "meta-data-utils": "*", "molgenis-components": "*", "vue": "3.4.15", "vue-router": "4.2.5", diff --git a/apps/schema/src/components/ColumnEditModal.vue b/apps/schema/src/components/ColumnEditModal.vue index 4296157142..6a950189ce 100644 --- a/apps/schema/src/components/ColumnEditModal.vue +++ b/apps/schema/src/components/ColumnEditModal.vue @@ -9,8 +9,8 @@

- Marked for deletion + + Marked for deletion {{ error }}
@@ -48,6 +48,7 @@ v-model="column.columnType" :options="columnTypes" label="columnType" + @update:modelValue="handleColumnTypeChanged" />
-
-
+
+
-
+
-
+
-
+
-
+
@@ -265,18 +267,18 @@ } -