From bdc0618d6cd2fd904fd761b5e1c5520c5be3accf Mon Sep 17 00:00:00 2001 From: Genaro Camele Date: Thu, 9 Nov 2023 15:21:17 -0300 Subject: [PATCH 1/7] Little improvements in performance of mirna-target-interactions endpoint + Added lot of docs and types + A lot of refactoring + Bumped version to 2.1.1 --- ModulectorBackend/settings.py | 2 +- ModulectorBackend/settings_dev.py | 2 +- ModulectorBackend/urls.py | 17 ---- modulector/mappers/gene_mapper.py | 2 +- modulector/processors/mirdip_processor.py | 2 +- modulector/processors/sequence_processor.py | 2 +- modulector/pubmed_job.py | 105 ++++++++++++-------- modulector/serializers.py | 22 ++-- modulector/services/pubmed_service.py | 59 ++++++----- modulector/views.py | 29 +++--- 10 files changed, 132 insertions(+), 110 deletions(-) diff --git a/ModulectorBackend/settings.py b/ModulectorBackend/settings.py index e19cb40..687e52a 100644 --- a/ModulectorBackend/settings.py +++ b/ModulectorBackend/settings.py @@ -13,7 +13,7 @@ import os # Modulector version -VERSION: str = '2.1.0' +VERSION: str = '2.1.1' # Default primary key field type # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field diff --git a/ModulectorBackend/settings_dev.py b/ModulectorBackend/settings_dev.py index aa8188b..1129c29 100644 --- a/ModulectorBackend/settings_dev.py +++ b/ModulectorBackend/settings_dev.py @@ -13,7 +13,7 @@ import os # Modulector version -VERSION: str = '2.1.0' +VERSION: str = '2.1.1' # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/ModulectorBackend/urls.py b/ModulectorBackend/urls.py index 42d7ab8..afb7d7f 100644 --- a/ModulectorBackend/urls.py +++ b/ModulectorBackend/urls.py @@ -1,23 +1,6 @@ -"""ModulectorBackend URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/3.0/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" from django.contrib import admin from django.urls import path - from modulector import views -from modulector.views import MethylationSites urlpatterns = [ path('admin/', admin.site.urls), diff --git a/modulector/mappers/gene_mapper.py b/modulector/mappers/gene_mapper.py index 7410663..bd45d2b 100644 --- a/modulector/mappers/gene_mapper.py +++ b/modulector/mappers/gene_mapper.py @@ -25,7 +25,7 @@ def process(): def load_translations(): df = pd.read_csv(filepath_or_buffer=path, delimiter="\t") - symbol_dic = dict() + symbol_dic = {} for index, item in df.iterrows(): ref_seq = item["RNA"].split(".")[0] symbol = item["Symbol"] diff --git a/modulector/processors/mirdip_processor.py b/modulector/processors/mirdip_processor.py index d4375d6..55b4d36 100644 --- a/modulector/processors/mirdip_processor.py +++ b/modulector/processors/mirdip_processor.py @@ -20,7 +20,7 @@ parent_dir = pathlib.Path(__file__).parent.absolute().parent -mirna_dict = dict() +mirna_dict = {} def process(): diff --git a/modulector/processors/sequence_processor.py b/modulector/processors/sequence_processor.py index 1b454dd..bb8772f 100644 --- a/modulector/processors/sequence_processor.py +++ b/modulector/processors/sequence_processor.py @@ -40,7 +40,7 @@ def process(): mirna_table = Mirna._meta.db_table update_template = "UPDATE {} set mirna_sequence= '{}' where mirna_code in {}" - mimat_map = dict() + mimat_map = {} # loading files #data = pd.read_excel(io=file_path, usecols=[0, 1, 2], names=['mimat', 'mirna', 'sequence']) data = pd.read_csv(file_path, usecols=[0, 1, 2], names=['mimat', 'mirna', 'sequence']) diff --git a/modulector/pubmed_job.py b/modulector/pubmed_job.py index 1474bc6..51e430a 100644 --- a/modulector/pubmed_job.py +++ b/modulector/pubmed_job.py @@ -2,19 +2,27 @@ import logging import sys from datetime import datetime +from typing import List from django.db.models import Count - +from ModulectorBackend.settings import DEBUG from modulector.models import SubscriptionItem, Subscription from modulector.services import mailing_service from modulector.services.pubmed_service import build_search_term, query_parse_and_build_pumbeds, get_pubmed_info +# Sets some logging configuration logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.DEBUG if DEBUG else logging.INFO) class MailDataItem: + """Represents a row in the mail table.""" + mirna: str + gene: str + pubmeds: List[str] + token: str + def __init__(self, mirna, gene, pubmeds, token): self.mirna = mirna self.gene = gene @@ -22,9 +30,16 @@ def __init__(self, mirna, gene, pubmeds, token): self.token = token -def should_include_pubmed(data, subscription_item, id): +def should_include_pubmed(data: str, subscription_item: SubscriptionItem, pubmed_id: int) -> bool: + """ + Checks if the pubmed should be included in the mail. + :param data: Data retrieved from the pubmed api. + :param subscription_item: SubscriptionItem instance. + :param pubmed_id: Pubmed id. + :return: True if the pubmed should be included, False otherwise. + """ json_object = json.loads(data) - date_string = json_object["result"][str(id)]["sortpubdate"] + date_string = json_object["result"][str(pubmed_id)]["sortpubdate"] parsed_date = datetime.strptime(date_string, '%Y/%m/%d %H:%M') record_date = subscription_item.record_date.replace(tzinfo=None) result = record_date < parsed_date @@ -33,50 +48,60 @@ def should_include_pubmed(data, subscription_item, id): def execute(): try: - # grouping the subscriptions by mirna and gene, null gene first - logger.info('Starting pubmed job') + # Grouping the subscriptions by mirna and gene, null gene first + logger.info('Starting pubmed job...') + result_set = SubscriptionItem.objects.values('mirna', 'gene').annotate(total=Count('id')) - # temporal maps - publications_map = dict() - data_for_mail = dict() - pubmed_cache = dict() - # iterate each row and query the api with the search term built + + # Temporal maps + publications_map = {} + data_for_mail = {} + pubmed_cache = {} + + # Iterates each row and query the api with the search term built for row in result_set: mirna = row['mirna'] gene = row['gene'] - logger.info('building search term') + + logger.info('Building search term...') term = build_search_term(mirna, gene) - # create list of pubmed ids - logger.info('creating pubmeds with term {}'.format(term)) - pubmeds = query_parse_and_build_pumbeds(term, mirna, gene, None) - logger.info('created pubmeds') + + # Creates list of pubmed ids + logger.info(f'Creating pubmeds with term {term}...') + pubmeds = query_parse_and_build_pumbeds(term, mirna, gene, timeout=None) + + logger.info('Created pubmeds') publications_map[term] = pubmeds - # retrieve all main subcriptions and iterate the items - for sub in Subscription.objects.all(): - subscription_items = SubscriptionItem.objects.filter(subscription=sub) - logger.info('subscriptions {}'.format(subscription_items)) - mail_rows = [] - for subscription_item in subscription_items: - term = build_search_term(subscription_item.mirna, subscription_item.gene) - # rebuild the search term for the subcription and retrieve the pubmeds from the map - pubmeds_ids = publications_map[term] - # object that represents a row in the table sent to the user - mail_data_item = MailDataItem(mirna=mirna, gene=gene, pubmeds=[], - token=subscription_item.unsubscribe_token) - logger.info('building each pubmed') - for index, pubmed_id in enumerate(pubmeds_ids): - data = get_pubmed_info(pubmed_id, pubmed_cache) - if should_include_pubmed(data, subscription_item, pubmed_id): - mail_data_item.pubmeds.append(pubmed_id) - if len(mail_data_item.pubmeds) > 0: - subscription_item.record_date = datetime.now() - subscription_item.save() - mail_rows.append(mail_data_item) - data_for_mail[sub.email] = mail_rows + + # Retrieve all main subscriptions and iterates the items + for sub in Subscription.objects.all(): + subscription_items = SubscriptionItem.objects.filter(subscription=sub) + logger.info('Subscriptions {}'.format(subscription_items)) + + mail_rows = [] + for subscription_item in subscription_items: + term = build_search_term(subscription_item.mirna, subscription_item.gene) + # Rebuilds the search term for the subscription and retrieve the pubmeds from the map + pubmeds_ids = publications_map[term] + + # Object that represents a row in the table sent to the user + mail_data_item = MailDataItem(mirna=mirna, gene=gene, pubmeds=[], + token=subscription_item.unsubscribe_token) + logger.info('building each pubmed') + for index, pubmed_id in enumerate(pubmeds_ids): + data = get_pubmed_info(pubmed_id, pubmed_cache) + if should_include_pubmed(data, subscription_item, pubmed_id): + mail_data_item.pubmeds.append(pubmed_id) + + if len(mail_data_item.pubmeds) > 0: + subscription_item.record_date = datetime.now() + subscription_item.save() + mail_rows.append(mail_data_item) + data_for_mail[sub.email] = mail_rows mailing_service.email_users(content=data_for_mail) logger.info('Finished pubmed job') except Exception as ex: - logger.error('Exception on job', exc_info=True) - logging.error(ex, exc_info=True) + logger.error('Exception on job') + logging.exception(ex) raise ex diff --git a/modulector/serializers.py b/modulector/serializers.py index d406a35..32d8e9a 100644 --- a/modulector/serializers.py +++ b/modulector/serializers.py @@ -1,11 +1,8 @@ import logging from typing import List, Optional, Dict, Set - from rest_framework import serializers - from ModulectorBackend.settings import USE_PUBMED_API, PUBMED_API_TIMEOUT -from modulector.models import MirnaXGene, MirnaSource, Mirna, MirnaColumns, MirbaseIdMirna, MirnaDisease, MirnaDrug, \ - GeneAliases +from modulector.models import MirnaXGene, MirnaSource, Mirna, MirnaColumns, MirbaseIdMirna, MirnaDisease, MirnaDrug from modulector.services import url_service, pubmed_service from modulector.utils import link_builder @@ -28,12 +25,15 @@ class Meta: def create(self, validated_data): columns = validated_data.pop('columns') source = MirnaSource.objects.create(**validated_data) + column_serializer = MirnaColumnsSerializer(many=True) for column in columns: column['mirna_source_id'] = source.id + self.columns = column_serializer.create(columns) for column in self.columnsCharField: source.mirnacolumns.add(column) + return source @@ -62,6 +62,7 @@ def get_pubmeds(self, mirna_gene_interaction: MirnaXGene) -> Set[str]: pubmed_urls.update( list(mirna_gene_interaction.pubmed.values_list('pubmed_url', flat=True))) + mirna = mirna_gene_interaction.mirna.mirna_code gene = mirna_gene_interaction.gene term = pubmed_service.build_search_term(mirna, gene) @@ -69,12 +70,11 @@ def get_pubmeds(self, mirna_gene_interaction: MirnaXGene) -> Set[str]: try: api_pubmeds = pubmed_service.query_parse_and_build_pumbeds(term=term, mirna=mirna, gene=gene, timeout=PUBMED_API_TIMEOUT) - for pubmed_id in api_pubmeds: - url = link_builder.build_pubmed_url(pubmed_id) - pubmed_urls.add(url) + + pubmed_urls.update([link_builder.build_pubmed_url(pubmed_id) for pubmed_id in api_pubmeds]) except Exception as ex: # we only handle exceptions in this case - logging.error('Errr getting PubMeds:') + logging.error('Error getting PubMeds:') logging.exception(ex) return set() @@ -112,9 +112,9 @@ class Meta: @staticmethod def get_links(mirna: Mirna) -> List[Dict]: """ - Gets a list of sources links for a specific miRNA - :param mirna: miRNA object - :return: List of sources links + Gets a list of sources links for a specific miRNA. + :param mirna: miRNA object. + :return: List of sources links. """ mirbase_id_obj = mirna.mirbase_accession_id if mirbase_id_obj is not None: diff --git a/modulector/services/pubmed_service.py b/modulector/services/pubmed_service.py index c81e783..ea7c0cd 100644 --- a/modulector/services/pubmed_service.py +++ b/modulector/services/pubmed_service.py @@ -1,25 +1,23 @@ import logging import sys -from typing import Optional +from typing import Optional, Dict, Final, Set from xml.etree import ElementTree import requests -from ModulectorBackend.settings import DEFAULT_FROM_EMAIL, NCBI_API_KEY +from ModulectorBackend.settings import DEFAULT_FROM_EMAIL, NCBI_API_KEY, DEBUG from modulector.models import Pubmed +# Sets some logging configuration logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) -logger.setLevel(logging.DEBUG) -tool = 'multiomix' -email = DEFAULT_FROM_EMAIL -search_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/' \ - 'esearch.fcgi?db=pmc&term={}&tool={}&email={}&api_key=' + NCBI_API_KEY +logger.setLevel(logging.DEBUG if DEBUG else logging.INFO) -pubmed_api_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id={}&retmode=json&tool={}'\ - '&email={}&api_key=' + NCBI_API_KEY +# Adds some constants +TOOL: Final[str] = 'multiomix' +SEARCH_URL: Final[str] = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/' \ + 'esearch.fcgi?db=pmc&term={}&tool={}&email={}&api_key=' + NCBI_API_KEY - -# pubmed_api_url = 'http://localhost:3000/?db=pubmed&id={}&retmode=json&tool={}&email={}' \ -# '&api_key=' + NCBI_API_KEY +PUBMED_API_URL: Final[str] = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id={}&' \ + 'retmode=json&tool={}&email={}&api_key=' + NCBI_API_KEY def build_search_term(mirna: str, gene: str) -> str: @@ -27,9 +25,17 @@ def build_search_term(mirna: str, gene: str) -> str: return mirna + ' AND ' + gene if gene else mirna -def query_parse_and_build_pumbeds(term: str, mirna: str, gene: str, timeout: Optional[int]): +def query_parse_and_build_pumbeds(term: str, mirna: str, gene: str, timeout: Optional[int]) -> Set[str]: + """ + Queries pubmed for a given term, parses the response and builds a list of pubmed ids. + :param term: Search term. + :param mirna: miRNA code. + :param gene: Gene name. + :param timeout: Timeout for the request. + :return: List of pubmed ids. + """ try: - response = requests.get(search_url.format(term, tool, email), timeout=timeout) + response = requests.get(SEARCH_URL.format(term, TOOL, DEFAULT_FROM_EMAIL), timeout=timeout) if response.status_code == 200: pubmed_ids = set() @@ -43,9 +49,7 @@ def query_parse_and_build_pumbeds(term: str, mirna: str, gene: str, timeout: Opt if gene: query = query.filter(gene=gene) - result = query.values('pubmed_id') - for value in result: - pubmed_ids.add(int(value['pubmed_id'])) + pubmed_ids.update(query.values_list('pubmed_id', flat=True)) return pubmed_ids else: raise Exception('Response code from pubmed query was not 200, it was {}'.format(response.status_code)) @@ -54,13 +58,20 @@ def query_parse_and_build_pumbeds(term: str, mirna: str, gene: str, timeout: Opt raise ex -def get_pubmed_info(id, cache): - if id in cache: - return cache.get(id) +def get_pubmed_info(pubmed_id: str, cache: Dict[str, str]) -> Optional[str]: + """ + Gets the pubmed info for a given pubmed id. + :param pubmed_id: Pubmed id. + :param cache: Dicts used as cache to store the results. + :return: Pubmed info. + :raises: Exception if the call to the pubmed api fails. + """ + if pubmed_id in cache: + return cache.get(pubmed_id) else: - response = requests.get(pubmed_api_url.format(id, tool, email)) + response = requests.get(PUBMED_API_URL.format(pubmed_id, TOOL, DEFAULT_FROM_EMAIL)) if response.status_code == 200: - cache[id] = response.text - return response.text + cache[pubmed_id] = response.text + return cache[pubmed_id] else: - raise Exception('pubmed info call failed for id {}'.format(id)) + raise Exception(f'Pubmed info call failed for id {pubmed_id}') diff --git a/modulector/views.py b/modulector/views.py index 80b3f90..53fe538 100644 --- a/modulector/views.py +++ b/modulector/views.py @@ -1,16 +1,17 @@ import re from concurrent.futures import ProcessPoolExecutor -from typing import List, Optional +from typing import List, Optional, Final from django.conf import settings from django.db.models.query_utils import Q -from django.http import Http404 +from django.http import Http404, HttpRequest from django.shortcuts import render from django_filters.rest_framework import DjangoFilterBackend from rest_framework import status, generics, filters, viewsets from rest_framework.exceptions import ParseError from rest_framework.response import Response from rest_framework.views import APIView -from modulector.models import MethylationUCSC_CPGIsland, MethylationUCSCRefGene, MirnaXGene, Mirna, MirbaseIdMirna, MirnaDisease, MirnaDrug, GeneAliases, MethylationEPIC +from modulector.models import MethylationUCSC_CPGIsland, MethylationUCSCRefGene, MirnaXGene, Mirna, MirbaseIdMirna, \ + MirnaDisease, MirnaDrug, GeneAliases, MethylationEPIC from modulector.pagination import StandardResultsSetPagination from modulector.serializers import MirnaXGenSerializer, MirnaSerializer, \ MirnaAliasesSerializer, MirnaDiseaseSerializer, MirnaDrugsSerializer, get_mirna_from_accession, \ @@ -20,13 +21,13 @@ regex = re.compile(r'-\d[a-z]') # Default page size for requests -DEFAULT_PAGE_SIZE: int = 50 +DEFAULT_PAGE_SIZE: Final[int] = 50 # Maximum page size for requests -MAX_PAGE_SIZE: int = 3000 +MAX_PAGE_SIZE: Final[int] = 3000 # Number of processes to use in Pool -PROCESS_POOL_WORKERS = settings.PROCESS_POOL_WORKERS +PROCESS_POOL_WORKERS: Final[int] = settings.PROCESS_POOL_WORKERS def get_methylation_epic_sites_names(input_id: str) -> List[str]: @@ -71,7 +72,6 @@ class MirnaTargetInteractions(generics.ListAPIView): @staticmethod def __get_gene_aliases(gene: str) -> List[str]: """Retrieves the aliases for a gene based on the gene provided""" - print() match_gene = GeneAliases.objects.filter( Q(alias=gene) | Q(gene_symbol=gene)).first() if match_gene is None: @@ -85,9 +85,10 @@ def __get_gene_aliases(gene: str) -> List[str]: return aliases def get_serializer_context(self): - include_pubmeds = self.request.GET.get("include_pubmeds") == "true" context = super(MirnaTargetInteractions, self).get_serializer_context() - context.update({'include_pubmeds': include_pubmeds}) + + include_pubmeds = self.request.GET.get("include_pubmeds") == "true" + context['include_pubmeds'] = include_pubmeds return context def get_queryset(self): @@ -378,7 +379,8 @@ def post(self, request): res = {} for methylation_name in methylation_sites: res[methylation_name] = [] - # For each CpG methylation site passed as a parameter... I look for its Identifier in the version of the EPIC v2 array: + # For each CpG methylation site passed as a parameter... I look for its Identifier in the version of + # the EPIC v2 array: epics_ids = self.__get_methylation_epic_sites_ids(methylation_name) for site_id in epics_ids: # For each identifier in the EPIC v2 array, I search for the genes involved: @@ -409,10 +411,10 @@ def get(self, _request): Q(name=methylation_site)).first() if epic_data: - # load name to response + # Loads name to response res["name"] = epic_data.name - # load chomosomic data + # Loads chromosome data if epic_data.strand_fr == "F": res["chromosome_position"] = epic_data.chr + \ ":" + str(epic_data.mapinfo) + " [+]" @@ -455,5 +457,6 @@ def get(self, _request): return Response(status=400, data={methylation_site + " is not a valid methylation site"}) -def index(request): +def index(request: HttpRequest): + """Returns the index.html page.""" return render(request, 'index.html', {'version': settings.VERSION}) From 16d6a400088b0a733fc355e7d94de69d4d689f23 Mon Sep 17 00:00:00 2001 From: Genaro Camele Date: Thu, 9 Nov 2023 15:30:46 -0300 Subject: [PATCH 2/7] More refactoring, docs and types --- modulector/exceptions/exceptions.py | 2 +- modulector/pubmed_job.py | 2 +- modulector/serializers.py | 24 +++++----- modulector/services/mailing_service.py | 64 ++++++++++++++------------ tools/healthcheck/check.py | 2 +- 5 files changed, 51 insertions(+), 43 deletions(-) diff --git a/modulector/exceptions/exceptions.py b/modulector/exceptions/exceptions.py index 633ef06..7216ad9 100644 --- a/modulector/exceptions/exceptions.py +++ b/modulector/exceptions/exceptions.py @@ -1,6 +1,6 @@ class CommandNotPresentException(Exception): def __init__(self, command, commands_list): - self.message = 'The command {} is not configured, the available commands are {}'.format(command, commands_list) + self.message = f'The command {command} is not configured, the available commands are {commands_list}' def __str__(self): return self.message diff --git a/modulector/pubmed_job.py b/modulector/pubmed_job.py index 51e430a..b6ff767 100644 --- a/modulector/pubmed_job.py +++ b/modulector/pubmed_job.py @@ -76,7 +76,7 @@ def execute(): # Retrieve all main subscriptions and iterates the items for sub in Subscription.objects.all(): subscription_items = SubscriptionItem.objects.filter(subscription=sub) - logger.info('Subscriptions {}'.format(subscription_items)) + logger.info(f'Subscriptions {subscription_items}') mail_rows = [] for subscription_item in subscription_items: diff --git a/modulector/serializers.py b/modulector/serializers.py index 32d8e9a..38cd5ce 100644 --- a/modulector/serializers.py +++ b/modulector/serializers.py @@ -51,9 +51,9 @@ class Meta: def get_pubmeds(self, mirna_gene_interaction: MirnaXGene) -> Set[str]: """ - Gets a list of related Pubmed URLs to a miRNA-Gene interaction - :param mirna_gene_interaction: miRNA-Gene interaction - :return: List of Pubmed URLs + Gets a list of related Pubmed URLs to a miRNA-Gene interaction. + :param mirna_gene_interaction: miRNA-Gene interaction. + :return: List of Pubmed URLs. """ pubmed_urls: Set[str] = set() @@ -61,8 +61,10 @@ def get_pubmeds(self, mirna_gene_interaction: MirnaXGene) -> Set[str]: return pubmed_urls pubmed_urls.update( - list(mirna_gene_interaction.pubmed.values_list('pubmed_url', flat=True))) + list(mirna_gene_interaction.pubmed.values_list('pubmed_url', flat=True)) + ) + # If the api call is enabled, it will query the API and get the pubmeds mirna = mirna_gene_interaction.mirna.mirna_code gene = mirna_gene_interaction.gene term = pubmed_service.build_search_term(mirna, gene) @@ -73,7 +75,7 @@ def get_pubmeds(self, mirna_gene_interaction: MirnaXGene) -> Set[str]: pubmed_urls.update([link_builder.build_pubmed_url(pubmed_id) for pubmed_id in api_pubmeds]) except Exception as ex: - # we only handle exceptions in this case + # Only handles exceptions in this case logging.error('Error getting PubMeds:') logging.exception(ex) return set() @@ -124,9 +126,9 @@ def get_links(mirna: Mirna) -> List[Dict]: @staticmethod def get_aliases(mirna: Mirna) -> List[str]: """ - Gets a miRNA aliases - :param mirna: miRNA object to get its aliases - :return: List of miRNA aliases + Gets a miRNA aliases. + :param mirna: miRNA object to get its aliases. + :return: List of miRNA aliases. """ return get_mirna_aliases(mirna.mirna_code) @@ -141,9 +143,9 @@ class Meta: @staticmethod def get_pubmed(disease: MirnaDisease) -> str: """ - Gets a PubMed URL for a miRNA-disease association - :param disease: MirnaDisease object - :return: Pubmed URL + Gets a PubMed URL for a miRNA-disease association. + :param disease: MirnaDisease object. + :return: Pubmed URL. """ return link_builder.build_pubmed_url(disease.pubmed_id) diff --git a/modulector/services/mailing_service.py b/modulector/services/mailing_service.py index 0cf188d..1662bca 100644 --- a/modulector/services/mailing_service.py +++ b/modulector/services/mailing_service.py @@ -2,52 +2,56 @@ import os import pathlib import sys - +from typing import Final, List, Dict, Union from django.core import mail from django.core.mail.message import EmailMessage from django.template.loader import render_to_string +from ModulectorBackend.settings import UNSUBSCRIBE_URL, DEFAULT_FROM_EMAIL, DEBUG -from ModulectorBackend.settings import UNSUBSCRIBE_URL, DEFAULT_FROM_EMAIL - +# Sets some logging configuration logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) -logger.setLevel(logging.INFO) +logger.setLevel(logging.DEBUG if DEBUG else logging.INFO) -parent_dir = pathlib.Path(__file__).parent.absolute().parent -html_template_path = os.path.join(parent_dir, "templates/email_template.html") -html_template_end = os.path.join(parent_dir, "templates/email_end.html") -html_rows_path = os.path.join(parent_dir, "templates/email_row.html") +parent_dir: Final[pathlib.Path] = pathlib.Path(__file__).parent.absolute().parent +html_template_path: Final[str] = os.path.join(parent_dir, "templates/email_template.html") +html_template_end: Final[str] = os.path.join(parent_dir, "templates/email_end.html") +html_rows_path: Final[str] = os.path.join(parent_dir, "templates/email_row.html") -html_template = render_to_string(html_template_path) -html_end = render_to_string(html_template_end) +html_template: Final[str] = render_to_string(html_template_path) +html_end: Final[str] = render_to_string(html_template_end) -def build_unsubscribe_url(unsubscribe_token): +def build_unsubscribe_url(unsubscribe_token: str) -> str: + """Generates an unsubscribe url for a given token.""" return UNSUBSCRIBE_URL + unsubscribe_token -def generate_link(link, text): - return '{}
'.format(link, text) +def generate_link(link: str, text: Union[str, int]) -> str: + """Generates an HTML link (as _blank).""" + return f'{text}
' -def build_url_to_send(pubmed_id): - return 'https://pubmed.ncbi.nlm.nih.gov/' + str(pubmed_id) +def build_url_to_send(pubmed_id: int) -> str: + """Generates a URL to send to the user.""" + return f'https://pubmed.ncbi.nlm.nih.gov/{pubmed_id}' -def build_pubmeds(pubmeds): - text = '' - for pubmed_id in pubmeds: - text = text + generate_link(build_url_to_send(pubmed_id), pubmed_id) - return text +def build_pubmeds(pubmeds: List[int]) -> str: + """Builds the HTML for the pubmeds. Generates some links concatenated.""" + return ''.join([ + generate_link(build_url_to_send(pubmed_id), pubmed_id) for pubmed_id in pubmeds + ]) -def email_users(content): +def email_users(content: Dict): subject = 'Update regarding your multiomix subscription' email_messages = [] + logger.info('Starting to build emails to send') for key in content: if len(content[key]) < 1: - logger.info('empty info for {}, skipping'.format(key)) + logger.info('Empty info for {}, skipping'.format(key)) continue rows = [] for mail_row in content[key]: @@ -57,11 +61,11 @@ def email_users(content): token = mail_row.token url = build_unsubscribe_url(token) url = generate_link(url, 'unsubscribe') - html_row = render_to_string(html_rows_path, - {"mirna": mirna, "gene": gene, "unsubscribe": "{{ unsubscribe }}" - , "pubmeds": "{{ pubmeds }}"}) - html_row = html_row.replace("{{ unsubscribe }}", url).replace("{{ pubmeds }}", - pubmeds) + html_row = render_to_string( + html_rows_path, + {"mirna": mirna, "gene": gene, "unsubscribe": "{{ unsubscribe }}", "pubmeds": "{{ pubmeds }}"} + ) + html_row = html_row.replace("{{ unsubscribe }}", url).replace("{{ pubmeds }}", pubmeds) rows.append(html_row) rows_data = ' '.join(rows) final_content = html_template + rows_data + html_end @@ -69,12 +73,14 @@ def email_users(content): message.content_subtype = "html" email_messages.append(message) logger.info('Appended email to {}'.format(message.to)) + for email in email_messages: logger.info(email) + if len(email_messages) > 0: - logger.info('Sending emails') + logger.info('Sending emails...') with mail.get_connection(fail_silently=False) as connection: connection.send_messages(email_messages) logger.info('Emails sent') else: - logger.info('skipping emails') + logger.info('Skipping emails...') diff --git a/tools/healthcheck/check.py b/tools/healthcheck/check.py index a250f1b..3ff9ef7 100644 --- a/tools/healthcheck/check.py +++ b/tools/healthcheck/check.py @@ -22,7 +22,7 @@ def notify(): tries = get_tries() tries = tries + 1 headers = {'Content-Type': 'application/json'} - message = "Modulector: Health check failed {} times.".format(tries) + message = f"Modulector: Health check failed {tries} times." if tries == 3: message = message + " The container has been marked as unhealthy." set_tries(0) From 4e929c578b00bf6655dfd3f5aefc1c1134e123d0 Mon Sep 17 00:00:00 2001 From: Genaro Camele Date: Thu, 9 Nov 2023 16:22:18 -0300 Subject: [PATCH 3/7] Updated dependencies --- config/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/requirements.txt b/config/requirements.txt index d50441d..a56d351 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -1,13 +1,13 @@ daphne==4.0.0 -Django==4.2.2 +Django==4.2.7 django-crontab==0.7.1 -django-filter==23.2 +django-filter==23.3 django-generate-secret-key==1.0.2 -django-stubs==4.2.1 +django-stubs==4.2.6 djangorestframework==3.14.0 -mypy==1.3.0 +mypy==1.6.1 pandas==2.0.2 -psycopg2-binary==2.9.6 +psycopg2-binary==2.9.9 requests==2.31.0 fasta_reader==3.0.1 tqdm==4.65.0 From 6243ea4bd24ad0c9b909c8fd8dba15b42b3ccbb8 Mon Sep 17 00:00:00 2001 From: Genaro Camele Date: Thu, 9 Nov 2023 16:22:22 -0300 Subject: [PATCH 4/7] Updated dependencies --- config/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/requirements.txt b/config/requirements.txt index a56d351..829f752 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -6,8 +6,8 @@ django-generate-secret-key==1.0.2 django-stubs==4.2.6 djangorestframework==3.14.0 mypy==1.6.1 -pandas==2.0.2 +pandas==2.1.2 psycopg2-binary==2.9.9 requests==2.31.0 -fasta_reader==3.0.1 -tqdm==4.65.0 +fasta_reader==3.0.2 +tqdm==4.66.1 From 66e8e63ac97a0a982e4cc064527e5f384fbcf65e Mon Sep 17 00:00:00 2001 From: Genaro Camele Date: Thu, 9 Nov 2023 16:22:34 -0300 Subject: [PATCH 5/7] Fixed ULRs in the README to prevent 301 redirections --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 57459fd..98d6c9b 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ If no gene symbol is entered, all mirna interactions are returned. If a mirna is - `sources`: miRNA-Gene interaction sources which publish this interaction. mirDIP score is based on the scores of those sources. This field is an array that contains the interaction score source names. - `score_class`: `L` (Low), `M` (Medium), `H` (High) or `V` (Very high) - Example: - - URL: http://localhost:8000/mirna-target-interactions?mirna=hsa-miR-891a-5p&gene=EGFR&include_pubmeds=true + - URL: http://localhost:8000/mirna-target-interactions/?mirna=hsa-miR-891a-5p&gene=EGFR&include_pubmeds=true - Response: ```json { @@ -177,7 +177,7 @@ Returns extra information of a miRNA. - `mirbase_accession_id`: miRNA accession ID (MIMAT). - `links` array of URLs with extra information about this miRNA. - Example: - - URL: http://localhost:8000/mirna?mirna=hsa-miR-548ai + - URL: http://localhost:8000/mirna/?mirna=hsa-miR-548ai - Response: ```json { @@ -218,7 +218,7 @@ Returns a paginated response with aliases of a miRNA. - `mirbase_accession_id`: miRNA mirBase accession ID (MIMAT). - `mature_mirna`: previous ID (according to mirBase). - Example: - - URL: http://localhost:8000/mirna-aliases?mirbase_accession_id=MIMAT0000062 + - URL: http://localhost:8000/mirna-aliases/?mirbase_accession_id=MIMAT0000062 - Response: ```json { @@ -255,7 +255,7 @@ Service that takes a string of any length and returns a list of miRNAs that cont - Code: 200 - Content: a list of miRNAs (IDs or accession IDs from miRbase DB) matching the search criteria. - Example: - - URL: http://localhost:8000/mirna-codes-finder?query=hsa-let-7a + - URL: http://localhost:8000/mirna-codes-finder/?query=hsa-let-7a - Response: ```json [ @@ -336,7 +336,7 @@ Service that takes a text string of any length and returns a list of methylation - Code: 200 - Content: a list of methylation sites from the Illumina 'Infinium MethylationEPIC 2.0' array matching the search criteria. - Example: - - URL: http://localhost:8000/methylation-sites-finder?query=cg25&limit=5 + - URL: http://localhost:8000/methylation-sites-finder/?query=cg25&limit=5 - Response: ```json [ @@ -466,7 +466,7 @@ Returns information of a methylation site. - `relation`: Relation of the site to the CpG island. The values it can take are: *Island*=within boundaries of a CpG Island, *N_Shore*=0-2kb 5' of Island, *N_Shelf*=2kb-4kb 5' of Island, *S_Shore*=0-2kb 3' of Island, *S_Shelf*=2kb-4kb 3' of Island. - `genes`: The value is a json where each key is a gene that is related to the methylation site. The values for each gene is a list that contains the region of the gene where the methylation site is located. These regions, according to the NCBI RefSeq database, can be: *5UTR*=5' untranslated region between the TSS and ATG start site, *3UTR*=3' untranslated region between stop codon and poly A signal, *exon_#*, *TSS200*=1-200 bp 5' the TSS, *TS1500*=200-1500 bp 5' of the TSS. - Example: - - URL: http://localhost:8000/methylation?methylation_site=cg22461615 + - URL: http://localhost:8000/methylation/?methylation_site=cg22461615 - Response: ```json { @@ -523,7 +523,7 @@ Returns a paginated response of diseases related to a miRNA. - `pubmed`: Pubmed URL. - `description`: description about why this miRNA is related to this disease. - Example: - - URL: http://localhost:8000/diseases?mirna=hsa-miR-9500 + - URL: http://localhost:8000/diseases/?mirna=hsa-miR-9500 - Response: ```json { @@ -574,7 +574,7 @@ Returns a paginated response of experimentally validated small molecules (or dru - `expression_pattern`: expression pattern of miRNA. - `support`: support information for this effect. - Example: - - URL: http://localhost:8000/drugs?mirna=miR-126* + - URL: http://localhost:8000/drugs/?mirna=miR-126* - Response: ```json { From 6552b64509b483c7603215a6bc3b7d418fe8c905 Mon Sep 17 00:00:00 2001 From: Genaro Camele Date: Thu, 9 Nov 2023 16:45:04 -0300 Subject: [PATCH 6/7] Fixed typos and missing link in the DEPLOYING.md --- DEPLOYING.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/DEPLOYING.md b/DEPLOYING.md index d11cae3..05d7047 100644 --- a/DEPLOYING.md +++ b/DEPLOYING.md @@ -19,22 +19,22 @@ Below are the steps to perform a production deploy. 1. Make a copy of `docker-compose_dist.yml` with the name `docker-compose.yml`. 1. Set the environment variables that are empty with data. They are listed below by category: - Django: - - `DJANGO_SETTINGS_MODULE`: indicates the `settings.py` file to read. In production we set in `docker-compose_dist.yml` the value `ModulectorBackend.settings_prod` which contains several production properties. + - `DJANGO_SETTINGS_MODULE`: indicates the `settings.py` file to read. In production, we set in `docker-compose_dist.yml` the value `ModulectorBackend.settings_prod` which contains several production properties. - `ALLOWED_HOSTS`: list of allowed host separated by commas. Default `['web', '.localhost', '127.0.0.1', '[::1]']`. - `ENABLE_SECURITY`: set the string `true` to enable Django's security mechanisms. In addition to this parameter, to have a secure site you must configure the HTTPS server, for more information on the latter see the section [Enable SSL/HTTPS](#enable-sslhttps). Default `false`. - - `CSRF_TRUSTED_ORIGINS`: in Django >= 4.x, it's mandatory to define this in production when you are using Daphne through NGINX. The value is a single host or list of hosts separated by commas. 'http://', 'https://' prefixes are mandatory. Examples of values: 'http://127.0.0.1', 'http://127.0.0.1,https://127.0.0.1:8000', etc. You can read more [here](#csrf-trusted-issue). + - `CSRF_TRUSTED_ORIGINS`: in Django >= 4.x, it's mandatory to define this in production when you are using Daphne through NGINX. The value is a single host or list of hosts separated by commas. 'http://', 'https://' prefixes are mandatory. Examples of values: 'http://127.0.0.1', 'http://127.0.0.1,https://127.0.0.1:8000', etc. You can read more [here][csrf-trusted-issue]. - `SECRET_KEY`: Django's secret key. If not specified, one is generated with [generate-secret-key application](https://github.com/MickaelBergem/django-generate-secret-key) automatically. - `MEDIA_ROOT`: absolute path where will be stored the uploaded files. By default `/uploads`. - `MEDIA_URL`: URL of the `MEDIA_ROOT` folder. By default `/media/`. - `ALLOWED_HOSTS`: list of allowed hosts (separated by commas) to access to Modulector. Default `web,localhost,127.0.0.1,::1'` - `PROCESS_POOL_WORKERS`: some request uses parallelized queries using ProcessPoolExecutor to improve performance. This parameter indicates the number of workers to be used. By default `4`. - Postgres: - - `POSTGRES_USERNAME` : Database username. By default the docker image uses `modulector`. - - `POSTGRES_PASSWORD` : Database username's password. By default the docker image uses `modulector`. - - `POSTGRES_PORT` : Database server listen port. By default the docker image uses `5432`. - - `POSTGRES_DB` : Database name to be used. By default the docker image uses `modulector`. + - `POSTGRES_USERNAME` : Database username. By default, the docker image uses `modulector`. + - `POSTGRES_PASSWORD` : Database username's password. By default, the docker image uses `modulector`. + - `POSTGRES_PORT` : Database server listen port. By default, the docker image uses `5432`. + - `POSTGRES_DB` : Database name to be used. By default, the docker image uses `modulector`. - Health-checks and alerts: - - `HEALTH_URL` : indicates the url that will be requested on Docker health-checks. By default it is http://localhost:8000/drugs/. The healthcheck makes a GET request on it. Any HTTP code value greater or equals than 400 is considered an error. + - `HEALTH_URL` : indicates the url that will be requested on Docker health-checks. By default, it is http://localhost:8000/drugs/. The healthcheck makes a GET request on it. Any HTTP code value greater or equals than 400 is considered an error. - `HEALTH_ALERT_URL` : if you want to receive an alert when health-checks failed, you can set this variable to a webhook endpoint that will receive a POST request and a JSON body with the field **content** that contains the fail message. 1. Go back to the project's root folder and run the following commands: - Docker Compose: @@ -44,7 +44,7 @@ Below are the steps to perform a production deploy. - Start: `docker stack deploy --compose-file docker-compose.yml modulector` - Stop: `docker stack rm modulector` 1. Import all the data following the instructions detailed in the [Import section](#import). -1. (Optional) Create a super user to access to the admin panel (`/admin`). +1. (Optional) Create a superuser to access to the admin panel (`/admin`). 1. Enter the running container: `docker container exec -it bash`. The name is usually `modulector_web_1` but you can check it with `docker container ps`. 1. Run: `python3 manage.py createsuperuser` 1. Exit the container: `exit` @@ -52,7 +52,7 @@ Below are the steps to perform a production deploy. ### Start delays -Due to the database restoration in the first start, the container `db_modulector` may take a while to be up an ready. We can follow the status of the startup process in the logs by doing: `docker compose logs --follow`. +Due to the database restoration in the first start, the container `db_modulector` may take a while to be up a ready. We can follow the status of the startup process in the logs by doing: `docker compose logs --follow`. Sometimes this delay makes django server throws database connection errors. If it is still down and not automatically fixed when database is finally up, we can restart the services by doing: `docker compose up -d`. @@ -64,7 +64,7 @@ To enable HTTPS, follow the steps below: 1. Copy the file `config/nginx/multiomics_intermediate_safe_dist.conf` and paste it into `config/nginx/conf.d/` with the name `multiomics_intermediate.conf`. 1. Get the `.crt` and `.pem` files for both the certificate and the private key and put them in the `config/nginx/certificates` folder. 1. Edit the `multiomics_intermediate.conf` file that we pasted in point 2. Uncomment the lines where both `.crt` and `.pem` files must be specified. -1. Edit the `docker-compose.yml` file so that the `nginx` service exposes both port 8000 and 443. Also you need to add `certificates` folder to `volumes` section. It should look something like this: +1. Edit the `docker-compose.yml` file so that the `nginx` service exposes both port 8000 and 443. Also, you need to add `certificates` folder to `volumes` section. It should look something like this: ```yaml ... @@ -91,7 +91,7 @@ Django provides in its official documentation a configuration checklist that mus docker container exec modulector_backend python3 manage.py check --deploy --settings ModulectorBackend.settings_prod ``` -Otherwise you could set all the mandatory variables found in `settings_prod.py` and run directly without the need to pick up any service: +Otherwise, you could set all the mandatory variables found in `settings_prod.py` and run directly without the need to pick up any service: ``` python3 manage.py check --deploy --settings ModulectorBackend.settings_prod @@ -151,7 +151,7 @@ You can use set Modulector DB in two ways. **For the EPIC Methylation array**: - Go to the [Illumina product files web page](https://support.illumina.com/downloads/infinium-methylationepic-v2-0-product-files.html) and download the ZIP file called "*Infinium MethylationEPIC v2.0 Product Files (ZIP Format)*". - Unzip the file. - - Within the unzipped files you will find one called "*EPIC-8v2-0_A1.csv*". Move this file to the directory **"modulor/files/"**. + - Within the unzipped files you will find one called "*EPIC-8v2-0_A1.csv*". Move this file to the directory **"modulector/files/"**. - **NOTE:** the total weight of both files is about 5 GB. **For the mirBase database**: this database is embedded as it weighs only a few MBs. Its data is processed in Django migrations during the execution of the `python3 manage.py migrate` command. So, you don't have to do manual steps to incorporate mirBase data inside Modulector. @@ -162,7 +162,7 @@ You can use set Modulector DB in two ways. ## Update databases Modulector currently works with the mirDIP (version 5.2) and miRBase (version 22.1) databases for miRNA data, and with information from the Illumina 'Infinium MethylationEPIC 2.0' array for information about methylation sites. -If new versions are released for these databases and you want to update them, follow these steps: +If new versions are released for these databases, and you want to update them, follow these steps: - For **mirDIP** and **Illumina EPIC array** you must follow the same steps described in the [Regenerating the data manually](#regenerating-the-data-manually) section, replacing the named files with the most recent versions that have been published on their sites . - For **miRBase**, follow the instructions below: @@ -177,7 +177,7 @@ If new versions are released for these databases and you want to update them, fo ## Configure your API key -When we notify user about updates of pubmeds they are subscribed to we interact with a ncbi api that uses an API_KEY, by default, we left a random API_KEY pre configured in our settings file, you should replace it with your own. +When we notify user about updates of pubmeds they are subscribed to we interact with a ncbi api that uses an API_KEY, by default, we left a random API_KEY pre-configured in our settings file, you should replace it with your own. ## Cron job configuration @@ -185,3 +185,4 @@ For cron jobs we use the following [library](https://github.com/kraiz/django-cro [mirbase-download-page]: https://www.mirbase.org/ftp.shtml +[csrf-trusted-issue]: https://docs.djangoproject.com/en/4.2/ref/csrf/ From 206fff52baa0dcfcfb54844c176c1d16b793b131 Mon Sep 17 00:00:00 2001 From: Genaro Camele Date: Fri, 10 Nov 2023 14:34:20 -0300 Subject: [PATCH 7/7] Removed unused files --- ModulectorBackend/settings_dev.py | 164 ------------------------------ tools/use_settings.sh | 13 --- 2 files changed, 177 deletions(-) delete mode 100644 ModulectorBackend/settings_dev.py delete mode 100755 tools/use_settings.sh diff --git a/ModulectorBackend/settings_dev.py b/ModulectorBackend/settings_dev.py deleted file mode 100644 index 1129c29..0000000 --- a/ModulectorBackend/settings_dev.py +++ /dev/null @@ -1,164 +0,0 @@ -""" -Django settings for ModulectorBackend project. - -Generated by 'django-admin startproject' using Django 3.0.8. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/3.0/ref/settings/ -""" - -import os - -# Modulector version -VERSION: str = '2.1.1' - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'm)95!xjla3k1h)1#ya_rp^7#7^l_laaa6x$ik2_vx#gc0ll=e(' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1', '*'] - -# Modulector unsubscribe endpoint -UNSUBSCRIBE_URL = 'http://localhost:8000/unsubscribe-pubmeds/?token=' -DEFAULT_FROM_EMAIL = 'multiomix@gmail.com' -NCBI_API_KEY = '61a2e5671d5d57b8dde8a61cad17eec99709' -# to enable and disable api call to pubmeds in the mirna interactions and mirna target interactions endpoints -USE_PUBMED_API = True -PUBMED_API_TIMEOUT = 1 -# Application definition - -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django_crontab', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'modulector.apps.ModulectorConfig', - 'rest_framework', - 'django_filters', - 'django_generate_secret_key' -] - -CRONJOBS = [ - ('0 0 * * SAT', 'modulector.pubmed_job.execute', - '>> ' + BASE_DIR + '/jobs_log.log') -] -CRONTAB_LOCK_JOBS = True -ALLOW_PARALLEL_RUNS = False -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -REST_FRAMEWORK = { - # Use Django's standard `django.contrib.auth` permissions, - # or allow read-only access for unauthenticated users. - # 'DEFAULT_PERMISSION_CLASSES': ( - # 'rest_framework.permissions.DjangoModelPermissions', - # ) -} - -ROOT_URLCONF = 'ModulectorBackend.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, 'templates')], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -WSGI_APPLICATION = 'ModulectorBackend.wsgi.application' - -# Database -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'USER': os.getenv('POSTGRES_USERNAME', 'root'), - 'PASSWORD': os.getenv('POSTGRES_PASSWORD', 'admin'), - 'HOST': os.getenv('POSTGRES_HOST', '127.0.0.1'), - 'PORT': os.getenv('POSTGRES_PORT', 5432), - 'NAME': os.getenv('POSTGRES_DB', 'modulector'), - }, -} -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'unique-snowflake', - } -} -# Password validation -# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - -# Internationalization -# https://docs.djangoproject.com/en/3.0/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.0/howto/static-files/ -STATIC_ROOT = os.path.join(BASE_DIR, 'static') -STATIC_URL = '/static/' - -# Media files -MEDIA_ROOT = os.getenv('MEDIA_ROOT', os.path.join(BASE_DIR, '')) -MEDIA_URL = os.getenv('MEDIA_URL', '/media/') - -# Email Server -EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' -# This email configuration is what postfix uses, for production, use your own -EMAIL_HOST = 'localhost' -EMAIL_PORT = '25' -EMAIL_HOST_PASSWORD = '' -EMAIL_HOST_PASSWORD = '' -EMAIL_USE_TLS = False diff --git a/tools/use_settings.sh b/tools/use_settings.sh deleted file mode 100755 index 83dacd5..0000000 --- a/tools/use_settings.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -case $1 in - "dev") - cp -vf $BASEDIR/ModulectorBackend/settings_dev.py $BASEDIR/ModulectorBackend/settings.py - ;; - "prod") - cp -vf $BASEDIR/ModulectorBackend/settings_prod.py $BASEDIR/ModulectorBackend/settings.py - ;; - "ci") - cp -vf $BASEDIR/ModulectorBackend/settings_ci.py $BASEDIR/ModulectorBackend/settings.py - ;; -esac