setup.py

########
#### setup.py: script to build and help development of the Vulnerability catalog.
#### Date: 2018-02-18
#### Version: 1.0
#### Author: Daniel Avelino https://daavelino.github.io
########

import platform
import sys
import shutil
import os
import subprocess
# import secrets # Will be imported after check if python version is > 3.6.
import getpass
import re
import json
import pprint

from pathlib import Path

global MINIMAL_PYTHON_VERSION
global PROJECT_NAME
global APP_NAME
global HOME_DIR

MINIMAL_PYTHON_VERSION = (3, 6, 0)
PROJECT_NAME = 'base'
APP_NAME = 'catalog'
HOME_DIR = Path(os.getcwd())
BASE_DIR = Path(os.path.join(HOME_DIR, PROJECT_NAME,))


######### DRY functions:
def tuple_comparison(a:tuple, b:tuple):
    '''
    Evaluate tuples as an ordered list of numbers 
    and returns True if a >= b
    '''
    control = False
    for i in range(0, len(b)):
        if int(a[i]) > int(b[i]):
            control = True
            break
        elif int(a[i]) < int(b[i]):
            control = False
            break
        else:
            control = True
    return(control)

def get_req_version(resource, filename):
    '''
    return a list representing the resource version 
    as indicated in filename, ['resource', 'version'], 
    or None otherwise. Filename should be the pip file 
    requirements.txt.
    '''
    version = None
    if not os.path.isfile(filename):
        print('No such file or directory.')
        return(version)
    else:
        f = open(filename,'r')
        line = f.readline()
        while line != '':
            if re.search(resource, line):
                regex = r'\W+' # \W+ matches any non-word character.
                splitter = re.search(regex, line, re.DOTALL | re.IGNORECASE)
                if splitter: # https://docs.python.org/3/library/re.html#re.search
                    splitter = splitter.group()
                    version = line.split(splitter)
                    for i in range(0, len(version)):
                        tmp = version[i]
                        version[i] = tmp.rstrip(os.linesep)
                break
            else:
                line = f.readline()
        f.close()
        if version:
            tmp = version[1].split('.')
            version = tuple(tmp)
        return(version)
######### End DRY functions

######## System properties:
def get_environment():
    '''Returns a dictionary with relevant information about OS environment'''
    env = dict()
    env['system'] = platform.system()

    if env['system'] == 'Linux':
        env['system version'] = platform.linux_distribution()
    
    if env['system'] == 'Windows':
        env['system version'] = platform.linux_distribution()

    return(env)

def check_parameters():
    allowed_params = {
        'build': "setup and builds the project from the scratch (with test data).",
        'build-only': "Just build project, without load data or create users.",
        'build-novenv': "same as 'build', but do not uses Python venv.",
        'clean': "remove all project related files.",
        'createstartscript': "creates the platform specific Catalog start script",
        'createsuperuser': "creates an admin user to the catalog.",
        'createvenv': "creates a functional Python's Virtual Environment at current directory.",
        'database': "Setup an alternative database instead of default sqlite.",
        'deep-clean': "remove all project related files and venv structure.",
        'loadtestdata': "Add some test data into database.",
        'templates': "updates project's Templates, forms and static files only.",
        'urlsviews': "updates project's Urls and Views only."   
    }

    # one and only one allowed parameter has to be passed.
    if (len(sys.argv) == 2) and (sys.argv[1] in allowed_params.keys()):
        param = sys.argv[1] # Because argv[0] is the file name.
        return param
    else:
        print('\nUsage:', sys.argv[0], '<options>, where <options> are:\n')
        for i in allowed_params.items():
            print('  ' + i[0] + ':    ' + i[1])
        print('\nExiting.')
        sys.exit(1)
        
def check_venv():
    venv_dir = Path('venv',)
    env = get_environment()
    
    if not venv_dir.is_dir():
        print('Missing venv structure. Creating...')
        os.makedirs("venv")
        os.system('python -m venv venv')
        
        print('\n[Warning]    Virtual Environment not load yet. '
              'Please, load venv by running:')
        if env['system'] == 'Linux':
            print('\n    source venv/bin/activate\n')       
        if env['system'] == 'Windows':
            print('\n    venv\\Scripts\\activate.bat\n')
        param = check_parameters()
        print('and run\n\n    python setup.py', param,'\n\nscript again.')
        sys.exit(0)       
    else:
        if not 'VIRTUAL_ENV' in os.environ.keys(): # venv is not loaded:
            print('\n[Warning]    Virtual Environment not load yet. '
                  'Please, load venv by running:')
            if env['system'] == 'Linux': 
                print('\n    source venv/bin/activate\n')
            if env['system'] == 'Windows':
                print('\n    venv\\Scripts\\activate.bat\n')
            sys.exit(0)
        else:
            pass

def check_python_version():
    '''
    Ensure that Python version is greater than MINIMAL_PYTHON_VERSION
    It also loads secrets module, new in Python 3.6 version. 
    https://docs.python.org/3/library/secrets.html
    '''

    #### Checking installed Python version:
    python_version = (sys.version_info.major, sys.version_info.minor, \
                      sys.version_info.micro)
    control = tuple_comparison(python_version, MINIMAL_PYTHON_VERSION)
    
    if not control:
        print('\n[Warning]    Missing Python ', end='')
        for i in range(0, len(MINIMAL_PYTHON_VERSION)):
            print(str(MINIMAL_PYTHON_VERSION[i]) \
              + '.', end='' )
        print(' (or greater).\n')
        print('Please, get it at (https://www.python.org/downloads/).\nExiting.\n')
        sys.exit(1)
    else:
        print('Importing secrets module')
        global secrets
        import secrets # secrets is New in Python 3.6.

def check_pip_version():
    'Check if pip is installed. If not, install it properly.'
    try:
        import pip
    except ImportError: # Exit, since it is a required dependency.
        print('\n[Warning]    Missing pip.\n')
        print('Please, install it first (https://pip.pypa.io).\nExiting.\n')
        sys.exit(1)

def check_django_version():
    'Check if Django is installed. If not, install it properly.'
    try: 
        from django.core.management import execute_from_command_line
    except ImportError: # so, install it properly:
        print('\n[Warning]    Missing Django.\n')

        django_min_version = ''
        for i in range(0, len(MINIMAL_DJANGO_VERSION)):
            django_min_version = django_min_version \
                               + str(MINIMAL_DJANGO_VERSION[i]) \
                               + '.'
        django_min_version = django_min_version[:-1]

        print('Using\n\n    pip install django==' \
              + django_min_version +'\n\nto fix this dependency...\n')
        os.system('pip install django==' + django_min_version)
        print('Done.')

    #### Check Django version:
    #### We opt not to update Django version here to avoid unecessary
    #### complications at development environment. Since this script
    #### install Django using its correct version from the scratch, if
    #### it find Django in an older version, things should be tested...
    try:
        import django
    except ImportError:
        pass 
        # It must already be present since we installed it previously.
    django_version = django.VERSION[0:3]
    django_min_version = ''
    for i in range(0, len(MINIMAL_DJANGO_VERSION)):
        django_min_version = django_min_version \
                           + str(MINIMAL_DJANGO_VERSION[i]) \
                           + '.'
    django_min_version = django_min_version[:-1]
    
    if not tuple_comparison(django_version, MINIMAL_DJANGO_VERSION):
        print('\n[Warning]    Django', django.__version__, 
              '(actual version) requires an update.\n')
        print('Please, upgrade Django to', django_min_version, \
              'version by running:\n')
        print('    pip install django==' \
            + django_min_version + '\n\nExiting.\n')
        sys.exit(1)

def check_whitenoise_version():
    try:
        import whitenoise # http://whitenoise.evans.io
    except ImportError:
        print('\n[Warning]    Missing WhiteNoise.\n')
        print('Using\n\n    pip install whitenoise\n'
              '\nto fix this dependency...\n')  
        os.system('pip install whitenoise')
        print('Done.')

def check_system_reqs():
    check_python_version()
    check_pip_version()
    check_django_version()
    check_whitenoise_version()

def cleaning_old_stuff(control):
    '''Cleaning created projects files'''
    env = get_environment()
    print('Cleaning old project structure...')
    target = Path(PROJECT_NAME)
    if target.is_dir():
        shutil.rmtree(target)

    if control == 'deep-clean':
        if 'venv' in sys.prefix: # venv is loaded:
            if env['system'] == 'Linux':
                #os.system('deactivate')
                # it will not work since deactivate is defined at python's parent process.
                pass
            if env['system'] == 'Windows':
                os.system('venv\Scripts\deactivate.bat')
        target = Path(os.path.join(os.path.curdir, 'venv'))
        if target.is_dir():
            shutil.rmtree(target)
    
        print('Done.')

######## End of System properties.

######## Django properties:
def start_django_project():
    '''Builds the project.'''
    print('Starting creating Django structure...')
    #### Starting project creation:
    os.system('django-admin startproject' + ' ' + PROJECT_NAME)
    BASEDIR = os.path.abspath(os.path.curdir) 
    os.chdir(PROJECT_NAME)

    #### Introducing APP_NAME into the project:
    os.system('python manage.py startapp' + ' ' + APP_NAME)
    print('Done.')

    os.chdir(BASEDIR)

def importing_settings():
    '''Get settings from metadata/settings/settings.py'''
    print('Copy settings.py from metadata...')
    src_path = os.path.join(os.path.curdir, \
                            'metadata', \
                            'settings', \
                            'settings.py')
    dst_path = os.path.join(os.path.curdir, 'base', 'base', 'settings.py')

    shutil.copy(src_path, dst_path)
    print('Done.')

def set_datamodel():
    '''Enabling catalog data models into Django structure.'''
    env = get_environment()
    print('Copy data models...')
    src_path = os.path.join(os.path.curdir, \
                            'metadata', \
                            'models', \
                            'catalog', \
                            'models.py')
    dst_path = os.path.join(os.path.curdir, 'base', 'catalog', 'models.py')

    shutil.copy(src_path, dst_path)
    print('Done.')

    #### Applying data models:
    print('Applying data models...')
    BASEDIR = os.path.abspath(os.path.curdir)
    os.chdir(PROJECT_NAME)
    os.system('python manage.py makemigrations' + ' ' + APP_NAME)
    os.system('python manage.py sqlmigrate' \
              + ' ' \
              + APP_NAME \
              + ' ' \
              + '0001')
    os.system('python manage.py migrate')
    os.chdir(BASEDIR)
    print('Done.')

def set_urls():
    print('Setting Urls...')
    # PROJECT_URLS:
    src_path = os.path.join(os.path.curdir, \
                            'metadata', \
                            'urls', \
                            'admin', \
                            'urls.py')
    dst_path = os.path.join(os.path.curdir, 'base', 'base', 'urls.py')    
    shutil.copy(src_path, dst_path)
   
    # APP_URLS:
    src_path = os.path.join(os.path.curdir,\
                            'metadata', \
                            'urls', \
                            'catalog', \
                            'urls.py')
    dst_path = os.path.join(os.path.curdir, 'base', 'catalog', 'urls.py')
    shutil.copy(src_path, dst_path)
    print('Done.')

def set_views():
    print('Setting Views...')
    src_path = os.path.join(os.path.curdir,'metadata', 'views', 'catalog', )
    dst_path = os.path.join(os.path.curdir, 'base', 'catalog', )

    for i in os.listdir(src_path):
        fsrc = os.path.join(src_path, i)
        shutil.copy(fsrc, dst_path)
    print('Done.')

def set_templates():
    print('Setting templates...')
    files = ['index.html', 
             'home.html', 
             'detail.html', 
             'add.html', 
             'update.html', 
             'delete.html', 
             'panorama.html', 
             'fastupdate.html', 
             'search.html', 
             'risk.html', 
             'cvss.html', 
             'upload.html'
            ]

    tmpl_srcdir = os.path.join(os.path.curdir, \
                               'metadata', \
                               'templates', \
                               'catalog',)
    tmpl_dstdir = os.path.join(os.path.curdir, \
                               'base', \
                               'catalog', \
                               'templates', \
                               'catalog',)
    tmpl_main_path = os.path.join(tmpl_srcdir, 'tmpl_main.html')

    # ensuring tmpl_dstdir exists:
    if not Path(tmpl_dstdir).is_dir():
        os.makedirs(tmpl_dstdir)

    # reading tmpl_main.html data content:
    fd = open(tmpl_main_path, 'rb') # 'rb' to avoid encoding problems.
    tmpl_main_data = fd.read()
    fd.close()

    for i in files:
        tmp = ''
        tmp = i.split('.')
        context = tmp[0]
        
        # the template files with custom content are <context_custom_content.html>:
        content_file = context + '_custom_content.html'
        content_file = os.path.join(tmpl_srcdir, content_file)

        fd = open(content_file, 'rb') # 'rb' to avoid encoding problems.
        custom_data = fd.read()
        fd.close()

        # all these templates are wrapped to tmpl_main.html (tmpl_main_data)
        tmpl_final_file = os.path.join(tmpl_dstdir, i)
        fd = open(tmpl_final_file, 'wb') # 'wb' to avoid encoding problems.
        data = tmpl_main_data.replace(b'__INSERT_CUSTOM_CONTENT__', \
                                      bytes(custom_data))
        fd.write(data)
        fd.close()

    print('Done.')

def set_login_template():
    print('Setting login template...')
    tmpl_srcdir = os.path.join(os.path.curdir, \
                               'metadata', \
                               'templates', \
                               'catalog', \
                               'login.html')
    tmpl_dstdir = os.path.join(os.path.curdir, \
                               'base', \
                               'catalog', \
                               'templates', \
                               'catalog',)

    # ensuring tmpl_dstdir exists:
    if not Path(tmpl_dstdir).is_dir():
        os.makedirs(tmpl_dstdir)

    shutil.copy(tmpl_srcdir, tmpl_dstdir)
    print('Done.')

def set_admin_template():
    # we just need to put static files in the right place.
    print('Setting admin template confs:')
    CURR_DIR = os.path.abspath(os.path.curdir) 
    os.chdir(PROJECT_NAME)
    os.system('python manage.py collectstatic --noinput')
    os.chdir(CURR_DIR)  
    print('Done.')

def set_forms():
    print('Setting Forms...')
    CURR_DIR = Path(os.getcwd())
    os.chdir(HOME_DIR)
    src_path = os.path.join(os.path.curdir,'metadata', 'forms', 'catalog', )
    dst_path = os.path.join(os.path.curdir, 'base', 'catalog', )

    for i in os.listdir(src_path):
        fsrc = os.path.join(src_path,i)
        shutil.copy(fsrc, dst_path)
    os.chdir(CURR_DIR)
    print('Done.')

def set_static_files():
    print('Setting Static Files...')
    CURR_DIR = os.path.curdir
    os.chdir(HOME_DIR)
    src_path = os.path.join(os.path.curdir,'metadata', 'static', 'catalog', )
    dst_path = os.path.join(os.path.curdir, 'base', 'catalog', 'static', )

    if Path(dst_path).is_dir():
        shutil.rmtree(dst_path)
    shutil.copytree(src_path, dst_path)
    os.chdir(CURR_DIR) 
    print('Done.')

def deployment_checklist():
    print('Deployment checklist...')
    # https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/
    key = secrets.token_hex(64)
    src_path = os.path.join(os.path.curdir, \
                            'metadata', \
                            'settings', \
                            'settings.py')
    dst_path = os.path.join(os.path.curdir, 'base', 'base', 'settings.py')

    # Open template file and copy its content to avoid data appending:
    fd = open(src_path, 'r')
    data = fd.read()
    fd.close()

    # https://goo.gl/PtCXNN
    fd = open(dst_path, 'w')
    data = data.replace('__SECRET_KEY__', key)
    fd.write(data)
    fd.close()
    print('Done.')

def create_test_environment():
    print('Creating test environment...')
    env = get_environment()
    os.chdir(HOME_DIR)
    BASEDIR = os.path.abspath(os.path.join(os.path.curdir, PROJECT_NAME,))
    src_path = os.path.join(os.path.curdir, \
                            'test', \
                            'data', \
                            'initialcatalogdata.json')
    fixturename_path = os.path.join(HOME_DIR, \
                                    'base', \
                                    'catalog', \
                                    'fixturename',)
    
    if not Path(fixturename_path).is_dir():
        os.mkdir(fixturename_path)
    shutil.copy(src_path, fixturename_path)

    os.chdir(PROJECT_NAME)
    fixturename_file = os.path.join(os.path.pardir, \
                                    fixturename_path, \
                                    'initialcatalogdata.json')
    os.system('python manage.py loaddata' + ' ' + fixturename_file)
    os.chdir(BASEDIR)  
    print('Done.')

def create_superuser():
    env = get_environment()
    BASEDIR = os.path.abspath(os.path.curdir) 
    os.chdir(PROJECT_NAME)
    os.system('python manage.py createsuperuser')
    os.chdir(BASEDIR)

def run_new_project():
    env = get_environment()
    print('Please access http://127.0.0.1:8000/admin to start app.')
    BASEDIR = os.path.abspath(os.path.curdir) 
    os.chdir(PROJECT_NAME)
    os.system('python manage.py runserver')
    os.chdir(BASEDIR)

def set_database():
    # https://docs.djangoproject.com/en/2.0/ref/settings/#databases
    env = get_environment()
    BASEDIR = os.path.abspath(os.path.curdir) 

    default_database = {
        'default': {
            'ENGINE': '',
            'NAME': '',
            'USER': '',
            'PASSWORD':'',
            'HOST': '',
            'PORT':''
        }
    }

    available_databases = {
    #   'key': ['DB friendly name', 'ENGINE', 'NAME', 'USER', 'PASS', HOST', 'PORT', 'db binding']
        '1': ['PostgreSQL', 'django.db.backends.postgresql', '', '', '', '127.0.0.1', '5432', 'psycopg2'],
        '2': ['MySQL', 'django.db.backends.mysql', '', '', '', '127.0.0.1', '3306', 'mysqlclient'],
        '3': ['Oracle', 'django.db.backends.oracle', '', '', '', '127.0.0.1', '1521', 'cx_Oracle'],
        '4': ['SQLite3', 'django.db.backends.sqlite3', "os.path.join(BASE_DIR, 'db.sqlite3')", '', '', '', '', None]
    }

    print('\nAvailable databases:\n')
    for i in available_databases.keys():
        print(i, '-', available_databases[i][0])    

    chosen_db = input('\nWhich one would you like to use? ')
    while chosen_db not in available_databases.keys():
        chosen_db = input('Choose one of the numbers above: ')

    print('\nLet us set', available_databases[chosen_db][0], 'database:')
    default_database['default']['ENGINE'] = available_databases[chosen_db][1]
    default_database['default']['NAME'] = input('Database name: ' \
                                                + available_databases[chosen_db][2]) \
                                                or available_databases[chosen_db][2]
    default_database['default']['USER'] = input('Database user name: ' \
                                                + available_databases[chosen_db][3]) \
                                                or available_databases[chosen_db][3]
    default_database['default']['PASSWORD'] = getpass.getpass('User password:')
    pwd_verify = getpass.getpass('User password (again):')
    while default_database['default']['PASSWORD'] != pwd_verify:
        print('Password mismatch.')
        default_database['default']['PASSWORD'] = getpass.getpass('User password:')
        pwd_verify = getpass.getpass('User password (again):')
    default_database['default']['HOST'] = input('Database Host address (' \
                                                + available_databases[chosen_db][5] \
                                                + '):') \
                                                or available_databases[chosen_db][5]
    default_database['default']['PORT'] = input('Database Port (' \
                                                + available_databases[chosen_db][6] \
                                                + '):') \
                                                or available_databases[chosen_db][6]

    #### Altering settings.py DATABASE entry:
    settings_path = os.path.join(os.curdir, 'base', 'base', 'settings.py')
    f = open(settings_path, 'r')
    data = f.read()
    f.close()

    regex = r"DATABASES = (.*)}\n" # Thanks to https://regex101.com/r/lH0jK9/1
    subst = json.dumps(default_database, indent=4)
    subst = subst.replace('"', '\'')
    subst = 'DATABASES = ' + subst + '\n'
      
    result = re.sub(regex, subst, data, 0, re.DOTALL)
    ### Since 'NAME': value is a path, it could not be treated as a string.
    if available_databases[chosen_db][0] == 'SQLite3':
        result = result.replace("'NAME': 'os.path.join(BASE_DIR, 'db.sqlite3')',", \
                                "'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),")

    f = open(settings_path, 'w')
    f.write(result)
    f.close()
    
    #### Check if Database bindings is installed:
    db_binding = available_databases[chosen_db][7]
    try:
        import db_binding
    except ImportError:          
        os.system('pip install ' + db_binding)
    
    set_datamodel()
#    create_superuser()
      
def create_startscripts():
    env = get_environment()
    if env['system'] == 'Linux':
        pass       
    if env['system'] == 'Windows':
        filename = 'run.bat'
        f = open(filename, 'w')
        data = '@echo off\n\ncall venv\\Scripts\\activate.bat\npython run.py\npause'
        f.write(data)
        f.close()


def run():
    param = check_parameters()
    global MINIMAL_DJANGO_VERSION
    MINIMAL_DJANGO_VERSION = get_req_version('django', 'requirements.txt')
         
    if param == 'build':
        check_parameters()
        get_environment()
        check_venv()
        check_system_reqs()
        cleaning_old_stuff('none')
        start_django_project()
        importing_settings()
        set_datamodel()
        set_urls()
        set_views()
        set_templates()
        set_login_template()
        set_forms()
        set_static_files()
        set_admin_template()
        deployment_checklist()
        create_superuser()
        create_startscripts()
    
    if param == 'build-only':
        check_parameters()
        get_environment()
        check_venv()
        check_system_reqs()
        cleaning_old_stuff('none')
        start_django_project()
        importing_settings()
        set_datamodel()
        set_urls()
        set_views()
        set_templates()
        set_login_template()
        set_forms()
        set_static_files()
        set_admin_template() 
        deployment_checklist()
        create_startscripts()

    if param == 'build-novenv':
        check_parameters()
        get_environment()
        #check_venv()
        check_system_reqs()
        cleaning_old_stuff('none')
        start_django_project()
        importing_settings()
        set_datamodel()
        set_urls()
        set_views()
        set_templates()
        set_login_template()
        set_forms()
        set_static_files()
        set_admin_template()
        deployment_checklist()
        create_superuser()
        
    if param == 'clean':
        cleaning_old_stuff('none')
    
    if param == 'createsuperuser':
        create_superuser()

    if param == 'database':
        set_database()

    if param == 'deep-clean':
        cleaning_old_stuff('deep-clean')
        # Cleaning also venv dir.

    if param == 'createvenv':
       check_venv()

    if param == 'templates':
       set_templates()
       set_login_template()
       set_forms()
       set_static_files()
       set_admin_template()

    if param == 'urlsviews':
        set_urls()
        set_views()
        
    if param == 'loadtestdata':
        create_test_environment()

    if param == 'createstartscript':
        create_startscripts()

run()