PLAT-1985 Better dependency management

.travis.yml

@@ -6,8 +6,7 @@ env:
 - TOXENV=django110
 - TOXENV=django111
 install:
-- pip install tox
-- pip install coveralls
+- pip install -r requirements/travis.txt
 script:
 - tox
 after_success: coveralls

MANIFEST.in

@@ -1,6 +1,5 @@
-include requirements.txt
-include test-requirements.txt
-include django-requirements.txt
+include requirements/base.in
+include requirements/test.in
 include LICENSE
 include AUTHORS
 include README.rst

Makefile

@@ -0,0 +1,65 @@
+.PHONY: clean coverage docs help quality requirements selfcheck test test-all upgrade validate
+
+.DEFAULT_GOAL := help
+
+define BROWSER_PYSCRIPT
+import os, webbrowser, sys
+try:
+	from urllib import pathname2url
+except:
+	from urllib.request import pathname2url
+
+webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+endef
+export BROWSER_PYSCRIPT
+BROWSER := python -c "$$BROWSER_PYSCRIPT"
+
+help: ## display this help message
+	@echo "Please use \`make <target>' where <target> is one of"
+	@perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
+
+clean: ## remove generated byte code, coverage reports, and build artifacts
+	find . -name '__pycache__' -exec rm -rf {} +
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	coverage erase
+	rm -fr build/
+	rm -fr dist/
+	rm -fr *.egg-info
+
+coverage: clean ## generate and view HTML coverage report
+	coverage report html
+	$(BROWSER) htmlcov/index.html
+
+upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+	pip install -q pip-tools
+	pip-compile --upgrade -o requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in requirements/test.in requirements/travis.in
+	pip-compile --upgrade -o requirements/quality.txt requirements/base.in requirements/quality.in requirements/test.in
+	pip-compile --upgrade -o requirements/test.txt requirements/base.in requirements/test.in
+	pip-compile --upgrade -o requirements/travis.txt requirements/travis.in
+	# Let tox control the Django version for tests
+	sed '/django==/d' requirements/test.txt > requirements/test.tmp
+	mv requirements/test.tmp requirements/test.txt
+
+quality: ## check coding style with pycodestyle and pylint
+	tox -e quality
+
+requirements: ## install development environment requirements
+	pip install -qr requirements/dev.txt --exists-action w
+	pip-sync requirements/dev.txt requirements/private.*
+
+test: clean ## run tests in the current virtualenv
+	python manage.py test
+
+diff_cover: test ## find diff lines that need test coverage
+	diff-cover coverage.xml
+
+test-all: ## run tests on every supported Python/Django combination
+	tox -e quality
+	tox
+
+validate: quality test ## run tests and quality checks
+
+selfcheck: ## check that the Makefile is well-formed
+	@echo "The Makefile is well-formed."

README.rst

@@ -0,0 +1,49 @@
+edx-val (Video Abstraction Layer)
+=================================
+
+edx-val is a django app that creates and retrieves metadata for videos and subtitles. When creating video entries, they can be assigned to preset profiles such as 'high_quality' or 'mobile_only'. When requesting a video, the client does not need to know which profile to retrieve, but only the `edx_video_id` of that video. Since all the different profiles for that particular video is returned, the client can decide which profiles they want to use. 
+
+Example:
+Retrieve all profiles for a video with `edx_video_id`="example"
+
+.. code-block:: python
+
+    >>> get_video_info("example")
+    {
+        'url' : '/edxval/videos/example',
+        'edx_video_id': u'example',
+        'duration': 111.0,
+        'client_video_id': u'The example video',
+        'encoded_videos': [
+            {
+                'url': u'http://www.example.com/example_mobile_video.mp4',
+                'file_size': 25556,
+                'bitrate': 9600,
+                'profile': u'mobile'
+            },
+            {
+                'url': u'http://www.example.com/example_desktop_video.mp4',
+                'file_size': 43096734,
+                'bitrate': 64000,
+                'profile': u'desktop'
+            }
+        ]
+    }
+
+ 
+Developing
+----------
+
+First, create a virtual environment:
+
+.. code-block:: bash
+
+    virtualenv venvs/val
+    source venvs/val/bin/activate
+
+
+To run tests:
+
+.. code-block:: bash
+
+    make test

requirements.txt → requirements/base.in

@@ -1,9 +1,12 @@
+# Core requirements for using this application
+
+boto
+Django>=1.8,<2
+django-model-utils
 -e git+https://github.com/edx/[email protected]#egg=django-oauth2-provider==0.2.7-fork-edx-6
+-e git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
 -e git+https://github.com/edx/django-rest-framework-oauth.git@f0b503fda8c254a38f97fef802ded4f5fe367f7a#egg=djangorestframework-oauth
-pillow==4.2.1
-boto==2.46.1
-django-model-utils==2.3.1
-django-storages==1.5.2
-enum34==1.1.6
-git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
-lxml==3.8.0
+django-storages
+enum34
+lxml
+pillow

requirements/dev.in

@@ -0,0 +1,4 @@
+# Additional requirements for development of this application
+
+diff-cover                # Changeset diff test coverage
+pip-tools                 # Requirements file management

requirements/dev.txt

@@ -0,0 +1,81 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in requirements/test.in requirements/travis.in
+#
+-e git+https://github.com/edx/[email protected]#egg=django-oauth2-provider==0.2.7-fork-edx-6
+-e git+https://github.com/edx/django-rest-framework-oauth.git@f0b503fda8c254a38f97fef802ded4f5fe367f7a#egg=djangorestframework-oauth
+-e git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
+appdirs==1.4.3            # via fs
+argparse==1.4.0           # via caniusepython3
+asn1crypto==0.24.0        # via cryptography
+astroid==1.5.2            # via edx-lint, pylint, pylint-celery, pylint-plugin-utils
+backports.functools-lru-cache==1.5  # via astroid, caniusepython3, pylint
+boto==2.48.0
+caniusepython3==6.0.0
+certifi==2018.1.18        # via requests, urllib3
+cffi==1.11.5              # via cryptography
+chardet==3.0.4            # via requests
+click-log==0.1.8          # via edx-lint
+click==6.7                # via click-log, edx-lint, pip-tools
+configparser==3.5.0       # via pylint
+coverage==4.5.1
+coveralls==1.2.0
+cryptography==2.1.4       # via pyopenssl, urllib3
+ddt==1.1.1
+diff-cover==1.0.2
+distlib==0.2.6            # via caniusepython3
+django-model-utils==3.1.1
+django-nose==1.4.5
+django-oauth-plus==2.2.9
+django-storages==1.6.5
+django==1.11.10
+docopt==0.6.2             # via coveralls
+edx-lint==0.5.5
+enum34==1.1.6
+first==2.0.1              # via pip-tools
+fs==2.0.18
+funcsigs==1.0.2           # via mock
+futures==3.2.0            # via caniusepython3, isort
+httplib2==0.10.3          # via oauth2
+idna==2.6                 # via cryptography, requests, urllib3
+inflect==0.2.5            # via jinja2-pluralize
+ipaddress==1.0.19         # via cryptography, urllib3
+isort==4.3.4
+jinja2-pluralize==0.3.0   # via diff-cover
+jinja2==2.10              # via diff-cover, jinja2-pluralize
+lazy-object-proxy==1.3.1  # via astroid
+lxml==4.1.1
+markupsafe==1.0           # via jinja2
+mccabe==0.6.1             # via pylint
+mock==2.0.0
+nose==1.3.7               # via django-nose
+oauth2==1.9.0.post1       # via django-oauth-plus
+packaging==17.1           # via caniusepython3
+pbr==3.1.1                # via mock
+pillow==5.0.0
+pip-tools==1.11.0
+pluggy==0.6.0             # via tox
+py==1.5.2                 # via tox
+pycodestyle==2.3.1
+pycparser==2.18           # via cffi
+pydocstyle==2.1.1
+pygments==2.2.0           # via diff-cover
+pylint-celery==0.3        # via edx-lint
+pylint-django==0.7.2      # via edx-lint
+pylint-plugin-utils==0.2.6  # via pylint-celery, pylint-django
+pylint==1.7.1             # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
+pyopenssl==17.5.0         # via urllib3
+pyparsing==2.2.0          # via packaging
+pytz==2018.3              # via django, fs
+requests==2.18.4          # via caniusepython3, coveralls
+shortuuid==0.3
+singledispatch==3.4.0.3   # via astroid, pylint
+six==1.11.0               # via astroid, cryptography, diff-cover, edx-lint, fs, mock, packaging, pip-tools, pydocstyle, pylint, pyopenssl, singledispatch, tox
+snowballstemmer==1.2.1    # via pydocstyle
+tox-battery==0.5
+tox==2.9.1
+urllib3[secure]==1.22     # via coveralls, requests
+virtualenv==15.1.0        # via tox
+wrapt==1.10.11            # via astroid

requirements/private.readme

@@ -0,0 +1,15 @@
+# If there are any Python packages you want to keep in your virtualenv beyond
+# those listed in the official requirements files, create a "private.in" file
+# and list them there.  Generate the corresponding "private.txt" file pinning
+# all of their indirect dependencies to specific versions as follows:
+
+#     pip-compile private.in
+
+# This allows you to use "pip-sync" without removing these packages:
+
+#     pip-sync requirements/*.txt
+
+# "private.in" and "private.txt" aren't checked into git to avoid merge
+# conflicts, and the presence of this file allows "private.*" to be
+# included in scripted pip-sync usage without requiring that those files be
+# created first.

requirements/quality.in

@@ -0,0 +1,7 @@
+# Requirements for code quality checks
+
+caniusepython3            # Additional Python 3 compatibility pylint checks
+edx-lint                  # edX pylint rules and plugins
+isort                     # to standardize order of imports
+pycodestyle               # PEP 8 compliance validation
+pydocstyle                # PEP 257 compliance validation

requirements/quality.txt

@@ -0,0 +1,60 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/quality.txt requirements/base.in requirements/quality.in requirements/test.in
+#
+-e git+https://github.com/edx/[email protected]#egg=django-oauth2-provider==0.2.7-fork-edx-6
+-e git+https://github.com/edx/django-rest-framework-oauth.git@f0b503fda8c254a38f97fef802ded4f5fe367f7a#egg=djangorestframework-oauth
+-e git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
+appdirs==1.4.3            # via fs
+argparse==1.4.0           # via caniusepython3
+astroid==1.5.2            # via edx-lint, pylint, pylint-celery, pylint-plugin-utils
+backports.functools-lru-cache==1.5  # via astroid, caniusepython3, pylint
+boto==2.48.0
+caniusepython3==6.0.0
+certifi==2018.1.18        # via requests
+chardet==3.0.4            # via requests
+click-log==0.1.8          # via edx-lint
+click==6.7                # via click-log, edx-lint
+configparser==3.5.0       # via pylint
+coverage==4.5.1
+ddt==1.1.1
+distlib==0.2.6            # via caniusepython3
+django-model-utils==3.1.1
+django-nose==1.4.5
+django-oauth-plus==2.2.9
+django-storages==1.6.5
+django==1.11.10
+edx-lint==0.5.5
+enum34==1.1.6
+fs==2.0.18
+funcsigs==1.0.2           # via mock
+futures==3.2.0            # via caniusepython3, isort
+httplib2==0.10.3          # via oauth2
+idna==2.6                 # via requests
+isort==4.3.4
+lazy-object-proxy==1.3.1  # via astroid
+lxml==4.1.1
+mccabe==0.6.1             # via pylint
+mock==2.0.0
+nose==1.3.7               # via django-nose
+oauth2==1.9.0.post1       # via django-oauth-plus
+packaging==17.1           # via caniusepython3
+pbr==3.1.1                # via mock
+pillow==5.0.0
+pycodestyle==2.3.1
+pydocstyle==2.1.1
+pylint-celery==0.3        # via edx-lint
+pylint-django==0.7.2      # via edx-lint
+pylint-plugin-utils==0.2.6  # via pylint-celery, pylint-django
+pylint==1.7.1             # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
+pyparsing==2.2.0          # via packaging
+pytz==2018.3              # via django, fs
+requests==2.18.4          # via caniusepython3
+shortuuid==0.3
+singledispatch==3.4.0.3   # via astroid, pylint
+six==1.11.0               # via astroid, edx-lint, fs, mock, packaging, pydocstyle, pylint, singledispatch
+snowballstemmer==1.2.1    # via pydocstyle
+urllib3==1.22             # via requests
+wrapt==1.10.11            # via astroid

requirements/test.in

@@ -0,0 +1,7 @@
+# Requirements for test runs.
+
+coverage
+ddt
+django-nose
+fs
+mock