[WIP] Activate doctest (qgis#3503)

* Create virtual environment on the fly

* Activate doctest with one testcode directive.

.dockerignore

@@ -0,0 +1,10 @@
+*
+!i18n/
+!output/
+!qgis-projects/
+!scripts/
+!source/
+!static/
+!themes/
+!Makefile
+!REQUIREMENTS.txt

.gitignore

@@ -18,3 +18,4 @@ live
 virtualenv
 source/i18n/*/conf.py
 source/static/
+venv

Makefile

@@ -3,8 +3,8 @@
 
 # as long as this branch is testing, we only build for english:
 LANG          = en
-SPHINXBUILD   = sphinx-build
-SPHINXINTL    = sphinx-intl
+SPHINXBUILD   ?= sphinx-build
+SPHINXINTL    ?= sphinx-intl
 PAPER         =
 SOURCEDIR     = source
 RESOURCEDIR   = static
@@ -17,11 +17,6 @@ VERSION       = testing
 # needed for python2 -> python3 migration?
 export LC_ALL=C.UTF-8
 
-# User-friendly check for sphinx-build
-ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
-$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
-endif
-
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
@@ -43,6 +38,7 @@ help:
 	@echo "  pretranslate to gather all strings from sources, put in .pot files"
 	@echo "                  AND merge them with available .po files"
 	@echo "  transifex_push (only for transifex Maintainers!): renew source files and push to transifex"
+	@echo "  doctest     to run all doctests embedded in the documentation (if enabled)"
 	@echo "  "
 	@echo "OPTION: use LANG=xx to do it only for one language, eg: make html LANG=de"
 	@echo "  "
@@ -245,3 +241,9 @@ fasthtml: updatestatic
 	# cancelling the build
 	# No internationalization is performed
 	$(SPHINXBUILD) -n -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html/$(LANG)
+
+.PHONY: doctest
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."

docker.mk

@@ -0,0 +1,8 @@
+# Makefile to run targets in docker containers
+
+doctest:
+	docker build -t qgis/documentation:latest -f doctest.dockerfile .
+	docker run --rm \
+		--user=$(shell id -u) \
+		--volume=$(PWD):/documentation \
+		qgis/documentation:latest

doctest.dockerfile

@@ -0,0 +1,9 @@
+FROM qgis/qgis:latest
+
+# Install requirement first to use caching
+COPY REQUIREMENTS.txt /documentation/REQUIREMENTS.txt
+RUN pip3 install -r /documentation/REQUIREMENTS.txt
+
+WORKDIR /documentation
+
+CMD make doctest

readme.rst

@@ -77,44 +77,16 @@ Building the documentation using Make
 
 Building is only tested on Linux systems using make, on windows we now started a Paver setup (see below)
 
-To be able to run localisation targets you will need Sphinx 1.2 which comes with pip.
-Sphinx coming with most distro's is just 1.1.3. You will get an gettext error with those.
+* ``make -f venv.mk html`` to build the english language
+* ``make -f venv.mk LANG=nl html`` to build the dutch version
 
-Best to run the make file in a virtual env ( http://www.virtualenv.org/ ):
+Note that with option ``-f venv.mk``, ``make`` will create and use a Python3 virtual environment with
+required dependencies in ``/venv`` folder on the fly. Once created you can activate this virtual environment
+using::
 
-Move to a directory (~/myvirtualenvs/) and create a virtualenv enabled dir::
+   source venv/bin/activate
 
-    virtualenv sphinx  # one time action, only to create the environment
-    cd sphinx
-
-And activate this virtualenv::
-
-    source bin/activate
-    # now you will see sphinx before your prompt:
-    (sphinx)richard@mymachine
-
-Now always activate your environment before building. To deactivate, you can do::
-
-    deactivate
-
-You can install all the tools in one go via the REQUIREMENTS.txt here in root of this repo::
-
-    pip install -r REQUIREMENTS.txt
-
-Alternatively do it one by one:
-
-Install sphinx 1.2 now in your virtual env::
-
-    pip install sphinx==1.2
-
-Sphinx intl extention ( https://pypi.python.org/pypi/sphinx-intl )::
-
-    pip install sphinx-intl
-
-Then build:
-
-* ``make html`` to build the english language
-* ``make LANG=nl html`` to build the dutch version
+No need to use option ``-f venv.mk`` after that.
 
 Speed up the documentation build
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -376,3 +348,48 @@ Team Lead) and supported by numerous contributors.
 
   * Contributor: Sorin Călinică < sorin.calinica[at]gmail.com >
   * Contributor: Tudor Bărăscu < tudor.barascu[at]qtibia.ro >
+
+
+Testing Python snippets
+***********************
+
+To test Python code snippets, you need a *QGIS* installation, for this there are many options:
+
+You can use your system *QGIS* installation with *Sphinx* from Python virtual environment::
+
+   make -f venv.mk doctest
+
+You can use a manually built installation of *QGIS*, to do so, you need to create a custom ``Makefile``
+extension on top of the ``venv.mk`` file, for example a ``user.mk`` file with the following content::
+
+   # Root installation forder
+   QGIS_PREFIX_PATH = /home/user/apps/qgis-master
+
+   # Or build output folder
+   QGIS_PREFIX_PATH = /home/user/dev/QGIS-build-master/output
+
+   include venv.mk
+
+Then use it to run target ``doctest``::
+
+   make -f user.mk doctest
+
+Or you can run target ``doctest`` inside the official *QGIS* docker image::
+
+   make -f docker.mk doctest
+
+Note that only code blocks with directive ``testcode`` are tested and it is possible to run tests setup code
+which does not appear in documentation with directive ``testsetup``, for example::
+
+   .. testsetup::
+
+      from qgis.core import QgsCoordinateReferenceSystem
+
+   .. testcode::
+
+      # PostGIS SRID 4326 is allocated for WGS84
+      crs = QgsCoordinateReferenceSystem(4326, QgsCoordinateReferenceSystem.PostgisCrsId)
+      assert crs.isValid()
+
+For more information see *Sphinx* doctest extension documentation:
+https://www.sphinx-doc.org/en/master/usage/extensions/doctest.html

source/conf.py

@@ -25,8 +25,11 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.imgmath', 'sphinx.ext.intersphinx']
-
+extensions = [
+    'sphinx.ext.imgmath',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.doctest',
+]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['../themes/qgis-theme']
@@ -522,3 +525,21 @@
                   ('py:meth', 'qgis.core.QgsMarkerSymbolLayer.renderPolygon'),
                   ('py:meth', 'qgis.core.QgsMarkerSymbolLayer.renderPolyline'),
                   ]
+
+doctest_global_setup = '''
+import os
+import sys
+from qgis.testing import start_app
+
+
+def start_qgis():
+    save_stdout = sys.stdout
+    try:
+        with open(os.devnull, 'w') as f:
+            sys.stdout = f
+            start_app()
+    finally:
+        sys.stdout = save_stdout
+    sys.stdout = sys.stderr
+'''
+doctest_test_doctest_blocks = ''

source/docs/pyqgis_developer_cookbook/crs.rst

@@ -24,18 +24,23 @@ created in several different ways:
 
 * specify CRS by its ID
 
-  ::
+.. testsetup::
+
+   from qgis.core import QgsCoordinateReferenceSystem
+
+.. testcode::
 
-    # PostGIS SRID 4326 is allocated for WGS84
-    crs = QgsCoordinateReferenceSystem(4326, QgsCoordinateReferenceSystem.PostgisCrsId)
+   # PostGIS SRID 4326 is allocated for WGS84
+   crs = QgsCoordinateReferenceSystem(4326, QgsCoordinateReferenceSystem.PostgisCrsId)
+   assert crs.isValid()
 
-  QGIS uses three different IDs for every reference system:
+QGIS uses three different IDs for every reference system:
 
   * :const:`PostgisCrsId` --- IDs used within PostGIS databases.
   * :const:`InternalCrsId` --- IDs internally used in QGIS database.
   * :const:`EpsgCrsId` --- IDs assigned by the EPSG organization
 
-  If not specified otherwise in second parameter, PostGIS SRID is used by default.
+If not specified otherwise in second parameter, PostGIS SRID is used by default.
 
 * specify CRS by its well-known text (WKT)
 

venv.mk

@@ -0,0 +1,43 @@
+# Makefile to create and use a python virtual environment on the fly
+
+# QGIS_PREFIX_PATH, used to determine relative paths.
+# Default to system QGIS installation
+QGIS_PREFIX_PATH ?= /usr
+
+# Include system packages (needed for PyQt)
+PYTHONPATH ?= /usr/lib/python3/dist-packages
+
+# Include QGIS binaries and Python binding
+ifneq ($(QGIS_PREFIX_PATH), "")
+LD_LIBRARY_PATH := $(QGIS_PREFIX_PATH)/lib:$(LD_LIBRARY_PATH)
+# For install folder
+PYTHONPATH := $(QGIS_PREFIX_PATH)/share/qgis/python:$(PYTHONPATH)
+# For build output folder
+PYTHONPATH := $(QGIS_PREFIX_PATH)/python:$(PYTHONPATH)
+endif
+
+# Use sphinx from virtualenv
+SPHINXBUILD   ?= QGIS_PREFIX_PATH=$(QGIS_PREFIX_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH) PYTHONPATH=$(PYTHONPATH) venv/bin/sphinx-build
+SPHINXINTL    ?= venv/bin/sphinx-intl
+
+# Delete all including the virtual environment
+cleanall: springclean
+	rm -rf venv
+
+# Create the venv and a timestamp file
+venv/venv.timestamp:
+	python3 -m venv venv
+	touch $@
+
+# Install requirements and create a timestamp file
+venv/REQUIREMENTS.timestamp: venv/venv.timestamp REQUIREMENTS.txt
+	venv/bin/pip install -r REQUIREMENTS.txt
+	touch $@
+
+include Makefile
+
+# Add dependencies to targets that require sphinx
+html: venv/REQUIREMENTS.timestamp
+gettext: venv/REQUIREMENTS.timestamp
+fasthtml: venv/REQUIREMENTS.timestamp
+doctest: venv/REQUIREMENTS.timestamp