Feature/py3 py2 conditional

.circleci/config.yml

@@ -10,17 +10,19 @@ jobs:
             pip install pip==20.3.3
             pip install flake8
             flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-  
+
   test_local_docker:
     working_directory: ~/ckanext-datajson
     machine: true
+    environment:
+      COMPOSE_FILE: docker-compose.legacy.yml
     steps:
       - checkout
       - run:
           name: Start CKAN
           command: |
-            docker-compose build
-            docker-compose up -d
+            make build
+            docker-compose -f $COMPOSE_FILE up -d
       - run:
           name: install dockerize
           command: wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
@@ -32,100 +34,15 @@ jobs:
       - run:
           name: Test extension
           command: |
-            docker-compose logs db
-            docker-compose logs ckan
-            docker-compose exec ckan /bin/bash -c "nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src_extensions/datajson/ckanext/datajson/tests"
-
-  build_ckan_28:
-    working_directory: ~/ckanext-datajson
-    machine:
-      image: circleci/classic:201708-01
-    steps:
-      - checkout
-      - run:
-          name: install and test
-          command: bin/travis-build-2.8.bash
-
-  build_ckan_23:
-    working_directory: ~/ckanext-datajson
-    machine:
-      image: circleci/classic:201708-01
-    environment:
-      CKANVERSION=2.3
-    steps:
-      - checkout
-      - run:
-          name: install
-          command: bin/travis-build.bash
-      - run:
-          name: test
-          command: bin/travis-run.sh
-
-  build_gsa_ckan_inventory:
-    working_directory: ~/ckanext-datajson
-    machine:
-      image: circleci/classic:201708-01
-    environment:
-      CKANVERSION=inventory
-    steps:
-      - checkout
-      - run:
-          name: install
-          command: bin/travis-build-inventory.bash
-      - run:
-          name: test
-          command: bin/travis-run-inventory.sh
-
-  build_gsa_ckan_inventory_next:
-    working_directory: ~/ckanext-datajson
-    machine:
-      image: circleci/classic:201708-01
-    environment:
-      CKANVERSION=inventory-next
-    steps:
-      - checkout
-      - run:
-          name: install
-          command: bin/travis-build-inventory.bash
-      - run:
-          name: test
-          command: bin/travis-run-inventory.sh
-
-  build_gsa_ckan_datagov:
-    working_directory: ~/ckanext-datajson
-    machine:
-      image: circleci/classic:201708-01
-    environment:
-      CKANVERSION=datagov
-    steps:
-      - checkout
-      - run:
-          name: install
-          command: bin/travis-build.bash
-      - run:
-          name: test
-          command: bin/travis-run.sh
+            docker-compose -f $COMPOSE_FILE logs db
+            docker-compose -f $COMPOSE_FILE logs ckan
+            make test-legacy
 
 workflows:
   version: 2
   build_and_test:
     jobs:
       - lint
-      - build_gsa_ckan_inventory_next:
-          requires:
-              - lint
       - test_local_docker:
           requires:
             - lint
-      - build_ckan_28:
-          requires:
-            - lint
-      - build_ckan_23:
-          requires:
-            - lint
-      - build_gsa_ckan_inventory:
-          requires:
-            - lint
-      - build_gsa_ckan_datagov:
-          requires:
-            - lint

.env

@@ -32,16 +32,14 @@ TEST_CKAN_REDIS_URL=redis://redis:6379/1
 
 # Core settings
 CKAN__STORAGE_PATH=/var/lib/ckan
+CKAN__PLUGINS=datajson harvest datajson_harvest
 
 CKAN_SMTP_SERVER=smtp.corporateict.domain:25
 CKAN_SMTP_STARTTLS=True
 CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost
 
-# Extensions
-CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher datagov_harvest ckan_harvester geodatagov datajson datajson_harvest geodatagov_miscs z3950_harvester arcgis_harvester geodatagov_geoportal_harvester waf_harvester_collection geodatagov_csw_harvester geodatagov_doc_harvester geodatagov_waf_harvester spatial_metadata spatial_query report qa archiver datagovtheme datagovcatalog googleanalyticsbasic dcat dcat_json_interface structured_data
-
 # Harvest settings
 CKAN__HARVEST__MQ__TYPE=redis
 CKAN__HARVEST__MQ__HOSTNAME=redis
@@ -67,5 +65,5 @@ CKAN___BROKER_HOST=redis://redis/1
 CKAN___CELERY_RESULT_BACKEND=redis
 CKAN___REDIS_HOST=redis
 CKAN___REDIS_PORT=6379
-CKAN___REDIS_DB=0
+CKAN___REDIS_DB=1
 CKAN___REDIS_CONNECT_RETRY=True

.github/workflows/test.yml

@@ -0,0 +1,68 @@
+name: Tests
+on: [pull_request]
+env:
+  CODE_COVERAGE_THRESHOLD_REQUIRED: 30
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan,nose
+      - name: Run flake8
+        run: flake8 . --count --show-source --statistics --exclude ckan,nose
+
+  test:
+    strategy:
+      matrix:
+        ckan-version: [2.9, 2.8]
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+    services:
+      solr:
+        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+          image: redis:3
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install requirements
+      run: |
+        pip install -r requirements.txt -r dev-requirements.txt -e .
+    - name: Setup extension (CKAN >= 2.9)
+      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
+      run: |
+        ckan -c test.ini db init
+    - name: Setup extension (CKAN < 2.9)
+      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
+      run: |
+        paster --plugin=ckan db init -c test.ini
+    - name: Run tests
+      run: |
+        set -o pipefail
+        pytest --ckan-ini=test.ini --cov=ckanext.datajson --cov-fail-under=${CODE_COVERAGE_THRESHOLD_REQUIRED} --disable-warnings ckanext/datajson/tests | tee pytest-coverage.txt
+    - name: Report test coverage
+      if: ${{ always() }}
+      uses: coroo/[email protected]

.gitignore

@@ -1,5 +1,6 @@
 **.pyc
 build
+.coverage
 ckanext_datajson.egg-info
 .DS_Store
 ckanext/datajson/export_map/*.map.json

Dockerfile

@@ -0,0 +1,8 @@
+ARG CKAN_VERSION=2.8
+FROM openknowledge/ckan-dev:${CKAN_VERSION}
+
+COPY . /app
+WORKDIR /app
+
+# python cryptography takes a while to build
+RUN pip install -r requirements.txt -r dev-requirements.txt -e .

Makefile

@@ -0,0 +1,39 @@
+CKAN_VERSION ?= 2.8
+COMPOSE_FILE ?= docker-compose.yml
+COMPOSE_LEGACY_FILE ?= docker-compose.legacy.yml
+
+build: ## Build the docker containers
+	CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) build
+
+lint: ## Lint the code
+	@# our linting only runs with python3
+	@# TODO use CKAN_VERSION make variable once 2.8 is deprecated
+	CKAN_VERSION=2.9 docker-compose -f docker-compose.yml run --rm app flake8 . --count --show-source --statistics --exclude ckan,nose
+
+clean: ## Clean workspace and containers
+	find . -name *.pyc -delete
+	CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) down -v --remove-orphan
+
+test: ## Run tests in a new container
+	CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) run --rm app ./test.sh
+
+test-legacy: ## Run legacy nose tests in an existing container
+	@# TODO wait for CKAN to be up; use docker-compose run instead
+	docker-compose -f docker-compose.legacy.yml exec ckan /bin/bash -c "nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src_extensions/datajson/ckanext/datajson/tests/nose"
+
+up: ## Start the containers
+ifeq ($(CKAN_VERSION), 2.8)
+	CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_LEGACY_FILE) up
+else
+	CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) up
+endif
+
+
+
+.DEFAULT_GOAL := help
+.PHONY: build clean help lint test test-legacy up
+
+# Output documentation for top-level targets
+# Thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+help: ## This help
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-10s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)