upgraded to clld 4.x

.gitignore

@@ -1,6 +1,5 @@
 data
 *.py[cod]
-development.ini
 
 # C extensions
 *.so

alembic.ini

@@ -11,7 +11,7 @@ script_location = migrations
 # the 'revision' command, regardless of autogenerate
 # revision_environment = false
 
-sqlalchemy.url = postgresql://robert@/ids
+sqlalchemy.url = postgresql://postgres@/ids
 
 [production]
 script_location = migrations

sqlite.ini → development.ini

@@ -3,30 +3,20 @@ use = egg:ids
 
 pyramid.reload_templates = true
 pyramid.debug_authorization = false
-pyramid.debug_notfound = true
+pyramid.debug_notfound = false
 pyramid.debug_routematch = false
 pyramid.default_locale_name = en
 pyramid.includes =
     pyramid_tm
-
-sqlalchemy.url = sqlite:///%(here)s/ids.sqlite
-
-###
-# wsgi server configuration
-###
+sqlalchemy.url = postgresql://postgres@/ids
 
 [server:main]
 use = egg:waitress#main
-host = 127.0.0.1
+host = 0.0.0.0
 port = 6543
 
-###
-# logging configuration
-# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
-###
-
 [loggers]
-keys = root, ids, sqlalchemy
+keys = root, ids
 
 [handlers]
 keys = console
@@ -43,14 +33,6 @@ level = DEBUG
 handlers =
 qualname = ids
 
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-# "level = INFO" logs SQL queries.
-# "level = DEBUG" logs SQL queries and results.
-# "level = WARN" logs neither.  (Recommended for production systems.)
-
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)

ids/adapters.py

@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
 from clld.db.meta import DBSession
-from clld.db.models.common import Language, Identifier
+from clld.db.models.common import Language
 from clld.web.adapters.geojson import GeoJsonParameter, GeoJsonLanguages
-from clld.web.adapters.cldf import CldfDataset
-from clld.interfaces import IParameter, IContribution, IIndex, ICldfDataset
+from clld.interfaces import IParameter, IContribution, IIndex
 
 
 class GeoJsonMeaning(GeoJsonParameter):
@@ -21,48 +20,6 @@ def feature_iterator(self, ctx, req):
         return DBSession.query(Language)
 
 
-class CldfDictionary(CldfDataset):
-    def columns(self, req):
-        return [
-            'ID',
-            {
-                'name': 'Language_ID',
-                'valueUrl': Identifier(type='glottolog', name='{Language_ID}').url()},
-            'Language_name',
-            {
-                'name': 'Parameter_ID',
-                'valueUrl': 'http://concepticon.clld.org/parameters/{Parameter_ID}'},
-            'Value',
-            'Transcription',
-            'Concept',
-            'Source',
-            'Comment',
-            'AlternativeValue',
-            'AlternativeTranscription',
-        ]
-
-    def refs_and_sources(self, req, value):
-        if not hasattr(self, '_refs_and_sources'):
-            self._refs_and_sources = CldfDataset.refs_and_sources(self, req, self.obj)
-        return self._refs_and_sources
-
-    def row(self, req, value, refs):
-        return [
-            value.id,
-            self.obj.language.glottocode,
-            self.obj.language.name,
-            value.valueset.parameter.concepticon_id,
-            value.word.name,
-            self.obj.default_representation,
-            value.valueset.parameter.name,
-            refs,
-            value.valueset.comment or '',
-            value.valueset.alt_representation or '',
-            self.obj.alt_representation or '',
-        ]
-
-
 def includeme(config):
     config.register_adapter(GeoJsonMeaning, IParameter)
-    config.register_adapter(CldfDictionary, IContribution, ICldfDataset, name='cldf')
     config.register_adapter(GeoJsonDictionaries, IContribution, IIndex)

ids/static/downloads.json

@@ -0,0 +1,20 @@
+{
+    "1.0": {
+        "bitstreams": [
+            {
+                "last-modified": 1516218189703,
+                "checksum": "f3cae525bb1cd246747e6b1c67c0ea65",
+                "created": 1516218189038,
+                "checksum-algorithm": "MD5",
+                "bitstreamid": "ids_dataset.cldf.zip",
+                "filesize": 9184372,
+                "content-type": "application/zip"
+            }
+        ],
+        "oid": "EAEA0-1575-B9D3-4DAF-0",
+        "metadata": {
+            "title": "ids 1.0 - downloads",
+            "creator": "pycdstar"
+        }
+    }
+}

ids/tests/test_functional.py

@@ -1,40 +1,31 @@
-from clldutils.path import Path
-from clld.tests.util import TestWithApp
-
-import ids
-
-
-class Tests(TestWithApp):
-    __cfg__ = Path(ids.__file__).parent.joinpath('..', 'development.ini').resolve()
-    __setup_db__ = False
-
-    def test_home(self):
-        self.app.get_html('/')
-
-    def test_contribution(self):
-        self.app.get_html('/contributions')
-        self.app.get_html('/contributions/500')
-        self.app.get_html('/contributions.geojson')
-        self.app.get_dt('/contributions')
-        self.app.get_dt('/languages')
-        self.app.get_html('/contributions/215')
-        self.app.get_dt('/values?contribution=220&iSortingCols=1&iSortCol_0=0')
-        self.app.get_dt('/values?contribution=220&iSortingCols=1&iSortCol_0=2')
-        self.app.get_dt('/values?contribution=220&sSearch_0=1&sSearch_2=4')
-
-    def test_contributor(self):
-        self.app.get_html('/contributors')
-        self.app.get_dt('/contributors')
-
-    def test_parameter(self):
-        self.app.get_html('/parameters')
-        r = self.app.get_dt('/parameters')
-        assert r 
-        self.app.get_dt('/parameters?chapter=1')
-        self.app.get_html('/parameters/1-222')
-        self.app.get_json('/parameters/1-222.geojson')
-        self.app.get_dt('/values?parameter=1-222')
-
-    def test_language(self):
-        self.app.get_html('/languages/182.snippet.html')
-        self.app.get_html('/languages/128.snippet.html?parameter=962')
+import pytest
+
+pytest_plugins = ['clld']
+
+
+@pytest.mark.parametrize(
+    "method,path",
+    [
+        ('get_html', '/'),
+        ('get_html', '/contributions'),
+        ('get_html', '/contributions/500'),
+        ('get_html', '/contributions.geojson'),
+        ('get_dt', '/contributions'),
+        ('get_dt', '/languages'),
+        ('get_html', '/contributions/215'),
+        ('get_dt', '/values?contribution=220&iSortingCols=1&iSortCol_0=0'),
+        ('get_dt', '/values?contribution=220&iSortingCols=1&iSortCol_0=2'),
+        ('get_dt', '/values?contribution=220&sSearch_0=1&sSearch_2=4'),
+        ('get_html', '/contributors'),
+        ('get_dt', '/contributors'),
+        ('get_html', '/parameters'),
+        ('get_dt', '/parameters'),
+        ('get_dt', '/parameters?chapter=1'),
+        ('get_html', '/parameters/1-222'),
+        ('get_json', '/parameters/1-222.geojson'),
+        ('get_dt', '/values?parameter=1-222'),
+        ('get_html', '/languages/182.snippet.html'),
+        ('get_html', '/languages/128.snippet.html?parameter=962'),
+    ])
+def test_pages(app, method, path):
+    getattr(app, method)(path)

migrations/versions/7ecffd0a3d77_update_unique_null.py

@@ -0,0 +1,165 @@
+"""update unique null
+
+Revision ID: 7ecffd0a3d77
+Revises: 28accca7dc14
+Create Date: 2018-01-17 20:48:11.182247
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '7ecffd0a3d77'
+down_revision = '28accca7dc14'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+UNIQUE_NULL = [
+    ('contributioncontributor',
+        ['contribution_pk', 'contributor_pk'], []),
+    ('contributionreference',
+        ['contribution_pk', 'source_pk', 'description'],
+        ['description']),
+    ('domainelement',
+        ['parameter_pk', 'name'],
+        ['name']),
+    ('domainelement',
+        ['parameter_pk', 'number'],
+        ['number']),
+    ('editor',
+        ['dataset_pk', 'contributor_pk'], []),
+    ('languageidentifier',
+        ['language_pk', 'identifier_pk'], []),
+    ('languagesource',
+        ['language_pk', 'source_pk'], []),
+    ('sentencereference',
+        ['sentence_pk', 'source_pk', 'description'],
+        ['description']),
+    ('unit',
+        ['language_pk', 'id'],
+        ['id']),
+    ('unitdomainelement',
+        ['unitparameter_pk', 'name'],
+        ['name']),
+    ('unitdomainelement',
+        ['unitparameter_pk', 'ord'],
+        ['ord']),
+    # NOTE: <unit, unitparameter, contribution> can have multiple values and also multiple unitdomainelements
+    ('unitvalue',
+        ['unit_pk', 'unitparameter_pk', 'contribution_pk', 'name', 'unitdomainelement_pk'],
+        ['contribution_pk', 'name', 'unitdomainelement_pk']),
+    # NOTE: <language, parameter, contribution> can have multiple values and also multiple domainelements
+    ('value',
+        ['valueset_pk', 'name', 'domainelement_pk'],
+        ['name', 'domainelement_pk']),
+    ('valuesentence',
+        ['value_pk', 'sentence_pk'], []),
+    ('valueset',
+        ['language_pk', 'parameter_pk', 'contribution_pk'],
+        ['contribution_pk']),
+    ('valuesetreference',
+        ['valueset_pk', 'source_pk', 'description'],
+        ['description']),
+]
+
+
+class DryRunException(Exception):
+    """Raised at the end of a dry run so the database transaction is not comitted."""
+
+
+def upgrade(dry=False, verbose=True):
+    conn = op.get_bind()
+
+    assert conn.dialect.name == 'postgresql'
+
+    def delete_null_duplicates(tablename, columns, notnull, returning=sa.text('*')):
+        assert columns
+        table = sa.table(tablename, *map(sa.column, ['pk'] + columns))
+        any_null = sa.or_(table.c[n] == sa.null() for n in notnull)
+        yield table.delete(bind=conn).where(any_null).returning(returning)
+        other = table.alias()
+        yield table.delete(bind=conn).where(~any_null).returning(returning)\
+            .where(sa.exists()
+                .where(sa.and_(table.c[c] == other.c[c] for c in columns))
+                .where(table.c.pk > other.c.pk))
+
+    def print_rows(rows, verbose=verbose):
+        if not verbose:
+            return
+        for r in rows:
+            print('    %r' % dict(r))
+
+    class regclass(sa.types.UserDefinedType):
+        def get_col_spec(self):
+            return 'regclass'
+
+    pga = sa.table('pg_attribute', *map(sa.column, ['attrelid', 'attname', 'attnum', 'attnotnull']))
+
+    select_nullable = sa.select([pga.c.attname], bind=conn)\
+        .where(pga.c.attrelid == sa.cast(sa.bindparam('table'), regclass))\
+        .where(pga.c.attname == sa.func.any(sa.bindparam('notnull')))\
+        .where(~pga.c.attnotnull)\
+        .order_by(pga.c.attnum)
+
+    pgco = sa.table('pg_constraint', *map(sa.column,
+                    ['oid', 'conname', 'contype', 'conrelid', 'conkey']))
+
+    sq = sa.select([
+            pgco.c.conname.label('name'),
+            sa.func.pg_get_constraintdef(pgco.c.oid).label('definition'),
+            sa.func.array(
+                sa.select([sa.cast(pga.c.attname, sa.Text)])
+                .where(pga.c.attrelid == pgco.c.conrelid)
+                .where(pga.c.attnum == sa.func.any(pgco.c.conkey))
+                .as_scalar()).label('names'),
+        ]).where(pgco.c.contype == 'u')\
+        .where(pgco.c.conrelid == sa.cast(sa.bindparam('table'), regclass))\
+        .alias()
+
+    select_const = sa.select([sq.c.name, sq.c.definition], bind=conn)\
+        .where(sq.c.names.op('@>')(sa.bindparam('cols')))\
+        .where(sq.c.names.op('<@')(sa.bindparam('cols')))
+
+    for table, unique, null in UNIQUE_NULL:
+        print(table)
+        notnull = [u for u in unique if u not in null]
+        delete_null, delete_duplicates = delete_null_duplicates(table, unique, notnull)
+
+        nulls = delete_null.execute().fetchall()
+        if nulls:
+            print('%s delete %d row(s) violating NOT NULL(%s)' % (table, len(nulls), ', '.join(notnull)))
+            print_rows(nulls)
+
+        duplicates = delete_duplicates.execute().fetchall()
+        if duplicates:
+            print('%s delete %d row(s) violating UNIQUE(%s)' % (table, len(duplicates), ', '.join(unique)))
+            print_rows(duplicates)
+
+        for col, in select_nullable.execute(table=table, notnull=notnull):
+            print('%s alter column %s NOT NULL' % (table, col))
+            op.alter_column(table, col, nullable=False)
+
+        constraint = 'UNIQUE (%s)' % ', '.join(unique)
+        matching = select_const.execute(table=table, cols=unique).fetchall()
+        if matching:
+            assert len(matching) == 1
+            (name, definition), = matching
+            if definition == constraint:
+                print('%s keep constraint %s %s\n' % (table, name, definition))
+                continue
+            print('%s drop constraint %s %s' % (table, name, definition))
+            op.drop_constraint(name, table)
+        name = '%s_%s_key' % (table, '_'.join(unique))
+        print('%s create constraint %s %s' % (table, name, constraint))
+        op.create_unique_constraint(name, table, unique)
+        print('')
+
+    if dry:
+        raise DryRunException('set dry=False to apply these changes')
+
+
+
+def downgrade():
+    pass