Initial commit

Signed-off-by: Leonardo Rossi <[email protected]>

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+source = dojson

.dockerignore

@@ -0,0 +1,5 @@
+.git
+*.pyc
+__pycache__/
+.tox
+.cache

.gitignore

@@ -0,0 +1,50 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+bin/
+build/
+develop-eggs/
+dist/
+.eggs
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/

.travis.yml

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of DoJSON
+# Copyright (C) 2015 CERN.
+#
+# DoJSON is free software; you can redistribute it and/or modify
+# it under the terms of the Revised BSD License; see LICENSE file for
+# more details.
+
+sudo: false
+
+language: python
+
+python:
+  - "2.6"
+  - "2.7"
+# FIXME esmre is not Python 3 compatible
+#  - "3.3"
+#  - "3.4"
+
+install:
+  - pip install --upgrade pip
+  - pip install coveralls pep257
+  - pip install pytest pytest-pep8 pytest-cov pytest-cache
+  - pip install -e .[docs,tests]
+
+script:
+  - pep257 --match-dir='dojson'
+  - "sphinx-build -qnNW docs docs/_build/html"
+  - python setup.py test
+
+after_success:
+  - coveralls
+
+notifications:
+  email: false

AUTHORS.rst

@@ -0,0 +1,8 @@
+Authors
+=======
+
+Active contributors:
+
+* Esteban J. G. Gabancho <[email protected]>
+* Harris Tzovanakis <[email protected]>
+* Leonardo Rossi <[email protected]>

CONTRIBUTING.rst

@@ -0,0 +1,30 @@
+Contributing
+============
+
+Bug reports, feature requests, and other contributions are welcome.
+If you find a demonstrable problem that is caused by the code of this
+library, please:
+
+1. Search for `already reported problems
+   <https://github.com/CERNDocumentServer/cds_dojson/issues>`_.
+2. Check if the issue has been fixed or is still reproducible on the
+   latest `master` branch.
+3. Create an issue with **a test case**.
+
+If you create a feature branch, you can run the tests to ensure everything is
+operating correctly:
+
+.. code-block:: console
+
+    $ ./run-tests.sh
+
+    ...
+
+    ====== 31 passed, 23 skipped in 1.37 seconds ======
+
+You can also test your feature branch using Docker::
+
+  $ docker-compose build
+  $ docker-compose run web python setup.py test
+  $ docker-compose run web python setup.py build_sphinx
+  $ docker-compose run web pep257 --match-dir='dojson'

Dockerfile

@@ -0,0 +1,40 @@
+# This file is part of DoJSON
+# Copyright (C) 2015 CERN.
+#
+# DoJSON is free software; you can redistribute it and/or
+# modify it under the terms of the Revised BSD License; see LICENSE
+# file for more details.
+
+# Use Python-2.7:
+FROM python:2.7
+
+# Install some prerequisites ahead of `setup.py` in order to profit
+# from the docker build cache:
+RUN pip install coveralls \
+                esmre \
+                ipython \
+                lxml \
+                mock \
+                pep257 \
+                pytest \
+                pytest-cache \
+                pytest-cov \
+                pytest-pep8 \
+                six \
+                sphinx_rtd_theme
+
+# Add sources to `code` and work there:
+WORKDIR /code
+ADD . /code
+
+# Install dojson:
+RUN pip install -e .[docs]
+
+# Run container as user `dojson` with UID `1000`, which should match
+# current host user in most situations:
+RUN adduser --uid 1000 --disabled-password --gecos '' dojson && \
+    chown -R dojson:dojson /code
+
+# Run test suite instead of starting the application:
+USER dojson
+CMD ["python", "setup.py", "test"]

LICENSE

@@ -0,0 +1,38 @@
+DoJSON is free software; you can redistribute it and/or modify it
+under the terms of the Revised BSD License quoted below.
+
+Copyright (C) 2015 CERN.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+In applying this license, CERN does not waive the privileges and
+immunities granted to it by virtue of its status as an
+Intergovernmental Organization or submit itself to any jurisdiction.

MANIFEST.in

@@ -0,0 +1,23 @@
+# This file is part of DoJSON
+# Copyright (C) 2015 CERN.
+#
+# DoJSON is free software; you can redistribute it and/or
+# modify it under the terms of the Revised BSD License; see LICENSE
+# file for more details.
+
+include *.rst
+include *.sh
+include *.yml
+include .coveragerc
+include .travis.yml
+include .dockerignore
+include Dockerfile
+include LICENSE
+include pytest.ini
+include tox.ini
+recursive-include docs *.py
+recursive-include docs *.rst
+recursive-include docs Makefile
+recursive-include dojson *.py
+recursive-include dojson *.json
+recursive-include tests *.py

README.rst

@@ -0,0 +1,43 @@
+========
+ DoJSON
+========
+
+.. image:: https://img.shields.io/travis/CERNDocumentServer/cds_dojson.svg
+        :target: https://travis-ci.org/CERNDocumentServer/cds_dojson
+
+.. image:: https://img.shields.io/coveralls/CERNDocumentServer/cds_dojson.svg
+        :target: https://coveralls.io/r/CERNDocumentServer/cds_dojson
+
+.. image:: https://img.shields.io/github/tag/CERNDocumentServer/cds_dojson.svg
+        :target: https://github.com/CERNDocumentServer/cds_dojson/releases
+
+.. image:: https://img.shields.io/pypi/dm/dojson.svg
+        :target: https://pypi.python.org/pypi/dojson
+
+.. image:: https://img.shields.io/github/license/CERNDocumentServer/cds_dojson.svg
+        :target: https://github.com/CERNDocumentServer/cds_dojson/blob/master/LICENSE
+
+
+About
+=====
+
+
+
+Installation
+============
+
+
+Documentation
+=============
+
+Documentation can be built using Sphinx: ::
+
+  pip install cds_dojson[docs]
+  python setup.py build_sphinx
+
+Testing
+=======
+
+Running the test suite is as simple as: ::
+
+  python setup.py test

cds_dojson/__init__.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of CERN Document Server.
+# Copyright (C) 2015 CERN.
+#
+# Invenio is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

cds_dojson/marc21/__init__.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of CERN Document Server.
+# Copyright (C) 2015 CERN.
+#
+# Invenio is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""Marc21 init."""
+
+from __future__ import absolute_import
+
+import pkg_resources
+import logging
+
+from invenio_utils.datastructures import SmartDict
+
+from ..query import Query
+from .models.default import model as marc21_default_model
+
+
+def convert_cdsmarcxml(source):
+    """Convert CDS to JSON."""
+    from dojson.contrib.marc21.utils import create_record, split_blob
+
+    for data in split_blob(source.read()):
+        record = create_record(data)
+        yield query_matcher(record).do(record)
+
+
+def query_matcher(record):
+    """Record query matcher.
+
+    :param record: :func:`dojson.contrib.marc21.utils.create_record` object.
+
+    :returns: a model instance
+    :rtype: :class:`~cds_dojson.marc21.translations.default.CDSMarc21`
+    """
+    logger = logging.getLogger(__name__ + ".query_matcher")
+
+    _smart_dict_record = SmartDict(dict(record))
+    _matches = []
+    for entry_point in pkg_resources.iter_entry_points(
+            'cds_dojson.marc21.models'):
+        name = entry_point.name
+        model = entry_point.load()
+        query = Query(model.__query__)
+
+        if query.match(_smart_dict_record):
+            logger.info("Model `{0}` found matching the query {1}.".format(
+                name, model
+            ))
+            _matches.append([name, model])
+
+    try:
+        if len(_matches) > 1:
+            logger.error(
+                ("Found more than one matches `{0}`, now it'll fallback to {1}"
+                 " for record {2}.").format(
+                    _matches, _matches[0], _smart_dict_record
+                )
+            )
+        return _matches[0][1]
+    except IndexError:
+        logger.warning(
+            "Model *not* found, fallback to default {0} for record {1}".format(
+                marc21_default_model, _smart_dict_record
+            )
+        )
+        return marc21_default_model