Feature #222: Glob-pattern Includes (#236)

* #222: Added glob-pattern includes

* #222: Fix logging for merging config includes

* #222: Fix missing branch for loading non-pattern file configs

* #222: Code style & Python 2.7 support

* #222: Added tests for new include styles

Co-authored-by: Peter Zaitcev <[email protected]>

pyhocon/config_parser.py

@@ -6,7 +6,9 @@
 import os
 import re
 import socket
+import sys
 from datetime import timedelta
+from glob import glob
 
 from pyparsing import (Forward, Group, Keyword, Literal, Optional,
                        ParserElement, ParseSyntaxException, QuotedString,
@@ -37,6 +39,14 @@
     basestring = str
     unicode = str
 
+if sys.version_info < (3, 5):
+    def glob(pathname, recursive=False):
+        if recursive and '**' in pathname:
+            import warnings
+            warnings.warn('This version of python (%s) does not support recursive import' % sys.version)
+        from glob import glob as _glob
+        return _glob(pathname)
+
 logger = logging.getLogger(__name__)
 
 #
@@ -338,13 +348,45 @@ def include_config(instring, loc, token):
                 )
             elif file is not None:
                 path = file if basedir is None else os.path.join(basedir, file)
-                logger.debug('Loading config from file %s', path)
-                obj = ConfigFactory.parse_file(
-                    path,
-                    resolve=False,
-                    required=required,
-                    unresolved_value=NO_SUBSTITUTION
-                )
+
+                def _make_prefix(path):
+                    return ('<root>' if path is None else '[%s]' % path).ljust(55).replace('\\', '/')
+                _prefix = _make_prefix(path)
+
+                def _load(path):
+                    _prefix = _make_prefix(path)
+                    logger.debug('%s Loading config from file %r', _prefix, path)
+                    obj = ConfigFactory.parse_file(
+                        path,
+                        resolve=False,
+                        required=required,
+                        unresolved_value=NO_SUBSTITUTION
+                    )
+                    logger.debug('%s Result: %s', _prefix, obj)
+                    return obj
+
+                if '*' in path or '?' in path:
+                    paths = glob(path, recursive=True)
+                    obj = None
+
+                    def _merge(a, b):
+                        if a is None or b is None:
+                            return a or b
+                        elif isinstance(a, ConfigTree) and isinstance(b, ConfigTree):
+                            return ConfigTree.merge_configs(a, b)
+                        elif isinstance(a, list) and isinstance(b, list):
+                            return a + b
+                        else:
+                            raise ConfigException('Unable to make such include (merging unexpected types: {a} and {b}', a=type(a), b=type(b))
+                    logger.debug('%s Loading following configs: %s', _prefix, paths)
+                    for p in paths:
+                        obj = _merge(obj, _load(p))
+                    logger.debug('%s Result: %s', _prefix, obj)
+
+                else:
+                    logger.debug('%s Loading single config: %s', _prefix, path)
+                    obj = _load(path)
+
             else:
                 raise ConfigException('No file or URL specified at: {loc}: {instring}', loc=loc, instring=instring)
 

samples/all_animals.conf

@@ -0,0 +1,3 @@
+animals: {
+    include "animals.d/*.conf"
+}

samples/all_bars.conf

@@ -0,0 +1,4 @@
+bars:
+[
+    include "bars.d/*.conf"
+]

samples/animals.conf

@@ -1,9 +1,9 @@
 {
   cat : {
-    include "cat.conf"
+    include "animals.d/cat.conf"
   }
 
   dog: {
-    include "dog.conf"
+    include "animals.d/dog.conf"
   }
 }

samples/bars.d/bar_night_city.conf

@@ -0,0 +1,5 @@
+[
+    { name: "Bloody Mary", type: "cocktail" }
+    { name: "Mohito", type: "cocktail" }
+    { name: "CyberBeer", type: "beer" }
+]

samples/bars.d/bar_springfield.conf

@@ -0,0 +1,5 @@
+[
+    { name: "Bud №5", type: "beer" }
+    { name: "Bud №7", type: "beer" }
+    { name: "Homer's favorite coffee", type: "coffee" }
+]

samples/bars.d/cafe.conf

@@ -0,0 +1,6 @@
+[
+    { name: "Cappuccino", type: "coffee" }
+    { name: "Caffè Americano", type: "coffee" }
+    { name: "Green Tea", type: "tea" }
+    { name: "Milk 3.5%", type: "milk" }
+]

tests/test_config_parser.py

@@ -1066,6 +1066,19 @@ def test_include_dict_from_samples(self):
         assert config.get('cat.garfield.say') == 'meow'
         assert config.get('dog.mutt.hates.garfield.say') == 'meow'
 
+    def test_include_glob_dict_from_samples(self):
+        config = ConfigFactory.parse_file("samples/all_animals.conf")
+        assert config.get('animals.garfield.say') == 'meow'
+        assert config.get('animals.mutt.hates.garfield.say') == 'meow'
+
+    def test_include_glob_list_from_samples(self):
+        config = ConfigFactory.parse_file("samples/all_bars.conf")
+        bars = config.get_list('bars')
+        assert len(bars) == 10
+        assert bars[0].get('name') == 'Bloody Mary'
+        assert bars[5].get('name') == 'Homer\'s favorite coffee'
+        assert bars[9].get('type') == 'milk'
+
     def test_list_of_dicts(self):
         config = ConfigFactory.parse_string(
             """
@@ -1203,7 +1216,7 @@ def test_include_required_file(self):
         config = ConfigFactory.parse_string(
             """
             a {
-                include required("samples/cat.conf")
+                include required("samples/animals.d/cat.conf")
                 t = 2
             }
             """
@@ -1221,7 +1234,7 @@ def test_include_required_file(self):
         config2 = ConfigFactory.parse_string(
             """
             a {
-                include required(file("samples/cat.conf"))
+                include required(file("samples/animals.d/cat.conf"))
                 t = 2
             }
             """