Add include_file statement

README.md

@@ -174,7 +174,9 @@ assertEquals(onePlusOne, anotherOnePlusOne)
 // Top level
 type_universe ::= <stmt>...
 definition ::= '(' 'define' symbol <domain_definition> ')'
-stmt ::=  <definition> | <transform>
+stmt ::=  <definition> | <transform> | <include_file>
+
+include_file ::= `(include_file <path-to-file>)`
 
 // Domain
 domain_definition ::= <domain> | <permute_domain>
@@ -400,6 +402,43 @@ Unlike record elements, product element defintions must include identifiers.
 (product int_pair first::int second::int)
 ```
 
+#### Type Domain Includes
+
+It is possible to split type universes among multiple files, which allows type domains defined in another project to be
+permuted.  For example:
+
+```
+// root.ion:
+(include_file "sibling.ion")
+(include_file "sub-dir/thing.ion")
+(include_file "/other-project/toy-ast.ion")
+```
+
+While discussing further details, it is helpful to introduce two terms: an "includer" is a file which includes another
+using `include_file`, and the "includee" is a file which is included.
+
+The `root.ion` universe will contain all type domains from all includees and may still define additional type domains
+of its own.
+
+`include_file` statements are allowed in includees. Any attempt to include a file that has already been seen will be 
+ignored.
+
+When resolving the file to include, the following locations are searched:
+
+- The directory containing the includer (if the path to the includee does not start with `/`)
+- The directory containing the "main" type universe that was passed to `pig` on the command-line.
+- Any directories specified with the `--include` or `-I` arguments, in the order they were specified.
+
+The first matching file found wins and any other matching files ignored.
+
+Note that paths starting with `/` do not actually refer to the root of any file system, but instead are treated as 
+relative to the include directories.
+
+Paths specified with `include_file` may only contain alphanumeric or one of the following characters: 
+`-`, `_`, `.` or `/`.  Additionally, two consecutive periods ".." (i.e. a parent directory) are not allowed.
+
+Lastly, `include_file` can only be used at the top-level within a `.ion` file.  It is not allowed anywhere within a 
+`(domain ...)` clause.
 
 #### Using PIG In Your Project
 
@@ -408,16 +447,14 @@ Unlike record elements, product element defintions must include identifiers.
 At build time and before compilation of your application or library, the following should be executed:
 
 ```
-pig \
-    -u <type universe.ion> \
-    -t kotlin \ 
-    -n <namespace> \ 
-    -o path/to/package/<output file>
+pig -u <type universe.ion> -t kotlin -n <namespace> -o <path/to/output_file> [ -I <path-to-include-dir> ]  
 ```
 
 - `<type universe.ion>`:  path to the Ion text file containing the type universe
-- `<output file>`: path to the file for the generated code 
+- `<path/to/output_file>`: path to the file for the generated code 
 - `<namespace>`: the name used in the `package` statement at the top of the output file
+- `<path-to-include-dir>`: search path to external include directory (optional).  Can be specified more than once,
+i.e. `pig ... -I <dir1> -I <dir2> -I <dir3>`
 
 Execute: `pig --help` for all command-line options.
 

pig/build.gradle

@@ -43,6 +43,7 @@ dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.6.2'
     testImplementation 'org.junit.jupiter:junit-jupiter-params:5.6.2'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.6.2'
+    testImplementation 'com.google.jimfs:jimfs:1.2'
 }
 
 application {

pig/src/org/partiql/pig/cmdline/Command.kt

@@ -15,10 +15,36 @@
 
 package org.partiql.pig.cmdline
 
-import java.io.File
+import java.nio.file.Path
 
+/** Represents command line options specified by the user. */
 sealed class Command {
+
+    /** The `--help` command. */
     object ShowHelp : Command()
+
+    /**
+     * Returned by [CommandLineParser] when the user has specified invalid command-line arguments
+     *
+     * - [message]: an error message to be displayed to the user.
+     */
     data class InvalidCommandLineArguments(val message: String) : Command()
-    data class Generate(val typeUniverseFile: File, val outputFile: File, val target: TargetLanguage) : Command()
-}
+
+    /**
+     * Contains the details of a *valid* command-line specified by the user.
+     *
+     * - [typeUniverseFilePath]: the path to the type universe file.
+     * - [outputFilePath]: the path to the output file.  (This makes the assumption that there is only one output file.)
+     * - [includePaths]: directories to be searched when looking for files included with `include_file`.
+     * - [target]: specifies the target language and any other parameters unique to the target language.
+     */
+    data class Generate(
+        val typeUniverseFilePath: Path,
+        val outputFilePath: Path,
+        val includePaths: List<Path>,
+        val target: TargetLanguage
+    ) : Command()
+}
+
+
+

pig/src/org/partiql/pig/cmdline/CommandLineParser.kt

@@ -16,8 +16,9 @@
 package org.partiql.pig.cmdline
 
 import joptsimple.*
-import java.io.File
 import java.io.PrintStream
+import java.nio.file.Path
+import java.nio.file.Paths
 
 
 class CommandLineParser {
@@ -30,7 +31,7 @@ class CommandLineParser {
         HTML
     }
 
-    private object languageTargetTypeValueConverter : ValueConverter<LanguageTargetType> {
+    private object LanguageTargetTypeValueConverter : ValueConverter<LanguageTargetType> {
         private val lookup = LanguageTargetType.values().associateBy { it.name.toLowerCase() }
 
         override fun convert(value: String?): LanguageTargetType {
@@ -47,6 +48,12 @@ class CommandLineParser {
         }
     }
 
+    private object PathValueConverter : ValueConverter<Path> {
+        override fun convert(value: String?): Path = Paths.get(value).toAbsolutePath().normalize()
+        override fun valueType(): Class<out Path> = Path::class.java
+        override fun valuePattern(): String? = null
+    }
+
     private val formatter = object : BuiltinHelpFormatter(120, 2) {
         override fun format(options: MutableMap<String, out OptionDescriptor>?): String {
             return """PartiQL I.R. Generator
@@ -56,6 +63,8 @@ class CommandLineParser {
                 |
                 |  --target=kotlin requires --namespace=<ns>
                 |  --target=custom requires --template=<path-to-template>
+                |  All paths specified in these command-line options are relative to the current working
+                |  directory by default.
                 | 
                 |Examples:
                 |
@@ -65,26 +74,32 @@ class CommandLineParser {
         """.trimMargin()
         }
     }
-    private val optParser = OptionParser().also { it.formatHelpWith(formatter) }
+    private val optParser = OptionParser().apply {
+        formatHelpWith(formatter)
+    }
 
 
     private val helpOpt = optParser.acceptsAll(listOf("help", "h", "?"), "prints this help")
         .forHelp()
 
     private val universeOpt = optParser.acceptsAll(listOf("universe", "u"), "Type universe input file")
         .withRequiredArg()
-        .ofType(File::class.java)
+        .withValuesConvertedBy(PathValueConverter)
         .required()
 
+    private val includeSearchRootOpt = optParser.acceptsAll(listOf("include", "I"), "Include search path")
+        .withRequiredArg()
+        .withValuesConvertedBy(PathValueConverter)
+        .describedAs("Search path for files included with include_file. May be specified multiple times.")
+
     private val outputOpt = optParser.acceptsAll(listOf("output", "o"), "Generated output file")
         .withRequiredArg()
-        .ofType(File::class.java)
+        .withValuesConvertedBy(PathValueConverter)
         .required()
 
     private val targetTypeOpt = optParser.acceptsAll(listOf("target", "t"), "Target language")
         .withRequiredArg()
-        //.ofType(LanguageTargetType::class.java)
-        .withValuesConvertedBy(languageTargetTypeValueConverter)
+        .withValuesConvertedBy(LanguageTargetTypeValueConverter)
         .required()
 
     private val namespaceOpt = optParser.acceptsAll(listOf("namespace", "n"), "Namespace for generated code")
@@ -93,7 +108,7 @@ class CommandLineParser {
 
     private val templateOpt = optParser.acceptsAll(listOf("template", "e"), "Path to an Apache FreeMarker template")
         .withOptionalArg()
-        .ofType(File::class.java)
+        .withValuesConvertedBy(PathValueConverter)
 
 
     /**
@@ -116,9 +131,15 @@ class CommandLineParser {
                 optSet.has(helpOpt) -> Command.ShowHelp
                 else -> {
                     // !! is fine in this case since we define these options as .required() above.
-                    val typeUniverseFile: File = optSet.valueOf(universeOpt)!!
+                    val typeUniverseFile: Path = optSet.valueOf(universeOpt)!!
                     val targetType = optSet.valueOf(targetTypeOpt)!!
-                    val outputFile: File = optSet.valueOf(outputOpt)!!
+                    val outputFile: Path = optSet.valueOf(outputOpt)!!
+
+                    // Always add the parent of the file containing the main type universe as an include root.
+                    val includeSearchRoots = listOf(
+                        typeUniverseFile.parent,
+                        *optSet.valuesOf(includeSearchRootOpt)!!.toTypedArray()
+                    )
 
                     if (targetType.requireNamespace) {
                         if (!optSet.has(namespaceOpt)) {
@@ -145,13 +166,11 @@ class CommandLineParser {
                         LanguageTargetType.CUSTOM -> TargetLanguage.Custom(optSet.valueOf(templateOpt))
                     }
 
-                    Command.Generate(typeUniverseFile, outputFile, target)
+                    Command.Generate(typeUniverseFile, outputFile, includeSearchRoots, target)
                 }
             }
         } catch(ex: OptionException) {
             Command.InvalidCommandLineArguments(ex.message!!)
         }
-
     }
-
-}
+}

pig/src/org/partiql/pig/cmdline/TargetLanguage.kt

@@ -16,9 +16,10 @@
 package org.partiql.pig.cmdline
 
 import java.io.File
+import java.nio.file.Path
 
 sealed class TargetLanguage {
     data class Kotlin(val namespace: String) : TargetLanguage()
-    data class Custom(val templateFile: File) : TargetLanguage()
+    data class Custom(val templateFile: Path) : TargetLanguage()
     object Html : TargetLanguage()
 }

pig/src/org/partiql/pig/domain/include/IncludeCycleHandler.kt

@@ -0,0 +1,71 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+package org.partiql.pig.domain.include
+
+import org.partiql.pig.domain.model.Statement
+import org.partiql.pig.domain.parser.TypeUniverseParser
+import java.nio.file.FileSystem
+import java.nio.file.Files
+import java.nio.file.Path
+
+/**
+ * Prevents cycles in files included with the `include_file` statement from becoming a problem.
+ *
+ * This is accomplished by keeping track of all files "seen" by the parser and then making any attempt to
+ * include a file that was previously seen a no-op.
+ *
+ * @param mainTypeUniversePath The path to the main type universe that was passed on the command-line. This is the
+ * first "seen" file and does not require resolution because the user gave an explicit path to its location.
+ *
+ * @param resolver For identifying the full path to the file to be included.
+ *
+ * @see IncludeResolver
+ * @see FileSystem
+ */
+internal class IncludeCycleHandler(
+    mainTypeUniversePath: Path,
+    private val resolver: IncludeResolver
+) {
+    private val seenFiles = HashSet<Path>().apply { add(mainTypeUniversePath.toAbsolutePath().normalize()) }
+
+    /**
+     * Parses a universe file included with `include_file`.
+     *
+     * The return value is a [List] of [Statement]s that make up the type universe file.
+     *
+     * This function becomes a no-op in the event that the [includee] has been seen previously: an
+     * empty [List] is is returned instead of the file being parsed again.
+     *
+     * @param includeePath The file requested to be included.
+     * @param includerPath The file in which the includee is to be included.
+     */
+    fun parseIncludedTypeUniverse(includeePath: String, includerPath: Path): List<Statement> {
+
+        val resolvedIncludeFile = resolver.resolve(resolver.fileSystem.getPath(includeePath), includerPath)
+
+        return if(!seenFiles.contains(resolvedIncludeFile)) {
+            seenFiles.add(resolvedIncludeFile)
+            Files.newInputStream(resolvedIncludeFile).use {
+                val source = InputSource(resolvedIncludeFile, it)
+                val parser = TypeUniverseParser(source, this)
+                parser.parse()
+            }.statements
+        } else {
+            listOf()
+        }
+    }
+}
+

pig/src/org/partiql/pig/domain/include/IncludeResolutionException.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+package org.partiql.pig.domain.include
+
+import java.lang.Exception
+
+/** Thrown by [IncludeResolver] to indicate that it cannot locate an include file. */
+class IncludeResolutionException(
+    val inputFilePathString: String,
+    val consideredFilePaths: List<String>
+) : Exception("Could not locate include file '$inputFilePathString' in any considered path")
+