Add include_file statement

README.md

@@ -174,7 +174,9 @@ assertEquals(onePlusOne, anotherOnePlusOne)
 // Top level
 type_universe ::= <stmt>...
 definition ::= '(' 'define' symbol <domain_definition> ')'
-stmt ::=  <definition> | <transform>
+stmt ::=  <definition> | <transform> | <include_file>
+
+include_file ::= `(include_file <path-to-file>)`
 
 // Domain
 domain_definition ::= <domain> | <permute_domain>
@@ -400,6 +402,32 @@ Unlike record elements, product element defintions must include identifiers.
 (product int_pair first::int second::int)
 ```
 
+#### Type Domain Includes
+
+It is possible to split type universes among multiple files:
+
+```
+// root.ion:
+(include_file "a.ion")
+(include_file "b.ion")
+```
+
+The resulting type universe will contain all type domains from both `a.ion` and `b.ion`. `root.ion` may also define
+additional type domains.  The primary purpose of this is to be able to permute domains defined in another file.
+
+`include_file` is recursive, so any type domains in any files included by `a.ion` and `b.ion` will also be present.  Any 
+attempt to include a file that has already been seen will be ignored.
+
+Paths are always relative to the directory containing the current file.  The working directory at the time `pig` 
+is executed is irrelevant.
+
+```
+(include_file "subdir/partiql.ion")
+(include_file "../other/some-universe.ion")
+```
+
+Lastly, `include_file` can only be used at the top-level within a `.ion` file.  It is not allowed within a 
+`(domain ...)` clause.
 
 #### Using PIG In Your Project
 

pig/src/org/partiql/pig/cmdline/Command.kt

@@ -17,8 +17,26 @@ package org.partiql.pig.cmdline
 
 import java.io.File
 
+/** Represents command line options specified by the user. */
 sealed class Command {
+
+    /** The `--help` command. */
     object ShowHelp : Command()
+
+    /**
+     * Returned by [CommandLineParser] when the user has specified invalid command-line arguments
+     *
+     * - [message]: an error message to be displayed to the user.
+     */
     data class InvalidCommandLineArguments(val message: String) : Command()
-    data class Generate(val typeUniverseFile: File, val outputFile: File, val target: TargetLanguage) : Command()
-}
+
+    /**
+     * Contains the details of a *valid* command-line specified by the user.
+     *
+     * - [typeUniverseFilePath]: the path to the type universe file.
+     * - [outputFilePath]: the path to the output file.  (This makes the assumption that there is only one output file.)
+     * - [target]: specifies the target language and any other parameters unique to the target language.
+     */
+    data class Generate(val typeUniverseFilePath: File, val outputFilePath: File, val target: TargetLanguage) : Command()
+}
+

pig/src/org/partiql/pig/domain/model/SemanticErrorContext.kt

@@ -15,12 +15,12 @@
 
 package org.partiql.pig.domain.model
 
-import com.amazon.ionelement.api.IonLocation
 import com.amazon.ionelement.api.MetaContainer
-import com.amazon.ionelement.api.location
-import org.partiql.pig.errors.PigException
+import org.partiql.pig.domain.parser.SourceLocation
+import org.partiql.pig.domain.parser.sourceLocation
 import org.partiql.pig.errors.ErrorContext
 import org.partiql.pig.errors.PigError
+import org.partiql.pig.errors.PigException
 
 /**
  * Encapsulates all error context information in an easily testable way.
@@ -109,9 +109,9 @@ sealed class SemanticErrorContext(val msgFormatter: () -> String): ErrorContext
  * Shortcut for throwing [PigException] with the specified metas and [PigError].
  */
 fun semanticError(blame: MetaContainer, context: ErrorContext): Nothing =
-    semanticError(blame.location, context)
+    semanticError(blame.sourceLocation, context)
 /**
  * Shortcut for throwing [PigException] with the specified metas and [PigError].
  */
-fun semanticError(blame: IonLocation?, context: ErrorContext): Nothing =
+fun semanticError(blame: SourceLocation?, context: ErrorContext): Nothing =
     throw PigException(PigError(blame, context))

pig/src/org/partiql/pig/domain/parser/InputStreamSource.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+package org.partiql.pig.domain.parser
+
+import java.io.FileInputStream
+import java.io.InputStream
+
+
+/**
+ * A simple abstraction for opening [InputStream], allows for type domains to be loaded from a file system or an
+ * in-memory data structure.  The latter is useful for unit testing.
+ */
+internal interface InputStreamSource {
+    /** Opens an input stream for the given filename name. */
+    fun openInputStream(fileName: String): InputStream
+}
+
+/** An [InputStreamSource] that constructs [FileInputStream] instances. */
+internal val FILE_SYSTEM_INPUT_STREAM_SOURCE = object : InputStreamSource {
+    override fun openInputStream(fileName: String) = FileInputStream(fileName)
+}
+

pig/src/org/partiql/pig/domain/parser/ParserErrorContext.kt

@@ -15,14 +15,11 @@
 
 package org.partiql.pig.domain.parser
 
-import com.amazon.ionelement.api.IonElement
-import com.amazon.ionelement.api.IonLocation
-import com.amazon.ionelement.api.location
 import com.amazon.ionelement.api.ElementType
 import com.amazon.ionelement.api.IonElementException
-import org.partiql.pig.errors.PigException
 import org.partiql.pig.errors.ErrorContext
 import org.partiql.pig.errors.PigError
+import org.partiql.pig.errors.PigException
 
 /**
  * Variants of [ParserErrorContext] contain details about various parse errors that can be encountered
@@ -33,7 +30,7 @@ import org.partiql.pig.errors.PigError
 sealed class ParserErrorContext(val msgFormatter: () -> String): ErrorContext {
     override val message: String get() = msgFormatter()
 
-    /** Indicates that an []IonElectrolyteException] was thrown during parsing of a type universe. */
+    /** Indicates that an [IonElementException] was thrown during parsing of a type universe. */
     data class IonElementError(val ex: IonElementException)
         : ParserErrorContext({ ex.message!! }) {
         // This is for unit tests... we don't include IonElectrolyteException here since it doesn't implement
@@ -42,6 +39,9 @@ sealed class ParserErrorContext(val msgFormatter: () -> String): ErrorContext {
         override fun hashCode(): Int = 0
     }
 
+    data class CouldNotFindIncludedFile(val tag: String)
+        : ParserErrorContext({ "Included file missing: $tag" })
+
     data class UnknownConstructor(val tag: String)
         : ParserErrorContext({ "Unknown constructor: '$tag' (expected constructors are 'domain' or 'permute_domain')" })
 
@@ -79,8 +79,7 @@ sealed class ParserErrorContext(val msgFormatter: () -> String): ErrorContext {
         : ParserErrorContext({ "Element has multiple name annotations"})
 }
 
-
-fun parseError(blame: IonLocation?, context: ErrorContext): Nothing =
+fun parseError(blame: SourceLocation?, context: ErrorContext): Nothing =
     PigError(blame, context).let {
         throw when (context) {
             is ParserErrorContext.IonElementError -> {
@@ -91,8 +90,3 @@ fun parseError(blame: IonLocation?, context: ErrorContext): Nothing =
         }
     }
 
-fun parseError(blame: IonElement, context: ErrorContext): Nothing {
-    val loc = blame.metas.location
-    parseError(loc, context)
-}
-

pig/src/org/partiql/pig/domain/parser/SourceLocation.kt

@@ -0,0 +1,34 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+package org.partiql.pig.domain.parser
+
+import com.amazon.ionelement.api.IonLocation
+import com.amazon.ionelement.api.MetaContainer
+
+/**
+ * Used to construct helpful error messages for the end-user, who will be able to know the file, line & column of
+ * a given error.
+ */
+data class SourceLocation(val path: String, val location: IonLocation) {
+    override fun toString(): String {
+        return "$path:$location"
+    }
+}
+
+internal const val SOURCE_LOCATION_META_TAG = "\$pig_source_location"
+
+internal val MetaContainer.sourceLocation
+    get() = this[SOURCE_LOCATION_META_TAG] as? SourceLocation