Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Nim to TranslatorSpec and fix various errors found thanks to that #292

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
106 changes: 89 additions & 17 deletions jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion shared/src/main/scala/io/kaitai/struct/JSON.scala
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ object JSON extends CommonLiterals {
}
}

/** octal escapes (which [[translators.CommonLiterals.strLiteralGenericCC]] uses by default) are not allowed in JSON */
override def strLiteralGenericCC(code: Char): String = strLiteralUnicode(code)

def stringToJson(str: String): String =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ trait CommonLiterals {
/**
* Handle ASCII character conversion for inlining into string literals.
* Default implementation consults [[asciiCharQuoteMap]] first, then
* just dumps it as is if it's a printable ASCII charcter, or calls
* just dumps it as is if it's a printable ASCII character, or calls
* [[strLiteralGenericCC]] if it's a control character.
* @param code character code to convert into string for inclusion in
* a string literal
Expand All @@ -53,18 +53,14 @@ trait CommonLiterals {

/**
* Converts generic control character code into something that's allowed
* inside a string literal. Default implementation uses octal encoding,
* inside a string literal. Default implementation uses hex encoding,
* which is ok for most C-derived languages.
*
* Note that we use strictly 3 octal digits to work around potential
* problems with following decimal digits, i.e. "\0" + "2" that would be
* parsed as single character "\02" = "\x02", instead of two characters
* "\x00\x32".
* @param code character code to represent
* @return string literal representation of given code
*/
def strLiteralGenericCC(code: Char): String =
"\\%03o".format(code.toInt)
"\\x%02X".format(code.toInt)

/**
* Converts Unicode (typically, non-ASCII) character code into something
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,21 @@ class CppTranslator(provider: TypeProvider, importListSrc: CppImportList, import
}
}

/**
* Hex escapes in C++ does not limited in length, so we use octal, as they are shorter.
*
* Note that we use strictly 3 octal digits to work around potential
* problems with following decimal digits, i.e. "\0" + "2" that would be
* parsed as single character "\02" = "\x02", instead of two characters
* "\x00\x32".
*
* @see https://en.cppreference.com/w/cpp/language/escape
* @param code character code to represent
* @return string literal representation of given code
*/
override def strLiteralGenericCC(code: Char): String =
"\\%03o".format(code.toInt)

override def genericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr, extPrec: Int) = {
(detectType(left), detectType(right), op) match {
case (_: IntType, _: IntType, Ast.operator.Mod) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,6 @@ class JavaScriptTranslator(provider: TypeProvider, importList: ImportList) exten
override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String =
s"new Uint8Array([${elts.map(translate).mkString(", ")}])"

/**
* JavaScript rendition of common control character that would use hex form,
* not octal. "Octal" control character string literals might be accepted
* in non-strict JS mode, but in strict mode only hex or unicode are ok.
* Here we'll use hex, as they are shorter.
*
* @see https://github.com/kaitai-io/kaitai_struct/issues/279
* @param code character code to represent
* @return string literal representation of given code
*/
override def strLiteralGenericCC(code: Char): String =
"\\x%02x".format(code.toInt)

override def genericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr, extPrec: Int) = {
(detectType(left), detectType(right), op) match {
case (_: IntType, _: IntType, Ast.operator.Div) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,21 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas
override def doByteArrayNonLiteral(elts: Seq[expr]): String =
s"new byte[] { ${elts.map(translate).mkString(", ")} }"

/**
* Java does not support two-digit hex escape sequences, so use octal, as they are shorter.
*
* Note that we use strictly 3 octal digits to work around potential
* problems with following decimal digits, i.e. "\0" + "2" that would be
* parsed as single character "\02" = "\x02", instead of two characters
* "\x00\x32".
*
* @see https://docs.oracle.com/javase/specs/jls/se7/html/jls-3.html#jls-3.10.6
* @param code character code to represent
* @return string literal representation of given code
*/
override def strLiteralGenericCC(code: Char): String =
"\\%03o".format(code.toInt)

override def genericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr, extPrec: Int) = {
(detectType(left), detectType(right), op) match {
case (_: IntType, _: IntType, Ast.operator.Mod) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ class LuaTranslator(provider: TypeProvider, importList: ImportList) extends Base
'\b' -> "\\b",
'\u000b' -> "\\v",
'\f' -> "\\f",
'\u001b' -> "\\027"
)

override def strLiteralUnicode(code: Char): String =
Expand Down Expand Up @@ -71,7 +70,7 @@ class LuaTranslator(provider: TypeProvider, importList: ImportList) extends Base
override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String =
"{" + value.map((v) => translate(v)).mkString(", ") + "}"
override def doByteArrayLiteral(arr: Seq[Byte]): String =
"\"" + decEscapeByteArray(arr) + "\""
"\"" + Utils.hexEscapeByteArray(arr) + "\""
override def doByteArrayNonLiteral(values: Seq[Ast.expr]): String =
// It is assumed that every expression produces integer in the range [0; 255]
"string.char(" + values.map(translate).mkString(", ") + ")"
Expand Down Expand Up @@ -186,17 +185,7 @@ class LuaTranslator(provider: TypeProvider, importList: ImportList) extends Base
case Ast.boolop.And => "and"
}
override def unaryOp(op: Ast.unaryop): String = op match {
case Ast.unaryop.Not => "not"
case Ast.unaryop.Not => "not "
case _ => super.unaryOp(op)
}

/**
* Converts byte array (Seq[Byte]) into decimal-escaped Lua-style literal
* characters (i.e. like \255).
*
* @param arr byte array to escape
* @return array contents decimal-escaped as string
*/
private def decEscapeByteArray(arr: Seq[Byte]): String =
arr.map((x) => "\\%03d".format(x & 0xff)).mkString
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,15 @@ import io.kaitai.struct.format.{EnumSpec, Identifier}
import io.kaitai.struct.languages.NimCompiler.{ksToNim, namespaced, camelCase}

class NimTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) {
override def genericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr, extPrec: Int) = {
(detectType(left), detectType(right), op) match {
case (_: IntType, _: IntType, Ast.operator.Div) =>
genericBinOpStr(left, op, "div", right, extPrec)
case _ =>
super.genericBinOp(left, op, right, extPrec)
}
}

// Members declared in io.kaitai.struct.translators.BaseTranslator
override def bytesToStr(bytesExpr: String, encoding: String): String = {
s"""encode($bytesExpr, ${doStringLiteral(encoding)})"""
Expand Down Expand Up @@ -38,18 +47,28 @@ class NimTranslator(provider: TypeProvider, importList: ImportList) extends Base
case _ => s"this.${doName(s)}"
}
override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
s"(if ${translate(condition)}: ${translate(ifTrue)} else: ${translate(ifFalse)})"
s"if ${translate(condition)}: ${translate(ifTrue)} else: ${translate(ifFalse)}"
override def arraySubscript(container: expr, idx: expr): String =
s"${translate(container)}[${translate(idx)}]"
s"${translate(container, METHOD_PRECEDENCE)}[${translate(idx)}]"

override def strConcat(left: expr, right: expr, extPrec: Int) = "($" + s"${translate(left)} & " + "$" + s"${translate(right)})"
override def strConcat(left: expr, right: expr, extPrec: Int): String = {
val thisPrec = OPERATOR_PRECEDENCE(Ast.operator.Add)
// $ is a special method which converts everything to a string, so use METHOD_PRECEDENCE
val leftStr = "$" + translate(left, METHOD_PRECEDENCE)
val rightStr = "$" + translate(right, METHOD_PRECEDENCE)
if (thisPrec <= extPrec) {
s"($leftStr & $rightStr)"
} else {
s"$leftStr & $rightStr"
}
}

// Members declared in io.kaitai.struct.translators.CommonMethods

override def unaryOp(op: Ast.unaryop): String = op match {
case Ast.unaryop.Invert => "not"
case Ast.unaryop.Invert => "not "
case Ast.unaryop.Minus => "-"
case Ast.unaryop.Not => "not"
case Ast.unaryop.Not => "not "
}

override def booleanOp(op: Ast.boolop): String = op match {
Expand All @@ -62,7 +81,7 @@ class NimTranslator(provider: TypeProvider, importList: ImportList) extends Base
case Ast.operator.Add => "+"
case Ast.operator.Sub => "-"
case Ast.operator.Mult => "*"
case Ast.operator.Div => "div"
case Ast.operator.Div => "/"
case Ast.operator.Mod => "%%%"
case Ast.operator.BitAnd => "and"
case Ast.operator.BitOr => "or"
Expand All @@ -75,9 +94,9 @@ class NimTranslator(provider: TypeProvider, importList: ImportList) extends Base
typeName match {
case at: ArrayType => {
importList.add("sequtils")
s"${translate(value)}.mapIt(it.${ksToNim(at.elType)})"
s"${translate(value, METHOD_PRECEDENCE)}.mapIt(it.${ksToNim(at.elType)})"
}
case _ => s"(${ksToNim(typeName)}(${translate(value)}))"
case _ => s"${ksToNim(typeName)}(${translate(value)})"
}
override def doIntLiteral(n: BigInt): String = {
if (n <= -2147483649L) { // -9223372036854775808..-2147483649
Expand Down Expand Up @@ -107,8 +126,8 @@ class NimTranslator(provider: TypeProvider, importList: ImportList) extends Base
}
override def doByteArrayNonLiteral(elts: Seq[expr]): String =
s"@[${elts.map(translate).mkString(", ")}]"
override def arrayFirst(a: expr): String = s"${translate(a)}[0]"
override def arrayLast(a: expr): String = s"${translate(a)}[^1]"
override def arrayFirst(a: expr): String = s"${translate(a, METHOD_PRECEDENCE)}[0]"
override def arrayLast(a: expr): String = s"${translate(a, METHOD_PRECEDENCE)}[^1]"
override def arrayMax(a: expr): String = s"max(${translate(a)})"
override def arrayMin(a: expr): String = s"min(${translate(a)})"
override def arraySize(a: expr): String = s"len(${translate(a)})"
Expand All @@ -124,9 +143,9 @@ class NimTranslator(provider: TypeProvider, importList: ImportList) extends Base
s"reversed(${translate(s)})"
}
override def strSubstring(s: expr, from: expr, to: expr): String =
s"${translate(s)}.substr(${translate(from)}, ${translate(to)} - 1)"
s"${translate(s, METHOD_PRECEDENCE)}.substr(${translate(from)}, ${translate(to)} - 1)"
override def strToInt(s: expr, base: expr): String =
s"${translate(s)}.parseInt(${translate(base)})"
s"${translate(s, METHOD_PRECEDENCE)}.parseInt(${translate(base)})"

override def doInterpolatedStringLiteral(exprs: Seq[Ast.expr]): String =
if (exprs.isEmpty) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig)
override def strLiteralGenericCC(code: Char): String =
strLiteralUnicode(code)

/**
* Hex escapes in form `\xHH` in Rust allows only codes in the range 0x00 - 0x7f.
*
* @see https://doc.rust-lang.org/reference/tokens.html#examples
* @param code character code to represent
* @return string literal representation of given code
*/
override def strLiteralUnicode(code: Char): String =
"\\u{%x}".format(code.toInt)

Expand Down
Loading