Skip to content

Commit

Permalink
Fix alignment, More Comprehensive test
Browse files Browse the repository at this point in the history
  • Loading branch information
@zeotuan
zeotuan committed Nov 17, 2024
1 parent e410165 commit 3a3dc3b
Show file tree
Hide file tree
Showing 2 changed files with 237 additions and 6 deletions.
10 changes: 6 additions & 4 deletions core/src/main/scala/com/github/mrpowers/spark/fast/tests/SchemaComparer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ object SchemaComparer {

def depthToIndentStr(depth: Int): String = Range(0, depth).map(_ => "| ").mkString + "|--"
val treeSpaces = 6
val (treeFieldPair1, headerGap) = flattenStrucType(actualSchema, 0)
val (treeFieldPair1, tree1MaxWidth) = flattenStrucType(actualSchema, 0)
val (treeFieldPair2, _) = flattenStrucType(expectedSchema, 0)
val (treePair, maxWidth) = treeFieldPair1
.zipAll(treeFieldPair2, (0, null), (0, null))
Expand Down Expand Up @@ -74,25 +74,27 @@ object SchemaComparer {
val name = Red(field1.name)
val dtype = Red(field1.dataType.typeName)
val nullable = Red(field1.nullable.toString)
s"$prefix1 $name : $dtype (nullable = $nullable)"
s"$sprefix1 $name : $dtype (nullable = $nullable)"
} else ""

val structString2 = if (field2 != null) {
val name = Green(field2.name)
val dtype = Green(field2.dataType.typeName)
val nullable = Green(field2.nullable.toString)
s"$prefix2 $name : $dtype (nullable = $nullable)"
s"$sprefix2 $name : $dtype (nullable = $nullable)"
} else ""
(structString1, structString2)
}
(acc :+ pair, math.max(maxWidth, pair._1.length))
}

val schemaGap = maxWidth + treeSpaces
val headerGap = tree1MaxWidth + treeSpaces
treePair
.foldLeft(new StringBuilder("\nActual Schema".padTo(headerGap, ' ') + "Expected Schema\n")) { case (sb, (s1, s2)) =>
val gap = if (s1.isEmpty) headerGap else schemaGap
sb.append(s1.padTo(gap, ' ') + s2 + "\n")
val s = if (s2.isEmpty) s1 else s1.padTo(gap, ' ')
sb.append(s + s2 + "\n")
}
.toString()
}
Expand Down
233 changes: 231 additions & 2 deletions core/src/test/scala/com/github/mrpowers/spark/fast/tests/SchemaComparerTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ class SchemaComparerTest extends AnyFreeSpec {
assert(SchemaComparer.equals(s1, s2))
}

"display schema diff as tree" in {
"display schema diff as tree with different depth" in {
val s1 = StructType(
Seq(
StructField("array", ArrayType(StringType, containsNull = true), true),
Expand Down Expand Up @@ -308,7 +308,7 @@ class SchemaComparerTest extends AnyFreeSpec {
}
val expectedMessage = """Diffs
|
|Actual Schema Expected Schema
|Actual Schema Expected Schema
|\u001b[90m|--\u001b[39m \u001b[90marray\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m|--\u001b[39m \u001b[90marray\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m|--\u001b[39m \u001b[31mmap\u001b[39m : \u001b[31mmap\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m|--\u001b[39m \u001b[32msomething\u001b[39m : \u001b[32mstring\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m|--\u001b[39m \u001b[31msomething\u001b[39m : \u001b[31mstring\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m|--\u001b[39m \u001b[32mmap\u001b[39m : \u001b[32mmap\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
Expand All @@ -319,6 +319,235 @@ class SchemaComparerTest extends AnyFreeSpec {
|\u001b[90m| | |--\u001b[39m \u001b[31mmood2\u001b[39m : \u001b[31marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m| | |--\u001b[39m \u001b[32mmood3\u001b[39m : \u001b[32marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[31m| | |--\u001b[39m \u001b[31msomething2\u001b[39m : \u001b[90mstring\u001b[39m (nullable = \u001b[90mfalse\u001b[39m) \u001b[32m|--\u001b[39m \u001b[32mnorma2\u001b[39m : \u001b[90mstring\u001b[39m (nullable = \u001b[90mfalse\u001b[39m)
|""".stripMargin

assert(e.getMessage == expectedMessage)
}

"display schema diff for wide tree" in {
val s1 = StructType(
Seq(
StructField("array", ArrayType(StringType, containsNull = true), true),
StructField("map", MapType(StringType, StringType, valueContainsNull = false), true),
StructField("something", StringType, true),
StructField(
"struct",
StructType(
StructType(
Seq(
StructField("mood", ArrayType(StringType, containsNull = false), true),
StructField("something", StringType, false),
StructField(
"something2",
StructType(
Seq(
StructField("mood2", ArrayType(DoubleType, containsNull = false), true),
StructField(
"something2",
StructType(
Seq(
StructField("mood", ArrayType(StringType, containsNull = false), true),
StructField("something", StringType, false),
StructField(
"something2",
StructType(
Seq(
StructField("mood2", ArrayType(DoubleType, containsNull = false), true),
StructField("something2", StringType, false)
)
),
false
)
)
),
false
)
)
),
false
)
)
)
),
true
)
)
)
val s2 = StructType(
Seq(
StructField("array", ArrayType(StringType, containsNull = true), true),
StructField("something", StringType, true),
StructField("map", MapType(StringType, StringType, valueContainsNull = false), true),
StructField(
"struct",
StructType(
StructType(
Seq(
StructField("something", StringType, false),
StructField("mood", ArrayType(StringType, containsNull = false), true),
StructField(
"something3",
StructType(
Seq(
StructField("mood2", ArrayType(DoubleType, containsNull = false), true),
StructField(
"something2",
StructType(
Seq(
StructField("mood", ArrayType(StringType, containsNull = false), true),
StructField("something", StringType, false),
StructField(
"something2",
StructType(
Seq(
StructField("mood2", ArrayType(DoubleType, containsNull = false), true),
StructField("something2", StringType, false)
)
),
false
)
)
),
false
)
)
),
false
)
)
)
),
true
),
StructField("norma2", StringType, false)
)
)

val e = intercept[DatasetSchemaMismatch] {
SchemaComparer.assertSchemaEqual(s1, s2, ignoreColumnOrder = false, outputFormat = SchemaDiffOutputFormat.Tree)
}
val expectedMessage = """Diffs
|
|Actual Schema Expected Schema
|\u001b[90m|--\u001b[39m \u001b[90marray\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m|--\u001b[39m \u001b[90marray\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m|--\u001b[39m \u001b[31mmap\u001b[39m : \u001b[31mmap\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m|--\u001b[39m \u001b[32msomething\u001b[39m : \u001b[32mstring\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m|--\u001b[39m \u001b[31msomething\u001b[39m : \u001b[31mstring\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m|--\u001b[39m \u001b[32mmap\u001b[39m : \u001b[32mmap\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m|--\u001b[39m \u001b[90mstruct\u001b[39m : \u001b[31mstruct\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m|--\u001b[39m \u001b[90mstruct\u001b[39m : \u001b[32mstruct\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m| |--\u001b[39m \u001b[31mmood\u001b[39m : \u001b[31marray\u001b[39m (nullable = \u001b[31mtrue\u001b[39m) \u001b[90m| |--\u001b[39m \u001b[32msomething\u001b[39m : \u001b[32mstring\u001b[39m (nullable = \u001b[32mfalse\u001b[39m)
|\u001b[90m| |--\u001b[39m \u001b[31msomething\u001b[39m : \u001b[31mstring\u001b[39m (nullable = \u001b[31mfalse\u001b[39m) \u001b[90m| |--\u001b[39m \u001b[32mmood\u001b[39m : \u001b[32marray\u001b[39m (nullable = \u001b[32mtrue\u001b[39m)
|\u001b[90m| |--\u001b[39m \u001b[31msomething2\u001b[39m : \u001b[90mstruct\u001b[39m (nullable = \u001b[90mfalse\u001b[39m) \u001b[90m| |--\u001b[39m \u001b[32msomething3\u001b[39m : \u001b[90mstruct\u001b[39m (nullable = \u001b[90mfalse\u001b[39m)
|\u001b[90m| | |--\u001b[39m \u001b[90mmood2\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m| | |--\u001b[39m \u001b[90mmood2\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m| | |--\u001b[39m \u001b[90msomething2\u001b[39m : \u001b[90mstruct\u001b[39m (nullable = \u001b[90mfalse\u001b[39m) \u001b[90m| | |--\u001b[39m \u001b[90msomething2\u001b[39m : \u001b[90mstruct\u001b[39m (nullable = \u001b[90mfalse\u001b[39m)
|\u001b[90m| | | |--\u001b[39m \u001b[90mmood\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m| | | |--\u001b[39m \u001b[90mmood\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m| | | |--\u001b[39m \u001b[90msomething\u001b[39m : \u001b[90mstring\u001b[39m (nullable = \u001b[90mfalse\u001b[39m) \u001b[90m| | | |--\u001b[39m \u001b[90msomething\u001b[39m : \u001b[90mstring\u001b[39m (nullable = \u001b[90mfalse\u001b[39m)
|\u001b[90m| | | |--\u001b[39m \u001b[90msomething2\u001b[39m : \u001b[90mstruct\u001b[39m (nullable = \u001b[90mfalse\u001b[39m) \u001b[90m| | | |--\u001b[39m \u001b[90msomething2\u001b[39m : \u001b[90mstruct\u001b[39m (nullable = \u001b[90mfalse\u001b[39m)
|\u001b[90m| | | | |--\u001b[39m \u001b[90mmood2\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m| | | | |--\u001b[39m \u001b[90mmood2\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m| | | | |--\u001b[39m \u001b[90msomething2\u001b[39m : \u001b[90mstring\u001b[39m (nullable = \u001b[90mfalse\u001b[39m) \u001b[90m| | | | |--\u001b[39m \u001b[90msomething2\u001b[39m : \u001b[90mstring\u001b[39m (nullable = \u001b[90mfalse\u001b[39m)
| \u001b[90m|--\u001b[39m \u001b[32mnorma2\u001b[39m : \u001b[32mstring\u001b[39m (nullable = \u001b[32mfalse\u001b[39m)
|""".stripMargin

assert(e.getMessage == expectedMessage)
}

"display schema diff as tree with more actual Column 2" in {
val s1 = StructType(
Seq(
StructField("array", ArrayType(StringType, containsNull = true), true),
StructField("map", MapType(StringType, StringType, valueContainsNull = false), true),
StructField("something", StringType, true),
StructField(
"struct",
StructType(
StructType(
Seq(
StructField("mood", ArrayType(StringType, containsNull = false), true),
StructField("something", StringType, false),
StructField(
"something2",
StructType(
Seq(
StructField("mood2", ArrayType(DoubleType, containsNull = false), true),
StructField(
"something2",
StructType(
Seq(
StructField("mood2", ArrayType(DoubleType, containsNull = false), true),
StructField(
"something2",
StructType(
Seq(
StructField("mood2", ArrayType(DoubleType, containsNull = false), true),
StructField("something2", StringType, false)
)
),
false
)
)
),
false
)
)
),
false
)
)
)
),
true
)
)
)
val s2 = StructType(
Seq(
StructField("array", ArrayType(StringType, containsNull = true), true),
StructField("something", StringType, true),
StructField(
"struct",
StructType(
StructType(
Seq(
StructField("something", StringType, false),
StructField("mood", ArrayType(StringType, containsNull = false), true),
StructField(
"something3",
StructType(
Seq(
StructField("mood3", ArrayType(StringType, containsNull = false), true)
)
),
false
)
)
)
),
true
)
)
)

val e = intercept[DatasetSchemaMismatch] {
SchemaComparer.assertSchemaEqual(s1, s2, ignoreColumnOrder = false, outputFormat = SchemaDiffOutputFormat.Tree)
}

val expectedMessage = """Diffs
|
|Actual Schema Expected Schema
|\u001b[90m|--\u001b[39m \u001b[90marray\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m|--\u001b[39m \u001b[90marray\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m|--\u001b[39m \u001b[31mmap\u001b[39m : \u001b[31mmap\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m|--\u001b[39m \u001b[32msomething\u001b[39m : \u001b[32mstring\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m|--\u001b[39m \u001b[31msomething\u001b[39m : \u001b[31mstring\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m|--\u001b[39m \u001b[32mstruct\u001b[39m : \u001b[32mstruct\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[31m|--\u001b[39m \u001b[31mstruct\u001b[39m : \u001b[31mstruct\u001b[39m (nullable = \u001b[31mtrue\u001b[39m) \u001b[32m| |--\u001b[39m \u001b[32msomething\u001b[39m : \u001b[32mstring\u001b[39m (nullable = \u001b[32mfalse\u001b[39m)
|\u001b[90m| |--\u001b[39m \u001b[90mmood\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m) \u001b[90m| |--\u001b[39m \u001b[90mmood\u001b[39m : \u001b[90marray\u001b[39m (nullable = \u001b[90mtrue\u001b[39m)
|\u001b[90m| |--\u001b[39m \u001b[31msomething\u001b[39m : \u001b[31mstring\u001b[39m (nullable = \u001b[90mfalse\u001b[39m) \u001b[90m| |--\u001b[39m \u001b[32msomething3\u001b[39m : \u001b[32mstruct\u001b[39m (nullable = \u001b[90mfalse\u001b[39m)
|\u001b[31m| |--\u001b[39m \u001b[31msomething2\u001b[39m : \u001b[31mstruct\u001b[39m (nullable = \u001b[31mfalse\u001b[39m) \u001b[32m| | |--\u001b[39m \u001b[32mmood3\u001b[39m : \u001b[32marray\u001b[39m (nullable = \u001b[32mtrue\u001b[39m)
|\u001b[31m| | |--\u001b[39m \u001b[31mmood2\u001b[39m : \u001b[31marray\u001b[39m (nullable = \u001b[31mtrue\u001b[39m)
|\u001b[31m| | |--\u001b[39m \u001b[31msomething2\u001b[39m : \u001b[31mstruct\u001b[39m (nullable = \u001b[31mfalse\u001b[39m)
|\u001b[31m| | | |--\u001b[39m \u001b[31mmood2\u001b[39m : \u001b[31marray\u001b[39m (nullable = \u001b[31mtrue\u001b[39m)
|\u001b[31m| | | |--\u001b[39m \u001b[31msomething2\u001b[39m : \u001b[31mstruct\u001b[39m (nullable = \u001b[31mfalse\u001b[39m)
|\u001b[31m| | | | |--\u001b[39m \u001b[31mmood2\u001b[39m : \u001b[31marray\u001b[39m (nullable = \u001b[31mtrue\u001b[39m)
|\u001b[31m| | | | |--\u001b[39m \u001b[31msomething2\u001b[39m : \u001b[31mstring\u001b[39m (nullable = \u001b[31mfalse\u001b[39m)
|""".stripMargin
val actual = e.getMessage
println(actual)
println(expectedMessage)
assert(e.getMessage == expectedMessage)
}
}
Expand Down

0 comments on commit 3a3dc3b

Please sign in to comment.