From 318a8ea2a2788aa40b859349124def8d70adb026 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Mon, 26 Aug 2024 02:17:00 -0700 Subject: [PATCH 01/13] Add conformance testing DSL implementation, CI/CD runner, and user targettable test --- Cargo.toml | 4 + ion-tests | 2 +- tests/conformance.rs | 58 ++++++ tests/conformance_dsl/clause.rs | 125 ++++++++++++ tests/conformance_dsl/context.rs | 127 ++++++++++++ tests/conformance_dsl/continuation.rs | 260 +++++++++++++++++++++++++ tests/conformance_dsl/document.rs | 121 ++++++++++++ tests/conformance_dsl/fragment.rs | 139 +++++++++++++ tests/conformance_dsl/mod.rs | 269 ++++++++++++++++++++++++++ tests/conformance_dsl/model.rs | 157 +++++++++++++++ tests/conformance_tests.rs | 72 +++++++ tests/ion_data_consistency.rs | 28 +-- 12 files changed, 1347 insertions(+), 15 deletions(-) create mode 100644 tests/conformance.rs create mode 100644 tests/conformance_dsl/clause.rs create mode 100644 tests/conformance_dsl/context.rs create mode 100644 tests/conformance_dsl/continuation.rs create mode 100644 tests/conformance_dsl/document.rs create mode 100644 tests/conformance_dsl/fragment.rs create mode 100644 tests/conformance_dsl/mod.rs create mode 100644 tests/conformance_dsl/model.rs create mode 100644 tests/conformance_tests.rs diff --git a/Cargo.toml b/Cargo.toml index 166771ba..1c65cbdc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,3 +94,7 @@ codegen-units = 1 [profile.profiling] inherits = "release" debug = true + +[[test]] +name = "conformance" +harness = false diff --git a/ion-tests b/ion-tests index 32722650..3770633f 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit 327226503e84198ba97b88734a83430c620c4f46 +Subproject commit 3770633f05a845430938e53f6ee75e9c7b1ff855 diff --git a/tests/conformance.rs b/tests/conformance.rs new file mode 100644 index 00000000..cef6a089 --- /dev/null +++ b/tests/conformance.rs @@ -0,0 +1,58 @@ + +#[cfg(feature = "experimental-reader-writer")] +mod conformance_dsl; + +#[cfg(feature = "experimental-reader-writer")] +pub fn main() { + use crate::conformance_dsl::prelude::*; + + let test_paths = std::env::args().skip(1).collect::>(); + let mut errors: Vec<(String, String, conformance_dsl::ConformanceError)> = vec!(); + + // Formatting: Get max test name length. + + println!("Testing {} conformance collections.\n", test_paths.len()); + + let mut failures = 0; + + for test_path in test_paths { + println!("\nRunning tests: {} ========================", test_path); + let collection = TestCollection::load(&test_path).expect("unable to load test file"); + let name_len = collection.iter().fold(0, |acc, d| std::cmp::max(acc, d.name.as_ref().map_or(0, |n| n.len()))); + + for doc in collection.iter() { + match doc.name.as_ref() { + Some(n) => print!(" {: print!(" {:", width = name_len), + } + + print!(" ... "); + match doc.run() { + Err(e) => { + println!("[FAILED]"); + failures += 1; + errors.push((test_path.to_owned(), doc.name.as_deref().unwrap_or("").to_owned(), e.clone())); + } + Ok(_) => println!("[OK]"), + } + } + } + + // println!("\nConformance Summary: {} Succeeded, {} Failed", collection.len() - failures, failures); + + for (test_path, test_name, err) in errors { + println!("-------------------------"); + println!("File: {}", test_path); + println!("Test: {}", test_name); + println!("Error: {:?}", err); + } + + if failures > 0 { + panic!("Conformance test(s) failed"); + } +} + +#[cfg(not(feature = "experimental-reader-writer"))] +pub fn main() { + println!("Needs feature experimental-reader-writer"); +} diff --git a/tests/conformance_dsl/clause.rs b/tests/conformance_dsl/clause.rs new file mode 100644 index 00000000..389c1a47 --- /dev/null +++ b/tests/conformance_dsl/clause.rs @@ -0,0 +1,125 @@ +use std::str::FromStr; + +use ion_rs::{Element, Sequence}; + +use super::*; + +#[allow(non_camel_case_types)] +#[derive(Debug)] +pub(crate) enum ClauseType { + Ion1_0, + Ion1_1, + Ion1_X, + Text, + Binary, + Ivm, + TopLevel, + Encoding, + MacTab, + Produces, + Denotes, + Signals, + And, + Not, + Bytes, + Then, + Document, + Each, + Absent, +} + +impl FromStr for ClauseType { + type Err = ConformanceErrorKind; + + fn from_str(s: &str) -> InnerResult { + use ClauseType::*; + + match s { + "ion_1_0" => Ok(Ion1_0), + "ion_1_1" => Ok(Ion1_1), + "ion_1_x" => Ok(Ion1_X), + "document" => Ok(Document), + "toplevel" => Ok(TopLevel), + "produces" => Ok(Produces), + "denotes" => Ok(Denotes), + "text" => Ok(Text), + "binary" => Ok(Binary), + "and" => Ok(And), + "not" => Ok(Not), + "then" => Ok(Then), + "each" => Ok(Each), + "absent" => Ok(Absent), + "ivm" => Ok(Ivm), + "signals" => Ok(Signals), + _ => Err(ConformanceErrorKind::UnknownClause(s.to_owned())), + } + } +} + +impl ClauseType { + pub fn is_fragment(&self) -> bool { + use ClauseType::*; + matches!(self, Text | Binary | Ivm | TopLevel | Encoding | MacTab) + } + + pub fn is_expectation(&self) -> bool { + use ClauseType::*; + matches!(self, Produces | Denotes | Signals | And | Not) + } +} + +#[derive(Debug)] +pub(crate) struct Clause { + pub tpe: ClauseType, + pub body: Vec, +} + +impl TryFrom<&Sequence> for Clause { + type Error = ConformanceErrorKind; + + fn try_from(other: &Sequence) -> InnerResult { + let clause_type = other + .iter() + .next() + .ok_or(ConformanceErrorKind::UnexpectedEndOfDocument)? + .as_symbol() + .ok_or(ConformanceErrorKind::ExpectedDocumentClause)?; + + let tpe = ClauseType::from_str(clause_type.text().ok_or(ConformanceErrorKind::ExpectedDocumentClause)?)?; + let body: Vec = other.iter().skip(1).cloned().collect(); + + Ok(Clause { + tpe, + body, + }) + } +} + +impl TryFrom for Clause { + type Error = ConformanceErrorKind; + + fn try_from(other: Sequence) -> InnerResult { + Self::try_from(&other) + } +} + +impl TryFrom<&[Element]> for Clause { + type Error = ConformanceErrorKind; + + fn try_from(other: &[Element]) -> InnerResult { + let clause_type = other + .iter() + .next() + .ok_or(ConformanceErrorKind::UnexpectedEndOfDocument)? + .as_symbol() + .ok_or(ConformanceErrorKind::ExpectedDocumentClause)?; + + let tpe = ClauseType::from_str(clause_type.text().ok_or(ConformanceErrorKind::ExpectedDocumentClause)?)?; + let body: Vec = other.iter().skip(1).cloned().collect(); + + Ok(Clause { + tpe, + body, + }) + } +} diff --git a/tests/conformance_dsl/context.rs b/tests/conformance_dsl/context.rs new file mode 100644 index 00000000..adbe0fe5 --- /dev/null +++ b/tests/conformance_dsl/context.rs @@ -0,0 +1,127 @@ +use crate::conformance_dsl::*; + +use ion_rs::{Element, ElementReader, Sequence, Reader, IonSlice}; +use ion_rs::{v1_0, v1_1}; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Context<'a> { + version: IonVersion, + encoding: IonEncoding, + fragments: &'a Vec, + parent_ctx: Option<&'a Context<'a>>, +} + +impl<'a> Context<'a> { + pub fn new(version: IonVersion, encoding: IonEncoding, fragments: &'a Vec) -> Self { + Self { version, encoding, fragments, parent_ctx: None} + } + + pub fn extend(parent: &'a Context, fragments: &'a Vec) -> Self { + Self { + version: parent.version, + encoding: parent.encoding, + parent_ctx: Some(parent), + fragments, + } + } + + pub fn version(&self) -> IonVersion { + let parent_ver = self.parent_ctx.map(|c| c.version()).unwrap_or(IonVersion::Unspecified); + let frag_ver = self.fragment_version(); + let my_ver = if frag_ver == IonVersion::Unspecified { + self.version + } else { + frag_ver + }; + + match (parent_ver, my_ver) { + (IonVersion::Unspecified, v) => v, + (v, IonVersion::Unspecified) => v, + (a, b) if a == b => a, + _ => panic!("Mismatched versions"), + } + } + + pub fn set_version(&mut self, version: IonVersion) { + self.version = version; + } + + pub fn encoding(&self) -> IonEncoding { + let parent_enc = self.parent_ctx.map(|c| c.encoding).unwrap_or(IonEncoding::Unspecified); + Self::resolve_encoding(parent_enc, self.encoding) + } + + pub fn fragment_version(&self) -> IonVersion { + match self.fragments.first() { + Some(Fragment::Ivm(1, 0)) => IonVersion::V1_0, + Some(Fragment::Ivm(1, 1)) => IonVersion::V1_1, + _ => IonVersion::Unspecified, + } + } + + pub fn fragment_encoding(&self) -> IonEncoding { + let enc = self.fragments.iter().find(|f| matches!(f, Fragment::Text(_) | Fragment::Binary(_))); + match enc { + Some(Fragment::Text(_)) => IonEncoding::Text, + Some(Fragment::Binary(_)) => IonEncoding::Binary, + _ => IonEncoding::Unspecified, + } + } + + pub fn set_encoding(&mut self, enc: IonEncoding) { + self.encoding = enc; + } + + fn resolve_encoding(parent: IonEncoding, child: IonEncoding) -> IonEncoding { + match (parent, child) { + (a, b) if a == b => a, + (IonEncoding::Unspecified, n) => n, + (n, IonEncoding::Unspecified) => n, + _ => panic!("Mismatched encodings for nested contexts"), // TODO: Bubble error. + } + } + + pub fn input(&self, child_encoding: IonEncoding) -> InnerResult<(Vec, IonEncoding)> { + let encoding = Self::resolve_encoding(self.encoding(), child_encoding); + let (data, data_encoding) = match encoding { + IonEncoding::Text => (to_text(self, self.fragments.iter())?, encoding), + IonEncoding::Binary => (to_binary(self, self.fragments.iter())?, encoding), + IonEncoding::Unspecified => (to_binary(self, self.fragments.iter())?, IonEncoding::Binary), + }; + Ok((data, data_encoding)) + } + + pub fn read_all(&self, encoding: IonEncoding) -> InnerResult { + let (data, data_encoding) = self.input(encoding)?; + let data_slice = IonSlice::new(data); + + if self.fragments.is_empty() { + let empty: Vec = vec!(); + return Ok(empty.into()); + } + + let version = match self.version() { + IonVersion::Unspecified => IonVersion::V1_0, + v => v, + }; + + // Ok(Reader::new(AnyEncoding, data_slice)?.read_all_elements()?) + + match (version, data_encoding) { + (IonVersion::V1_0, IonEncoding::Binary) => + Ok(Reader::new(v1_0::Binary, data_slice)?.read_all_elements()?), + (IonVersion::V1_0, IonEncoding::Text) => + Ok(Reader::new(v1_0::Text, data_slice)?.read_all_elements()?), + (IonVersion::V1_0, IonEncoding::Unspecified) => + Ok(Reader::new(v1_0::Binary, data_slice)?.read_all_elements()?), + (IonVersion::V1_1, IonEncoding::Binary) => + Ok(Reader::new(v1_1::Binary, data_slice)?.read_all_elements()?), + (IonVersion::V1_1, IonEncoding::Text) => + Ok(Reader::new(v1_1::Text, data_slice)?.read_all_elements()?), + (IonVersion::V1_1, IonEncoding::Unspecified) => + Ok(Reader::new(v1_1::Binary, data_slice)?.read_all_elements()?), + _ => unreachable!(), + } + } + +} diff --git a/tests/conformance_dsl/continuation.rs b/tests/conformance_dsl/continuation.rs new file mode 100644 index 00000000..82e9f7f3 --- /dev/null +++ b/tests/conformance_dsl/continuation.rs @@ -0,0 +1,260 @@ +use super::*; +use super::context::Context; +use super::model::ModelValue; + +use ion_rs::{Element, Sequence}; + +#[derive(Clone, Debug)] +pub(crate) enum Continuation { + // expectations + Produces(Vec), + Denotes(Vec), + Signals(String), + // extensions + Extensions(Vec), + Then(Box), + Each(Vec, Box), + And(Vec), + Not(Box), +} + +impl Continuation { + pub fn evaluate(&self, ctx: &Context) -> InnerResult<()> { + match self { + // Produces is terminal, so we can evaluate. + Continuation::Produces(expected_elems) => { + let elems = ctx.read_all(ctx.encoding())?; + if expected_elems.len() != elems.len() { + Err(ConformanceErrorKind::MismatchedProduce) + } else { + let zip = expected_elems.iter().zip(elems.iter()); + match zip.fold(true, |acc, (x, y)| acc && (*x == *y)) { + true => Ok(()), + false => Err(ConformanceErrorKind::MismatchedProduce), + } + } + } + Continuation::Not(inner) => match inner.evaluate(ctx) { + Err(_) => Ok(()), + Ok(_) => Err(ConformanceErrorKind::UnknownError), + }, + Continuation::And(inners) => { + for c in inners { + c.evaluate(ctx)?; + } + Ok(()) + } + Continuation::Then(then) => then.evaluate(ctx), + Continuation::Denotes(expected_vals) => { + let elems = ctx.read_all(ctx.encoding())?; + if expected_vals.len() != elems.len() { + Err(ConformanceErrorKind::MismatchedDenotes) + } else { + let zip = expected_vals.iter().zip(elems.iter()); + match zip.fold(true, |acc, (x, y)| acc && (*x == *y)) { + true => Ok(()), + false => Err(ConformanceErrorKind::MismatchedDenotes), + } + } + } + Continuation::Extensions(exts) => { + for ext in exts { + ext.evaluate(ctx)?; + } + Ok(()) + } + Continuation::Each(branches, continuation) => { + for branch in branches { + let frags = vec!(branch.fragment.clone()); + let mut new_context = Context::extend(ctx, &frags); + new_context.set_encoding(branch.fragment.required_encoding()); + continuation.evaluate(&new_context)?; + } + Ok(()) + } + Continuation::Signals(msg) => { + match ctx.read_all(ctx.encoding()) { + Err(_e) => Ok(()), + Ok(_) => Err(ConformanceErrorKind::ExpectedSignal(msg.to_owned()))?, + } + } + } + } +} + +impl Default for Continuation { + fn default() -> Self { + Continuation::Produces(vec!()) + } +} + +pub fn parse_continuation(clause: Clause) -> InnerResult { + let continuation = match clause.tpe { + ClauseType::Produces => { + Continuation::Produces(clause.body.clone()) + } + ClauseType::And => { + if !clause.body.is_empty() { + let mut args = vec!(); + for elem in clause.body { + if let Some(seq) = elem.as_sequence() { + let clause = Clause::try_from(seq)?; + if clause.tpe.is_expectation() { + let continuation = parse_continuation(clause)?; + args.push(continuation); + } else { + return Err(ConformanceErrorKind::ExpectedExpectation); + } + } else { + return Err(ConformanceErrorKind::ExpectedExpectation); + } + } + Continuation::And(args) + } else { + return Err(ConformanceErrorKind::ExpectedExpectation) + } + } + ClauseType::Not => { + if clause.body.len() == 1 { + let inner_elem = clause.body.first().unwrap(); // SAFETY: Just tested len(). + if let Some(inner_seq) = inner_elem.as_sequence() { + let inner_clause = Clause::try_from(inner_seq)?; + if inner_clause.tpe.is_expectation() { + let continuation = parse_continuation(inner_clause)?; + return Ok(Continuation::Not(Box::new(continuation))); + } + } + } + return Err(ConformanceErrorKind::ExpectedExpectation); + } + ClauseType::Then => { + let then: Then = parse_document_like(&clause)?; + Continuation::Then(Box::new(then)) + } + ClauseType::Denotes => { + let mut values: Vec = vec!(); + for elem in clause.body { + if let Some(seq) = elem.as_sequence() { + let model_value = ModelValue::try_from(seq)?; + values.push(model_value); + } else { + return Err(ConformanceErrorKind::ExpectedModelValue); + } + } + Continuation::Denotes(values) + } + ClauseType::Each => { + let mut parsing_branches = true; + let mut sequence_idx = 0; + let mut branches: Vec = vec!(); + loop { + if sequence_idx >= clause.body.len() { + return Err(ConformanceErrorKind::ExpectedClause); + } + if parsing_branches { + let mut name: Option = None; + // Branch: name-string? fragment + // Check for name-string.. + if let Some(elem) = clause.body.get(sequence_idx).filter(|e| e.ion_type() == IonType::String) { + name = elem.as_string().map(|s| s.to_string()); + sequence_idx += 1; + } + + let seq = clause.body.get(sequence_idx) + .and_then(|e| e.as_sequence()) + .ok_or(ConformanceErrorKind::ExpectedModelValue)?; + let mut seq_iter = seq.iter().peekable(); + + let fragment = match Fragment::try_from(Sequence::new(seq_iter)) { + Ok(frag) => frag, + Err(ConformanceErrorKind::ExpectedFragment) => { + parsing_branches = false; + continue; + } + Err(x) => return Err(x), + }; + branches.push(EachBranch { + name, + fragment, + }); + } else { + let seq = clause.body.get(sequence_idx) + .and_then(|e| e.as_sequence()) + .ok_or(ConformanceErrorKind::ExpectedModelValue)?; + let clause = Clause::try_from(seq.clone())?; + match continuation::parse_continuation(clause) { + Ok(c) => return Ok(Continuation::Each(branches, Box::new(c))), + Err(e) => return Err(e), + } + } + sequence_idx += 1; + } + } + ClauseType::Signals => { + let msg = clause.body.first() + .and_then(|e| e.as_string()) + .ok_or(ConformanceErrorKind::ExpectedString)? + .to_string(); + Continuation::Signals(msg) + } + _ => unreachable!(), + }; + + + Ok(continuation) +} + +#[derive(Clone, Debug)] +pub(crate) struct EachBranch { + name: Option, + fragment: Fragment, +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct Then { + pub test_name: Option, + pub fragments: Vec, + pub continuation: Continuation, +} + +impl Then { + pub fn evaluate(&self, ctx: &Context) -> InnerResult<()> { + // We need to create a new context for the Then scope. + let mut then_ctx = Context::extend(ctx, &self.fragments); + then_ctx.set_encoding(self.fragment_encoding()); + then_ctx.set_version(self.fragment_version()); + + self.continuation.evaluate(&then_ctx) + } + + fn fragment_encoding(&self) -> IonEncoding { + let enc = self.fragments.iter().find(|f| matches!(f, Fragment::Text(_) | Fragment::Binary(_))); + match enc { + Some(Fragment::Text(_)) => IonEncoding::Text, + Some(Fragment::Binary(_)) => IonEncoding::Binary, + _ => IonEncoding::Unspecified, + } + } + + fn fragment_version(&self) -> IonVersion { + match self.fragments.first() { + Some(Fragment::Ivm(1, 0)) => IonVersion::V1_0, + Some(Fragment::Ivm(1, 1)) => IonVersion::V1_1, + _ => IonVersion::Unspecified, + } + } +} + +impl DocumentLike for Then { + fn set_name(&mut self, name: &str) { + self.test_name = Some(name.to_owned()); + } + + fn add_fragment(&mut self, frag: Fragment) { + self.fragments.push(frag); + } + + fn set_continuation(&mut self, continuation: Continuation) { + self.continuation = continuation; + } +} diff --git a/tests/conformance_dsl/document.rs b/tests/conformance_dsl/document.rs new file mode 100644 index 00000000..4ae99a1d --- /dev/null +++ b/tests/conformance_dsl/document.rs @@ -0,0 +1,121 @@ +use std::str::FromStr; + +use super::*; +use super::context::Context; +use super::continuation::*; + +use ion_rs::{Element, Sequence}; + +pub(crate) fn to_binary<'a, T: IntoIterator>(ctx: &'a Context, fragments: T) -> InnerResult> { + let mut bin_encoded = vec!(); + for frag in fragments { + let bin = frag.to_binary(ctx)?; + bin_encoded.extend(bin); + } + Ok(bin_encoded) +} + +pub(crate) fn to_text<'a, T: IntoIterator>(ctx: &'a Context, fragments: T) -> InnerResult> { + let mut txt_encoded = vec!(); + for frag in fragments { + let txt = frag.to_text(ctx)?; + txt_encoded.extend(txt); + txt_encoded.push(0x20); // Text fragments need to be separated by whitespace. + } + Ok(txt_encoded) +} + +#[derive(Debug, Default)] +pub(crate) struct Document { + pub name: Option, + pub fragments: Vec, + pub continuation: Continuation, +} + +impl Document { + pub fn run(&self) -> Result<()> { + let ctx = Context::new(IonVersion::Unspecified, self.encoding(), &self.fragments); + self.continuation.evaluate(&ctx)?; + Ok(()) + } + + fn encoding(&self) -> IonEncoding { + match self.fragments.iter().fold((false,false), |acc, f| { + (acc.0 || matches!(f, Fragment::Text(_)), acc.1 || matches!(f, Fragment::Binary(_))) + }) { + (true, false) => IonEncoding::Text, + (false, true) => IonEncoding::Binary, + (false, false) => IonEncoding::Unspecified, + (true, true) => panic!("Both binary and text fragments specified"), // TODO: Make + // error. + } + } +} + +impl DocumentLike for Document { + fn set_name(&mut self, name: &str) { + self.name = Some(name.to_owned()); + } + + fn add_fragment(&mut self, frag: Fragment) { + self.fragments.push(frag); + } + + fn set_continuation(&mut self, continuation: Continuation) { + self.continuation = continuation; + } +} + +impl FromStr for Document { + type Err = ConformanceError; + + fn from_str(other: &str) -> std::result::Result { + let element = Element::read_first(other)? + .ok_or(ConformanceErrorKind::ExpectedDocumentClause)?; + let Some(seq) = element.as_sequence() else { + return Err(ConformanceErrorKind::ExpectedDocumentClause.into()); + }; + Document::try_from(seq.clone()).map_err(|x| x.into()) + } +} + +impl TryFrom for Document { + type Error = ConformanceErrorKind; + + fn try_from(other: Sequence) -> InnerResult { + let clause: Clause = Clause::try_from(other)?; + + let mut doc: Document = parse_document_like(&clause)?; + let continuation = match clause.tpe { + ClauseType::Ion1_X => { + Continuation::Extensions(vec!( + Continuation::Then(Box::new(Then { + test_name: None, + fragments: [vec!(Fragment::Ivm(1, 0)), doc.fragments.clone()].concat(), + continuation: doc.continuation.clone(), + })), + Continuation::Then(Box::new(Then { + test_name: None, + fragments: [vec!(Fragment::Ivm(1, 1)), doc.fragments].concat(), + continuation: doc.continuation.clone(), + })), + )) + } + ClauseType::Ion1_0 => Continuation::Then(Box::new(Then { + test_name: None, + fragments: [vec!(Fragment::Ivm(1, 0)), doc.fragments].concat(), + continuation: doc.continuation.clone(), + })), + ClauseType::Ion1_1 => Continuation::Then(Box::new(Then { + test_name: None, + fragments: [vec!(Fragment::Ivm(1, 1)), doc.fragments].concat(), + continuation: doc.continuation.clone(), + })), + ClauseType::Document => return Ok(doc), + _ => return Err(ConformanceErrorKind::ExpectedDocumentClause), + }; + doc.continuation = continuation; + doc.fragments = vec!(); + Ok(doc) + } +} diff --git a/tests/conformance_dsl/fragment.rs b/tests/conformance_dsl/fragment.rs new file mode 100644 index 00000000..709b3030 --- /dev/null +++ b/tests/conformance_dsl/fragment.rs @@ -0,0 +1,139 @@ +use ion_rs::{Element, Sequence}; +use ion_rs::{v1_0, v1_1, WriteConfig, Encoding}; + +use super::*; +use super::context::Context; + +trait FragmentImpl { + fn encode(&self, config: impl Into>) -> InnerResult>; +} + + +#[derive(Clone, Debug)] +pub(crate) enum Fragment { + Binary(Vec), + Each(Vec), + Ivm(i64, i64), + Text(String), + TopLevel(TopLevel), + MacTab, // TODO: Implement. + Encoding, // TODO: Implement. +} + +static EMPTY_TOPLEVEL: Fragment = Fragment::TopLevel(TopLevel { elems: vec!() }); + +impl Fragment { + pub fn to_binary(&self, ctx: &Context) -> InnerResult> { + match ctx.version() { + IonVersion::V1_1 => self.write_as_binary(ctx, v1_1::Binary), + _ => self.write_as_binary(ctx, v1_0::Binary), + } + } + + pub fn to_text(&self, ctx: &Context) -> InnerResult> { + match ctx.version() { + IonVersion::V1_1 => self.write_as_text(ctx, v1_1::Text), + _ => self.write_as_text(ctx, v1_0::Text), + } + } + + fn write_as_binary(&self, _ctx: &Context, config: impl Into>) -> InnerResult> { + match self { + Fragment::TopLevel(toplevel) => toplevel.encode(config), + Fragment::Binary(bin) => Ok(bin.clone()), + Fragment::Text(_) => unreachable!(), + Fragment::Ivm(maj, min) => Ok([0xE0, *maj as u8, *min as u8, 0xEA].to_vec()), + _ => unimplemented!(), + } + } + + fn write_as_text(&self, _ctx: &Context, config: impl Into>) -> InnerResult> { + match self { + Fragment::TopLevel(toplevel) => toplevel.encode(config), + Fragment::Text(txt) => { + let bytes = txt.as_bytes(); + Ok(bytes.to_owned()) + } + Fragment::Binary(_) => unreachable!(), + Fragment::Ivm(maj, min) => return Ok(format!("$ion_{}_{}", maj, min).as_bytes().to_owned()), + _ => unimplemented!(), + } + } + + + pub fn required_encoding(&self) -> IonEncoding { + match self { + Fragment::Text(_) => IonEncoding::Text, + Fragment::Binary(_) => IonEncoding::Binary, + _ => IonEncoding::Unspecified, + } + } +} + +impl TryFrom for Fragment { + type Error = ConformanceErrorKind; + + fn try_from(other: Clause) -> InnerResult { + let frag = match other.tpe { + ClauseType::Text => { + // TODO: grammar is "(" "text" string* ")", we need to handle 0+ strings. + let txt = match other.body.first() { + Some(txt) if txt.ion_type() == IonType::String => txt.as_string().unwrap().to_owned(), + Some(_) => return Err(ConformanceErrorKind::UnexpectedValue), + None => String::from(""), + }; + Fragment::Text(txt) + } + ClauseType::Binary => { + // TODO: Support string of hex values. + let mut bytes: Vec = vec!(); + for elem in other.body { + if let Some(byte) = elem.as_i64() { + if (0..=255).contains(&byte) { + bytes.push(byte as u8); + } + } + } + Fragment::Binary(bytes) + } + ClauseType::Ivm => { + // IVM: (ivm ) + let maj = other.body.first().map(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?.unwrap(); + let min = other.body.get(1).map(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?.unwrap(); + Fragment::Ivm(maj, min) + } + ClauseType::TopLevel => Fragment::TopLevel(TopLevel { elems: other.body }), + ClauseType::Encoding => Fragment::Encoding, + ClauseType::MacTab => Fragment::MacTab, + _ => return Err(ConformanceErrorKind::ExpectedFragment), + }; + Ok(frag) + } +} + +impl TryFrom for Fragment { + type Error = ConformanceErrorKind; + + fn try_from(other: Sequence) -> InnerResult { + let clause = Clause::try_from(other)?; + Fragment::try_from(clause) + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct TopLevel { + elems: Vec, +} + +impl FragmentImpl for TopLevel { + fn encode(&self, config: impl Into>) -> InnerResult> { + use ion_rs::Writer; + let mut buffer = Vec::with_capacity(1024); + let mut writer = Writer::new(config, buffer)?; + for elem in self.elems.as_slice() { + writer.write(elem)?; + } + buffer = writer.close()?; + Ok(buffer) + } +} diff --git a/tests/conformance_dsl/mod.rs b/tests/conformance_dsl/mod.rs new file mode 100644 index 00000000..1dc45f36 --- /dev/null +++ b/tests/conformance_dsl/mod.rs @@ -0,0 +1,269 @@ +#![allow(dead_code)] + +mod context; +mod document; +mod continuation; +mod model; +mod fragment; +mod clause; + +use std::io::Read; +use std::path::{Path, PathBuf}; + +use ion_rs::{Element, IonError, IonStream, IonType}; + +use clause::*; +use document::*; +use fragment::*; + +#[allow(unused)] +pub(crate) mod prelude { + pub(crate) use super::document::Document; + pub(crate) use super::TestCollection; + pub(crate) use super::IonVersion; +} + +#[derive(Clone, Default, Debug)] +pub(crate) enum ConformanceErrorKind { + #[default] + UnknownError, + IoError(std::io::ErrorKind), + IonError(IonError), + UnexpectedEndOfDocument, + UnknownClause(String), + ExpectedDocumentClause, + ExpectedClause, + ExpectedFragment, + ExpectedExpectation, + ExpectedModelValue, + ExpectedFloatString, + ExpectedAsciiCodepoint, + ExpectedSymbolType, + ExpectedInteger, + ExpectedSignal(String), + ExpectedString, + MismatchedProduce, + MismatchedDenotes, + UnexpectedValue, + UnknownVersion, +} + +impl From for ConformanceErrorKind { + fn from(other: std::io::Error) -> Self { + ConformanceErrorKind::IoError(other.kind()) + } +} + +impl From for ConformanceErrorKind { + fn from(other: IonError) -> Self { + ConformanceErrorKind::IonError(other) + } +} + +#[derive(Clone, Default, Debug)] +struct ConformanceErrorImpl { + file: PathBuf, + test_name: String, + kind: ConformanceErrorKind, +} + +#[derive(Clone, Default, Debug)] +pub struct ConformanceError(Box); + +impl From for ConformanceError { + fn from(inner: ConformanceErrorImpl) -> Self { + ConformanceError(Box::new(inner)) + } +} + +impl From for ConformanceError { + fn from(other: std::io::Error) -> Self { + ConformanceErrorImpl { + kind: ConformanceErrorKind::IoError(other.kind()), + ..Default::default() + }.into() + } +} + +impl From for ConformanceError { + fn from(other: IonError) -> Self { + ConformanceErrorImpl { + kind: ConformanceErrorKind::IonError(other), + ..Default::default() + }.into() + } +} + +impl From for ConformanceError { + fn from(other: ConformanceErrorKind) -> Self { + ConformanceErrorImpl { + kind: other, + ..Default::default() + }.into() + } +} + +// Used for internal error handling. +type InnerResult = std::result::Result; + +// Used for public conformance API error handling. +pub(crate) type Result = std::result::Result; + +// Encoding captures whether an encoding is forced by including a text, or binary clause. +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) enum IonEncoding { + Text, // Text clause used. + Binary, // Binary clause used. + Unspecified, // No encoding specific clauses. +} + + +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Default, Debug, PartialEq)] +pub(crate) enum IonVersion { + #[default] + Unspecified, + V1_0, + V1_1, + V1_X, +} + +/// A document-like is anything that matches the grammar of a document. Currently this includes +/// both Document, and Then clauses. +pub(crate) trait DocumentLike: Default { + fn set_name(&mut self, name: &str); + fn add_fragment(&mut self, frag: Fragment); + fn set_continuation(&mut self, continuation: continuation::Continuation); +} + +pub(crate) fn parse_document_like(clause: &Clause) -> InnerResult { + // let clause: Clause = Clause::try_from(seq)?; + let mut doc_like = T::default(); + let mut sequence_idx = 0; + + // We have an optional name as the second argument.. + if let Some(elem) = clause.body.first().filter(|e| e.ion_type() == IonType::String) { + if let Some(s) = elem.as_string() { + doc_like.set_name(s); + } + sequence_idx += 1; + } + + let mut handle_fragments = true; + loop { + if sequence_idx >= clause.body.len() { + return Err(ConformanceErrorKind::ExpectedClause); + } + let element = clause.body.get(sequence_idx).expect("unwrapping element"); + if handle_fragments { + let Some(seq) = element.as_sequence() else { + return Err(ConformanceErrorKind::ExpectedClause) + }; + let fragment = match Fragment::try_from(seq.clone()) { + Ok(frag) => frag, + Err(ConformanceErrorKind::ExpectedFragment) => { + handle_fragments = false; + continue; + } + Err(x) => return Err(x), + }; + doc_like.add_fragment(fragment); + sequence_idx += 1 + } else { + let Some(seq) = element.as_sequence() else { + return Err(ConformanceErrorKind::ExpectedClause) + }; + let clause: Clause = seq.clone().try_into().expect("unable to convert to clause"); + match continuation::parse_continuation(clause) { + Ok(c) => doc_like.set_continuation(c), + Err(e) => return Err(e), + } + break; + } + } + Ok(doc_like) +} + + + +/// A collection of Tests, usually stored together in a file. +#[derive(Debug)] +pub(crate) struct TestCollection { + documents: Vec, +} + +impl TestCollection { + /// Loads a TestCollection from a file at the provided path. + pub fn load>(path: P) -> Result { + let test_file = std::fs::File::open(&path)?; + match Self::load_from(test_file) { + Err(e) => Err(ConformanceErrorImpl { + file: path.as_ref().to_owned(), + ..*e.0 + }.into()), + Ok(t) => Ok(t), + } + } + + pub fn load_from(reader: R) -> Result { + let iter = Element::iter(IonStream::new(reader))?; + let mut docs: Vec = vec!(); + + for element in iter { + let element = element?; + match element.ion_type() { + IonType::SExp => { + let seq = element.as_sexp().unwrap(); + let doc = match Document::try_from(seq.clone()) { + Err(kind) => return Err(ConformanceErrorImpl { + kind, + ..Default::default() + }.into()), + Ok(doc) => doc, + }; + docs.push(doc); + } + _ => todo!(), + } + } + + let collection = TestCollection{ + documents: docs, + }; + + Ok(collection) + } + + pub fn run(&self) -> Result<()> { + for test in self.documents.iter() { + test.run()?; + } + Ok(()) + } + + pub fn len(&self) -> usize { + self.documents.len() + } + + pub fn iter(&self) -> impl Iterator { + self.documents.iter() + } + +} + +pub(crate) fn parse_text_exp<'a, I: IntoIterator>(elems: I) -> InnerResult { + let bytes: Vec> = elems.into_iter().map(|v| match v.ion_type() { + IonType::String => v.as_string().map(|s| Ok(s.as_bytes().to_vec())).unwrap(), + IonType::Int => { + match v.as_i64() { + Some(i) if i < 256 => Ok(vec!(i as u8)), + _ => Err(ConformanceErrorKind::ExpectedAsciiCodepoint), + } + } + _ => Err(ConformanceErrorKind::ExpectedModelValue), + }).collect::>>>()?; + + let val_string = bytes.iter().map(|v| unsafe { String::from_utf8_unchecked(v.to_vec()) }).collect(); + Ok(val_string) + +} diff --git a/tests/conformance_dsl/model.rs b/tests/conformance_dsl/model.rs new file mode 100644 index 00000000..ef70196a --- /dev/null +++ b/tests/conformance_dsl/model.rs @@ -0,0 +1,157 @@ +use ion_rs::{Decimal, Element, IonType, Sequence}; +use super::{Clause, ClauseType, ConformanceErrorKind, InnerResult, parse_text_exp}; + +use std::collections::HashMap; + +#[derive(Debug, Clone)] +pub(crate) enum SymTok { + Text(String), + Offset(i64), + Absent(String, i64), +} + +#[derive(Debug, Clone)] +pub(crate) enum ModelValue { + Null(IonType), + Bool(bool), + Int(i64), + Float(f64), + Decimal(i64, i64), + // TODO: Timestamp + String(String), + Symbol(SymTok), + List(Vec), + Sexp(Vec), + Struct(HashMap), + Blob(Vec), + Clob(Vec), +} + +impl TryFrom<&Sequence> for ModelValue { + type Error = ConformanceErrorKind; + + fn try_from(other: &Sequence) -> InnerResult { + let elems: Vec = other.iter().cloned().collect(); + let tpe_sym = elems.first().ok_or(ConformanceErrorKind::ExpectedModelValue)?.as_symbol().ok_or(ConformanceErrorKind::ExpectedModelValue)?; + let tpe = tpe_sym.text().ok_or(ConformanceErrorKind::ExpectedModelValue)?; + match tpe { + "Null" => { + let type_elem = elems.get(1).ok_or(ConformanceErrorKind::ExpectedModelValue)?; + let type_str = type_elem.as_symbol() + .and_then(|s| s.text()) + .ok_or(ConformanceErrorKind::ExpectedModelValue)?; + + match ion_type_from_str(type_str) { + Some(tpe) => Ok(ModelValue::Null(tpe)), + None => Err(ConformanceErrorKind::ExpectedModelValue), + } + } + "Bool" => { + let value = elems.get(1) + .and_then(|e| e.as_bool()) + .ok_or(ConformanceErrorKind::ExpectedModelValue)?; + Ok(ModelValue::Bool(value)) + } + "Int" => { + let value = elems.get(1) + .and_then(|e| e.as_i64()) + .ok_or(ConformanceErrorKind::ExpectedModelValue)?; + Ok(ModelValue::Int(value)) + } + "Float" => { + let value_str = elems.get(1) + .and_then(|e| e.as_string()) + .ok_or(ConformanceErrorKind::ExpectedModelValue)?; + match value_str.parse::() { + Ok(f) => Ok(ModelValue::Float(f)), + Err(_) => Err(ConformanceErrorKind::ExpectedFloatString), + } + } + "Decimal" => todo!(), + "String" => { + let string = parse_text_exp(elems.iter().skip(1))?; + Ok(ModelValue::String(string)) + } + "Symbol" => { + let value = elems.get(1).ok_or(ConformanceErrorKind::ExpectedSymbolType)?; + match value.ion_type() { + IonType::String => Ok(ModelValue::Symbol(SymTok::Text(value.as_string().unwrap().to_owned()))), + IonType::Int => Ok(ModelValue::Symbol(SymTok::Offset(value.as_i64().unwrap()))), + IonType::SExp => { + let clause: Clause = value.as_sequence().unwrap().try_into()?; + + match clause.tpe { + ClauseType::Text => { + let text = parse_text_exp(clause.body.iter())?; + Ok(ModelValue::Symbol(SymTok::Text(text))) + }, + ClauseType::Absent => { + let text = clause.body.get(1).and_then(|v| v.as_string()).ok_or(ConformanceErrorKind::ExpectedSymbolType)?; + let offset = clause.body.get(2).and_then(|v| v.as_i64()).ok_or(ConformanceErrorKind::ExpectedSymbolType)?; + Ok(ModelValue::Symbol(SymTok::Absent(text.to_string(), offset))) + } + _ => unreachable!(), + } + } + _ => Err(ConformanceErrorKind::ExpectedSymbolType), + } + } + "List" => todo!(), + "Sexp" => todo!(), + "Struct" => todo!(), + "Blob" => todo!(), + "Clob" => todo!(), + _ => unreachable!(), + } + } +} + +impl PartialEq for ModelValue { + fn eq(&self, other: &Element) -> bool { + match self { + ModelValue::Null(tpe) => other.ion_type() == *tpe && other.is_null(), + ModelValue::Bool(val) => other.as_bool() == Some(*val), + ModelValue::Int(val) => other.as_i64() == Some(*val), + ModelValue::Float(val) => other.as_float() == Some(*val), + ModelValue::Decimal(coef, exp) => other.as_decimal() == Some(Decimal::new(*coef, *exp)), + // TODO: Timestamp + ModelValue::String(val) => other.as_string() == Some(val), + ModelValue::List(_vals) => todo!(), + ModelValue::Sexp(_vals) => todo!(), + ModelValue::Struct(_fields) => todo!(), + ModelValue::Blob(data) => other.as_blob() == Some(data.as_slice()), + ModelValue::Clob(data) => other.as_clob() == Some(data.as_slice()), + ModelValue::Symbol(sym) => { + if let Some(other_sym) = other.as_symbol() { + match sym { + SymTok::Text(text) => Some(text.as_ref()) == other_sym.text(), + SymTok::Offset(_offset) => todo!(), + SymTok::Absent(_text, _offset) => todo!(), + } + } else { + false + } + } + _ => todo!(), + } + } +} + +fn ion_type_from_str(name: &str) -> Option { + match name { + "null" => Some(IonType::Null), + "bool" => Some(IonType::Bool), + "int" => Some(IonType::Int), + "float" => Some(IonType::Float), + "decimal" => Some(IonType::Decimal), + "timestamp" => Some(IonType::Timestamp), + "string" => Some(IonType::String), + "symbol" => Some(IonType::Symbol), + "list" => Some(IonType::List), + "sexp" => Some(IonType::SExp), + "struct" => Some(IonType::Struct), + "blob" => Some(IonType::Blob), + "clob" => Some(IonType::Clob), + _ => None, + } +} diff --git a/tests/conformance_tests.rs b/tests/conformance_tests.rs new file mode 100644 index 00000000..dd2c48cd --- /dev/null +++ b/tests/conformance_tests.rs @@ -0,0 +1,72 @@ +#![cfg(feature = "experimental-reader-writer")] +mod conformance_dsl; +use conformance_dsl::prelude::*; + +use test_generator::test_resources; + +use std::str::FromStr; + + + +mod implementation { + use super::*; + + // #[test] + fn toplevel_absent_symbols() { + let test = r#" + (ion_1_0 (toplevel '#$1') (produces $ion)) + "#; + + Document::from_str(test) + .unwrap_or_else(|e| panic!("Failed to load document: <<{}>>\n{:?}", test, e)) + .run() + .unwrap_or_else(|e| panic!("Test failed: <<{}>>\n{:?}", test, e)); + } + + #[test] + fn test_simple_docs() { + let tests: &[&str] = &[ + "(document (produces ))", + "(document (toplevel a) (produces a))", + "(document (text \"a\") (produces a))", + "(ion_1_0 (produces ))", + "(ion_1_1 (produces ))", + "(document (and (produces ) (produces )))", + "(document (text \"a\") (not (and (produces b) (produces c))))", + "(ion_1_1 (binary 0x60 0x61 0x01 0xEB 0x01) (produces 0 1 null.int))", + r#"(ion_1_0 (then (text "a") (produces a)))"#, + r#"(ion_1_1 (text "a") (text "b") (text "c") (produces a b c))"#, + r#"(ion_1_1 (text "\"Hello\" null.int false") (denotes (String "Hello") (Null int) (Bool false)))"#, + r#"(ion_1_1 (each + (text "0") + (binary 0x60) + (denotes (Int 0))) + )"#, + r#"(document (ivm 1 2) (signals "Invalid Version"))"#, + ]; + for test in tests { + println!("Testing: {}", test); + Document::from_str(test) + .unwrap_or_else(|e| panic!("Failed to load document: <<{}>>\n{:?}", test, e)) + .run() + .unwrap_or_else(|e| panic!("Test failed for simple doc: <<{}>>\n{:?}", test, e)); + } + } +} + +mod ion_tests { + use super::*; + + #[test_resources("ion-tests/conformance/null.ion")] + #[test_resources("ion-tests/conformance/core/typed_null.ion")] + #[test_resources("ion-tests/conformance/core/string_symbol.ion")] + #[test_resources("ion-tests/conformance/core/empty_document.ion")] + // #[test_resources("ion-tests/conformance/ivm.ion")] + fn conformance(file_name: &str) { + println!("Testing: {}", file_name); + let collection = TestCollection::load(file_name).expect("unable to load test file"); + + println!("Collection: {:?}", collection); + collection.run().expect("failed to run collection"); + } +} diff --git a/tests/ion_data_consistency.rs b/tests/ion_data_consistency.rs index 39aebff8..6add979a 100644 --- a/tests/ion_data_consistency.rs +++ b/tests/ion_data_consistency.rs @@ -20,23 +20,23 @@ fn contains_path(paths: &[&str], file_name: &str) -> bool { } const SKIP_LIST: &[&str] = &[ - "ion-tests/iontestdata_1_0/good/equivs/localSymbolTableAppend.ion", - "ion-tests/iontestdata_1_0/good/equivs/localSymbolTableNullSlots.ion", - "ion-tests/iontestdata_1_0/good/equivs/nonIVMNoOps.ion", + "ion-tests/iontestdata/good/equivs/localSymbolTableAppend.ion", + "ion-tests/iontestdata/good/equivs/localSymbolTableNullSlots.ion", + "ion-tests/iontestdata/good/equivs/nonIVMNoOps.ion", // Integers outside the i128 range - "ion-tests/iontestdata_1_0/good/intBigSize16.10n", - "ion-tests/iontestdata_1_0/good/intBigSize256.ion", - "ion-tests/iontestdata_1_0/good/intBigSize256.10n", - "ion-tests/iontestdata_1_0/good/intBigSize512.ion", - "ion-tests/iontestdata_1_0/good/intBigSize1201.10n", - "ion-tests/iontestdata_1_0/good/equivs/bigInts.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarInt.ion", - "ion-tests/iontestdata_1_0/good/equivs/intsLargePositive3.10n", - "ion-tests/iontestdata_1_0/good/equivs/intsLargeNegative3.10n", + "ion-tests/iontestdata/good/intBigSize16.10n", + "ion-tests/iontestdata/good/intBigSize256.ion", + "ion-tests/iontestdata/good/intBigSize256.10n", + "ion-tests/iontestdata/good/intBigSize512.ion", + "ion-tests/iontestdata/good/intBigSize1201.10n", + "ion-tests/iontestdata/good/equivs/bigInts.ion", + "ion-tests/iontestdata/good/subfieldVarInt.ion", + "ion-tests/iontestdata/good/equivs/intsLargePositive3.10n", + "ion-tests/iontestdata/good/equivs/intsLargeNegative3.10n", ]; -#[test_resources("ion-tests/iontestdata_1_0/good/equivs/**/*.ion")] -#[test_resources("ion-tests/iontestdata_1_0/good/equivs/**/*.10n")] +#[test_resources("ion-tests/iontestdata/good/equivs/**/*.ion")] +#[test_resources("ion-tests/iontestdata/good/equivs/**/*.10n")] fn ion_data_eq_ord_consistency(file_name: &str) { // Best-effort tests to check that Eq and Ord are consistent. From 2c3808d87e4ce9fe4a344de8aeb2f28d0c33b818 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Mon, 26 Aug 2024 02:36:30 -0700 Subject: [PATCH 02/13] Fix remaining paths for ion 1.0 iontestdata --- tests/detect_incomplete_text.rs | 2 +- tests/element_display.rs | 42 ++++++------ tests/ion_tests/lazy_element_ion_tests.rs | 12 ++-- tests/ion_tests/mod.rs | 78 +++++++++++------------ 4 files changed, 67 insertions(+), 67 deletions(-) diff --git a/tests/detect_incomplete_text.rs b/tests/detect_incomplete_text.rs index abea7abf..c98c176a 100644 --- a/tests/detect_incomplete_text.rs +++ b/tests/detect_incomplete_text.rs @@ -37,7 +37,7 @@ static SKIP_LIST_1_1: LazyLock> = LazyLock::new(|| { .collect() }); -#[test_resources("ion-tests/iontestdata_1_0/good/**/*.ion")] +#[test_resources("ion-tests/iontestdata/good/**/*.ion")] fn detect_incomplete_input_1_0(file_name: &str) { incomplete_text_detection_test(&SKIP_LIST_1_0, file_name).unwrap() } diff --git a/tests/element_display.rs b/tests/element_display.rs index a2aca08a..a269f297 100644 --- a/tests/element_display.rs +++ b/tests/element_display.rs @@ -10,34 +10,34 @@ mod ion_tests; const TO_STRING_SKIP_LIST: &[&str] = &[ // These tests have shared symbol table imports in them, which the Reader does not // yet support. - "ion-tests/iontestdata_1_0/good/subfieldVarInt.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt15bit.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt16bit.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt32bit.ion", + "ion-tests/iontestdata/good/subfieldVarInt.ion", + "ion-tests/iontestdata/good/subfieldVarUInt.ion", + "ion-tests/iontestdata/good/subfieldVarUInt15bit.ion", + "ion-tests/iontestdata/good/subfieldVarUInt16bit.ion", + "ion-tests/iontestdata/good/subfieldVarUInt32bit.ion", // This test requires the reader to be able to read symbols whose ID is encoded // with more than 8 bytes. Having a symbol table with more than 18 quintillion // symbols is not very practical. - "ion-tests/iontestdata_1_0/good/typecodes/T7-large.10n", - "ion-tests/iontestdata_1_0/good/item1.10n", - "ion-tests/iontestdata_1_0/good/localSymbolTableImportZeroMaxId.ion", - "ion-tests/iontestdata_1_0/good/testfile35.ion", + "ion-tests/iontestdata/good/typecodes/T7-large.10n", + "ion-tests/iontestdata/good/item1.10n", + "ion-tests/iontestdata/good/localSymbolTableImportZeroMaxId.ion", + "ion-tests/iontestdata/good/testfile35.ion", // These files are encoded in utf16 and utf32; the reader currently assumes utf8. - "ion-tests/iontestdata_1_0/good/utf16.ion", - "ion-tests/iontestdata_1_0/good/utf32.ion", + "ion-tests/iontestdata/good/utf16.ion", + "ion-tests/iontestdata/good/utf32.ion", // Test files that include Int values outside the range supported by i128 - "ion-tests/iontestdata_1_0/good/intBigSize16.10n", - "ion-tests/iontestdata_1_0/good/intBigSize256.ion", - "ion-tests/iontestdata_1_0/good/intBigSize256.10n", - "ion-tests/iontestdata_1_0/good/intBigSize512.ion", - "ion-tests/iontestdata_1_0/good/intBigSize1201.10n", - "ion-tests/iontestdata_1_0/good/equivs/bigInts.ion", - "ion-tests/iontestdata_1_0/good/equivs/intsLargePositive3.10n", - "ion-tests/iontestdata_1_0/good/equivs/intsLargeNegative3.10n", + "ion-tests/iontestdata/good/intBigSize16.10n", + "ion-tests/iontestdata/good/intBigSize256.ion", + "ion-tests/iontestdata/good/intBigSize256.10n", + "ion-tests/iontestdata/good/intBigSize512.ion", + "ion-tests/iontestdata/good/intBigSize1201.10n", + "ion-tests/iontestdata/good/equivs/bigInts.ion", + "ion-tests/iontestdata/good/equivs/intsLargePositive3.10n", + "ion-tests/iontestdata/good/equivs/intsLargeNegative3.10n", ]; -#[test_resources("ion-tests/iontestdata_1_0/good/**/*.ion")] -#[test_resources("ion-tests/iontestdata_1_0/good/**/*.10n")] +#[test_resources("ion-tests/iontestdata/good/**/*.ion")] +#[test_resources("ion-tests/iontestdata/good/**/*.10n")] fn test_to_string(file_name: &str) { if contains_path(TO_STRING_SKIP_LIST, file_name) { println!("IGNORING: {file_name}"); diff --git a/tests/ion_tests/lazy_element_ion_tests.rs b/tests/ion_tests/lazy_element_ion_tests.rs index 81edd3f6..fbdde512 100644 --- a/tests/ion_tests/lazy_element_ion_tests.rs +++ b/tests/ion_tests/lazy_element_ion_tests.rs @@ -56,22 +56,22 @@ good_round_trip! { fn pretty_lines(Format::Text(TextFormat::Pretty), Format::Text(TextFormat::Lines)); } -#[test_resources("ion-tests/iontestdata_1_0/bad/**/*.ion")] -#[test_resources("ion-tests/iontestdata_1_0/bad/**/*.10n")] +#[test_resources("ion-tests/iontestdata/bad/**/*.ion")] +#[test_resources("ion-tests/iontestdata/bad/**/*.10n")] fn lazy_bad(file_name: &str) { bad(LazyReaderElementApi, file_name) } -#[test_resources("ion-tests/iontestdata_1_0/good/equivs/**/*.ion")] -#[test_resources("ion-tests/iontestdata_1_0/good/equivs/**/*.10n")] +#[test_resources("ion-tests/iontestdata/good/equivs/**/*.ion")] +#[test_resources("ion-tests/iontestdata/good/equivs/**/*.10n")] fn lazy_equivs(file_name: &str) { equivs(LazyReaderElementApi, file_name) } -#[test_resources("ion-tests/iontestdata_1_0/good/non-equivs/**/*.ion")] +#[test_resources("ion-tests/iontestdata/good/non-equivs/**/*.ion")] // no binary files exist and the macro doesn't like empty globs... // see frehberg/test-generator#12 -//#[test_resources("ion-tests/iontestdata_1_0/good/non-equivs/**/*.10n")] +//#[test_resources("ion-tests/iontestdata/good/non-equivs/**/*.10n")] fn lazy_non_equivs(file_name: &str) { non_equivs(LazyReaderElementApi, file_name) } diff --git a/tests/ion_tests/mod.rs b/tests/ion_tests/mod.rs index f273fdbc..f3ae35ff 100644 --- a/tests/ion_tests/mod.rs +++ b/tests/ion_tests/mod.rs @@ -320,9 +320,9 @@ macro_rules! good_round_trip { (use $ElementApiImpl:ident; $(fn $test_name:ident($format1:expr, $format2:expr);)+) => { mod good_round_trip_tests { use super::*; $( - #[test_resources("ion-tests/iontestdata_1_0/good/**/*.ion")] + #[test_resources("ion-tests/iontestdata/good/**/*.ion")] //#[test_resources("ion-tests/iontestdata_1_1/good/**/*.ion")] - #[test_resources("ion-tests/iontestdata_1_0/good/**/*.10n")] + #[test_resources("ion-tests/iontestdata/good/**/*.10n")] fn $test_name(file_name: &str) { $ElementApiImpl::assert_file($ElementApiImpl::global_skip_list(), file_name, || { $ElementApiImpl::assert_three_way_round_trip(file_name, $format1, $format2) @@ -381,61 +381,61 @@ pub fn non_equivs(_element_api: E, file_name: &str) { pub const ELEMENT_GLOBAL_SKIP_LIST: SkipList = &[ // The binary reader does not check whether nested values are longer than their // parent container. - "ion-tests/iontestdata_1_0/bad/listWithValueLargerThanSize.10n", + "ion-tests/iontestdata/bad/listWithValueLargerThanSize.10n", // ROUND TRIP // These tests have shared symbol table imports in them, which the Reader does not // yet support. - "ion-tests/iontestdata_1_0/good/subfieldVarInt.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt15bit.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt16bit.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt32bit.ion", + "ion-tests/iontestdata/good/subfieldVarInt.ion", + "ion-tests/iontestdata/good/subfieldVarUInt.ion", + "ion-tests/iontestdata/good/subfieldVarUInt15bit.ion", + "ion-tests/iontestdata/good/subfieldVarUInt16bit.ion", + "ion-tests/iontestdata/good/subfieldVarUInt32bit.ion", // This test requires the reader to be able to read symbols whose ID is encoded // with more than 8 bytes. Having a symbol table with more than 18 quintillion // symbols is not very practical. - "ion-tests/iontestdata_1_0/good/typecodes/T7-large.10n", + "ion-tests/iontestdata/good/typecodes/T7-large.10n", // --- // Requires importing shared symbol tables - "ion-tests/iontestdata_1_0/good/item1.10n", - "ion-tests/iontestdata_1_0/good/localSymbolTableImportZeroMaxId.ion", + "ion-tests/iontestdata/good/item1.10n", + "ion-tests/iontestdata/good/localSymbolTableImportZeroMaxId.ion", // Requires importing shared symbol tables - "ion-tests/iontestdata_1_0/good/testfile35.ion", + "ion-tests/iontestdata/good/testfile35.ion", // These files are encoded in utf16 and utf32; the reader currently assumes utf8. - "ion-tests/iontestdata_1_0/good/utf16.ion", - "ion-tests/iontestdata_1_0/good/utf32.ion", + "ion-tests/iontestdata/good/utf16.ion", + "ion-tests/iontestdata/good/utf32.ion", // NON-EQUIVS - "ion-tests/iontestdata_1_0/good/non-equivs/localSymbolTableWithAnnotations.ion", - "ion-tests/iontestdata_1_0/good/non-equivs/symbolTablesUnknownText.ion", + "ion-tests/iontestdata/good/non-equivs/localSymbolTableWithAnnotations.ion", + "ion-tests/iontestdata/good/non-equivs/symbolTablesUnknownText.ion", // Integers outside the i128 range - "ion-tests/iontestdata_1_0/good/intBigSize16.10n", - "ion-tests/iontestdata_1_0/good/intBigSize256.ion", - "ion-tests/iontestdata_1_0/good/intBigSize256.10n", - "ion-tests/iontestdata_1_0/good/intBigSize512.ion", - "ion-tests/iontestdata_1_0/good/intBigSize1201.10n", - "ion-tests/iontestdata_1_0/good/equivs/bigInts.ion", - "ion-tests/iontestdata_1_0/good/equivs/intsLargePositive3.10n", - "ion-tests/iontestdata_1_0/good/equivs/intsLargeNegative3.10n", + "ion-tests/iontestdata/good/intBigSize16.10n", + "ion-tests/iontestdata/good/intBigSize256.ion", + "ion-tests/iontestdata/good/intBigSize256.10n", + "ion-tests/iontestdata/good/intBigSize512.ion", + "ion-tests/iontestdata/good/intBigSize1201.10n", + "ion-tests/iontestdata/good/equivs/bigInts.ion", + "ion-tests/iontestdata/good/equivs/intsLargePositive3.10n", + "ion-tests/iontestdata/good/equivs/intsLargeNegative3.10n", ]; pub const ELEMENT_ROUND_TRIP_SKIP_LIST: SkipList = &[ - "ion-tests/iontestdata_1_0/good/item1.10n", - "ion-tests/iontestdata_1_0/good/localSymbolTableImportZeroMaxId.ion", - "ion-tests/iontestdata_1_0/good/notVersionMarkers.ion", - "ion-tests/iontestdata_1_0/good/subfieldInt.ion", - "ion-tests/iontestdata_1_0/good/subfieldUInt.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarInt.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt15bit.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt16bit.ion", - "ion-tests/iontestdata_1_0/good/subfieldVarUInt32bit.ion", - "ion-tests/iontestdata_1_0/good/utf16.ion", - "ion-tests/iontestdata_1_0/good/utf32.ion", + "ion-tests/iontestdata/good/item1.10n", + "ion-tests/iontestdata/good/localSymbolTableImportZeroMaxId.ion", + "ion-tests/iontestdata/good/notVersionMarkers.ion", + "ion-tests/iontestdata/good/subfieldInt.ion", + "ion-tests/iontestdata/good/subfieldUInt.ion", + "ion-tests/iontestdata/good/subfieldVarInt.ion", + "ion-tests/iontestdata/good/subfieldVarUInt.ion", + "ion-tests/iontestdata/good/subfieldVarUInt15bit.ion", + "ion-tests/iontestdata/good/subfieldVarUInt16bit.ion", + "ion-tests/iontestdata/good/subfieldVarUInt32bit.ion", + "ion-tests/iontestdata/good/utf16.ion", + "ion-tests/iontestdata/good/utf32.ion", ]; pub const ELEMENT_EQUIVS_SKIP_LIST: SkipList = &[ - "ion-tests/iontestdata_1_0/good/equivs/localSymbolTableAppend.ion", - "ion-tests/iontestdata_1_0/good/equivs/localSymbolTableNullSlots.ion", - "ion-tests/iontestdata_1_0/good/equivs/nonIVMNoOps.ion", + "ion-tests/iontestdata/good/equivs/localSymbolTableAppend.ion", + "ion-tests/iontestdata/good/equivs/localSymbolTableNullSlots.ion", + "ion-tests/iontestdata/good/equivs/nonIVMNoOps.ion", ]; /// An implementation of `io::Read` that only yields a single byte on each From ad2b1ac16a11c827efc63de9ebeda67200c3d01e Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Tue, 27 Aug 2024 17:31:06 -0700 Subject: [PATCH 03/13] Handle typed nulls in read_resolved --- src/lazy/binary/raw/v1_1/value.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lazy/binary/raw/v1_1/value.rs b/src/lazy/binary/raw/v1_1/value.rs index 8765c72c..445c16f3 100644 --- a/src/lazy/binary/raw/v1_1/value.rs +++ b/src/lazy/binary/raw/v1_1/value.rs @@ -194,7 +194,13 @@ impl<'top> LazyRawValue<'top, BinaryEncoding_1_1> for &'top LazyRawBinaryValue_1 return Ok(ValueRef::Int(int)); } if self.is_null() { - return Ok(ValueRef::Null(self.ion_type())); + let ion_type = if self.encoded_value.header.ion_type_code == OpcodeType::TypedNull { + let body = self.value_body(); + ION_1_1_TYPED_NULL_TYPES[body[0] as usize] + } else { + IonType::Null + }; + return Ok(ValueRef::Null(ion_type)); } // Anecdotally, string and integer values are very common in Ion streams. This `match` creates // an inlineable fast path for them while other types go through the general case impl. From 1743ea45bde4620832b0f684c69aeb1acba08bbf Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 28 Aug 2024 00:14:54 -0700 Subject: [PATCH 04/13] Add support for more data model clauses --- tests/conformance_dsl/continuation.rs | 2 +- tests/conformance_dsl/mod.rs | 26 ++++ tests/conformance_dsl/model.rs | 165 +++++++++++++++++++++++--- tests/conformance_tests.rs | 1 + 4 files changed, 179 insertions(+), 15 deletions(-) diff --git a/tests/conformance_dsl/continuation.rs b/tests/conformance_dsl/continuation.rs index 82e9f7f3..f4356020 100644 --- a/tests/conformance_dsl/continuation.rs +++ b/tests/conformance_dsl/continuation.rs @@ -163,7 +163,7 @@ pub fn parse_continuation(clause: Clause) -> InnerResult { let seq = clause.body.get(sequence_idx) .and_then(|e| e.as_sequence()) .ok_or(ConformanceErrorKind::ExpectedModelValue)?; - let mut seq_iter = seq.iter().peekable(); + let seq_iter = seq.iter().peekable(); let fragment = match Fragment::try_from(Sequence::new(seq_iter)) { Ok(frag) => frag, diff --git a/tests/conformance_dsl/mod.rs b/tests/conformance_dsl/mod.rs index 1dc45f36..39ca3fe4 100644 --- a/tests/conformance_dsl/mod.rs +++ b/tests/conformance_dsl/mod.rs @@ -42,6 +42,8 @@ pub(crate) enum ConformanceErrorKind { ExpectedInteger, ExpectedSignal(String), ExpectedString, + InvalidByte, + InvalidHexString, MismatchedProduce, MismatchedDenotes, UnexpectedValue, @@ -251,6 +253,30 @@ impl TestCollection { } +pub(crate) fn parse_bytes_exp<'a, I: IntoIterator>(elems: I) -> InnerResult> { + // Bytes can be of the form int (0..255), and a string containing hexadecimal digits. + use std::result::Result; + let mut bytes: Vec = vec!(); + for elem in elems.into_iter() { + match elem.ion_type() { + IonType::Int => match elem.as_i64() { + Some(i) if (0..255).contains(&i) => bytes.push(i as u8), + _ => return Err(ConformanceErrorKind::InvalidByte), + } + IonType::String => { + let hex = elem.as_string().ok_or(ConformanceErrorKind::ExpectedString)?.replace(" ", ""); + let hex_bytes = (0..hex.len()).step_by(2).map(|i| u8::from_str_radix(&hex[i..i+2], 16)).collect::, _>>(); + match hex_bytes { + Err(_) => return Err(ConformanceErrorKind::InvalidHexString), + Ok(v) => bytes.extend_from_slice(v.as_slice()), + } + } + _ => return Err(ConformanceErrorKind::InvalidByte), + } + } + Ok(bytes) +} + pub(crate) fn parse_text_exp<'a, I: IntoIterator>(elems: I) -> InnerResult { let bytes: Vec> = elems.into_iter().map(|v| match v.ion_type() { IonType::String => v.as_string().map(|s| Ok(s.as_bytes().to_vec())).unwrap(), diff --git a/tests/conformance_dsl/model.rs b/tests/conformance_dsl/model.rs index ef70196a..973f5e01 100644 --- a/tests/conformance_dsl/model.rs +++ b/tests/conformance_dsl/model.rs @@ -1,22 +1,51 @@ use ion_rs::{Decimal, Element, IonType, Sequence}; -use super::{Clause, ClauseType, ConformanceErrorKind, InnerResult, parse_text_exp}; +use ion_rs::decimal::coefficient::Coefficient; +use super::{Clause, ClauseType, ConformanceErrorKind, InnerResult, parse_text_exp, parse_bytes_exp}; use std::collections::HashMap; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, Hash, PartialEq)] pub(crate) enum SymTok { Text(String), Offset(i64), Absent(String, i64), } +impl TryFrom<&Element> for SymTok { + type Error = ConformanceErrorKind; + + fn try_from(other: &Element) -> InnerResult { + match other.ion_type() { + IonType::String => Ok(SymTok::Text(other.as_string().unwrap().to_owned())), + IonType::Int => Ok(SymTok::Offset(other.as_i64().unwrap())), + IonType::SExp => { + let clause: Clause = other.as_sequence().unwrap().try_into()?; + + match clause.tpe { + ClauseType::Text => { + let text = parse_text_exp(clause.body.iter())?; + Ok(SymTok::Text(text)) + }, + ClauseType::Absent => { + let text = clause.body.get(1).and_then(|v| v.as_string()).ok_or(ConformanceErrorKind::ExpectedSymbolType)?; + let offset = clause.body.get(2).and_then(|v| v.as_i64()).ok_or(ConformanceErrorKind::ExpectedSymbolType)?; + Ok(SymTok::Absent(text.to_string(), offset)) + } + _ => unreachable!(), + } + } + _ => Err(ConformanceErrorKind::ExpectedSymbolType), + } + } +} + #[derive(Debug, Clone)] pub(crate) enum ModelValue { Null(IonType), Bool(bool), Int(i64), Float(f64), - Decimal(i64, i64), + Decimal(Decimal), // TODO: Timestamp String(String), Symbol(SymTok), @@ -67,7 +96,28 @@ impl TryFrom<&Sequence> for ModelValue { Err(_) => Err(ConformanceErrorKind::ExpectedFloatString), } } - "Decimal" => todo!(), + "Decimal" => { + let (first, second) = (elems.get(1), elems.get(2)); + match (first.map(|e| e.ion_type()), second.map(|e| e.ion_type())) { + (Some(IonType::String), Some(IonType::Int)) => { + let (first, second) = (first.unwrap(), second.unwrap()); // SAFETY: We have non-None types. + if let Some("negative_0") = first.as_string() { + let exp = second.as_i64().ok_or(ConformanceErrorKind::ExpectedModelValue)?; + Ok(ModelValue::Decimal(Decimal::new(Coefficient::NEGATIVE_ZERO, exp))) + } else { + Err(ConformanceErrorKind::ExpectedModelValue) + } + } + (Some(IonType::Int), Some(IonType::Int)) => { + let (first, second) = (first.unwrap(), second.unwrap()); // SAFETY: We have non-None types. + Ok(ModelValue::Decimal(Decimal::new( + first.as_i64().ok_or(ConformanceErrorKind::ExpectedModelValue)?, + second.as_i64().ok_or(ConformanceErrorKind::ExpectedModelValue)?, + ))) + } + _ => Err(ConformanceErrorKind::ExpectedModelValue), + } + } "String" => { let string = parse_text_exp(elems.iter().skip(1))?; Ok(ModelValue::String(string)) @@ -96,11 +146,53 @@ impl TryFrom<&Sequence> for ModelValue { _ => Err(ConformanceErrorKind::ExpectedSymbolType), } } - "List" => todo!(), - "Sexp" => todo!(), - "Struct" => todo!(), - "Blob" => todo!(), - "Clob" => todo!(), + "List" => { + let mut list = vec!(); + for elem in elems.iter().skip(1) { + if let Some(seq) = elem.as_sequence() { + list.push(ModelValue::try_from(seq)?); + } + } + Ok(ModelValue::List(list)) + } + "Sexp" => { + let mut sexp = vec!(); + for elem in elems.iter().skip(1) { + if let Some(seq) = elem.as_sequence() { + sexp.push(ModelValue::try_from(seq)?); + } + } + Ok(ModelValue::Sexp(sexp)) + } + "Struct" => { + let mut fields = HashMap::new(); + for elem in elems.iter().skip(1) { + if let Some(seq) = elem.as_sequence() { + // Each elem should be a model symtok followed by a model value. + let (first, second) = (seq.get(0), seq.get(1)); + let field_sym = first.map(SymTok::try_from).ok_or(ConformanceErrorKind::ExpectedSymbolType)?.unwrap(); + let value = match second.map(|e| e.ion_type()) { + Some(IonType::String) => { + let string = second.unwrap().as_string().unwrap(); + ModelValue::String(string.to_string()) + } + Some(IonType::Int) => { + let int_val = second.unwrap().as_i64().unwrap(); + ModelValue::Int(int_val) + } + Some(IonType::SExp) => { + let seq = second.unwrap().as_sequence().unwrap(); + ModelValue::try_from(seq)? + } + _ => return Err(ConformanceErrorKind::ExpectedModelValue), + }; + fields.insert(field_sym, value); + } + } + Ok(ModelValue::Struct(fields)) + } + "Blob" => Ok(ModelValue::Blob(parse_bytes_exp(elems.iter().skip(1))?)), + "Clob" => Ok(ModelValue::Clob(parse_bytes_exp(elems.iter().skip(1))?)), _ => unreachable!(), } } @@ -113,12 +205,58 @@ impl PartialEq for ModelValue { ModelValue::Bool(val) => other.as_bool() == Some(*val), ModelValue::Int(val) => other.as_i64() == Some(*val), ModelValue::Float(val) => other.as_float() == Some(*val), - ModelValue::Decimal(coef, exp) => other.as_decimal() == Some(Decimal::new(*coef, *exp)), + ModelValue::Decimal(dec) => other.as_decimal() == Some(*dec), // TODO: Timestamp ModelValue::String(val) => other.as_string() == Some(val), - ModelValue::List(_vals) => todo!(), - ModelValue::Sexp(_vals) => todo!(), - ModelValue::Struct(_fields) => todo!(), + ModelValue::List(vals) => { + if other.ion_type() != IonType::List { + return false; + } + let other_seq = other.as_sequence().unwrap(); // SAFETY: Confirmed it's a list. + if other_seq.len() != vals.len() { + return false; + } + + for (ours, others) in vals.iter().zip(other_seq) { + if ours != others { + return false; + } + } + true + } + ModelValue::Sexp(vals) => { + if other.ion_type() != IonType::SExp { + return false; + } + let other_seq = other.as_sequence().unwrap(); // SAFETY: Confirmed it's a list. + if other_seq.len() != vals.len() { + return false; + } + + for (ours, others) in vals.iter().zip(other_seq) { + if ours != others { + return false; + } + } + true + } + ModelValue::Struct(fields) => { + if other.ion_type() != IonType::Struct { + false + } else { + let struct_val = other.as_struct().unwrap(); + for (field, val) in struct_val.fields() { + let denoted_field = field.text().unwrap(); // TODO: Symbol IDs + let denoted_symtok = SymTok::Text(denoted_field.to_string()); + if let Some(expected_val) = fields.get(&denoted_symtok) { + if expected_val != val { + return false; + } + } + } + true + } + } ModelValue::Blob(data) => other.as_blob() == Some(data.as_slice()), ModelValue::Clob(data) => other.as_clob() == Some(data.as_slice()), ModelValue::Symbol(sym) => { @@ -132,7 +270,6 @@ impl PartialEq for ModelValue { false } } - _ => todo!(), } } } diff --git a/tests/conformance_tests.rs b/tests/conformance_tests.rs index dd2c48cd..38b6fd1e 100644 --- a/tests/conformance_tests.rs +++ b/tests/conformance_tests.rs @@ -61,6 +61,7 @@ mod ion_tests { #[test_resources("ion-tests/conformance/core/typed_null.ion")] #[test_resources("ion-tests/conformance/core/string_symbol.ion")] #[test_resources("ion-tests/conformance/core/empty_document.ion")] + #[test_resources("ion-tests/conformance/core/toplevel_produces.ion")] // #[test_resources("ion-tests/conformance/ivm.ion")] fn conformance(file_name: &str) { println!("Testing: {}", file_name); From a951e27bc0094e34432768968f460859cc0dc31d Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 28 Aug 2024 00:56:36 -0700 Subject: [PATCH 05/13] Remove left over comments; fix byte range; support multiple string in text clause --- tests/conformance.rs | 4 ---- tests/conformance_dsl/document.rs | 3 +-- tests/conformance_dsl/fragment.rs | 27 +++++++++------------------ tests/conformance_dsl/mod.rs | 2 +- 4 files changed, 11 insertions(+), 25 deletions(-) diff --git a/tests/conformance.rs b/tests/conformance.rs index cef6a089..f7c85a9c 100644 --- a/tests/conformance.rs +++ b/tests/conformance.rs @@ -9,8 +9,6 @@ pub fn main() { let test_paths = std::env::args().skip(1).collect::>(); let mut errors: Vec<(String, String, conformance_dsl::ConformanceError)> = vec!(); - // Formatting: Get max test name length. - println!("Testing {} conformance collections.\n", test_paths.len()); let mut failures = 0; @@ -38,8 +36,6 @@ pub fn main() { } } - // println!("\nConformance Summary: {} Succeeded, {} Failed", collection.len() - failures, failures); - for (test_path, test_name, err) in errors { println!("-------------------------"); println!("File: {}", test_path); diff --git a/tests/conformance_dsl/document.rs b/tests/conformance_dsl/document.rs index 4ae99a1d..f1fb5434 100644 --- a/tests/conformance_dsl/document.rs +++ b/tests/conformance_dsl/document.rs @@ -46,8 +46,7 @@ impl Document { (true, false) => IonEncoding::Text, (false, true) => IonEncoding::Binary, (false, false) => IonEncoding::Unspecified, - (true, true) => panic!("Both binary and text fragments specified"), // TODO: Make - // error. + (true, true) => panic!("Both binary and text fragments specified"), } } } diff --git a/tests/conformance_dsl/fragment.rs b/tests/conformance_dsl/fragment.rs index 709b3030..7aec785f 100644 --- a/tests/conformance_dsl/fragment.rs +++ b/tests/conformance_dsl/fragment.rs @@ -76,26 +76,17 @@ impl TryFrom for Fragment { fn try_from(other: Clause) -> InnerResult { let frag = match other.tpe { ClauseType::Text => { - // TODO: grammar is "(" "text" string* ")", we need to handle 0+ strings. - let txt = match other.body.first() { - Some(txt) if txt.ion_type() == IonType::String => txt.as_string().unwrap().to_owned(), - Some(_) => return Err(ConformanceErrorKind::UnexpectedValue), - None => String::from(""), - }; - Fragment::Text(txt) - } - ClauseType::Binary => { - // TODO: Support string of hex values. - let mut bytes: Vec = vec!(); - for elem in other.body { - if let Some(byte) = elem.as_i64() { - if (0..=255).contains(&byte) { - bytes.push(byte as u8); - } - } + let mut text = String::from(""); + for elem in other.body.iter() { + let txt = match elem.ion_type() { + IonType::String => elem.as_string().unwrap().to_owned(), + _ => return Err(ConformanceErrorKind::UnexpectedValue), + }; + text = text + " " + &txt; } - Fragment::Binary(bytes) + Fragment::Text(text) } + ClauseType::Binary => Fragment::Binary(parse_bytes_exp(other.body.iter())?), ClauseType::Ivm => { // IVM: (ivm ) let maj = other.body.first().map(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?.unwrap(); diff --git a/tests/conformance_dsl/mod.rs b/tests/conformance_dsl/mod.rs index 39ca3fe4..6880c8a2 100644 --- a/tests/conformance_dsl/mod.rs +++ b/tests/conformance_dsl/mod.rs @@ -260,7 +260,7 @@ pub(crate) fn parse_bytes_exp<'a, I: IntoIterator>(elems: I) - for elem in elems.into_iter() { match elem.ion_type() { IonType::Int => match elem.as_i64() { - Some(i) if (0..255).contains(&i) => bytes.push(i as u8), + Some(i) if (0..=255).contains(&i) => bytes.push(i as u8), _ => return Err(ConformanceErrorKind::InvalidByte), } IonType::String => { From a8e2538e70a3fc0ab51e83f3a293356844be0d84 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 28 Aug 2024 11:22:43 -0700 Subject: [PATCH 06/13] Update 1.1 skip list to address ion-test path changes --- tests/detect_incomplete_text.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detect_incomplete_text.rs b/tests/detect_incomplete_text.rs index c98c176a..2f72d1d9 100644 --- a/tests/detect_incomplete_text.rs +++ b/tests/detect_incomplete_text.rs @@ -33,7 +33,7 @@ static SKIP_LIST_1_0: LazyLock> = static SKIP_LIST_1_1: LazyLock> = LazyLock::new(|| { CANONICAL_FILE_NAMES .iter() - .map(|file_1_0| file_1_0.replace("_1_0", "_1_1")) + .map(|file_1_0| file_1_0.replace("/iontestdata/", "/iontestdata_1_1/")) .collect() }); From 703bba0e2f8f36429bc84d9f6de52d930a9fea09 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 28 Aug 2024 11:32:17 -0700 Subject: [PATCH 07/13] Address clippy checks, remove unused test --- tests/conformance_tests.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/conformance_tests.rs b/tests/conformance_tests.rs index 38b6fd1e..613bdbd0 100644 --- a/tests/conformance_tests.rs +++ b/tests/conformance_tests.rs @@ -11,18 +11,6 @@ use std::str::FromStr; mod implementation { use super::*; - // #[test] - fn toplevel_absent_symbols() { - let test = r#" - (ion_1_0 (toplevel '#$1') (produces $ion)) - "#; - - Document::from_str(test) - .unwrap_or_else(|e| panic!("Failed to load document: <<{}>>\n{:?}", test, e)) - .run() - .unwrap_or_else(|e| panic!("Test failed: <<{}>>\n{:?}", test, e)); - } - #[test] fn test_simple_docs() { let tests: &[&str] = &[ From cf5dd98e260c17e3947a4a23d27d0637f0cb3166 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 28 Aug 2024 11:47:18 -0700 Subject: [PATCH 08/13] Drop path separators from path replacement so the canonicalized windows version can match --- tests/detect_incomplete_text.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detect_incomplete_text.rs b/tests/detect_incomplete_text.rs index 2f72d1d9..0492d725 100644 --- a/tests/detect_incomplete_text.rs +++ b/tests/detect_incomplete_text.rs @@ -33,7 +33,7 @@ static SKIP_LIST_1_0: LazyLock> = static SKIP_LIST_1_1: LazyLock> = LazyLock::new(|| { CANONICAL_FILE_NAMES .iter() - .map(|file_1_0| file_1_0.replace("/iontestdata/", "/iontestdata_1_1/")) + .map(|file_1_0| file_1_0.replace("iontestdata", "iontestdata_1_1")) .collect() }); From 6811fbfac66c8f3ecc1b0bbd5944674d7806e118 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 28 Aug 2024 12:54:57 -0700 Subject: [PATCH 09/13] Remove conformance cli from default tests --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 1c65cbdc..f521fd1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,3 +98,4 @@ debug = true [[test]] name = "conformance" harness = false +test = false From 51b75f3d0d67c6f257fa7545d867edeff781f17a Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Thu, 29 Aug 2024 03:16:07 -0700 Subject: [PATCH 10/13] Add encoding and mactab fragments --- tests/conformance_dsl/clause.rs | 2 ++ tests/conformance_dsl/fragment.rs | 28 ++++++++++++++++++++++++---- tests/conformance_tests.rs | 14 ++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/conformance_dsl/clause.rs b/tests/conformance_dsl/clause.rs index 389c1a47..c28d0b92 100644 --- a/tests/conformance_dsl/clause.rs +++ b/tests/conformance_dsl/clause.rs @@ -51,6 +51,8 @@ impl FromStr for ClauseType { "absent" => Ok(Absent), "ivm" => Ok(Ivm), "signals" => Ok(Signals), + "encoding" => Ok(Encoding), + "mactab" => Ok(MacTab), _ => Err(ConformanceErrorKind::UnknownClause(s.to_owned())), } } diff --git a/tests/conformance_dsl/fragment.rs b/tests/conformance_dsl/fragment.rs index 7aec785f..ba97bdf8 100644 --- a/tests/conformance_dsl/fragment.rs +++ b/tests/conformance_dsl/fragment.rs @@ -16,8 +16,6 @@ pub(crate) enum Fragment { Ivm(i64, i64), Text(String), TopLevel(TopLevel), - MacTab, // TODO: Implement. - Encoding, // TODO: Implement. } static EMPTY_TOPLEVEL: Fragment = Fragment::TopLevel(TopLevel { elems: vec!() }); @@ -94,8 +92,30 @@ impl TryFrom for Fragment { Fragment::Ivm(maj, min) } ClauseType::TopLevel => Fragment::TopLevel(TopLevel { elems: other.body }), - ClauseType::Encoding => Fragment::Encoding, - ClauseType::MacTab => Fragment::MacTab, + ClauseType::Encoding => { + // Rather than treat Encoding special, we expand it to a (toplevel ..) as described + // in the spec. + let inner: Element = SExp(Sequence::new(other.body)).into(); + let inner = inner.with_annotations(["$ion_encoding"]); + Fragment::TopLevel(TopLevel { elems: vec!(inner) }) + } + ClauseType::MacTab => { + // Like encoding, MacTab is expanded into a TopLevel fragment. + let mut mac_table_elems: Vec = vec!(Symbol::from("macro_table").into()); + for elem in other.body { + mac_table_elems.push(elem); + } + let mac_table: Element = SExp(Sequence::new(mac_table_elems)).into(); + let module: Element = SExp(ion_seq!( + Symbol::from("module"), + Symbol::from("M"), + mac_table, + SExp(ion_seq!(Symbol::from("macro_table"), Symbol::from("M"))), + )).into(); + let encoding: Element = SExp(ion_seq!(module)).into(); + let encoding = encoding.with_annotations(["$ion_encoding"]); + Fragment::TopLevel(TopLevel { elems: vec!(encoding) }) + } _ => return Err(ConformanceErrorKind::ExpectedFragment), }; Ok(frag) diff --git a/tests/conformance_tests.rs b/tests/conformance_tests.rs index 613bdbd0..9ade4d28 100644 --- a/tests/conformance_tests.rs +++ b/tests/conformance_tests.rs @@ -11,6 +11,20 @@ use std::str::FromStr; mod implementation { use super::*; + #[test] + fn test_encoding() { + let test: &str = r#" + (ion_1_1 + (encoding (macro_table (macro m () 1))) + (text "(:m)") + (produces 1) + )"#; + Document::from_str(test) + .unwrap_or_else(|e| panic!("Failed to load document:\n{:?}", e)) + .run() + .unwrap_or_else(|e| panic!("Test failed: {:?}", e)); + } + #[test] fn test_simple_docs() { let tests: &[&str] = &[ From 5204862c3176ee4cc574ae66fd552d8f2ce5c564 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Thu, 29 Aug 2024 03:24:44 -0700 Subject: [PATCH 11/13] Add timestamp to denote data model --- tests/conformance_dsl/mod.rs | 2 + tests/conformance_dsl/model.rs | 167 ++++++++++++++++++++++++++++----- tests/conformance_tests.rs | 19 ++++ 3 files changed, 163 insertions(+), 25 deletions(-) diff --git a/tests/conformance_dsl/mod.rs b/tests/conformance_dsl/mod.rs index 6880c8a2..4421f985 100644 --- a/tests/conformance_dsl/mod.rs +++ b/tests/conformance_dsl/mod.rs @@ -42,6 +42,8 @@ pub(crate) enum ConformanceErrorKind { ExpectedInteger, ExpectedSignal(String), ExpectedString, + ExpectedTimestampPrecision, + ExpectedTimestampOffset, InvalidByte, InvalidHexString, MismatchedProduce, diff --git a/tests/conformance_dsl/model.rs b/tests/conformance_dsl/model.rs index 973f5e01..2d9b50ec 100644 --- a/tests/conformance_dsl/model.rs +++ b/tests/conformance_dsl/model.rs @@ -1,4 +1,4 @@ -use ion_rs::{Decimal, Element, IonType, Sequence}; +use ion_rs::{Decimal, Element, IonType, Sequence, Timestamp}; use ion_rs::decimal::coefficient::Coefficient; use super::{Clause, ClauseType, ConformanceErrorKind, InnerResult, parse_text_exp, parse_bytes_exp}; @@ -46,7 +46,7 @@ pub(crate) enum ModelValue { Int(i64), Float(f64), Decimal(Decimal), - // TODO: Timestamp + Timestamp(Timestamp), String(String), Symbol(SymTok), List(Vec), @@ -96,28 +96,7 @@ impl TryFrom<&Sequence> for ModelValue { Err(_) => Err(ConformanceErrorKind::ExpectedFloatString), } } - "Decimal" => { - let (first, second) = (elems.get(1), elems.get(2)); - match (first.map(|e| e.ion_type()), second.map(|e| e.ion_type())) { - (Some(IonType::String), Some(IonType::Int)) => { - let (first, second) = (first.unwrap(), second.unwrap()); // SAFETY: We have non-None types. - if let Some("negative_0") = first.as_string() { - let exp = second.as_i64().ok_or(ConformanceErrorKind::ExpectedModelValue)?; - Ok(ModelValue::Decimal(Decimal::new(Coefficient::NEGATIVE_ZERO, exp))) - } else { - Err(ConformanceErrorKind::ExpectedModelValue) - } - } - (Some(IonType::Int), Some(IonType::Int)) => { - let (first, second) = (first.unwrap(), second.unwrap()); // SAFETY: We have non-None types. - Ok(ModelValue::Decimal(Decimal::new( - first.as_i64().ok_or(ConformanceErrorKind::ExpectedModelValue)?, - second.as_i64().ok_or(ConformanceErrorKind::ExpectedModelValue)?, - ))) - } - _ => Err(ConformanceErrorKind::ExpectedModelValue), - } - } + "Decimal" => Ok(ModelValue::Decimal(parse_model_decimal(elems.iter().skip(1))?)), "String" => { let string = parse_text_exp(elems.iter().skip(1))?; Ok(ModelValue::String(string)) @@ -146,6 +125,7 @@ impl TryFrom<&Sequence> for ModelValue { _ => Err(ConformanceErrorKind::ExpectedSymbolType), } } + "Timestamp" => Ok(ModelValue::Timestamp(parse_timestamp(elems.iter().skip(1))?)), "List" => { let mut list = vec!(); for elem in elems.iter().skip(1) { @@ -206,7 +186,6 @@ impl PartialEq for ModelValue { ModelValue::Int(val) => other.as_i64() == Some(*val), ModelValue::Float(val) => other.as_float() == Some(*val), ModelValue::Decimal(dec) => other.as_decimal() == Some(*dec), - // TODO: Timestamp ModelValue::String(val) => other.as_string() == Some(val), ModelValue::List(vals) => { if other.ion_type() != IonType::List { @@ -270,7 +249,145 @@ impl PartialEq for ModelValue { false } } + ModelValue::Timestamp(ts) => other.as_timestamp() == Some(*ts), + } + } +} + +fn parse_timestamp<'a, I: IntoIterator>(elems: I) -> InnerResult { + let mut iter = elems.into_iter(); + let first = iter.next().and_then(|e| e.as_symbol()).and_then(|s| s.text()); + match first { + Some("year") => { + let year = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + Ok(Timestamp::with_year(year as u32).build()?) + } + Some("month") => { + let year = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let month = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let ts = Timestamp::with_year(year as u32) + .with_month(month as u32) + .build()?; + Ok(ts) + } + Some("day") => { + let year = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let month = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let day = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let ts = Timestamp::with_year(year as u32) + .with_month(month as u32) + .with_day(day as u32) + .build()?; + Ok(ts) + } + Some("minute") => { + let year = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let month = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let day = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + + let offset = parse_ts_offset(iter.next().and_then(|e| e.as_sequence()).ok_or(ConformanceErrorKind::ExpectedInteger)?)?; + + let hour = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let minute = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let ts = Timestamp::with_year(year as u32) + .with_month(month as u32) + .with_day(day as u32) + .with_hour_and_minute(hour as u32, minute as u32); + if let Some(offset) = offset { + let ts = ts.with_offset(offset as i32); + Ok(ts.build()?) + } else { + Ok(ts.build()?) + } + } + Some("second") => { + let year = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let month = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let day = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + + let offset = parse_ts_offset(iter.next().and_then(|e| e.as_sequence()).ok_or(ConformanceErrorKind::ExpectedInteger)?)?; + + let hour = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let minute = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let second = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let ts = Timestamp::with_year(year as u32) + .with_month(month as u32) + .with_day(day as u32) + .with_hour_and_minute(hour as u32, minute as u32) + .with_second(second as u32); + if let Some(offset) = offset { + let ts = ts.with_offset(offset as i32); + Ok(ts.build()?) + } else { + Ok(ts.build()?) + } + } + Some("fraction") => { + let year = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let month = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let day = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + + let offset = parse_ts_offset(iter.next().and_then(|e| e.as_sequence()).ok_or(ConformanceErrorKind::ExpectedInteger)?)?; + + let hour = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let minute = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let second = iter.next().and_then(|e| e.as_i64()).ok_or(ConformanceErrorKind::ExpectedInteger)?; + let fraction = parse_model_decimal(iter)?; + let ts = Timestamp::with_year(year as u32) + .with_month(month as u32) + .with_day(day as u32) + .with_hour_and_minute(hour as u32, minute as u32) + .with_second(second as u32) + .with_fractional_seconds(fraction); + if let Some(offset) = offset { + let ts = ts.with_offset(offset as i32); + Ok(ts.build()?) + } else { + Ok(ts.build()?) + } + } + _ => Err(ConformanceErrorKind::ExpectedTimestampPrecision), + } +} + +fn parse_ts_offset<'a, I: IntoIterator>(elems: I) -> InnerResult> { + let mut iter = elems.into_iter(); + match iter.next().and_then(|e| e.as_symbol()).and_then(|s| s.text()) { + Some("offset") => { + // Either an int or null.. + let offset = iter.next().ok_or(ConformanceErrorKind::ExpectedTimestampOffset)?; + if offset.is_null() { + Ok(None) + } else { + let offset = offset.as_i64().ok_or(ConformanceErrorKind::ExpectedInteger)?; + Ok(Some(offset)) + } + } + _ => Err(ConformanceErrorKind::ExpectedTimestampOffset), + } +} + +fn parse_model_decimal<'a, I: IntoIterator>(elems: I) -> InnerResult { + let mut iter = elems.into_iter(); + let (first, second) = (iter.next(), iter.next()); + match (first.map(|e| e.ion_type()), second.map(|e| e.ion_type())) { + (Some(IonType::String), Some(IonType::Int)) => { + let (first, second) = (first.unwrap(), second.unwrap()); // SAFETY: We have non-None types. + if let Some("negative_0") = first.as_string() { + let exp = second.as_i64().ok_or(ConformanceErrorKind::ExpectedModelValue)?; + Ok(Decimal::new(Coefficient::NEGATIVE_ZERO, exp)) + } else { + Err(ConformanceErrorKind::ExpectedModelValue) + } + } + (Some(IonType::Int), Some(IonType::Int)) => { + let (first, second) = (first.unwrap(), second.unwrap()); // SAFETY: We have non-None types. + Ok(Decimal::new( + first.as_i64().ok_or(ConformanceErrorKind::ExpectedModelValue)?, + second.as_i64().ok_or(ConformanceErrorKind::ExpectedModelValue)?, + )) } + _ => Err(ConformanceErrorKind::ExpectedModelValue), } } diff --git a/tests/conformance_tests.rs b/tests/conformance_tests.rs index 9ade4d28..c06d4f9a 100644 --- a/tests/conformance_tests.rs +++ b/tests/conformance_tests.rs @@ -11,6 +11,24 @@ use std::str::FromStr; mod implementation { use super::*; + #[test] + fn test_timestamps() { + let tests: &[&str] = &[ + r#"(ion_1_1 "Timestamp Year" (text "2023T") (denotes (Timestamp year 2023)))"#, + r#"(ion_1_1 "Timestamp Month" (text "2023-03T") (denotes (Timestamp month 2023 3)))"#, + r#"(ion_1_1 "Timestamp Day" (text "2023-03-23T") (denotes (Timestamp day 2023 3 23)))"#, + r#"(ion_1_1 "Timestamp Minute" (text "2023-03-23T10:12Z") (denotes (Timestamp minute 2023 3 23 (offset 0) 10 12)))"#, + r#"(ion_1_1 "Timestamp Second" (text "2023-03-23T10:12:21Z") (denotes (Timestamp second 2023 3 23 (offset 0) 10 12 21))) "#, + r#"(ion_1_1 "Timestamp Fractional" (text "2023-03-23T10:12:21.23Z") (denotes (Timestamp fraction 2023 3 23 (offset 0) 10 12 21 23 -2))) "#, + ]; + for test in tests { + Document::from_str(test) + .unwrap_or_else(|e| panic!("Failed to load document: <<{}>>\n{:?}", test, e)) + .run() + .unwrap_or_else(|e| panic!("Test failed for simple doc: <<{}>>\n{:?}", test, e)); + } + } + #[test] fn test_encoding() { let test: &str = r#" @@ -45,6 +63,7 @@ mod implementation { (denotes (Int 0))) )"#, r#"(document (ivm 1 2) (signals "Invalid Version"))"#, + r#"(ion_1_1 (text "2.3") (denotes (Decimal 23 -1)))"#, ]; for test in tests { println!("Testing: {}", test); From 2a69f4ed40533428874828ec459a9d0b7bd8127f Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Thu, 29 Aug 2024 03:28:49 -0700 Subject: [PATCH 12/13] Missed some uses when --patch'ing --- tests/conformance_dsl/fragment.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conformance_dsl/fragment.rs b/tests/conformance_dsl/fragment.rs index ba97bdf8..43100180 100644 --- a/tests/conformance_dsl/fragment.rs +++ b/tests/conformance_dsl/fragment.rs @@ -1,5 +1,5 @@ -use ion_rs::{Element, Sequence}; -use ion_rs::{v1_0, v1_1, WriteConfig, Encoding}; +use ion_rs::{Element, Sequence, SExp, Symbol}; +use ion_rs::{v1_0, v1_1, WriteConfig, Encoding, ion_seq}; use super::*; use super::context::Context; From 499e649ca33da7b556f323ffea9b75a0fa72d3ec Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Tue, 3 Sep 2024 03:06:46 -0700 Subject: [PATCH 13/13] Address PR feedback; Remove unused Each variant of Fragment since it is not a fragment --- src/lazy/binary/raw/v1_1/value.rs | 24 ++++++--------- tests/conformance_dsl/clause.rs | 35 +++++++++++++++++++++- tests/conformance_dsl/context.rs | 29 ++++++++++++++++-- tests/conformance_dsl/continuation.rs | 25 ++++++++++++++++ tests/conformance_dsl/document.rs | 7 +++++ tests/conformance_dsl/fragment.rs | 19 +++++++++--- tests/conformance_dsl/mod.rs | 19 +++++++++--- tests/conformance_dsl/model.rs | 43 ++++++++++++++++----------- tests/conformance_tests.rs | 1 - 9 files changed, 157 insertions(+), 45 deletions(-) diff --git a/src/lazy/binary/raw/v1_1/value.rs b/src/lazy/binary/raw/v1_1/value.rs index 445c16f3..7f0de9d0 100644 --- a/src/lazy/binary/raw/v1_1/value.rs +++ b/src/lazy/binary/raw/v1_1/value.rs @@ -119,7 +119,13 @@ impl<'top> HasRange for &'top LazyRawBinaryValue_1_1<'top> { impl<'top> LazyRawValue<'top, BinaryEncoding_1_1> for &'top LazyRawBinaryValue_1_1<'top> { fn ion_type(&self) -> IonType { - self.encoded_value.ion_type() + // Handle retrieving the type for a typed null. + if self.encoded_value.header.type_code() == OpcodeType::TypedNull { + let body = self.value_body(); + ION_1_1_TYPED_NULL_TYPES[body[0] as usize] + } else { + self.encoded_value.ion_type() + } } fn is_null(&self) -> bool { @@ -145,13 +151,7 @@ impl<'top> LazyRawValue<'top, BinaryEncoding_1_1> for &'top LazyRawBinaryValue_1 } if self.is_null() { - let ion_type = if self.encoded_value.header.ion_type_code == OpcodeType::TypedNull { - let body = self.value_body(); - ION_1_1_TYPED_NULL_TYPES[body[0] as usize] - } else { - IonType::Null - }; - return Ok(RawValueRef::Null(ion_type)); + return Ok(RawValueRef::Null(self.ion_type())); } match self.ion_type() { @@ -194,13 +194,7 @@ impl<'top> LazyRawValue<'top, BinaryEncoding_1_1> for &'top LazyRawBinaryValue_1 return Ok(ValueRef::Int(int)); } if self.is_null() { - let ion_type = if self.encoded_value.header.ion_type_code == OpcodeType::TypedNull { - let body = self.value_body(); - ION_1_1_TYPED_NULL_TYPES[body[0] as usize] - } else { - IonType::Null - }; - return Ok(ValueRef::Null(ion_type)); + return Ok(ValueRef::Null(self.ion_type())); } // Anecdotally, string and integer values are very common in Ion streams. This `match` creates // an inlineable fast path for them while other types go through the general case impl. diff --git a/tests/conformance_dsl/clause.rs b/tests/conformance_dsl/clause.rs index c28d0b92..cec7aed8 100644 --- a/tests/conformance_dsl/clause.rs +++ b/tests/conformance_dsl/clause.rs @@ -1,30 +1,59 @@ +//! A Clause represents the DSL's S-Expression operations for defining tests. Each possible +//! expression should come from a Clause. +//! +//! The grammar defining each of the clauses can be found [here][Grammar]. +//! +//! [Grammar]: https://github.com/amazon-ion/ion-tests/blob/master/conformance/README.md#grammar + use std::str::FromStr; use ion_rs::{Element, Sequence}; use super::*; +/// Represents each type of S-Expression Clause that we can have in the DSL. This currently does +/// not capture the Data Model clauses used in Denotes fragments. #[allow(non_camel_case_types)] #[derive(Debug)] pub(crate) enum ClauseType { + /// Start an ion 1.0 test document. Ion1_0, + /// Start an ion 1.1 test document. Ion1_1, + /// Start a test document that validates both ion 1.1 and 1.0 Ion1_X, + /// Provide a string as text ion, that will be inserted into the test document. Text, + /// Provide a sequence of bytes that is interpreted as binary ion, that will be inserted into + /// the document. Binary, + /// Provide a major and minor version that will be emitted into the document as an IVM. Ivm, + /// Specify a ion data to be inserted into the document, using inline ion syntax. TopLevel, + /// Provide ion data defining the contents of an '$ion_encoding' directive. Encoding, + /// Provide ion data defining the contents of a macro table wrapped by a module within an encoding directive. MacTab, + /// Define data that is expected to be produced by the test's document, using inline ion + /// syntax. Produces, + /// Define data that is expected to be produced by the test's document, using a clause-based + /// data model. Denotes, + /// Specify that the test should signal (fail). Signals, + /// Evaluate the logical conjunction of the clause's arguments. And, + /// Negate the evaluation of the clause's argument. Not, - Bytes, + /// A continuation that allows for the chaining of fragments and expectations. Then, + /// Specify the start of a test. Document, + /// Combine one or more continuations with a parent document separately. Each, + /// Define a symbol using both text and symbol id for testing in a denotes clause. Absent, } @@ -59,17 +88,21 @@ impl FromStr for ClauseType { } impl ClauseType { + + /// Utility function to test if the Clause is a fragment node. pub fn is_fragment(&self) -> bool { use ClauseType::*; matches!(self, Text | Binary | Ivm | TopLevel | Encoding | MacTab) } + /// Utility function to test if the Clause is an expectation node. pub fn is_expectation(&self) -> bool { use ClauseType::*; matches!(self, Produces | Denotes | Signals | And | Not) } } +/// Represents a valid clause accepted by the conformance DSL for specifying a test. #[derive(Debug)] pub(crate) struct Clause { pub tpe: ClauseType, diff --git a/tests/conformance_dsl/context.rs b/tests/conformance_dsl/context.rs index adbe0fe5..04f710e2 100644 --- a/tests/conformance_dsl/context.rs +++ b/tests/conformance_dsl/context.rs @@ -3,6 +3,9 @@ use crate::conformance_dsl::*; use ion_rs::{Element, ElementReader, Sequence, Reader, IonSlice}; use ion_rs::{v1_0, v1_1}; +/// A Context forms a scope for tracking all of the document fragments and any parent Contexts that +/// also need to be considered. Through this context the ability to generate the full test +/// document input, and evaluate any forced encodings or Ion versions, is provided. #[derive(Clone, Copy, Debug)] pub(crate) struct Context<'a> { version: IonVersion, @@ -12,10 +15,14 @@ pub(crate) struct Context<'a> { } impl<'a> Context<'a> { + /// Creates a new Context with the provided version, encoding and fragments. A parent context + /// is not set. pub fn new(version: IonVersion, encoding: IonEncoding, fragments: &'a Vec) -> Self { Self { version, encoding, fragments, parent_ctx: None} } + /// Creates a new Context with the provided fragments, based on the supplied `parent`. The + /// encoding and version for the new Context is inherited from `parent`. pub fn extend(parent: &'a Context, fragments: &'a Vec) -> Self { Self { version: parent.version, @@ -25,6 +32,10 @@ impl<'a> Context<'a> { } } + /// Determine the ion version used for this context. In the case of multi-version testing + /// (eg. through ion_1_x) multiple branches in the test are produced, one for each concrete + /// version. `IonVersion::Unspecified` will be returned only when no IVM is emitted in the test + /// document, and no version has been set for the context. pub fn version(&self) -> IonVersion { let parent_ver = self.parent_ctx.map(|c| c.version()).unwrap_or(IonVersion::Unspecified); let frag_ver = self.fragment_version(); @@ -42,15 +53,19 @@ impl<'a> Context<'a> { } } + /// Force an ion version to be used in this Context. Manually setting a version will be + /// overridden if the fragments for this context emit an IVM. pub fn set_version(&mut self, version: IonVersion) { self.version = version; } + /// Determine the encoding for all fragments in the path to this Context. pub fn encoding(&self) -> IonEncoding { let parent_enc = self.parent_ctx.map(|c| c.encoding).unwrap_or(IonEncoding::Unspecified); Self::resolve_encoding(parent_enc, self.encoding) } + /// Determine the version requirements for this Context's fragments. pub fn fragment_version(&self) -> IonVersion { match self.fragments.first() { Some(Fragment::Ivm(1, 0)) => IonVersion::V1_0, @@ -59,6 +74,7 @@ impl<'a> Context<'a> { } } + /// Determine the encoding requirements for this Context's fragments. pub fn fragment_encoding(&self) -> IonEncoding { let enc = self.fragments.iter().find(|f| matches!(f, Fragment::Text(_) | Fragment::Binary(_))); match enc { @@ -68,10 +84,14 @@ impl<'a> Context<'a> { } } + /// Force an ion encoding (text or binary) for this Context. All encodings through the path of + /// a test must match. pub fn set_encoding(&mut self, enc: IonEncoding) { self.encoding = enc; } + /// Given 2 encodings, one for a parent context, and one for the child, validate and return the + /// resulting encoding for the whole path. fn resolve_encoding(parent: IonEncoding, child: IonEncoding) -> IonEncoding { match (parent, child) { (a, b) if a == b => a, @@ -81,6 +101,8 @@ impl<'a> Context<'a> { } } + /// Returns a Vec containing the serialized data consisting of all fragments in the path + /// for this context. pub fn input(&self, child_encoding: IonEncoding) -> InnerResult<(Vec, IonEncoding)> { let encoding = Self::resolve_encoding(self.encoding(), child_encoding); let (data, data_encoding) = match encoding { @@ -88,9 +110,12 @@ impl<'a> Context<'a> { IonEncoding::Binary => (to_binary(self, self.fragments.iter())?, encoding), IonEncoding::Unspecified => (to_binary(self, self.fragments.iter())?, IonEncoding::Binary), }; - Ok((data, data_encoding)) + let (mut parent_input, _) = self.parent_ctx.map(|c| c.input(encoding)).unwrap_or(Ok((vec!(), encoding)))?; + parent_input.extend(data.clone()); + Ok((parent_input, data_encoding)) } + /// Returns the Sequence of Elements representing the test document. pub fn read_all(&self, encoding: IonEncoding) -> InnerResult { let (data, data_encoding) = self.input(encoding)?; let data_slice = IonSlice::new(data); @@ -105,8 +130,6 @@ impl<'a> Context<'a> { v => v, }; - // Ok(Reader::new(AnyEncoding, data_slice)?.read_all_elements()?) - match (version, data_encoding) { (IonVersion::V1_0, IonEncoding::Binary) => Ok(Reader::new(v1_0::Binary, data_slice)?.read_all_elements()?), diff --git a/tests/conformance_dsl/continuation.rs b/tests/conformance_dsl/continuation.rs index f4356020..360c3216 100644 --- a/tests/conformance_dsl/continuation.rs +++ b/tests/conformance_dsl/continuation.rs @@ -1,3 +1,7 @@ +//! Continuations are clauses which represent both Expectations (tests validating the expectations +//! of the test document when read) and Extensions (clauses that allow the chaining, or +//! permutations for document creation). + use super::*; use super::context::Context; use super::model::ModelValue; @@ -7,18 +11,31 @@ use ion_rs::{Element, Sequence}; #[derive(Clone, Debug)] pub(crate) enum Continuation { // expectations + + // Verify that reading the current document produces the expected data provided. Produces(Vec), + // Verify that reading the current document produces the expected data, provided through data + // model elements. Denotes(Vec), + // Verify that reading the current document produces an error. Signals(String), // extensions + // Internal. This continuation tracks multiple continuations that are allowed in a document. Extensions(Vec), + // Contiunue the document within a sub-branch of the test; this allows for multiple tests that + // deviate from the same start. Then(Box), + // Allows a single expectation to be evaluated for multiple fragments. Each(Vec, Box), + // Apply a logical-AND to the outcomes of each continuation provided. And(Vec), + // Negate the outcome of the provided continuation. Not(Box), } impl Continuation { + /// Test the outcome of the current continuation. This will generate the serialization of the + /// document and any other parent nodes. pub fn evaluate(&self, ctx: &Context) -> InnerResult<()> { match self { // Produces is terminal, so we can evaluate. @@ -88,6 +105,7 @@ impl Default for Continuation { } } +/// Parses a clause known to be a continuation into a proper Continuation instance. pub fn parse_continuation(clause: Clause) -> InnerResult { let continuation = match clause.tpe { ClauseType::Produces => { @@ -204,12 +222,16 @@ pub fn parse_continuation(clause: Clause) -> InnerResult { Ok(continuation) } +/// Represents a single branch in an Each clause. Each branch is allowed to be named (optionally) +/// and must contain a fragment. #[derive(Clone, Debug)] pub(crate) struct EachBranch { name: Option, fragment: Fragment, } +/// Represents a Then clause, it's optional name, the list of fragments, and continuation. A 'Then' +/// acts as almost a sub-document. #[derive(Clone, Debug, Default)] pub(crate) struct Then { pub test_name: Option, @@ -218,6 +240,7 @@ pub(crate) struct Then { } impl Then { + /// Evaluate the outcome of the Then clause. pub fn evaluate(&self, ctx: &Context) -> InnerResult<()> { // We need to create a new context for the Then scope. let mut then_ctx = Context::extend(ctx, &self.fragments); @@ -227,6 +250,7 @@ impl Then { self.continuation.evaluate(&then_ctx) } + /// Determine the encoding (text/binary) of the fragments contained within this Then clause. fn fragment_encoding(&self) -> IonEncoding { let enc = self.fragments.iter().find(|f| matches!(f, Fragment::Text(_) | Fragment::Binary(_))); match enc { @@ -236,6 +260,7 @@ impl Then { } } + /// Determine the ion version of the fragments contained within this Then clause. fn fragment_version(&self) -> IonVersion { match self.fragments.first() { Some(Fragment::Ivm(1, 0)) => IonVersion::V1_0, diff --git a/tests/conformance_dsl/document.rs b/tests/conformance_dsl/document.rs index f1fb5434..258ae57c 100644 --- a/tests/conformance_dsl/document.rs +++ b/tests/conformance_dsl/document.rs @@ -6,6 +6,7 @@ use super::continuation::*; use ion_rs::{Element, Sequence}; +/// Convert a collection of Fragments into a binary encoded ion stream. pub(crate) fn to_binary<'a, T: IntoIterator>(ctx: &'a Context, fragments: T) -> InnerResult> { let mut bin_encoded = vec!(); for frag in fragments { @@ -15,6 +16,7 @@ pub(crate) fn to_binary<'a, T: IntoIterator>(ctx: &'a Context Ok(bin_encoded) } +/// Convert a collection of Fragments into a text encoded ion stream. pub(crate) fn to_text<'a, T: IntoIterator>(ctx: &'a Context, fragments: T) -> InnerResult> { let mut txt_encoded = vec!(); for frag in fragments { @@ -25,6 +27,8 @@ pub(crate) fn to_text<'a, T: IntoIterator>(ctx: &'a Context, Ok(txt_encoded) } +/// The root clause for a test. A document contains an optional name, set of fragments, and a +/// continuation. All tests defined by this document are evaluated through the `run` function. #[derive(Debug, Default)] pub(crate) struct Document { pub name: Option, @@ -33,12 +37,15 @@ pub(crate) struct Document { } impl Document { + /// Execute the test by evaluating the document's continuation. pub fn run(&self) -> Result<()> { let ctx = Context::new(IonVersion::Unspecified, self.encoding(), &self.fragments); self.continuation.evaluate(&ctx)?; Ok(()) } + /// Determine the ion encoding (text/binary) of this document based on the fragments defined by + /// the document. fn encoding(&self) -> IonEncoding { match self.fragments.iter().fold((false,false), |acc, f| { (acc.0 || matches!(f, Fragment::Text(_)), acc.1 || matches!(f, Fragment::Binary(_))) diff --git a/tests/conformance_dsl/fragment.rs b/tests/conformance_dsl/fragment.rs index 43100180..10b69810 100644 --- a/tests/conformance_dsl/fragment.rs +++ b/tests/conformance_dsl/fragment.rs @@ -4,23 +4,30 @@ use ion_rs::{v1_0, v1_1, WriteConfig, Encoding, ion_seq}; use super::*; use super::context::Context; +/// Shared functionality for Fragments. trait FragmentImpl { + /// Encode the current fragment into ion given the provided `WriteConfig` fn encode(&self, config: impl Into>) -> InnerResult>; } +/// Fragments represent parts of the ion document read for testing. #[derive(Clone, Debug)] pub(crate) enum Fragment { + /// Provide ion data encoded as binary ion to be used as part of the test document. Binary(Vec), - Each(Vec), + /// Provide a major and minor version that should be used to emit an IVM for the document. Ivm(i64, i64), + /// Provide ion data encoded as text ion to be used as part of the test document. Text(String), + /// Provide ion data using ion literals to be used as part of the test document. TopLevel(TopLevel), } static EMPTY_TOPLEVEL: Fragment = Fragment::TopLevel(TopLevel { elems: vec!() }); impl Fragment { + /// Encode the fragment as binary ion. pub fn to_binary(&self, ctx: &Context) -> InnerResult> { match ctx.version() { IonVersion::V1_1 => self.write_as_binary(ctx, v1_1::Binary), @@ -28,6 +35,7 @@ impl Fragment { } } + /// Encode the fragment as text ion. pub fn to_text(&self, ctx: &Context) -> InnerResult> { match ctx.version() { IonVersion::V1_1 => self.write_as_text(ctx, v1_1::Text), @@ -35,16 +43,17 @@ impl Fragment { } } + /// Internal. Writes the fragment as binary ion using the provided WriteConfig. fn write_as_binary(&self, _ctx: &Context, config: impl Into>) -> InnerResult> { match self { Fragment::TopLevel(toplevel) => toplevel.encode(config), Fragment::Binary(bin) => Ok(bin.clone()), Fragment::Text(_) => unreachable!(), Fragment::Ivm(maj, min) => Ok([0xE0, *maj as u8, *min as u8, 0xEA].to_vec()), - _ => unimplemented!(), } } + /// Internal. Writes the fragment as text ion using the provided WriteConfig. fn write_as_text(&self, _ctx: &Context, config: impl Into>) -> InnerResult> { match self { Fragment::TopLevel(toplevel) => toplevel.encode(config), @@ -54,11 +63,11 @@ impl Fragment { } Fragment::Binary(_) => unreachable!(), Fragment::Ivm(maj, min) => return Ok(format!("$ion_{}_{}", maj, min).as_bytes().to_owned()), - _ => unimplemented!(), } } - + /// Returns the required encoding (binary/text) for the fragment if one is required, otherwise + /// `IonEncoding::Unspecified` is returned. pub fn required_encoding(&self) -> IonEncoding { match self { Fragment::Text(_) => IonEncoding::Text, @@ -131,12 +140,14 @@ impl TryFrom for Fragment { } } +/// Implments the TopLevel fragment. #[derive(Clone, Debug, Default)] pub(crate) struct TopLevel { elems: Vec, } impl FragmentImpl for TopLevel { + /// Encodes the provided ion literals into an ion stream, using the provided WriteConfig. fn encode(&self, config: impl Into>) -> InnerResult> { use ion_rs::Writer; let mut buffer = Vec::with_capacity(1024); diff --git a/tests/conformance_dsl/mod.rs b/tests/conformance_dsl/mod.rs index 4421f985..d1a468d7 100644 --- a/tests/conformance_dsl/mod.rs +++ b/tests/conformance_dsl/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod prelude { pub(crate) use super::IonVersion; } +/// Specific errors used during parsing and test evaluation. #[derive(Clone, Default, Debug)] pub(crate) enum ConformanceErrorKind { #[default] @@ -64,10 +65,14 @@ impl From for ConformanceErrorKind { } } +/// Error details for a user-facing error. #[derive(Clone, Default, Debug)] struct ConformanceErrorImpl { + /// Path to the file containing the test. file: PathBuf, + /// The document-level test name. test_name: String, + /// The specific error kind. kind: ConformanceErrorKind, } @@ -107,13 +112,13 @@ impl From for ConformanceError { } } -// Used for internal error handling. +/// Used for internal error handling. type InnerResult = std::result::Result; -// Used for public conformance API error handling. +/// Used for public conformance API error handling. pub(crate) type Result = std::result::Result; -// Encoding captures whether an encoding is forced by including a text, or binary clause. +/// Encoding captures whether an encoding is forced by including a text, or binary clause. #[derive(Clone, Copy, Debug, PartialEq)] pub(crate) enum IonEncoding { Text, // Text clause used. @@ -140,6 +145,8 @@ pub(crate) trait DocumentLike: Default { fn set_continuation(&mut self, continuation: continuation::Continuation); } +/// Parses a Clause that has the format of a Document clause. This includes, an optional name, +/// multiple fragments, followed by an expectation or multiple extensions. pub(crate) fn parse_document_like(clause: &Clause) -> InnerResult { // let clause: Clause = Clause::try_from(seq)?; let mut doc_like = T::default(); @@ -238,6 +245,7 @@ impl TestCollection { Ok(collection) } + /// Evaluates the tests in all of the test documents contained in the collection. pub fn run(&self) -> Result<()> { for test in self.documents.iter() { test.run()?; @@ -255,6 +263,9 @@ impl TestCollection { } +/// Parses a 'bytes*' expression. A bytes expression can be either an integer (0..255) or a string +/// containing hexadecimal digits (whitespace allowed). The `elems` provided should be all of the +/// arguments to be included in the bytes* expression. pub(crate) fn parse_bytes_exp<'a, I: IntoIterator>(elems: I) -> InnerResult> { // Bytes can be of the form int (0..255), and a string containing hexadecimal digits. use std::result::Result; @@ -279,6 +290,7 @@ pub(crate) fn parse_bytes_exp<'a, I: IntoIterator>(elems: I) - Ok(bytes) } +/// Parses a sequence of Elements that represent text data. pub(crate) fn parse_text_exp<'a, I: IntoIterator>(elems: I) -> InnerResult { let bytes: Vec> = elems.into_iter().map(|v| match v.ion_type() { IonType::String => v.as_string().map(|s| Ok(s.as_bytes().to_vec())).unwrap(), @@ -293,5 +305,4 @@ pub(crate) fn parse_text_exp<'a, I: IntoIterator>(elems: I) -> let val_string = bytes.iter().map(|v| unsafe { String::from_utf8_unchecked(v.to_vec()) }).collect(); Ok(val_string) - } diff --git a/tests/conformance_dsl/model.rs b/tests/conformance_dsl/model.rs index 2d9b50ec..f9d3aad7 100644 --- a/tests/conformance_dsl/model.rs +++ b/tests/conformance_dsl/model.rs @@ -4,32 +4,33 @@ use super::{Clause, ClauseType, ConformanceErrorKind, InnerResult, parse_text_ex use std::collections::HashMap; +/// Represents a symbol in the Data Model representation of ion data. #[derive(Debug, Clone, Eq, Hash, PartialEq)] -pub(crate) enum SymTok { +pub(crate) enum SymbolToken { Text(String), Offset(i64), Absent(String, i64), } -impl TryFrom<&Element> for SymTok { +impl TryFrom<&Element> for SymbolToken { type Error = ConformanceErrorKind; fn try_from(other: &Element) -> InnerResult { match other.ion_type() { - IonType::String => Ok(SymTok::Text(other.as_string().unwrap().to_owned())), - IonType::Int => Ok(SymTok::Offset(other.as_i64().unwrap())), + IonType::String => Ok(SymbolToken::Text(other.as_string().unwrap().to_owned())), + IonType::Int => Ok(SymbolToken::Offset(other.as_i64().unwrap())), IonType::SExp => { let clause: Clause = other.as_sequence().unwrap().try_into()?; match clause.tpe { ClauseType::Text => { let text = parse_text_exp(clause.body.iter())?; - Ok(SymTok::Text(text)) + Ok(SymbolToken::Text(text)) }, ClauseType::Absent => { let text = clause.body.get(1).and_then(|v| v.as_string()).ok_or(ConformanceErrorKind::ExpectedSymbolType)?; let offset = clause.body.get(2).and_then(|v| v.as_i64()).ok_or(ConformanceErrorKind::ExpectedSymbolType)?; - Ok(SymTok::Absent(text.to_string(), offset)) + Ok(SymbolToken::Absent(text.to_string(), offset)) } _ => unreachable!(), } @@ -39,6 +40,11 @@ impl TryFrom<&Element> for SymTok { } } +/// Data Model value representation. Implementation provides parsing of data model clauses and +/// comparison functionality for test evaluation. Each variant represents a single data model value +/// clause. +/// +/// [Grammar]: https://github.com/amazon-ion/ion-tests/tree/master/conformance#grammar #[derive(Debug, Clone)] pub(crate) enum ModelValue { Null(IonType), @@ -48,10 +54,10 @@ pub(crate) enum ModelValue { Decimal(Decimal), Timestamp(Timestamp), String(String), - Symbol(SymTok), + Symbol(SymbolToken), List(Vec), Sexp(Vec), - Struct(HashMap), + Struct(HashMap), Blob(Vec), Clob(Vec), } @@ -104,20 +110,20 @@ impl TryFrom<&Sequence> for ModelValue { "Symbol" => { let value = elems.get(1).ok_or(ConformanceErrorKind::ExpectedSymbolType)?; match value.ion_type() { - IonType::String => Ok(ModelValue::Symbol(SymTok::Text(value.as_string().unwrap().to_owned()))), - IonType::Int => Ok(ModelValue::Symbol(SymTok::Offset(value.as_i64().unwrap()))), + IonType::String => Ok(ModelValue::Symbol(SymbolToken::Text(value.as_string().unwrap().to_owned()))), + IonType::Int => Ok(ModelValue::Symbol(SymbolToken::Offset(value.as_i64().unwrap()))), IonType::SExp => { let clause: Clause = value.as_sequence().unwrap().try_into()?; match clause.tpe { ClauseType::Text => { let text = parse_text_exp(clause.body.iter())?; - Ok(ModelValue::Symbol(SymTok::Text(text))) + Ok(ModelValue::Symbol(SymbolToken::Text(text))) }, ClauseType::Absent => { let text = clause.body.get(1).and_then(|v| v.as_string()).ok_or(ConformanceErrorKind::ExpectedSymbolType)?; let offset = clause.body.get(2).and_then(|v| v.as_i64()).ok_or(ConformanceErrorKind::ExpectedSymbolType)?; - Ok(ModelValue::Symbol(SymTok::Absent(text.to_string(), offset))) + Ok(ModelValue::Symbol(SymbolToken::Absent(text.to_string(), offset))) } _ => unreachable!(), } @@ -150,7 +156,7 @@ impl TryFrom<&Sequence> for ModelValue { if let Some(seq) = elem.as_sequence() { // Each elem should be a model symtok followed by a model value. let (first, second) = (seq.get(0), seq.get(1)); - let field_sym = first.map(SymTok::try_from).ok_or(ConformanceErrorKind::ExpectedSymbolType)?.unwrap(); + let field_sym = first.map(SymbolToken::try_from).ok_or(ConformanceErrorKind::ExpectedSymbolType)?.unwrap(); let value = match second.map(|e| e.ion_type()) { Some(IonType::String) => { let string = second.unwrap().as_string().unwrap(); @@ -226,7 +232,7 @@ impl PartialEq for ModelValue { let struct_val = other.as_struct().unwrap(); for (field, val) in struct_val.fields() { let denoted_field = field.text().unwrap(); // TODO: Symbol IDs - let denoted_symtok = SymTok::Text(denoted_field.to_string()); + let denoted_symtok = SymbolToken::Text(denoted_field.to_string()); if let Some(expected_val) = fields.get(&denoted_symtok) { if expected_val != val { return false; @@ -241,9 +247,9 @@ impl PartialEq for ModelValue { ModelValue::Symbol(sym) => { if let Some(other_sym) = other.as_symbol() { match sym { - SymTok::Text(text) => Some(text.as_ref()) == other_sym.text(), - SymTok::Offset(_offset) => todo!(), - SymTok::Absent(_text, _offset) => todo!(), + SymbolToken::Text(text) => Some(text.as_ref()) == other_sym.text(), + SymbolToken::Offset(_offset) => todo!(), + SymbolToken::Absent(_text, _offset) => todo!(), } } else { false @@ -254,6 +260,7 @@ impl PartialEq for ModelValue { } } +/// Parses a Timestamp clause into an ion-rs Timestamp. fn parse_timestamp<'a, I: IntoIterator>(elems: I) -> InnerResult { let mut iter = elems.into_iter(); let first = iter.next().and_then(|e| e.as_symbol()).and_then(|s| s.text()); @@ -350,6 +357,7 @@ fn parse_timestamp<'a, I: IntoIterator>(elems: I) -> InnerResu } } +/// Parses a data-model value timestamp's 'offset' clause into an i64. fn parse_ts_offset<'a, I: IntoIterator>(elems: I) -> InnerResult> { let mut iter = elems.into_iter(); match iter.next().and_then(|e| e.as_symbol()).and_then(|s| s.text()) { @@ -367,6 +375,7 @@ fn parse_ts_offset<'a, I: IntoIterator>(elems: I) -> InnerResu } } +/// Parses a data-model value's Decimal clause into an ion-rs Decimal. fn parse_model_decimal<'a, I: IntoIterator>(elems: I) -> InnerResult { let mut iter = elems.into_iter(); let (first, second) = (iter.next(), iter.next()); diff --git a/tests/conformance_tests.rs b/tests/conformance_tests.rs index c06d4f9a..e091d60d 100644 --- a/tests/conformance_tests.rs +++ b/tests/conformance_tests.rs @@ -83,7 +83,6 @@ mod ion_tests { #[test_resources("ion-tests/conformance/core/string_symbol.ion")] #[test_resources("ion-tests/conformance/core/empty_document.ion")] #[test_resources("ion-tests/conformance/core/toplevel_produces.ion")] - // #[test_resources("ion-tests/conformance/ivm.ion")] fn conformance(file_name: &str) { println!("Testing: {}", file_name); let collection = TestCollection::load(file_name).expect("unable to load test file");