diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7298c55ff..788c9d851 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -28,10 +28,6 @@ updates: directory: "/stac-server" schedule: interval: "weekly" - - package-ecosystem: "cargo" - directory: "/stac-validate" - schedule: - interval: "weekly" - package-ecosystem: "pip" directory: "/scripts" schedule: diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml deleted file mode 100644 index e65f6bae5..000000000 --- a/.github/workflows/validate.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Validate - -on: - pull_request: - paths: - - validate/** - push: - branches: - - main - paths: - - validate/** - workflow_dispatch: - -defaults: - run: - working-directory: validate - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - CARGO_TERM_COLOR: always - CARGO_TERM_VERBOSE: true - -jobs: - test-validate: - strategy: - matrix: - os: - - ubuntu-latest - - macos-latest - - windows-latest - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Set up Rust cache - uses: Swatinem/rust-cache@v2 - - name: Test - run: cargo test diff --git a/Cargo.toml b/Cargo.toml index 2a9da38f3..5fe1ee759 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,4 @@ [workspace] resolver = "2" -members = [ - "api", - "cli", - "core", - "duckdb", - "pgstac", - "python", - "server", - "validate", -] -default-members = ["api", "cli", "core", "server", "validate"] +members = ["api", "cli", "core", "duckdb", "pgstac", "python", "server"] +default-members = ["api", "cli", "core", "server"] diff --git a/README.md b/README.md index fee685a8e..bb0dfa823 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,6 @@ This monorepo contains several crates: | [stac-api](./api/README.md) | Data structures for the [STAC API](https://github.com/radiantearth/stac-api-spec) specification | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/stac-utils/stac-rs/api.yml?style=flat-square&branch=main)](https://github.com/stac-utils/stac-rs/actions/workflows/api.yml)
[![docs.rs](https://img.shields.io/docsrs/stac-api?style=flat-square)](https://docs.rs/stac-api/latest/stac_api/)
[![Crates.io](https://img.shields.io/crates/v/stac-api?style=flat-square)](https://crates.io/crates/stac-api) | | [stac-cli](./cli/README.md)| Command line interface | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/stac-utils/stac-rs/cli.yml?style=flat-square&branch=main)](https://github.com/stac-utils/stac-rs/actions/workflows/cli.yml)
[![docs.rs](https://img.shields.io/docsrs/stac-cli?style=flat-square)](https://docs.rs/stac-cli/latest/stac_cli/)
[![Crates.io](https://img.shields.io/crates/v/stac-cli?style=flat-square)](https://crates.io/crates/stac-cli) | | [stac-server](./server/README.md)| STAC API server with multiple backends | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/stac-utils/stac-rs/server.yml?style=flat-square&branch=main)](https://github.com/stac-utils/stac-rs/actions/workflows/server.yml)
[![docs.rs](https://img.shields.io/docsrs/stac-server?style=flat-square)](https://docs.rs/stac-server/latest/stac_server/)
[![Crates.io](https://img.shields.io/crates/v/stac-server?style=flat-square)](https://crates.io/crates/stac-server) | -| [stac-validate](./validate/README.md) | Validate STAC data structures with [jsonschema](https://json-schema.org/) | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/stac-utils/stac-rs/validate.yml?style=flat-square&branch-main)](https://github.com/stac-utils/stac-rs/actions/workflows/validate.yml)
[![docs.rs](https://img.shields.io/docsrs/stac-validate?style=flat-square)](https://docs.rs/stac-validate/latest/stac-validate/)
[![Crates.io](https://img.shields.io/crates/v/stac-validate?style=flat-square)](https://crates.io/crates/stac-validate) | | [pgstac](./pgstac/README.md) | Bindings for [pgstac](https://github.com/stac-utils/pgstac) | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/stac-utils/stac-rs/pgstac.yml?style=flat-square&branch=main)](https://github.com/stac-utils/stac-rs/actions/workflows/pgstac.yml)
[![docs.rs](https://img.shields.io/docsrs/pgstac?style=flat-square)](https://docs.rs/pgstac/latest/pgstac/)
[![Crates.io](https://img.shields.io/crates/v/pgstac?style=flat-square)](https://crates.io/crates/pgstac) | | [stac-duckdb](./duckdb/README.md) | Experimental client for [duckdb](https://duckdb.org/) | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/stac-utils/stac-rs/duckdb.yml?style=flat-square&branch=main)](https://github.com/stac-utils/stac-rs/actions/workflows/duckdb.yml)
[![docs.rs](https://img.shields.io/docsrs/stac-duckdb?style=flat-square)](https://docs.rs/stac-duckdb/latest/stac_duckdb/)
[![Crates.io](https://img.shields.io/crates/v/stac-duckdb?style=flat-square)](https://crates.io/crates/stac-duckdb) | diff --git a/cli/Cargo.toml b/cli/Cargo.toml index bdfce5cd7..e0b29b982 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -32,11 +32,11 @@ stac = { version = "0.10.0", path = "../core", features = [ "geoparquet-compression", "object-store-all", "reqwest", + "validate", ] } stac-api = { version = "0.6.0", path = "../api", features = ["client"] } stac-duckdb = { version = "0.0.2", path = "../duckdb", optional = true } stac-server = { version = "0.3.1", path = "../server", features = ["axum"] } -stac-validate = { version = "0.3.0", path = "../validate" } thiserror = "1" tokio = { version = "1.23", features = [ "macros", diff --git a/cli/src/args/validate.rs b/cli/src/args/validate.rs index 17699c90a..29c339519 100644 --- a/cli/src/args/validate.rs +++ b/cli/src/args/validate.rs @@ -1,6 +1,6 @@ use super::{Input, Run}; use crate::{Error, Result, Value}; -use stac_validate::Validate; +use stac::Validate; use tokio::sync::mpsc::Sender; /// Arguments for the `validate` subcommand. @@ -17,7 +17,7 @@ impl Run for Args { async fn run(self, input: Input, _: Option>) -> Result> { let value = input.get().await?; let result = value.validate().await; - if let Err(stac_validate::Error::Validation(ref errors)) = result { + if let Err(stac::Error::Validation(ref errors)) = result { let message_base = match value { stac::Value::Item(item) => format!("[item={}] ", item.id), stac::Value::Catalog(catalog) => format!("[catalog={}] ", catalog.id), diff --git a/cli/src/error.rs b/cli/src/error.rs index 879dcf4f5..96fd0d25e 100644 --- a/cli/src/error.rs +++ b/cli/src/error.rs @@ -42,10 +42,6 @@ pub enum Error { #[error(transparent)] StacServer(#[from] stac_server::Error), - /// [stac_validate::Error] - #[error(transparent)] - StacValidate(#[from] stac_validate::Error), - /// [tokio::sync::mpsc::error::SendError] #[error(transparent)] TokioSend(#[from] tokio::sync::mpsc::error::SendError), diff --git a/core/CHANGELOG.md b/core/CHANGELOG.md index 6c290e7f5..4bcabdba3 100644 --- a/core/CHANGELOG.md +++ b/core/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +### Added + +- Validation, from the (now defunct) **stac-validate** ([#434](https://github.com/stac-utils/stac-rs/pull/434)) + ## [0.10.1] - 2024-09-20 ### Fixed diff --git a/core/Cargo.toml b/core/Cargo.toml index d0a503db7..ea8319a34 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -43,6 +43,8 @@ object-store-all = [ "object-store-http", ] reqwest = ["dep:reqwest"] +validate = ["dep:jsonschema", "dep:reqwest", "dep:tokio", "dep:tracing"] +validate-blocking = ["validate", "tokio/rt"] [dependencies] arrow-array = { version = "52", optional = true } @@ -57,6 +59,7 @@ geo = { version = "0.28", optional = true } geo-types = { version = "0.7", optional = true } geoarrow = { version = "0.3", optional = true } geojson = "0.24" +jsonschema = { version = "0.20", optional = true } log = "0.4" mime = "0.3" object_store = { version = "0.11", optional = true } @@ -65,6 +68,8 @@ reqwest = { version = "0.12", optional = true, features = ["json", "blocking"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } thiserror = "1" +tracing = { version = "0.1", optional = true } +tokio = { version = "1", optional = true } url = "2" [dev-dependencies] @@ -72,7 +77,7 @@ assert-json-diff = "2" bytes = "1" rstest = "0.22" tempdir = "0.3" -tokio = "1" +tokio = { version = "1", features = ["macros"] } tokio-test = "0.4" [package.metadata.docs.rs] diff --git a/core/src/error.rs b/core/src/error.rs index 2456d5b40..d7fe44243 100644 --- a/core/src/error.rs +++ b/core/src/error.rs @@ -1,4 +1,6 @@ use crate::{Value, Version}; +#[cfg(feature = "validate")] +use jsonschema::ValidationError; use serde_json::Value as JsonValue; use thiserror::Error; @@ -11,6 +13,11 @@ pub enum Error { #[cfg(feature = "geoarrow")] Arrow(#[from] arrow_schema::ArrowError), + /// Cannot validate a non-object, non-array + #[error("value is not an object or an array, cannot validate")] + #[cfg(feature = "validate")] + CannotValidate(serde_json::Value), + /// [chrono::ParseError] #[error(transparent)] ChronoParse(#[from] chrono::ParseError), @@ -69,16 +76,23 @@ pub enum Error { /// Returned when a geometry is missing but is required. #[error("no geometry field")] + #[deprecated(since = "0.10.2", note = "renamed to NoGeometry")] MissingGeometry, /// Returned when there is not a `type` field on a STAC object #[error("no \"type\" field in the JSON object")] + #[deprecated(since = "0.10.2", note = "renamed to NoType")] MissingType, /// Returned when an object is expected to have an href, but it doesn't. #[error("object has no href")] + #[deprecated(since = "0.10.2", note = "use to NoHref")] MissingHref, + /// There is no geometry. + #[error("no geometry")] + NoGeometry, + /// There are no items, when items are required. #[error("no items")] NoItems, @@ -87,6 +101,14 @@ pub enum Error { #[error("no href")] NoHref, + /// There is no type. + #[error("no type")] + NoType, + + /// No version field on an object. + #[error("no version field")] + NoVersion, + /// This value is not an item. #[error("value is not an item")] NotAnItem(Value), @@ -119,7 +141,7 @@ pub enum Error { Parquet(#[from] parquet::errors::ParquetError), /// [reqwest::Error] - #[cfg(feature = "reqwest")] + #[cfg(any(feature = "reqwest", feature = "validate"))] #[error(transparent)] Reqwest(#[from] reqwest::Error), @@ -127,6 +149,27 @@ pub enum Error { #[error(transparent)] SerdeJson(#[from] serde_json::Error), + /// [tokio::task::JoinError] + #[error(transparent)] + #[cfg(feature = "validate")] + TokioJoin(#[from] tokio::task::JoinError), + + /// [tokio::sync::mpsc::error::SendError] + #[error(transparent)] + #[cfg(feature = "validate")] + TokioSend( + #[from] + tokio::sync::mpsc::error::SendError<( + url::Url, + tokio::sync::oneshot::Sender>>, + )>, + ), + + /// [tokio::sync::oneshot::error::RecvError] + #[error(transparent)] + #[cfg(feature = "validate")] + TokioRecv(#[from] tokio::sync::oneshot::error::RecvError), + /// [std::num::TryFromIntError] #[error(transparent)] TryFromInt(#[from] std::num::TryFromIntError), @@ -154,4 +197,35 @@ pub enum Error { /// [url::ParseError] #[error(transparent)] Url(#[from] url::ParseError), + + /// A list of validation errors. + /// + /// Since we usually don't have the original [serde_json::Value] (because we + /// create them from the STAC objects), we need these errors to be `'static` + /// lifetime. + #[error("validation errors")] + #[cfg(feature = "validate")] + Validation(Vec>), +} + +#[cfg(feature = "validate")] +impl Error { + pub(crate) fn from_validation_errors<'a, I>(errors: I) -> Error + where + I: Iterator>, + { + use std::borrow::Cow; + + let mut error_vec = Vec::new(); + for error in errors { + // Cribbed from https://docs.rs/jsonschema/latest/src/jsonschema/error.rs.html#21-30 + error_vec.push(ValidationError { + instance_path: error.instance_path.clone(), + instance: Cow::Owned(error.instance.into_owned()), + kind: error.kind, + schema_path: error.schema_path, + }) + } + Error::Validation(error_vec) + } } diff --git a/core/src/lib.rs b/core/src/lib.rs index 41639afa3..17ec32150 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -176,9 +176,15 @@ mod migrate; pub mod mime; mod ndjson; mod statistics; +#[cfg(feature = "validate")] +mod validate; mod value; mod version; +#[cfg(feature = "validate-blocking")] +pub use validate::ValidateBlocking; +#[cfg(feature = "validate")] +pub use validate::{Validate, Validator}; pub use { asset::{Asset, Assets}, band::Band, diff --git a/validate/src/blocking.rs b/core/src/validate/blocking.rs similarity index 77% rename from validate/src/blocking.rs rename to core/src/validate/blocking.rs index 3df56b1e1..a70760bac 100644 --- a/validate/src/blocking.rs +++ b/core/src/validate/blocking.rs @@ -11,8 +11,7 @@ pub trait ValidateBlocking: Validate { /// # Examples /// /// ``` - /// use stac_validate::ValidateBlocking; - /// use stac::Item; + /// use stac::{ValidateBlocking, Item}; /// /// let mut item = Item::new("an-id"); /// item.validate_blocking().unwrap(); @@ -30,9 +29,8 @@ impl ValidateBlocking for T {} #[cfg(test)] mod tests { use super::ValidateBlocking; - use geojson::{Geometry, Value}; + use crate::{Catalog, Collection, Item}; use rstest as _; - use stac::{Catalog, Collection, Item}; #[test] fn item() { @@ -41,7 +39,10 @@ mod tests { } #[test] + #[cfg(feature = "geo")] fn item_with_geometry() { + use geojson::{Geometry, Value}; + let mut item = Item::new("an-id"); item.set_geometry(Geometry::new(Value::Point(vec![-105.1, 40.1]))) .unwrap(); @@ -51,7 +52,7 @@ mod tests { #[test] fn item_with_extensions() { let item: Item = - stac::read("examples/extensions-collection/proj-example/proj-example.json").unwrap(); + crate::read("examples/extensions-collection/proj-example/proj-example.json").unwrap(); item.validate_blocking().unwrap(); } @@ -69,14 +70,14 @@ mod tests { #[test] fn value() { - let value: stac::Value = stac::read("examples/simple-item.json").unwrap(); + let value: crate::Value = crate::read("examples/simple-item.json").unwrap(); value.validate_blocking().unwrap(); } #[test] fn item_collection() { - let item = stac::read("examples/simple-item.json").unwrap(); - let item_collection = stac::ItemCollection::from(vec![item]); + let item = crate::read("examples/simple-item.json").unwrap(); + let item_collection = crate::ItemCollection::from(vec![item]); item_collection.validate_blocking().unwrap(); } } diff --git a/core/src/validate/mod.rs b/core/src/validate/mod.rs new file mode 100644 index 000000000..9f11d71e2 --- /dev/null +++ b/core/src/validate/mod.rs @@ -0,0 +1,83 @@ +//! Validate STAC objects with [json-schema](https://json-schema.org/). +//! +//! # Examples +//! +//! Validation is provided via the [Validate] trait: +//! +//! ``` +//! use stac::{Item, Validate}; +//! +//! # tokio_test::block_on(async { +//! Item::new("an-id").validate().await.unwrap(); +//! # }) +//! ``` +//! +//! If you're working in a blocking context (not async), enable the `blocking` feature and use [ValidateBlocking]: +//! +//! ``` +//! #[cfg(feature = "blocking")] +//! { +//! use stac::{ValidateBlocking, Item}; +//! Item::new("an-id").validate_blocking().unwrap(); +//! } +//! ``` +//! +//! All fetched schemas are cached, so if you're you're doing multiple +//! validations, you should re-use the same [Validator]: +//! +//! ``` +//! # use stac::{Item, Validator}; +//! let mut items: Vec<_> = (0..10).map(|n| Item::new(format!("item-{}", n))).collect(); +//! # tokio_test::block_on(async { +//! let mut validator = Validator::new().await; +//! for item in items { +//! validator.validate(&item).await.unwrap(); +//! } +//! # }) +//! ``` +//! +//! [Validator] is cheap to clone, so you are encouraged to validate a large +//! number of objects at the same time if that's your use-case. + +use crate::Result; +use serde::Serialize; +use std::future::Future; + +#[cfg(feature = "validate-blocking")] +mod blocking; +mod validator; + +#[cfg(feature = "validate-blocking")] +pub use blocking::ValidateBlocking; +pub use validator::Validator; + +/// Validate any serializable object with [json-schema](https://json-schema.org/) +pub trait Validate: Serialize + Sized { + /// Validates this object. + /// + /// If the object fails validation, this will return an + /// [Error::Validation](crate::Error::Validation) which contains a vector of + /// all of the validation errors. + /// + /// If you're doing multiple validations, use [Validator::validate], which + /// will re-use cached schemas. + /// + /// # Examples + /// + /// ``` + /// use stac::{Item, Validate}; + /// + /// let mut item = Item::new("an-id"); + /// # tokio_test::block_on(async { + /// item.validate().await.unwrap(); + /// }); + /// ``` + fn validate(&self) -> impl Future> { + async { + let validator = Validator::new().await; + validator.validate(self).await + } + } +} + +impl Validate for T {} diff --git a/validate/schemas/geojson/Feature.json b/core/src/validate/schemas/geojson/Feature.json similarity index 100% rename from validate/schemas/geojson/Feature.json rename to core/src/validate/schemas/geojson/Feature.json diff --git a/validate/schemas/geojson/Geometry.json b/core/src/validate/schemas/geojson/Geometry.json similarity index 100% rename from validate/schemas/geojson/Geometry.json rename to core/src/validate/schemas/geojson/Geometry.json diff --git a/validate/schemas/json-schema/draft-07.json b/core/src/validate/schemas/json-schema/draft-07.json similarity index 100% rename from validate/schemas/json-schema/draft-07.json rename to core/src/validate/schemas/json-schema/draft-07.json diff --git a/validate/schemas/v1.0.0/basics.json b/core/src/validate/schemas/v1.0.0/basics.json similarity index 100% rename from validate/schemas/v1.0.0/basics.json rename to core/src/validate/schemas/v1.0.0/basics.json diff --git a/validate/schemas/v1.0.0/catalog.json b/core/src/validate/schemas/v1.0.0/catalog.json similarity index 100% rename from validate/schemas/v1.0.0/catalog.json rename to core/src/validate/schemas/v1.0.0/catalog.json diff --git a/validate/schemas/v1.0.0/collection.json b/core/src/validate/schemas/v1.0.0/collection.json similarity index 100% rename from validate/schemas/v1.0.0/collection.json rename to core/src/validate/schemas/v1.0.0/collection.json diff --git a/validate/schemas/v1.0.0/datetime.json b/core/src/validate/schemas/v1.0.0/datetime.json similarity index 100% rename from validate/schemas/v1.0.0/datetime.json rename to core/src/validate/schemas/v1.0.0/datetime.json diff --git a/validate/schemas/v1.0.0/instrument.json b/core/src/validate/schemas/v1.0.0/instrument.json similarity index 100% rename from validate/schemas/v1.0.0/instrument.json rename to core/src/validate/schemas/v1.0.0/instrument.json diff --git a/validate/schemas/v1.0.0/item.json b/core/src/validate/schemas/v1.0.0/item.json similarity index 100% rename from validate/schemas/v1.0.0/item.json rename to core/src/validate/schemas/v1.0.0/item.json diff --git a/validate/schemas/v1.0.0/licensing.json b/core/src/validate/schemas/v1.0.0/licensing.json similarity index 100% rename from validate/schemas/v1.0.0/licensing.json rename to core/src/validate/schemas/v1.0.0/licensing.json diff --git a/validate/schemas/v1.0.0/provider.json b/core/src/validate/schemas/v1.0.0/provider.json similarity index 100% rename from validate/schemas/v1.0.0/provider.json rename to core/src/validate/schemas/v1.0.0/provider.json diff --git a/validate/schemas/v1.1.0/bands.json b/core/src/validate/schemas/v1.1.0/bands.json similarity index 100% rename from validate/schemas/v1.1.0/bands.json rename to core/src/validate/schemas/v1.1.0/bands.json diff --git a/validate/schemas/v1.1.0/basics.json b/core/src/validate/schemas/v1.1.0/basics.json similarity index 100% rename from validate/schemas/v1.1.0/basics.json rename to core/src/validate/schemas/v1.1.0/basics.json diff --git a/validate/schemas/v1.1.0/catalog.json b/core/src/validate/schemas/v1.1.0/catalog.json similarity index 100% rename from validate/schemas/v1.1.0/catalog.json rename to core/src/validate/schemas/v1.1.0/catalog.json diff --git a/validate/schemas/v1.1.0/collection.json b/core/src/validate/schemas/v1.1.0/collection.json similarity index 100% rename from validate/schemas/v1.1.0/collection.json rename to core/src/validate/schemas/v1.1.0/collection.json diff --git a/validate/schemas/v1.1.0/common.json b/core/src/validate/schemas/v1.1.0/common.json similarity index 100% rename from validate/schemas/v1.1.0/common.json rename to core/src/validate/schemas/v1.1.0/common.json diff --git a/validate/schemas/v1.1.0/data-values.json b/core/src/validate/schemas/v1.1.0/data-values.json similarity index 100% rename from validate/schemas/v1.1.0/data-values.json rename to core/src/validate/schemas/v1.1.0/data-values.json diff --git a/validate/schemas/v1.1.0/datetime.json b/core/src/validate/schemas/v1.1.0/datetime.json similarity index 100% rename from validate/schemas/v1.1.0/datetime.json rename to core/src/validate/schemas/v1.1.0/datetime.json diff --git a/validate/schemas/v1.1.0/instrument.json b/core/src/validate/schemas/v1.1.0/instrument.json similarity index 100% rename from validate/schemas/v1.1.0/instrument.json rename to core/src/validate/schemas/v1.1.0/instrument.json diff --git a/validate/schemas/v1.1.0/item.json b/core/src/validate/schemas/v1.1.0/item.json similarity index 100% rename from validate/schemas/v1.1.0/item.json rename to core/src/validate/schemas/v1.1.0/item.json diff --git a/validate/schemas/v1.1.0/licensing.json b/core/src/validate/schemas/v1.1.0/licensing.json similarity index 100% rename from validate/schemas/v1.1.0/licensing.json rename to core/src/validate/schemas/v1.1.0/licensing.json diff --git a/validate/schemas/v1.1.0/provider.json b/core/src/validate/schemas/v1.1.0/provider.json similarity index 100% rename from validate/schemas/v1.1.0/provider.json rename to core/src/validate/schemas/v1.1.0/provider.json diff --git a/validate/src/validator.rs b/core/src/validate/validator.rs similarity index 90% rename from validate/src/validator.rs rename to core/src/validate/validator.rs index 0a1602e47..c659d70e4 100644 --- a/validate/src/validator.rs +++ b/core/src/validate/validator.rs @@ -1,11 +1,10 @@ -use crate::{Error, Result}; +use crate::{Error, Result, Type, Version}; use jsonschema::{ SchemaResolver, SchemaResolverError, ValidationOptions, Validator as JsonschemaValidator, }; use reqwest::Client; use serde::Serialize; use serde_json::{Map, Value}; -use stac::{Type, Version}; use std::{ collections::{HashMap, HashSet}, future::Future, @@ -46,7 +45,7 @@ impl Validator { /// # Examples /// /// ``` - /// use stac_validate::Validator; + /// use stac::Validator; /// /// # tokio_test::block_on(async { /// let validator = Validator::new().await; @@ -62,15 +61,14 @@ impl Validator { let mut validation_options = JsonschemaValidator::options(); let _ = validation_options.with_resolver(resolver); let (sender, receiver) = tokio::sync::mpsc::channel(BUFFER); - let _ = tokio::spawn(async move { get_urls(receiver).await }); - let validator = Validator { + drop(tokio::spawn(async move { get_urls(receiver).await })); + Validator { schemas: Arc::new(RwLock::new(schemas(&validation_options))), validation_options, cache, urls, sender, - }; - validator + } } /// Validates a single value. @@ -78,8 +76,7 @@ impl Validator { /// # Examples /// /// ``` - /// use stac_validate::Validator; - /// use stac::Item; + /// use stac::{Item, Validator}; /// /// let item = Item::new("an-id"); /// # tokio_test::block_on(async { @@ -174,7 +171,7 @@ impl Validator { } } while let Some(result) = join_set.join_next().await { - let _ = result??; + result??; } let object = if let Value::Object(o) = Arc::into_inner(value).unwrap() { o @@ -304,20 +301,20 @@ fn schemas(validation_options: &ValidationOptions) -> HashMap HashMap> { // General resolve!( "https://geojson.org/schema/Feature.json", - "../schemas/geojson/Feature.json" + "schemas/geojson/Feature.json" ); resolve!( "https://geojson.org/schema/Geometry.json", - "../schemas/geojson/Geometry.json" + "schemas/geojson/Geometry.json" ); resolve!( "http://json-schema.org/draft-07/schema", - "../schemas/json-schema/draft-07.json" + "schemas/json-schema/draft-07.json" ); // STAC v1.0.0 resolve!( "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/basics.json", - "../schemas/v1.0.0/basics.json" + "schemas/v1.0.0/basics.json" ); resolve!( "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json", - "../schemas/v1.0.0/datetime.json" + "schemas/v1.0.0/datetime.json" ); resolve!( "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/instrument.json", - "../schemas/v1.0.0/instrument.json" + "schemas/v1.0.0/instrument.json" ); resolve!( "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json", - "../schemas/v1.0.0/item.json" + "schemas/v1.0.0/item.json" ); resolve!( "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/licensing.json", - "../schemas/v1.0.0/licensing.json" + "schemas/v1.0.0/licensing.json" ); resolve!( "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/provider.json", - "../schemas/v1.0.0/provider.json" + "schemas/v1.0.0/provider.json" ); // STAC v1.1.0 resolve!( "https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/bands.json", - "../schemas/v1.1.0/bands.json" + "schemas/v1.1.0/bands.json" ); resolve!( "https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/basics.json", - "../schemas/v1.1.0/basics.json" + "schemas/v1.1.0/basics.json" ); resolve!( "https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/common.json", - "../schemas/v1.1.0/common.json" + "schemas/v1.1.0/common.json" ); resolve!( "https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/data-values.json", - "../schemas/v1.1.0/data-values.json" + "schemas/v1.1.0/data-values.json" ); resolve!( "https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/datetime.json", - "../schemas/v1.1.0/datetime.json" + "schemas/v1.1.0/datetime.json" ); resolve!( "https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/instrument.json", - "../schemas/v1.1.0/instrument.json" + "schemas/v1.1.0/instrument.json" ); resolve!( "https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/item.json", - "../schemas/v1.1.0/item.json" + "schemas/v1.1.0/item.json" ); resolve!( "https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/licensing.json", - "../schemas/v1.1.0/licensing.json" + "schemas/v1.1.0/licensing.json" ); resolve!( "https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/provider.json", - "../schemas/v1.1.0/provider.json" + "schemas/v1.1.0/provider.json" ); cache @@ -459,12 +456,12 @@ async fn get_urls(mut receiver: Receiver<(Url, OneshotSender>> tracing::debug!("getting url: {}", url); let local_sender = local_sender.clone(); let client = client.clone(); - let _ = tokio::spawn(async move { + drop(tokio::spawn(async move { match get(client, url.clone()).await { Ok(value) => local_sender.send((url, Ok(value))).await, Err(err) => local_sender.send((url, Err(err))).await, } - }); + })); Vec::new() }) .push(sender); @@ -488,7 +485,7 @@ async fn get(client: Client, url: Url) -> Result { #[cfg(test)] mod tests { use super::Validator; - use stac::Item; + use crate::Item; #[tokio::test] async fn validate_array() { diff --git a/validate/tests/examples.rs b/core/tests/examples.rs similarity index 88% rename from validate/tests/examples.rs rename to core/tests/examples.rs index cf0fbaec5..575da41c2 100644 --- a/validate/tests/examples.rs +++ b/core/tests/examples.rs @@ -1,6 +1,5 @@ use rstest::rstest; -use stac::Value; -use stac_validate::ValidateBlocking; +use stac::{ValidateBlocking, Value}; use std::path::PathBuf; #[rstest] diff --git a/core/tests/migrate.rs b/core/tests/migrate.rs index e5c31ba12..31ddaa792 100644 --- a/core/tests/migrate.rs +++ b/core/tests/migrate.rs @@ -1,9 +1,10 @@ use rstest::rstest; -use stac::{Migrate, Value, Version}; +use stac::{Migrate, ValidateBlocking, Value, Version}; use std::path::PathBuf; #[rstest] fn v1_0_0_to_v1_1_0(#[files("../spec-examples/v1.0.0/**/*.json")] path: PathBuf) { let value: Value = stac::read(path.to_str().unwrap()).unwrap(); - let _ = value.migrate(&Version::v1_1_0).unwrap(); + let value = value.migrate(&Version::v1_1_0).unwrap(); + value.validate_blocking().unwrap(); } diff --git a/docs/index.md b/docs/index.md index 68d531d0f..2ff95b120 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,5 @@ Our Python package is named **stacrs** and its documentation is available [here] - [stac](https://docs.rs/stac): The core Rust crate - [stac-api](https://docs.rs/stac-api): Data structures for a STAC API, and a client for searching one -- [stac-validate](https://docs.rs/stac-validate): Efficiently validate STAC objects with [json-schema](https://json-schema.org/) - [stac-server](https://docs.rs/stac-server): A STAC API server with multiple backends - [pgstac](https://docs.rs/pgstac): Rust bindings for [pgstac](https://github.com/stac-utils/pgstac) diff --git a/python/Cargo.toml b/python/Cargo.toml index f7e44ba38..2749ab7b2 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -22,7 +22,7 @@ stac = { path = "../core", features = [ "geoparquet-compression", "object-store-all", "reqwest", + "validate-blocking", ] } stac-api = { path = "../api", features = ["client"] } -stac-validate = { path = "../validate", features = ["blocking"] } tokio = { version = "1", features = ["rt"] } diff --git a/python/src/error.rs b/python/src/error.rs index 44e9118f6..395648beb 100644 --- a/python/src/error.rs +++ b/python/src/error.rs @@ -16,12 +16,6 @@ impl From for Error { } } -impl From for Error { - fn from(value: stac_validate::Error) -> Self { - Error(value.to_string()) - } -} - impl From for Error { fn from(value: geojson::Error) -> Self { Error(value.to_string()) diff --git a/python/src/validate.rs b/python/src/validate.rs index beeae69c5..061e5129d 100644 --- a/python/src/validate.rs +++ b/python/src/validate.rs @@ -1,7 +1,6 @@ use crate::{Error, Result}; use pyo3::{prelude::*, types::PyDict}; -use stac::Value; -use stac_validate::ValidateBlocking; +use stac::{ValidateBlocking, Value}; /// Validates a single href with json-schema. /// @@ -44,7 +43,7 @@ pub fn validate(value: &Bound<'_, PyDict>) -> PyResult<()> { fn validate_value(value: Value) -> Result<()> { if let Err(error) = value.validate_blocking() { match error { - stac_validate::Error::Validation(errors) => { + stac::Error::Validation(errors) => { let mut message = "Validation errors: ".to_string(); for error in errors { message.push_str(&format!("{}, ", error)); diff --git a/server/Cargo.toml b/server/Cargo.toml index bf601b4d1..64d1a2c51 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -37,7 +37,7 @@ url = "2" [dev-dependencies] serde_json = "1" -stac-validate = { version = "0.3", path = "../validate" } +stac = { version = "0.10", path = "../core", features = ["validate"] } tokio = { version = "1.37", features = ["macros"] } tokio-test = "0.4" tower = { version = "0.5", features = ["util"] } diff --git a/server/src/api.rs b/server/src/api.rs index 2f7fc1d02..577063602 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -387,9 +387,8 @@ mod tests { use super::Api; use crate::{Backend, MemoryBackend}; use http::Method; - use stac::{Catalog, Collection, Item, Links}; + use stac::{Catalog, Collection, Item, Links, Validate}; use stac_api::{Items, Search, ITEM_SEARCH_URI}; - use stac_validate::Validate; use std::collections::HashSet; macro_rules! assert_link { diff --git a/validate/CHANGELOG.md b/validate/CHANGELOG.md deleted file mode 100644 index 0d3c1147b..000000000 --- a/validate/CHANGELOG.md +++ /dev/null @@ -1,71 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -## [0.3.0] - 2024-09-19 - -### Added - -- STAC v1.1.0 schemas ([#399](https://github.com/stac-utils/stac-rs/pull/399)) - -### Changed - -- Moved to async-first, with a blocking interface ([#397](https://github.com/stac-utils/stac-rs/pull/397)) - -## [0.2.2] - 2024-09-06 - -### Added - -- json-schema draft 07 and item resolver ([#358](https://github.com/stac-utils/stac-rs/pull/358)) - -## [0.2.1] - 2024-09-05 - -### Changed - -- Bump **stac** version - -## [0.2.0] - 2024-08-12 - -### Added - -- `impl Default for Validator` ([#252](https://github.com/stac-utils/stac-rs/pull/252)) -- Support for validating versions other than v1.0.0 ([#293](https://github.com/stac-utils/stac-rs/pull/293)) - -### Changed - -- `ValidateCore::validate_core_json` now takes a mutable reference to the validator ([#293](https://github.com/stac-utils/stac-rs/pull/293)) - -### Removed - -- `ValidateCore::validate_core` ([#293](https://github.com/stac-utils/stac-rs/pull/293)) - -## [0.1.2] - 2024-04-29 - -### Changed - -- Updated **stac** version - -## [0.1.1] - 2023-10-09 - -### Added - -- Validation for `serde_json::Value` ([#190](https://github.com/stac-utils/stac-rs/pull/190)) - -## [0.1.0] - 2023-06-27 - -Initial release. - -[Unreleased]: https://github.com/stac-utils/stac-rs/compare/stac-validate-v0.3.0...main -[0.3.0]: https://github.com/stac-utils/stac-rs/compare/stac-validate-v0.2.2..stac-validate-v0.3.0 -[0.2.2]: https://github.com/stac-utils/stac-rs/compare/stac-validate-v0.2.1..stac-validate-v0.2.2 -[0.2.1]: https://github.com/stac-utils/stac-rs/compare/stac-validate-v0.2.0..stac-validate-v0.2.1 -[0.2.0]: https://github.com/stac-utils/stac-rs/compare/stac-validate-v0.1.2..stac-validate-v0.2.0 -[0.1.2]: https://github.com/stac-utils/stac-rs/compare/stac-validate-v0.1.1..stac-validate-v0.1.2 -[0.1.1]: https://github.com/stac-utils/stac-rs/compare/stac-validate-v0.1.0..stac-validate-v0.1.1 -[0.1.0]: https://github.com/stac-utils/stac-rs/releases/tag/stac-validate-v0.1.0 - - diff --git a/validate/Cargo.toml b/validate/Cargo.toml deleted file mode 100644 index 15f1ad06e..000000000 --- a/validate/Cargo.toml +++ /dev/null @@ -1,44 +0,0 @@ -[package] -name = "stac-validate" -version = "0.3.0" -authors = ["Pete Gadomski "] -edition = "2021" -description = "Validate STAC objects with jsonschema" -homepage = "https://github.com/stac-utils/stac-rs" -repository = "https://github.com/stac-utils/stac-rs" -license = "MIT OR Apache-2.0" -keywords = ["geospatial", "stac", "metadata", "geo", "raster"] -categories = ["science", "data-structures"] - -[features] -blocking = ["tokio/rt"] - -[dependencies] -jsonschema = "0.20" -reqwest = { version = "0.12", features = ["blocking", "json"] } -serde = "1" -serde_json = "1" -stac = { version = "0.10.0", path = "../core" } -thiserror = "1" -tokio = "1" -tracing = "0.1" -url = "2" - -[dev-dependencies] -geojson = "0.24" -stac = { version = "0.10.0", path = "../core", features = ["geo"] } -rstest = "0.22" -tokio = { version = "1", features = ["macros"] } -tokio-test = "0.4" - -[[test]] -name = "examples" -required-features = ["blocking"] - -[[test]] -name = "migrate" -required-features = ["blocking"] - -[package.metadata.docs.rs] -all-features = true -rustdoc-args = ["--cfg", "docsrs"] diff --git a/validate/README.md b/validate/README.md deleted file mode 100644 index bddb4f0a6..000000000 --- a/validate/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# stac-validate - -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/stac-utils/stac-rs/ci.yml?branch=main&style=for-the-badge)](https://github.com/stac-utils/stac-rs/actions/workflows/ci.yml) -[![docs.rs](https://img.shields.io/docsrs/stac-validate?style=for-the-badge)](https://docs.rs/stac-validate/latest/stac-validate/) -[![Crates.io](https://img.shields.io/crates/v/stac-validate?style=for-the-badge)](https://crates.io/crates/stac-validate) -![Crates.io](https://img.shields.io/crates/l/stac-validate?style=for-the-badge) -[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg?style=for-the-badge)](./CODE_OF_CONDUCT) - -Validate [STAC](https://stacspec.org/) with [json-schema](https://json-schema.org/). - -## Usage - -To use the library in your project: - -```toml -[dependencies] -stac-validate = "0.3" -``` - -## Examples - -```rust -use stac_validate::Validate; -let item: stac::Item = stac::read("examples/simple-item.json").unwrap(); -tokio_test::block_on(async { - item.validate().await.unwrap(); -}); -``` - -Please see the [documentation](https://docs.rs/stac-validate) for more usage examples. - -## Other info - -This crate is part of the [stac-rs](https://github.com/stac-utils/stac-rs) monorepo, see its README for contributing and license information. diff --git a/validate/examples b/validate/examples deleted file mode 120000 index dc8ca8578..000000000 --- a/validate/examples +++ /dev/null @@ -1 +0,0 @@ -../spec-examples/v1.1.0 \ No newline at end of file diff --git a/validate/src/error.rs b/validate/src/error.rs deleted file mode 100644 index 09c677f30..000000000 --- a/validate/src/error.rs +++ /dev/null @@ -1,88 +0,0 @@ -use jsonschema::ValidationError; -use serde_json::Value; -use std::{borrow::Cow, sync::Arc}; -use thiserror::Error; -use url::Url; - -/// Crate-specific error type. -#[derive(Debug, Error)] -#[non_exhaustive] -pub enum Error { - /// Cannot validate a non-object, non-array - #[error("value is not an object or an array, cannot validate")] - CannotValidate(Value), - - /// [std::io::Error] - #[error(transparent)] - Io(#[from] std::io::Error), - - /// No type field on an object. - #[error("no type field")] - NoType, - - /// No version field on an object. - #[error("no version field")] - NoVersion, - - /// [reqwest::Error] - #[error(transparent)] - Reqwest(#[from] reqwest::Error), - - /// [serde_json::Error] - #[error(transparent)] - SerdeJson(#[from] serde_json::Error), - - /// [stac::Error] - #[error(transparent)] - Stac(#[from] stac::Error), - - /// [tokio::task::JoinError] - #[error(transparent)] - TokioJoin(#[from] tokio::task::JoinError), - - /// [tokio::sync::mpsc::error::SendError] - #[error(transparent)] - TokioSend( - #[from] - tokio::sync::mpsc::error::SendError<( - Url, - tokio::sync::oneshot::Sender>>, - )>, - ), - - /// [tokio::sync::oneshot::error::RecvError] - #[error(transparent)] - TokioRecv(#[from] tokio::sync::oneshot::error::RecvError), - - /// [url::ParseError] - #[error(transparent)] - UrlParse(#[from] url::ParseError), - - /// A list of validation errors. - /// - /// Since we usually don't have the original [serde_json::Value] (because we - /// create them from the STAC objects), we need these errors to be `'static` - /// lifetime. - #[error("validation errors")] - Validation(Vec>), -} - -impl Error { - /// Creates an [crate::Error] from an iterator over [jsonschema::ValidationError]. - pub fn from_validation_errors<'a, I>(errors: I) -> Error - where - I: Iterator>, - { - let mut error_vec = Vec::new(); - for error in errors { - // Cribbed from https://docs.rs/jsonschema/latest/src/jsonschema/error.rs.html#21-30 - error_vec.push(ValidationError { - instance_path: error.instance_path.clone(), - instance: Cow::Owned(error.instance.into_owned()), - kind: error.kind, - schema_path: error.schema_path, - }) - } - Error::Validation(error_vec) - } -} diff --git a/validate/src/lib.rs b/validate/src/lib.rs deleted file mode 100644 index 39fc93358..000000000 --- a/validate/src/lib.rs +++ /dev/null @@ -1,102 +0,0 @@ -//! Validate STAC objects with [json-schema](https://json-schema.org/). -//! -//! # Examples -//! -//! Validation is provided via the [Validate] trait: -//! -//! ``` -//! use stac::Item; -//! use stac_validate::Validate; -//! -//! # tokio_test::block_on(async { -//! Item::new("an-id").validate().await.unwrap(); -//! # }) -//! ``` -//! -//! If you're working in a blocking context (not async), enable the `blocking` feature and use [ValidateBlocking]: -//! -//! ``` -//! # use stac::Item; -//! #[cfg(feature = "blocking")] -//! { -//! use stac_validate::ValidateBlocking; -//! Item::new("an-id").validate_blocking().unwrap(); -//! } -//! ``` -//! -//! All fetched schemas are cached, so if you're you're doing multiple -//! validations, you should re-use the same [Validator]: -//! -//! ``` -//! # use stac::Item; -//! # use stac_validate::Validator; -//! let mut items: Vec<_> = (0..10).map(|n| Item::new(format!("item-{}", n))).collect(); -//! # tokio_test::block_on(async { -//! let mut validator = Validator::new().await; -//! for item in items { -//! validator.validate(&item).await.unwrap(); -//! } -//! # }) -//! ``` -//! -//! [Validator] is cheap to clone, so you are encouraged to validate a large -//! number of objects at the same time if that's your use-case. - -#![deny( - elided_lifetimes_in_paths, - explicit_outlives_requirements, - keyword_idents, - macro_use_extern_crate, - meta_variable_misuse, - missing_abi, - missing_debug_implementations, - missing_docs, - non_ascii_idents, - noop_method_call, - rust_2021_incompatible_closure_captures, - rust_2021_incompatible_or_patterns, - rust_2021_prefixes_incompatible_syntax, - rust_2021_prelude_collisions, - single_use_lifetimes, - trivial_casts, - trivial_numeric_casts, - unreachable_pub, - unsafe_code, - unsafe_op_in_unsafe_fn, - unused_crate_dependencies, - unused_extern_crates, - unused_import_braces, - unused_lifetimes, - unused_qualifications, - unused_results -)] - -#[cfg(feature = "blocking")] -mod blocking; -mod error; -mod validate; -mod validator; - -#[cfg(feature = "blocking")] -pub use blocking::ValidateBlocking; -pub use {error::Error, validate::Validate, validator::Validator}; - -/// Crate-specific result type. -pub type Result = std::result::Result; - -#[cfg(test)] -use {geojson as _, rstest as _, tokio_test as _}; - -// From https://github.com/rust-lang/cargo/issues/383#issuecomment-720873790, -// may they be forever blessed. -#[cfg(doctest)] -mod readme { - macro_rules! external_doc_test { - ($x:expr) => { - #[doc = $x] - extern "C" {} - }; - } - - external_doc_test!(include_str!("../README.md")); -} diff --git a/validate/src/validate.rs b/validate/src/validate.rs deleted file mode 100644 index 891ba5df7..000000000 --- a/validate/src/validate.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::{Result, Validator}; -use serde::Serialize; -use std::future::Future; - -/// Validate any serializable object with [json-schema](https://json-schema.org/) -pub trait Validate: Serialize + Sized { - /// Validates this object. - /// - /// If the object fails validation, this will return an - /// [Error::Validation](crate::Error::Validation) which contains a vector of - /// all of the validation errors. - /// - /// If you're doing multiple validations, use [Validator::validate], which - /// will re-use cached schemas. - /// - /// # Examples - /// - /// ``` - /// use stac_validate::Validate; - /// use stac::Item; - /// - /// let mut item = Item::new("an-id"); - /// # tokio_test::block_on(async { - /// item.validate().await.unwrap(); - /// }); - /// ``` - fn validate(&self) -> impl Future> { - async { - let validator = Validator::new().await; - validator.validate(self).await - } - } -} - -impl Validate for T {} diff --git a/validate/tests/migrate.rs b/validate/tests/migrate.rs deleted file mode 100644 index 2791d4bb6..000000000 --- a/validate/tests/migrate.rs +++ /dev/null @@ -1,11 +0,0 @@ -use rstest::rstest; -use stac::{Migrate, Value, Version}; -use stac_validate::ValidateBlocking; -use std::path::PathBuf; - -#[rstest] -fn v1_0_0_to_v1_1_0(#[files("../spec-examples/v1.0.0/**/*.json")] path: PathBuf) { - let value: Value = stac::read(path.to_str().unwrap()).unwrap(); - let value = value.migrate(&Version::v1_1_0).unwrap(); - value.validate_blocking().unwrap(); -}