Skip to content

Commit

Permalink
[dg] Move check impl to lighter weight JSON validation
Browse files Browse the repository at this point in the history
  • Loading branch information
benpankow committed Jan 23, 2025
1 parent 185efb7 commit 00df599
Show file tree
Hide file tree
Showing 13 changed files with 430 additions and 353 deletions.
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import click
from dagster.version import __version__

from dagster_components.cli.check import check_cli
from dagster_components.cli.list import list_cli
from dagster_components.cli.scaffold import scaffold_cli
from dagster_components.core.component import BUILTIN_MAIN_COMPONENT_ENTRY_POINT
Expand All @@ -12,7 +11,6 @@ def create_dagster_components_cli():
commands = {
"scaffold": scaffold_cli,
"list": list_cli,
"check": check_cli,
}

@click.group(
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type: dagster_components.test.simple_asset

params:
asset_key: "test"
value: {}
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,14 @@ class MyComponentSchema(ComponentSchemaBaseModel):
@component_type
class MyComponent(Component):
name = "my_component"
params_schema = MyComponentSchema

@classmethod
def get_schema(cls):
return MyComponentSchema

@classmethod
def load(cls, context: ComponentLoadContext) -> Self:
context.load_params(cls.params_schema)
context.load_params(MyComponentSchema)
return cls()

def build_defs(self, context: ComponentLoadContext) -> Definitions:
Expand All @@ -40,11 +43,14 @@ class MyNestedComponentSchema(ComponentSchemaBaseModel):
@component_type
class MyNestedComponent(Component):
name = "my_nested_component"
params_schema = MyNestedComponentSchema

@classmethod
def get_schema(cls):
return MyNestedComponentSchema

@classmethod
def load(cls, context: ComponentLoadContext) -> Self:
context.load_params(cls.params_schema)
context.load_params(MyNestedComponentSchema)
return cls()

def build_defs(self, context: ComponentLoadContext) -> Definitions:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ class ComponentValidationTestCase:
"""

component_path: str
component_type_filepath: Path
component_type_filepath: Optional[Path]
should_error: bool
validate_error_msg: Optional[Callable[[str], None]] = None
validate_error_msg_additional_cli: Optional[Callable[[str], None]] = None
check_error_msg: Optional[Callable[[str], None]] = None


def msg_includes_all_of(*substrings: str) -> Callable[[str], None]:
Expand All @@ -31,15 +31,22 @@ def _validate_error_msg(msg: str) -> None:
validate_error_msg=msg_includes_all_of(
"component.yaml:5", "params.an_int", "Input should be a valid integer"
),
check_error_msg=msg_includes_all_of(
"component.yaml:5",
"params.an_int",
"{} is not of type 'integer'",
),
)

BASIC_MISSING_VALUE = ComponentValidationTestCase(
component_path="validation/basic_component_missing_value",
component_type_filepath=Path(__file__).parent / "basic_components.py",
should_error=True,
validate_error_msg=msg_includes_all_of("component.yaml:3", "params.an_int", "required"),
validate_error_msg_additional_cli=msg_includes_all_of(
"Field `an_int` is required but not provided"
check_error_msg=msg_includes_all_of(
"component.yaml:3",
"params",
"'an_int' is a required property",
),
)

Expand All @@ -51,13 +58,30 @@ def _validate_error_msg(msg: str) -> None:
),
BASIC_INVALID_VALUE,
BASIC_MISSING_VALUE,
ComponentValidationTestCase(
component_path="validation/simple_asset_invalid_value",
component_type_filepath=None,
should_error=True,
validate_error_msg=msg_includes_all_of(
"component.yaml:5", "params.value", "Input should be a valid string"
),
check_error_msg=msg_includes_all_of(
"component.yaml:5",
"params.value",
"{} is not of type 'string'",
),
),
ComponentValidationTestCase(
component_path="validation/basic_component_extra_value",
component_type_filepath=Path(__file__).parent / "basic_components.py",
should_error=True,
validate_error_msg=msg_includes_all_of(
"component.yaml:7", "params.a_bool", "Extra inputs are not permitted"
),
check_error_msg=msg_includes_all_of(
"component.yaml:3",
"'a_bool' was unexpected",
),
),
ComponentValidationTestCase(
component_path="validation/nested_component_invalid_values",
Expand All @@ -71,6 +95,14 @@ def _validate_error_msg(msg: str) -> None:
"params.nested.baz.a_string",
"Input should be a valid string",
),
check_error_msg=msg_includes_all_of(
"component.yaml:7",
"params.nested.foo.an_int",
"{} is not of type 'integer'",
"component.yaml:12",
"params.nested.baz.a_string",
"{} is not of type 'string'",
),
),
ComponentValidationTestCase(
component_path="validation/nested_component_missing_values",
Expand All @@ -79,8 +111,13 @@ def _validate_error_msg(msg: str) -> None:
validate_error_msg=msg_includes_all_of(
"component.yaml:5", "params.nested.foo.an_int", "required"
),
validate_error_msg_additional_cli=msg_includes_all_of(
"Field `a_string` is required but not provided"
check_error_msg=msg_includes_all_of(
"component.yaml:5",
"params.nested.foo",
"'an_int' is a required property",
"component.yaml:10",
"params.nested.baz",
"'a_string' is a required property",
),
),
ComponentValidationTestCase(
Expand All @@ -94,6 +131,14 @@ def _validate_error_msg(msg: str) -> None:
"component.yaml:15",
"params.nested.baz.another_bool",
),
check_error_msg=msg_includes_all_of(
"component.yaml:5",
"params.nested.foo",
"'a_bool' was unexpected",
"component.yaml:12",
"params.nested.baz",
"'another_bool' was unexpected",
),
),
ComponentValidationTestCase(
component_path="validation/invalid_component_file_model",
Expand All @@ -107,5 +152,13 @@ def _validate_error_msg(msg: str) -> None:
"params",
"Input should be an object",
),
check_error_msg=msg_includes_all_of(
"component.yaml:1",
"type",
"{} is not of type 'string'",
"component.yaml:3",
"params",
"'asdfasdf' is not of type 'object'",
),
),
]
Loading

0 comments on commit 00df599

Please sign in to comment.