Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BED-5464 - Additional CySQL Fixes #1189

Merged
merged 3 commits into from
Mar 4, 2025
Merged

BED-5464 - Additional CySQL Fixes #1189

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f1f8624
fix: BP-1396, BED-5464: support additional query components; rework t…
zinic Mar 3, 2025
828a1e5
chore: PFC
zinic Mar 4, 2025
3e1a04a
chore: cleanup
zinic Mar 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion packages/go/cypher/models/pgsql/test/translation_cases/multipart.sql
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions packages/go/cypher/models/pgsql/test/translation_cases/nodes.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,13 +169,13 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not exists (select 1 from edge e0 where e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id);

-- case: match (s) where not (s)-[]->()-[]->() return s
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id), s2 as (select s1.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.id = e1.end_id) select count(*) > 0 from s2);
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id), s2 as (select s1.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.id = e1.end_id) select count(*) > 0 from s2);

-- case: match (s) where not (s)-[{prop: 'a'}]-({name: 'n3'}) return s
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id or n1.id = e0.start_id where e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on (s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id or n1.id = e0.start_id where e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);

-- case: match (s) where not (s)<-[{prop: 'a'}]-({name: 'n3'}) return s
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id where n1.properties ->> 'name' = 'n3' and e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id where n1.properties ->> 'name' = 'n3' and e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);

-- case: match (n:NodeKind1) where n.distinguishedname = toUpper('admin') return n
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.properties ->> 'distinguishedname' = upper('admin')::text and n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]) select s0.n0 as n from s0;
Expand All @@ -190,7 +190,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.properties ->> 'distinguishedname' like '%' || upper('admin')::text and n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]) select s0.n0 as n from s0;

-- case: match (s) where not (s)-[{prop: 'a'}]->({name: 'n3'}) return s
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id where n1.properties ->> 'name' = 'n3' and e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id where n1.properties ->> 'name' = 'n3' and e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);

-- case: match (s) where not (s)-[]-() return id(s)
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (s0.n0).id from s0 where not exists (select 1 from edge e0 where e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id);
Expand Down
2 changes: 1 addition & 1 deletion packages/go/cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::e
with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where n1.kind_ids operator (pg_catalog.&&) array [2]::int2[] and e0.kind_id = any (array [3, 4]::int2[]) and n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]) select (s0.n0).properties -> 'name', (s0.n1).properties -> 'name' from s0;

-- case: match (s)-[r:EdgeKind1]->() where (s)-[r {prop: 'a'}]->() return s
with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.properties ->> 'prop' = 'a' and e0.kind_id = any (array [3]::int2[])) select s0.n0 as s from s0 where (with s1 as (select s0.e0 as e0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n0 on (s0.n0).id = (s0.e0).start_id join node n2 on n2.id = (s0.e0).end_id) select count(*) > 0 from s1);
with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.properties ->> 'prop' = 'a' and e0.kind_id = any (array [3]::int2[])) select s0.n0 as s from s0 where (with s1 as (select s0.e0 as e0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e0 on (s0.n0).id = (s0.e0).start_id join node n2 on n2.id = (s0.e0).end_id) select count(*) > 0 from s1);

-- case: match (s)-[r:EdgeKind1]->(e) where not (s.system_tags contains 'admin_tier_0') and id(e) = 1 return id(s), labels(s), id(r), type(r)
with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where n1.id = 1 and e0.kind_id = any (array [3]::int2[]) and not (coalesce(n0.properties ->> 'system_tags', '')::text like '%admin_tier_0%')) select (s0.n0).id, (s0.n0).kind_ids, (s0.e0).id, (s0.e0).kind_id from s0;
Expand Down
3 changes: 1 addition & 2 deletions packages/go/cypher/models/pgsql/test/translation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,11 @@ func TestTranslate(t *testing.T) {
)

if updateCases, varSet := os.LookupEnv("CYSQL_UPDATE_CASES"); varSet && strings.ToLower(strings.TrimSpace(updateCases)) == "true" {
if err := UpdateTranslationTestCases(kindMapper); err != nil {
if err := UpdateTranslationTestCases(kindMapper); err != nil {
fmt.Printf("Error updating cases: %v\n", err)
}
}


if testCases, err := ReadTranslationTestCases(); err != nil {
t.Fatal(err)
} else {
Expand Down
134 changes: 29 additions & 105 deletions packages/go/cypher/models/pgsql/translate/building.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,32 @@ package translate

import (
"errors"

"github.com/specterops/bloodhound/cypher/models/pgsql"
)

func (s *Translator) buildInlineProjection(part *QueryPart) (pgsql.Select, error) {
var sqlSelect pgsql.Select

if part.projections.Frame != nil {
sqlSelect.From = []pgsql.FromClause{{
Source: part.projections.Frame.Binding.Identifier,
}}
sqlSelect := pgsql.Select{
Where: part.projections.Constraints,
}

if projectionConstraint, err := s.treeTranslator.ConsumeAll(); err != nil {
return sqlSelect, err
} else {
sqlSelect.Where = projectionConstraint.Expression
// If there's a projection frame set, some additional negotiation is required to identify which frame the
// from-statement should be written to. Some of this would be better figured out during the translation
// of the projection where query scope and other components are not yet fully translated.
if part.projections.Frame != nil {
// Look up to see if there are CTE expressions registered. If there are then it is likely
// there was a projection between this CTE and the previous multipart query part
hasCTEs := part.Model.CommonTableExpressions != nil && len(part.Model.CommonTableExpressions.Expressions) > 0

if part.Frame.Previous == nil || hasCTEs {
sqlSelect.From = []pgsql.FromClause{{
Source: part.projections.Frame.Binding.Identifier,
}}
} else {
sqlSelect.From = []pgsql.FromClause{{
Source: part.Frame.Previous.Binding.Identifier,
}}
}
}

for _, projection := range part.projections.Items {
Expand Down Expand Up @@ -161,7 +171,7 @@ func (s *Translator) translateTraversalPatternPart(part *PatternPart, isolatedPr
}

func (s *Translator) translateTraversalPatternPartWithoutExpansion(isFirstTraversalStep bool, traversalStep *PatternSegment) error {
if constraints, err := s.patternConstraints(isFirstTraversalStep, nonRecursivePattern, traversalStep); err != nil {
if constraints, err := consumePatternConstraints(isFirstTraversalStep, nonRecursivePattern, traversalStep, s.treeTranslator.IdentifierConstraints); err != nil {
return err
} else {
if isFirstTraversalStep {
Expand Down Expand Up @@ -237,7 +247,7 @@ func (s *Translator) translateTraversalPatternPartWithoutExpansion(isFirstTraver
} else {
// Zip through all projected identifiers and update their last projected frame
for _, binding := range boundProjections.Bindings {
binding.LastProjection = traversalStep.Frame
binding.MaterializedBy(traversalStep.Frame)
}

traversalStep.Projection = boundProjections.Items
Expand All @@ -246,94 +256,8 @@ func (s *Translator) translateTraversalPatternPartWithoutExpansion(isFirstTraver
return nil
}

type PatternConstraints struct {
LeftNode *Constraint
Edge *Constraint
RightNode *Constraint
}

// OptimizePatternConstraintBalance considers the constraints that apply to a pattern segment's bound identifiers.
//
// If only the right side of the pattern segment is constrained, this could result in an imbalanced expansion where one side
// of the traversal has an extreme disparity in search space.
//
// In cases that match this heuristic, it's beneficial to begin the traversal with the most tightly constrained set
// of nodes. To accomplish this we flip the order of the traversal step.
func (s *PatternConstraints) OptimizePatternConstraintBalance(traversalStep *PatternSegment) {
var (
// If the left node is previously bound (query knows a set of IDs) the left node is considered to sill be constrained
leftNodeHasConstraints = traversalStep.LeftNodeBound || s.LeftNode.Expression != nil
rightNodeHasConstraints = s.RightNode.Expression != nil
)

// (a)-[*..]->(b:Constraint)
// (a)<-[*..]-(b:Constraint)
if !leftNodeHasConstraints && rightNodeHasConstraints {
traversalStep.FlipNodes()
s.FlipNodes()
}
}

func (s *PatternConstraints) FlipNodes() {
oldLeftNode := s.LeftNode
s.LeftNode = s.RightNode
s.RightNode = oldLeftNode
}

const (
recursivePattern = true
nonRecursivePattern = false
)

func (s *Translator) patternConstraints(isFirstTraversalStep, isRecursivePattern bool, traversalStep *PatternSegment) (PatternConstraints, error) {
var (
constraints PatternConstraints
err error
)

// Even if this isn't the first traversal and the node may be already bound, this should result in an empty
// constraint instead of a nil value for `leftNode`
if constraints.LeftNode, err = consumeConstraintsFrom(pgsql.AsIdentifierSet(traversalStep.LeftNode.Identifier), s.treeTranslator.IdentifierConstraints); err != nil {
return constraints, err
}

if isFirstTraversalStep {
// If this is the first traversal step then the left node is just coming into scope
traversalStep.Frame.Export(traversalStep.LeftNode.Identifier)
}

// Track the identifiers visible at this frame to correctly assign the remaining constraints
knownBindings := traversalStep.Frame.Known()

if isRecursivePattern {
// The exclusion below is done at this step in the process since the recursive descent portion of an expansion
// will no longer have a reference to the root node; any dependent interaction between the root and terminal
// nodes would require an additional join. By not consuming the remaining constraints for the root and terminal
// nodes, they become visible up in the outer select of the recursive CTE.
knownBindings.Remove(traversalStep.LeftNode.Identifier)
}

// Export the edge identifier first
traversalStep.Frame.Export(traversalStep.Edge.Identifier)
knownBindings.Add(traversalStep.Edge.Identifier)

if constraints.Edge, err = consumeConstraintsFrom(knownBindings, s.treeTranslator.IdentifierConstraints); err != nil {
return constraints, err
}

// Export the right node identifier last
traversalStep.Frame.Export(traversalStep.RightNode.Identifier)
knownBindings.Add(traversalStep.RightNode.Identifier)

if constraints.RightNode, err = consumeConstraintsFrom(knownBindings, s.treeTranslator.IdentifierConstraints); err != nil {
return constraints, err
}

return constraints, nil
}

func (s *Translator) translateTraversalPatternPartWithExpansion(isFirstTraversalStep bool, traversalStep *PatternSegment) error {
if constraints, err := s.patternConstraints(isFirstTraversalStep, recursivePattern, traversalStep); err != nil {
if constraints, err := consumePatternConstraints(isFirstTraversalStep, recursivePattern, traversalStep, s.treeTranslator.IdentifierConstraints); err != nil {
return err
} else {
// If one side of the expansion has constraints but the other does not this may be an opportunity to reorder the traversal
Expand Down Expand Up @@ -392,15 +316,15 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(isFirstTraversal
traversalStep.Expansion.Value.RecursiveConstraints = pgsql.OptionalAnd(traversalStep.Expansion.Value.ExpansionEdgeConstraints, expansionConstraints(expansionFrame.Binding.Identifier, traversalStep.Expansion.Value.MinDepth, traversalStep.Expansion.Value.MaxDepth))

// Remove the previous projections of the root and terminal node to reproject them after expansion
traversalStep.LeftNode.LastProjection = nil
traversalStep.RightNode.LastProjection = nil
traversalStep.LeftNode.Dematerialize()
traversalStep.RightNode.Dematerialize()

if boundProjections, err := buildVisibleProjections(s.query.Scope); err != nil {
return err
} else {
// Zip through all projected identifiers and update their last projected frame
for _, binding := range boundProjections.Bindings {
binding.LastProjection = expansionFrame
binding.MaterializedBy(expansionFrame)
}

traversalStep.Expansion.Value.Projection = boundProjections.Items
Expand All @@ -416,7 +340,7 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(isFirstTraversal
} else {
// Zip through all projected identifiers and update their last projected frame
for _, binding := range boundProjections.Bindings {
binding.LastProjection = traversalStep.Frame
binding.MaterializedBy(traversalStep.Frame)
}

traversalStep.Projection = boundProjections.Items
Expand All @@ -433,7 +357,7 @@ func (s *Translator) translateNonTraversalPatternPart(part *PatternPart) error {

nextFrame.Export(part.NodeSelect.Binding.Identifier)

if constraint, err := consumeConstraintsFrom(nextFrame.Known(), s.treeTranslator.IdentifierConstraints); err != nil {
if constraint, err := s.treeTranslator.IdentifierConstraints.ConsumeSet(nextFrame.Known()); err != nil {
return err
} else if err := RewriteFrameBindings(s.query.Scope, constraint.Expression); err != nil {
return err
Expand All @@ -446,7 +370,7 @@ func (s *Translator) translateNonTraversalPatternPart(part *PatternPart) error {
} else {
// Zip through all projected identifiers and update their last projected frame
for _, binding := range boundProjections.Bindings {
binding.LastProjection = nextFrame
binding.MaterializedBy(nextFrame)
}

part.NodeSelect.Select.Projection = boundProjections.Items
Expand Down
Loading
Loading