diff --git a/go.mod b/go.mod index b7c5a563..90265345 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 + github.com/brittonhayes/notionmd v0.6.1 + github.com/dstotijn/go-notion v0.11.0 github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/evanphx/go-hclog-slog v0.0.0-20240717231540-be48fc4c4df5 github.com/gobwas/glob v0.2.3 @@ -90,6 +92,7 @@ require ( github.com/go-swiss/fonts v0.0.0-20221219152310-0b267088f53d // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/gomarkdown/markdown v0.0.0-20240723152757-afa4a469d4f9 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect diff --git a/go.sum b/go.sum index 7100142d..f732d9c6 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEq github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/brittonhayes/notionmd v0.6.1 h1:yCeL1PLb5Zoksnq8BrNB08BEtmbIpHgzG5LprlF3lVI= +github.com/brittonhayes/notionmd v0.6.1/go.mod h1:r6V8qENKn8hyDEDZsUJFAnY9+fdyEqPA57vwIvEeMpA= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= @@ -74,6 +76,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dstotijn/go-notion v0.11.0 h1:v+ZUiyKd+UBk1SRkUSa86QOU5DP8ziSI4E7NFIS4rRU= +github.com/dstotijn/go-notion v0.11.0/go.mod h1:FWfmGRnE8Drm6CnNQQO7slXcu1lrKmRY2KfFgeq6Z2g= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow= @@ -106,6 +110,8 @@ github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17w github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/gomarkdown/markdown v0.0.0-20240723152757-afa4a469d4f9 h1:TRYrIWJziqvMVn1owO8bmkDJTlMQFYnf74yhD8LXfgU= +github.com/gomarkdown/markdown v0.0.0-20240723152757-afa4a469d4f9/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= diff --git a/internal/builtin/content_helpers.go b/internal/builtin/content_helpers.go index faeb5a86..739cd02f 100644 --- a/internal/builtin/content_helpers.go +++ b/internal/builtin/content_helpers.go @@ -22,6 +22,11 @@ func countDeclarations(data *plugin.ContentSection, name string) int { return count } +func ParseScope(datactx plugindata.Map) (document, section *plugin.ContentSection) { + return parseScope(datactx) +} + +// TODO:(britton) change signature of ParseScope to be exported func parseScope(datactx plugindata.Map) (document, section *plugin.ContentSection) { documentMap, ok := datactx["document"] if !ok { diff --git a/internal/notion/data_markdown.go b/internal/notion/data_markdown.go new file mode 100644 index 00000000..943e67a5 --- /dev/null +++ b/internal/notion/data_markdown.go @@ -0,0 +1,148 @@ +package notion + +import ( + "context" + "io" + "log/slog" + "os" + "path/filepath" + + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" + + "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/plugin" + "github.com/blackstork-io/fabric/plugin/dataspec" + "github.com/blackstork-io/fabric/plugin/plugindata" +) + +// TODO:(britton) markdown data source should come from shared local file source +func makeMarkdownDataSource() *plugin.DataSource { + return &plugin.DataSource{ + DataFunc: fetchMarkdownData, + Args: &dataspec.RootSpec{ + Attrs: []*dataspec.AttrSpec{ + { + Name: "glob", + Type: cty.String, + ExampleVal: cty.StringVal("path/to/file*.md"), + Doc: `A glob pattern to select MD files to read`, + }, + { + Name: "path", + Type: cty.String, + ExampleVal: cty.StringVal("path/to/file.md"), + Doc: `A file path to a MD file to read`, + }, + }, + }, + Doc: ` + Loads markdown files with the names that match a provided ` + "`glob`" + ` pattern or a single file from a provided path. + + Either ` + "`glob`" + ` or ` + "`path`" + ` argument must be set. + + When ` + "`path`" + ` argument is specified, the data source returns only the content of a file. + When ` + "`glob`" + ` argument is specified, the data source returns a list of dicts that contain the content of a file and file's metadata. For example: + ` + "```json" + ` + [ + { + "file_path": "path/file-a.md", + "file_name": "file-a.md", + "content": "foobar" + }, + { + "file_path": "path/file-b.md", + "file_name": "file-b.md", + "content": "x\\ny\\nz" + } + ] + ` + "```", + } +} + +func readMarkdownFile(path string) (plugindata.Data, error) { + f, err := os.Open(path) + if err != nil { + return nil, diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to open a file", + Detail: err.Error(), + }} + } + defer f.Close() + data, err := io.ReadAll(f) + if err != nil { + return nil, diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to read a file", + Detail: err.Error(), + }} + } + return plugindata.String(string(data)), nil +} + +func readMarkdownFiles(_ context.Context, pattern string) (plugindata.Data, error) { + paths, err := filepath.Glob(pattern) + if err != nil { + return nil, err + } + + result := make(plugindata.List, 0, len(paths)) + for _, path := range paths { + fileData, err := readMarkdownFile(path) + if err != nil { + return result, err + } + result = append(result, plugindata.Map{ + "file_path": plugindata.String(path), + "file_name": plugindata.String(filepath.Base(path)), + "content": fileData, + }) + } + return result, nil +} + +func fetchMarkdownData(ctx context.Context, params *plugin.RetrieveDataParams) (plugindata.Data, diagnostics.Diag) { + glob := params.Args.GetAttrVal("glob") + path := params.Args.GetAttrVal("path") + + if !path.IsNull() && path.AsString() != "" { + slog.Debug("Reading a file from the path", "path", path.AsString()) + data, err := readMarkdownFile(path.AsString()) + if err != nil { + slog.Error( + "Error while reading a file", + slog.String("path", path.AsString()), + slog.Any("error", err), + ) + return nil, diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to read a file", + Detail: err.Error(), + }} + } + return data, nil + } else if !glob.IsNull() && glob.AsString() != "" { + slog.Debug("Reading the files that match the glob pattern", "glob", glob.AsString()) + data, err := readMarkdownFiles(ctx, glob.AsString()) + if err != nil { + slog.Error( + "Error while reading the files", + slog.String("glob", glob.AsString()), + slog.Any("error", err), + ) + return nil, diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to read the files", + Detail: err.Error(), + }} + } + return data, nil + } + slog.Error("Either \"glob\" value or \"path\" value must be provided") + return nil, diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to parse provided arguments", + Detail: "Either \"glob\" value or \"path\" value must be provided", + }} +} diff --git a/internal/notion/plugin.go b/internal/notion/plugin.go new file mode 100644 index 00000000..fcafa2d5 --- /dev/null +++ b/internal/notion/plugin.go @@ -0,0 +1,21 @@ +package notion + +import ( + "log/slog" + + "github.com/blackstork-io/fabric/plugin" + "go.opentelemetry.io/otel/trace" +) + +func Plugin(version string, logger *slog.Logger, tracer trace.Tracer) *plugin.Schema { + return &plugin.Schema{ + Name: "blackstork/notion", + Version: version, + DataSources: plugin.DataSources{ + "md": makeMarkdownDataSource(), + }, + Publishers: plugin.Publishers{ + "notion_page": makeNotionPagePublisher(logger, tracer), + }, + } +} diff --git a/internal/notion/publish_notion_page.go b/internal/notion/publish_notion_page.go new file mode 100644 index 00000000..2914b831 --- /dev/null +++ b/internal/notion/publish_notion_page.go @@ -0,0 +1,156 @@ +package notion + +import ( + "bytes" + "context" + "io" + "log/slog" + + "github.com/blackstork-io/fabric/internal/builtin" + "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/plugin" + "github.com/blackstork-io/fabric/plugin/dataspec" + "github.com/blackstork-io/fabric/plugin/dataspec/constraint" + "github.com/blackstork-io/fabric/plugin/plugindata" + "github.com/blackstork-io/fabric/print/mdprint" + "github.com/brittonhayes/notionmd" + "github.com/dstotijn/go-notion" + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" + "go.opentelemetry.io/otel/trace" + nooptrace "go.opentelemetry.io/otel/trace/noop" +) + +func makeNotionPagePublisher(logger *slog.Logger, tracer trace.Tracer) *plugin.Publisher { + if logger == nil { + logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + } + if tracer == nil { + tracer = nooptrace.Tracer{} + } + + return &plugin.Publisher{ + Doc: "Publishes content to a Notion page", + Tags: []string{}, + Args: &dataspec.RootSpec{ + Attrs: []*dataspec.AttrSpec{ + { + Name: "title", + Doc: "Title of the Notion page", + Type: cty.String, + ExampleVal: cty.StringVal("My Notion Page"), + Constraints: constraint.Required, + }, + { + Name: "parent_page_id", + Doc: "Notion parent page ID", + Type: cty.String, + ExampleVal: cty.StringVal("1234567890"), + Constraints: constraint.Required, + }, + { + Name: "api_key", + Doc: "Notion API key", + Type: cty.String, + ExampleVal: cty.StringVal("secret_1234567890"), + Constraints: constraint.Required, + Secret: true, + }, + }, + }, + AllowedFormats: []plugin.OutputFormat{plugin.OutputFormatMD}, + PublishFunc: publishNotionPage(logger, tracer), + } +} + +func publishNotionPage(logger *slog.Logger, _ trace.Tracer) plugin.PublishFunc { + return func(ctx context.Context, params *plugin.PublishParams) diagnostics.Diag { + document, _ := builtin.ParseScope(params.DataContext) + if document == nil { + return diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to parse document", + Detail: "document is required", + }} + } + + datactx := params.DataContext + datactx["format"] = plugindata.String(params.Format.String()) + + titleAttr := params.Args.GetAttrVal("title") + if titleAttr.IsNull() || titleAttr.AsString() == "" { + return diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "title is required", + }} + } + + parentPageIDAttr := params.Args.GetAttrVal("parent_page_id") + if parentPageIDAttr.IsNull() || parentPageIDAttr.AsString() == "" { + return diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "parent_page_id is required", + }} + } + + apiKeyAttr := params.Args.GetAttrVal("api_key") + if apiKeyAttr.IsNull() || apiKeyAttr.AsString() == "" { + return diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "api_key is required", + }} + } + + writer := bytes.NewBuffer([]byte{}) + + printer := mdprint.New() + err := printer.Print(ctx, writer, document) + if err != nil { + return diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to print content", + Detail: err.Error(), + }} + } + + blocks, err := notionmd.Convert(writer.String()) + if err != nil { + return diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to convert content to Notion blocks", + Detail: err.Error(), + }} + } + + // Publish to Notion + logger.InfoContext(ctx, "Publishing to Notion", "title", titleAttr.AsString()) + client := notion.NewClient(apiKeyAttr.AsString()) + page, err := client.CreatePage(ctx, notion.CreatePageParams{ + ParentType: notion.ParentTypePage, + ParentID: parentPageIDAttr.AsString(), + Title: []notion.RichText{ + { + Type: notion.RichTextTypeText, + Text: ¬ion.Text{ + Content: titleAttr.AsString(), + }, + }, + }, + Children: blocks, + }) + if err != nil { + return diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to create a Notion page", + Detail: err.Error(), + }} + } + + logger.InfoContext(ctx, "Published to Notion", "page_id", page.ID) + + return nil + } +} diff --git a/internal/plugin_validity_test.go b/internal/plugin_validity_test.go index de2e0616..662e5d9c 100644 --- a/internal/plugin_validity_test.go +++ b/internal/plugin_validity_test.go @@ -13,6 +13,7 @@ import ( "github.com/blackstork-io/fabric/internal/hackerone" "github.com/blackstork-io/fabric/internal/microsoft" "github.com/blackstork-io/fabric/internal/nistnvd" + "github.com/blackstork-io/fabric/internal/notion" "github.com/blackstork-io/fabric/internal/openai" "github.com/blackstork-io/fabric/internal/opencti" "github.com/blackstork-io/fabric/internal/postgresql" @@ -46,6 +47,7 @@ func TestAllPluginSchemaValidity(t *testing.T) { nistnvd.Plugin(ver, nil), snyk.Plugin(ver, nil), microsoft.Plugin(ver, nil, nil), + notion.Plugin(ver, nil, nil), } for _, p := range plugins { p := p diff --git a/tools/docgen/main.go b/tools/docgen/main.go index 88aa18f6..c2da2e0d 100644 --- a/tools/docgen/main.go +++ b/tools/docgen/main.go @@ -22,6 +22,7 @@ import ( "github.com/blackstork-io/fabric/internal/hackerone" "github.com/blackstork-io/fabric/internal/microsoft" "github.com/blackstork-io/fabric/internal/nistnvd" + "github.com/blackstork-io/fabric/internal/notion" "github.com/blackstork-io/fabric/internal/openai" "github.com/blackstork-io/fabric/internal/opencti" "github.com/blackstork-io/fabric/internal/postgresql" @@ -278,6 +279,7 @@ func main() { nistnvd.Plugin(version, nil), snyk.Plugin(version, nil), microsoft.Plugin(version, nil, nil), + notion.Plugin(version, nil, nil), } // generate markdown for each plugin for _, p := range plugins {