(TF-18292) Load Stack component sources from metadata

.changes/unreleased/ENHANCEMENTS-20240717-114614.yaml

@@ -0,0 +1,6 @@
+kind: ENHANCEMENTS
+body: Load Stack component sources from metadata
+time: 2024-07-17T11:46:14.048412-04:00
+custom:
+    Issue: "1768"
+    Repository: terraform-ls

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/hashicorp/terraform-exec v0.21.0
 	github.com/hashicorp/terraform-json v0.22.1
 	github.com/hashicorp/terraform-registry-address v0.2.3
-	github.com/hashicorp/terraform-schema v0.0.0-20240717123934-4ae973d1b11b
+	github.com/hashicorp/terraform-schema v0.0.0-20240717154107-7d112f69e8e0
 	github.com/mcuadros/go-defaults v1.2.0
 	github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
 	github.com/mitchellh/cli v1.1.5

go.sum

@@ -229,8 +229,8 @@ github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7
 github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A=
 github.com/hashicorp/terraform-registry-address v0.2.3 h1:2TAiKJ1A3MAkZlH1YI/aTVcLZRu7JseiXNRHbOAyoTI=
 github.com/hashicorp/terraform-registry-address v0.2.3/go.mod h1:lFHA76T8jfQteVfT7caREqguFrW3c4MFSPhZB7HHgUM=
-github.com/hashicorp/terraform-schema v0.0.0-20240717123934-4ae973d1b11b h1:CI+WVmvNp63uPudDQy6d66y3ad2n8bY+gyePuwMeT0E=
-github.com/hashicorp/terraform-schema v0.0.0-20240717123934-4ae973d1b11b/go.mod h1:ar787Bv/qD6tlnjtzH8fQ1Yi6c/B5LsnpFlO8c92Atg=
+github.com/hashicorp/terraform-schema v0.0.0-20240717154107-7d112f69e8e0 h1:qgbO+ZzDng3AOKZ8GQ+BuoF24hYNM6EzaQMK8ICPbuw=
+github.com/hashicorp/terraform-schema v0.0.0-20240717154107-7d112f69e8e0/go.mod h1:ar787Bv/qD6tlnjtzH8fQ1Yi6c/B5LsnpFlO8c92Atg=
 github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ=
 github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc=
 github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo=

internal/eventbus/bus.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"log"
 	"sync"
+
+	"github.com/hashicorp/terraform-ls/internal/job"
 )
 
 const ChannelSize = 10
@@ -52,14 +54,16 @@ type Topic[T any] struct {
 	mutex       sync.Mutex
 }
 
+type DoneChannel <-chan job.IDs
+
 // Subscriber represents a subscriber to a topic
 type Subscriber[T any] struct {
 	// channel is the channel to which all events of the topic are sent
 	channel chan<- T
 
 	// doneChannel is an optional channel that the subscriber can use to signal
 	// that it is done processing the event
-	doneChannel <-chan struct{}
+	doneChannel DoneChannel
 }
 
 // NewTopic creates a new topic
@@ -70,7 +74,7 @@ func NewTopic[T any]() *Topic[T] {
 }
 
 // Subscribe adds a subscriber to a topic
-func (eb *Topic[T]) Subscribe(doneChannel <-chan struct{}) <-chan T {
+func (eb *Topic[T]) Subscribe(doneChannel DoneChannel) <-chan T {
 	channel := make(chan T, ChannelSize)
 	eb.mutex.Lock()
 	defer eb.mutex.Unlock()
@@ -83,7 +87,9 @@ func (eb *Topic[T]) Subscribe(doneChannel <-chan struct{}) <-chan T {
 }
 
 // Publish sends an event to all subscribers of a specific topic
-func (eb *Topic[T]) Publish(event T) {
+func (eb *Topic[T]) Publish(event T) job.IDs {
+	ids := make(job.IDs, 0)
+
 	eb.mutex.Lock()
 	defer eb.mutex.Unlock()
 
@@ -93,7 +99,10 @@ func (eb *Topic[T]) Publish(event T) {
 
 		if subscriber.doneChannel != nil {
 			// And wait until the subscriber is done processing it
-			<-subscriber.doneChannel
+			spawnedIds := <-subscriber.doneChannel
+			ids = append(ids, spawnedIds...)
 		}
 	}
+
+	return ids
 }

internal/eventbus/did_change.go

@@ -20,7 +20,7 @@ type DidChangeEvent struct {
 	LanguageID string
 }
 
-func (n *EventBus) OnDidChange(identifier string, doneChannel <-chan struct{}) <-chan DidChangeEvent {
+func (n *EventBus) OnDidChange(identifier string, doneChannel DoneChannel) <-chan DidChangeEvent {
 	n.logger.Printf("bus: %q subscribed to OnDidChange", identifier)
 	return n.didChangeTopic.Subscribe(doneChannel)
 }

internal/eventbus/did_change_watched.go

@@ -23,7 +23,7 @@ type DidChangeWatchedEvent struct {
 	ChangeType protocol.FileChangeType
 }
 
-func (n *EventBus) OnDidChangeWatched(identifier string, doneChannel <-chan struct{}) <-chan DidChangeWatchedEvent {
+func (n *EventBus) OnDidChangeWatched(identifier string, doneChannel DoneChannel) <-chan DidChangeWatchedEvent {
 	n.logger.Printf("bus: %q subscribed to OnDidChangeWatched", identifier)
 	return n.didChangeWatchedTopic.Subscribe(doneChannel)
 }

internal/eventbus/did_open.go

@@ -7,6 +7,7 @@ import (
 	"context"
 
 	"github.com/hashicorp/terraform-ls/internal/document"
+	"github.com/hashicorp/terraform-ls/internal/job"
 )
 
 // DidOpenEvent is an event to signal that a directory is open in the editor
@@ -23,12 +24,12 @@ type DidOpenEvent struct {
 	LanguageID string
 }
 
-func (n *EventBus) OnDidOpen(identifier string, doneChannel <-chan struct{}) <-chan DidOpenEvent {
+func (n *EventBus) OnDidOpen(identifier string, doneChannel DoneChannel) <-chan DidOpenEvent {
 	n.logger.Printf("bus: %q subscribed to OnDidOpen", identifier)
 	return n.didOpenTopic.Subscribe(doneChannel)
 }
 
-func (n *EventBus) DidOpen(e DidOpenEvent) {
+func (n *EventBus) DidOpen(e DidOpenEvent) job.IDs {
 	n.logger.Printf("bus: -> DidOpen %s", e.Dir)
-	n.didOpenTopic.Publish(e)
+	return n.didOpenTopic.Publish(e)
 }

internal/eventbus/discover.go

@@ -17,7 +17,7 @@ type DiscoverEvent struct {
 	Files []string
 }
 
-func (n *EventBus) OnDiscover(identifier string, doneChannel <-chan struct{}) <-chan DiscoverEvent {
+func (n *EventBus) OnDiscover(identifier string, doneChannel DoneChannel) <-chan DiscoverEvent {
 	n.logger.Printf("bus: %q subscribed to OnDiscover", identifier)
 	return n.discoverTopic.Subscribe(doneChannel)
 }

internal/eventbus/lockfile_change.go

@@ -19,7 +19,7 @@ type PluginLockChangeEvent struct {
 	ChangeType protocol.FileChangeType
 }
 
-func (n *EventBus) OnPluginLockChange(identifier string, doneChannel <-chan struct{}) <-chan PluginLockChangeEvent {
+func (n *EventBus) OnPluginLockChange(identifier string, doneChannel DoneChannel) <-chan PluginLockChangeEvent {
 	n.logger.Printf("bus: %q subscribed to OnPluginLockChange", identifier)
 	return n.pluginLockChangeTopic.Subscribe(doneChannel)
 }

internal/eventbus/manifest_change.go

@@ -19,7 +19,7 @@ type ManifestChangeEvent struct {
 	ChangeType protocol.FileChangeType
 }
 
-func (n *EventBus) OnManifestChange(identifier string, doneChannel <-chan struct{}) <-chan ManifestChangeEvent {
+func (n *EventBus) OnManifestChange(identifier string, doneChannel DoneChannel) <-chan ManifestChangeEvent {
 	n.logger.Printf("bus: %q subscribed to OnManifestChange", identifier)
 	return n.manifestChangeTopic.Subscribe(doneChannel)
 }

internal/features/modules/modules_feature.go

@@ -20,6 +20,7 @@ import (
 	"github.com/hashicorp/terraform-ls/internal/features/modules/hooks"
 	"github.com/hashicorp/terraform-ls/internal/features/modules/jobs"
 	"github.com/hashicorp/terraform-ls/internal/features/modules/state"
+	"github.com/hashicorp/terraform-ls/internal/job"
 	"github.com/hashicorp/terraform-ls/internal/langserver/diagnostics"
 	"github.com/hashicorp/terraform-ls/internal/registry"
 	globalState "github.com/hashicorp/terraform-ls/internal/state"
@@ -74,13 +75,13 @@ func (f *ModulesFeature) Start(ctx context.Context) {
 
 	discover := f.eventbus.OnDiscover("feature.modules", nil)
 
-	didOpenDone := make(chan struct{}, 10)
+	didOpenDone := make(chan job.IDs, 10)
 	didOpen := f.eventbus.OnDidOpen("feature.modules", didOpenDone)
 
-	didChangeDone := make(chan struct{}, 10)
+	didChangeDone := make(chan job.IDs, 10)
 	didChange := f.eventbus.OnDidChange("feature.modules", didChangeDone)
 
-	didChangeWatchedDone := make(chan struct{}, 10)
+	didChangeWatchedDone := make(chan job.IDs, 10)
 	didChangeWatched := f.eventbus.OnDidChangeWatched("feature.modules", didChangeWatchedDone)
 
 	go func() {
@@ -91,16 +92,16 @@ func (f *ModulesFeature) Start(ctx context.Context) {
 				f.discover(discover.Path, discover.Files)
 			case didOpen := <-didOpen:
 				// TODO? collect errors
-				f.didOpen(didOpen.Context, didOpen.Dir, didOpen.LanguageID)
-				didOpenDone <- struct{}{}
+				spawnedIds, _ := f.didOpen(didOpen.Context, didOpen.Dir, didOpen.LanguageID)
+				didOpenDone <- spawnedIds
 			case didChange := <-didChange:
 				// TODO? collect errors
-				f.didChange(didChange.Context, didChange.Dir)
-				didChangeDone <- struct{}{}
+				spawnedIds, _ := f.didChange(didChange.Context, didChange.Dir)
+				didChangeDone <- spawnedIds
 			case didChangeWatched := <-didChangeWatched:
 				// TODO? collect errors
-				f.didChangeWatched(didChangeWatched.Context, didChangeWatched.RawPath, didChangeWatched.ChangeType, didChangeWatched.IsDir)
-				didChangeWatchedDone <- struct{}{}
+				spawnedIds, _ := f.didChangeWatched(didChangeWatched.Context, didChangeWatched.RawPath, didChangeWatched.ChangeType, didChangeWatched.IsDir)
+				didChangeWatchedDone <- spawnedIds
 
 			case <-ctx.Done():
 				return

internal/features/rootmodules/root_modules_feature.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/terraform-ls/internal/eventbus"
 	"github.com/hashicorp/terraform-ls/internal/features/rootmodules/jobs"
 	"github.com/hashicorp/terraform-ls/internal/features/rootmodules/state"
+	"github.com/hashicorp/terraform-ls/internal/job"
 	globalState "github.com/hashicorp/terraform-ls/internal/state"
 	"github.com/hashicorp/terraform-ls/internal/telemetry"
 	"github.com/hashicorp/terraform-ls/internal/terraform/exec"
@@ -70,13 +71,13 @@ func (f *RootModulesFeature) Start(ctx context.Context) {
 
 	discover := f.eventbus.OnDiscover("feature.rootmodules", nil)
 
-	didOpenDone := make(chan struct{}, 10)
+	didOpenDone := make(chan job.IDs, 10)
 	didOpen := f.eventbus.OnDidOpen("feature.rootmodules", didOpenDone)
 
-	manifestChangeDone := make(chan struct{}, 10)
+	manifestChangeDone := make(chan job.IDs, 10)
 	manifestChange := f.eventbus.OnManifestChange("feature.rootmodules", manifestChangeDone)
 
-	pluginLockChangeDone := make(chan struct{}, 10)
+	pluginLockChangeDone := make(chan job.IDs, 10)
 	pluginLockChange := f.eventbus.OnPluginLockChange("feature.rootmodules", pluginLockChangeDone)
 
 	go func() {
@@ -87,16 +88,16 @@ func (f *RootModulesFeature) Start(ctx context.Context) {
 				f.discover(discover.Path, discover.Files)
 			case didOpen := <-didOpen:
 				// TODO? collect errors
-				f.didOpen(didOpen.Context, didOpen.Dir)
-				didOpenDone <- struct{}{}
+				spawnedIds, _ := f.didOpen(didOpen.Context, didOpen.Dir)
+				didOpenDone <- spawnedIds
 			case manifestChange := <-manifestChange:
 				// TODO? collect errors
-				f.manifestChange(manifestChange.Context, manifestChange.Dir, manifestChange.ChangeType)
-				manifestChangeDone <- struct{}{}
+				spawnedIds, _ := f.manifestChange(manifestChange.Context, manifestChange.Dir, manifestChange.ChangeType)
+				manifestChangeDone <- spawnedIds
 			case pluginLockChange := <-pluginLockChange:
 				// TODO? collect errors
-				f.pluginLockChange(pluginLockChange.Context, pluginLockChange.Dir)
-				pluginLockChangeDone <- struct{}{}
+				spawnedIds, _ := f.pluginLockChange(pluginLockChange.Context, pluginLockChange.Dir)
+				pluginLockChangeDone <- spawnedIds
 
 			case <-ctx.Done():
 				return

internal/features/stacks/events.go

@@ -9,14 +9,18 @@ import (
 	"path/filepath"
 
 	"github.com/hashicorp/terraform-ls/internal/document"
+	"github.com/hashicorp/terraform-ls/internal/eventbus"
 	"github.com/hashicorp/terraform-ls/internal/features/stacks/ast"
 	"github.com/hashicorp/terraform-ls/internal/features/stacks/jobs"
+	"github.com/hashicorp/terraform-ls/internal/features/stacks/state"
 	"github.com/hashicorp/terraform-ls/internal/job"
 	"github.com/hashicorp/terraform-ls/internal/lsp"
 	"github.com/hashicorp/terraform-ls/internal/protocol"
 	"github.com/hashicorp/terraform-ls/internal/schemas"
 	globalAst "github.com/hashicorp/terraform-ls/internal/terraform/ast"
 	"github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
+	tfaddr "github.com/hashicorp/terraform-registry-address"
+	"github.com/hashicorp/terraform-schema/module"
 )
 
 func (f *StacksFeature) discover(path string, files []string) error {
@@ -195,6 +199,20 @@ func (f *StacksFeature) decodeStack(ctx context.Context, dir document.DirHandle,
 		Type:        operation.OpTypeLoadStackMetadata.String(),
 		DependsOn:   job.IDs{parseId},
 		IgnoreState: ignoreState,
+		Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
+			if jobErr != nil {
+				f.logger.Printf("loading module metadata returned error: %s", jobErr)
+			}
+
+			spawnedIds, err := loadStackComponentSources(ctx, f.store, f.bus, path)
+
+			// while we now have the job ids in here, depending on the metaId job is not enough
+			// to await these spawned jobs, so we will need to move all depending jobs to this function
+			// as well. e.g. LoadStackComponentSources, PreloadEmbeddedSchema (because future ref collection jobs depend on it), etc.
+			// we might just move all in here for simplicity
+
+			return spawnedIds, err
+		},
 	})
 	if err != nil {
 		return ids, err
@@ -239,3 +257,54 @@ func (f *StacksFeature) removeIndexedStack(rawPath string) {
 		return
 	}
 }
+
+// loadStackComponentSources will trigger parsing the local terraform modules for a stack in the ModulesFeature
+func loadStackComponentSources(ctx context.Context, stackStore *state.StackStore, bus *eventbus.EventBus, stackPath string) (job.IDs, error) {
+	ids := make(job.IDs, 0)
+
+	record, err := stackStore.StackRecordByPath(stackPath)
+	if err != nil {
+		return ids, err
+	}
+
+	// iterate over each component in the stack and find local terraform modules
+	for _, component := range record.Meta.Components {
+		if component.Source == "" {
+			// no source recorded, skip
+			continue
+		}
+
+		var fullPath string
+		// detect if component.Source is a local module
+		switch component.SourceAddr.(type) {
+		case module.LocalSourceAddr:
+			fullPath = filepath.Join(stackPath, filepath.FromSlash(component.Source))
+		case tfaddr.Module:
+			continue
+		case module.RemoteSourceAddr:
+			continue
+		default:
+			// Unknown source address, we can't resolve the path
+			continue
+		}
+
+		if fullPath == "" {
+			// Unknown source address, we can't resolve the path
+			continue
+		}
+
+		dh := document.DirHandleFromPath(fullPath)
+
+		// notify the event bus that a new Component with a
+		// local modules has been opened
+		spawnedIds := bus.DidOpen(eventbus.DidOpenEvent{
+			Context:    ctx,
+			Dir:        dh,
+			LanguageID: lsp.Terraform.String(),
+		})
+
+		ids = append(ids, spawnedIds...)
+	}
+
+	return ids, nil
+}

internal/features/stacks/stacks_feature.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hashicorp/terraform-ls/internal/features/modules/jobs"
 	stackDecoder "github.com/hashicorp/terraform-ls/internal/features/stacks/decoder"
 	"github.com/hashicorp/terraform-ls/internal/features/stacks/state"
+	"github.com/hashicorp/terraform-ls/internal/job"
 	"github.com/hashicorp/terraform-ls/internal/langserver/diagnostics"
 	globalState "github.com/hashicorp/terraform-ls/internal/state"
 )
@@ -57,9 +58,9 @@ func (f *StacksFeature) Start(ctx context.Context) {
 
 	topic := "feature.stacks"
 
-	didOpenDone := make(chan struct{}, 10)
-	didChangeDone := make(chan struct{}, 10)
-	didChangeWatchedDone := make(chan struct{}, 10)
+	didOpenDone := make(chan job.IDs, 10)
+	didChangeDone := make(chan job.IDs, 10)
+	didChangeWatchedDone := make(chan job.IDs, 10)
 
 	discover := f.bus.OnDiscover(topic, nil)
 	didOpen := f.bus.OnDidOpen(topic, didOpenDone)
@@ -74,16 +75,16 @@ func (f *StacksFeature) Start(ctx context.Context) {
 				f.discover(discover.Path, discover.Files)
 			case didOpen := <-didOpen:
 				// TODO? collect errors
-				f.didOpen(didOpen.Context, didOpen.Dir, didOpen.LanguageID)
-				didOpenDone <- struct{}{}
+				spawnedIds, _ := f.didOpen(didOpen.Context, didOpen.Dir, didOpen.LanguageID)
+				didOpenDone <- spawnedIds
 			case didChange := <-didChange:
 				// TODO? collect errors
-				f.didChange(didChange.Context, didChange.Dir)
-				didChangeDone <- struct{}{}
+				spawnedIds, _ := f.didChange(didChange.Context, didChange.Dir)
+				didChangeDone <- spawnedIds
 			case didChangeWatched := <-didChangeWatched:
 				// TODO? collect errors
-				f.didChangeWatched(didChangeWatched.Context, didChangeWatched.RawPath, didChangeWatched.ChangeType, didChangeWatched.IsDir)
-				didChangeWatchedDone <- struct{}{}
+				spawnedIds, _ := f.didChangeWatched(didChangeWatched.Context, didChangeWatched.RawPath, didChangeWatched.ChangeType, didChangeWatched.IsDir)
+				didChangeWatchedDone <- spawnedIds
 
 			case <-ctx.Done():
 				return