From 65d76a72236c1a667526e8da4c40eb9d0b0021a6 Mon Sep 17 00:00:00 2001 From: Finn Arne Gangstad Date: Fri, 15 Sep 2023 12:42:59 +0200 Subject: [PATCH 1/3] remove recursion from moduleInfo.load() moduleInfo.load() used to load modules and depdencies recursively, and due to some unfortunate circumstances it would usually create a very deep call stack where each load() called another load() to load the next one. The terraform parsing uses enough stack space that this can become a problem, so for a slightly big project this sometimes caused atlantis to die from a stack overflow: runtime: goroutine stack exceeds 1000000000-byte limit runtime: sp=0xc021478380 stack=[0xc021478000, 0xc041478000] fatal error: stack overflow [...] github.com/runatlantis/atlantis/server/events.moduleInfo.load(...) ...atlantis/server/events/modules.go:108 +0x46b fp=0xc021478750 sp=0xc021478570 pc=0xfeaa6b github.com/runatlantis/atlantis/server/events.moduleInfo.load(...) ...atlantis/server/events/modules.go:108 +0x46b fp=0xc021478930 sp=0xc021478750 pc=0xfeaa6b github.com/runatlantis/atlantis/server/events.moduleInfo.load(...) ...atlantis/server/events/modules.go:108 +0x46b fp=0xc021478b10 sp=0xc021478930 pc=0xfeaa6b ... and so on, several hundred times. --- server/events/modules.go | 64 ++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/server/events/modules.go b/server/events/modules.go index 396cd59d8a..6f1bc11978 100644 --- a/server/events/modules.go +++ b/server/events/modules.go @@ -81,39 +81,51 @@ func (t tfFs) ReadDir(dirname string) ([]os.FileInfo, error) { var _ tfconfig.FS = tfFs{} func (m moduleInfo) load(files fs.FS, dir string, projects ...string) (_ *module, diags tfconfig.Diagnostics) { - if _, set := m[dir]; !set { - tfFiles := tfFs{files} - var mod *tfconfig.Module - mod, diags = tfconfig.LoadModuleFromFilesystem(tfFiles, dir) - - deps := make(map[string]bool) - if mod != nil { - for _, c := range mod.ModuleCalls { - mPath := path.Join(dir, c.Source) - if !tfconfig.IsModuleDirOnFilesystem(tfFiles, mPath) { - continue + dependenciesHandled := map[string]struct{}{dir: {}} // Only look at each dependency once + todo := []string{dir} + + for len(todo) > 0 { + dir := todo[len(todo)-1] // order of processing list not important, pick last for efficiency + todo = todo[:len(todo)-1] + + if _, set := m[dir]; !set { + tfFiles := tfFs{files} + var mod *tfconfig.Module + mod, diag := tfconfig.LoadModuleFromFilesystem(tfFiles, dir) + if diag != nil { + diags = append(diags, diag...) + } + + deps := make(map[string]bool) + if mod != nil { + for _, c := range mod.ModuleCalls { + mPath := path.Join(dir, c.Source) + if !tfconfig.IsModuleDirOnFilesystem(tfFiles, mPath) { + continue + } + deps[mPath] = true } - deps[mPath] = true + } + + m[dir] = &module{ + path: dir, + dependencies: deps, + projects: make(map[string]bool), } } - m[dir] = &module{ - path: dir, - dependencies: deps, - projects: make(map[string]bool), + // set projects on my dependencies + for dep := range m[dir].dependencies { + if _, exists := dependenciesHandled[dep]; !exists { + dependenciesHandled[dep] = struct{}{} + todo = append(todo, dep) + } } - } - // set projects on my dependencies - for dep := range m[dir].dependencies { - _, err := m.load(files, dep, projects...) - if err != nil { - diags = append(diags, err...) + // add projects to the list of dependant projects + for _, p := range projects { + m[dir].projects[p] = true } } - // add projects to the list of dependant projects - for _, p := range projects { - m[dir].projects[p] = true - } return m[dir], diags } From d4838446d0d971f769a0ad3f1a4c6094001cf440 Mon Sep 17 00:00:00 2001 From: Vincent De Smet Date: Tue, 20 Feb 2024 22:29:22 +0700 Subject: [PATCH 2/3] Fix: regenerate mocks --- .../mocks/mock_command_requirement_handler.go | 62 +++++++++++-------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/server/events/mocks/mock_command_requirement_handler.go b/server/events/mocks/mock_command_requirement_handler.go index ee156e0cd8..d5a36f20eb 100644 --- a/server/events/mocks/mock_command_requirement_handler.go +++ b/server/events/mocks/mock_command_requirement_handler.go @@ -44,12 +44,12 @@ func (mock *MockCommandRequirementHandler) ValidateApplyProject(repoDir string, return ret0, ret1 } -func (mock *MockCommandRequirementHandler) ValidateProjectDependencies(_param0 command.ProjectContext) (string, error) { +func (mock *MockCommandRequirementHandler) ValidateImportProject(repoDir string, ctx command.ProjectContext) (string, error) { if mock == nil { panic("mock must not be nil. Use myMock := NewMockCommandRequirementHandler().") } - params := []pegomock.Param{_param0} - result := pegomock.GetGenericMockFrom(mock).Invoke("ValidateProjectDependencies", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + params := []pegomock.Param{repoDir, ctx} + result := pegomock.GetGenericMockFrom(mock).Invoke("ValidateImportProject", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) var ret0 string var ret1 error if len(result) != 0 { @@ -63,12 +63,12 @@ func (mock *MockCommandRequirementHandler) ValidateProjectDependencies(_param0 c return ret0, ret1 } -func (mock *MockCommandRequirementHandler) ValidateImportProject(repoDir string, ctx command.ProjectContext) (string, error) { +func (mock *MockCommandRequirementHandler) ValidatePlanProject(repoDir string, ctx command.ProjectContext) (string, error) { if mock == nil { panic("mock must not be nil. Use myMock := NewMockCommandRequirementHandler().") } params := []pegomock.Param{repoDir, ctx} - result := pegomock.GetGenericMockFrom(mock).Invoke("ValidateImportProject", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + result := pegomock.GetGenericMockFrom(mock).Invoke("ValidatePlanProject", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) var ret0 string var ret1 error if len(result) != 0 { @@ -82,12 +82,12 @@ func (mock *MockCommandRequirementHandler) ValidateImportProject(repoDir string, return ret0, ret1 } -func (mock *MockCommandRequirementHandler) ValidatePlanProject(repoDir string, ctx command.ProjectContext) (string, error) { +func (mock *MockCommandRequirementHandler) ValidateProjectDependencies(ctx command.ProjectContext) (string, error) { if mock == nil { panic("mock must not be nil. Use myMock := NewMockCommandRequirementHandler().") } - params := []pegomock.Param{repoDir, ctx} - result := pegomock.GetGenericMockFrom(mock).Invoke("ValidatePlanProject", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + params := []pegomock.Param{ctx} + result := pegomock.GetGenericMockFrom(mock).Invoke("ValidateProjectDependencies", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) var ret0 string var ret1 error if len(result) != 0 { @@ -144,25 +144,6 @@ func (verifier *VerifierMockCommandRequirementHandler) ValidateApplyProject(repo return &MockCommandRequirementHandler_ValidateApplyProject_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} } -func (mock *MockCommandRequirementHandler) ValidateProjectDependencies(_param0 command.ProjectContext) (string, error) { - if mock == nil { - panic("mock must not be nil. Use myMock := NewMockCommandRequirementHandler().") - } - params := []pegomock.Param{_param0} - result := pegomock.GetGenericMockFrom(mock).Invoke("ValidateProjectDependencies", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) - var ret0 string - var ret1 error - if len(result) != 0 { - if result[0] != nil { - ret0 = result[0].(string) - } - if result[1] != nil { - ret1 = result[1].(error) - } - } - return ret0, ret1 -} - type MockCommandRequirementHandler_ValidateApplyProject_OngoingVerification struct { mock *MockCommandRequirementHandler methodInvocations []pegomock.MethodInvocation @@ -249,3 +230,30 @@ func (c *MockCommandRequirementHandler_ValidatePlanProject_OngoingVerification) } return } + +func (verifier *VerifierMockCommandRequirementHandler) ValidateProjectDependencies(ctx command.ProjectContext) *MockCommandRequirementHandler_ValidateProjectDependencies_OngoingVerification { + params := []pegomock.Param{ctx} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "ValidateProjectDependencies", params, verifier.timeout) + return &MockCommandRequirementHandler_ValidateProjectDependencies_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockCommandRequirementHandler_ValidateProjectDependencies_OngoingVerification struct { + mock *MockCommandRequirementHandler + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockCommandRequirementHandler_ValidateProjectDependencies_OngoingVerification) GetCapturedArguments() command.ProjectContext { + ctx := c.GetAllCapturedArguments() + return ctx[len(ctx)-1] +} + +func (c *MockCommandRequirementHandler_ValidateProjectDependencies_OngoingVerification) GetAllCapturedArguments() (_param0 []command.ProjectContext) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]command.ProjectContext, len(c.methodInvocations)) + for u, param := range params[0] { + _param0[u] = param.(command.ProjectContext) + } + } + return +} From fd0e70af1f803bbe8d6f3ba650f5a259fbc307f7 Mon Sep 17 00:00:00 2001 From: Vincent De Smet Date: Tue, 20 Feb 2024 22:30:31 +0700 Subject: [PATCH 3/3] chore: bump pre-release tag --- .github/workflows/release-fork.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-fork.yaml b/.github/workflows/release-fork.yaml index e13f179216..cab44ce78c 100644 --- a/.github/workflows/release-fork.yaml +++ b/.github/workflows/release-fork.yaml @@ -27,7 +27,7 @@ jobs: with: release-type: simple default-branch: release-fork - release-as: 0.27.0-pre.1 + release-as: 0.28.0-pre.1 changelog-types: ${{ steps.configure-changelog.outputs.result }} # https://github.com/google-github-actions/release-please-action#github-credentials token: ${{ secrets.VINCENT_PAT }}