Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Include caching improved #217

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

WIP: Include caching improved #217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4327e72
Pass debug level and logger to ObjFileIsUpToDate
matthijskooijman Mar 2, 2017
3a4083c
Let ObjFileIsUpToDate output verbose debug output
matthijskooijman Mar 2, 2017
824a5ff
ContainerFindIncludes: Add some temporary variables
matthijskooijman Mar 2, 2017
d4e9078
Generalize ObjFileIsUpToDate to BuildResultIsUpToDate
matthijskooijman Mar 2, 2017
f13ade8
Let include detection generate .d files too
matthijskooijman Mar 2, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 51 additions & 17 deletions src/arduino.cc/builder/builder_utils/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ import (
"arduino.cc/properties"
)

func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
objectFiles, err := CompileFiles(objectFiles, sourcePath, false, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) ([]string, error) {
objectFiles, err := CompileFiles(objectFiles, sourcePath, false, buildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger)
if err != nil {
return nil, i18n.WrapError(err)
}
Expand All @@ -55,7 +55,7 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st
}

for _, folder := range folders {
objectFiles, err = CompileFilesRecursive(objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes, verbose, warningsLevel, logger)
objectFiles, err = CompileFilesRecursive(objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes, verbose, warningsLevel, debugLevel, logger)
if err != nil {
return nil, i18n.WrapError(err)
}
Expand All @@ -64,28 +64,28 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st
return objectFiles, nil
}

func CompileFiles(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
objectFiles, err := compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN, verbose, warningsLevel, logger)
func CompileFiles(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) ([]string, error) {
objectFiles, err := compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN, verbose, warningsLevel, debugLevel, logger)
if err != nil {
return nil, i18n.WrapError(err)
}
objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN, verbose, warningsLevel, logger)
objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN, verbose, warningsLevel, debugLevel, logger)
if err != nil {
return nil, i18n.WrapError(err)
}
objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN, verbose, warningsLevel, logger)
objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN, verbose, warningsLevel, debugLevel, logger)
if err != nil {
return nil, i18n.WrapError(err)
}
return objectFiles, nil
}

func compileFilesWithExtensionWithRecipe(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
func compileFilesWithExtensionWithRecipe(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) ([]string, error) {
sources, err := findFilesInFolder(sourcePath, extension, recurse)
if err != nil {
return nil, i18n.WrapError(err)
}
return compileFilesWithRecipe(objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
return compileFilesWithRecipe(objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, debugLevel, logger)
}

func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]string, error) {
Expand Down Expand Up @@ -116,9 +116,9 @@ func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]str
return sources, nil
}

func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) ([]string, error) {
for _, source := range sources {
objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, debugLevel, logger)
if err != nil {
return nil, i18n.WrapError(err)
}
Expand All @@ -128,7 +128,7 @@ func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []s
return objectFiles, nil
}

func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) {
func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) (string, error) {
properties := buildProperties.Clone()
properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel]
properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE)
Expand All @@ -138,13 +138,14 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b
return "", i18n.WrapError(err)
}
properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = filepath.Join(buildPath, relativeSource+".o")
properties[constants.BUILD_PROPERTIES_DEP_FILE] = filepath.Join(buildPath, relativeSource+".d")

err = utils.EnsureFolderExists(filepath.Dir(properties[constants.BUILD_PROPERTIES_OBJECT_FILE]))
if err != nil {
return "", i18n.WrapError(err)
}

objIsUpToDate, err := ObjFileIsUpToDate(properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d"))
objIsUpToDate, err := BuildResultIsUpToDate(properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d"), debugLevel, logger)
if err != nil {
return "", i18n.WrapError(err)
}
Expand All @@ -161,19 +162,27 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b
return properties[constants.BUILD_PROPERTIES_OBJECT_FILE], nil
}

func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, error) {
func BuildResultIsUpToDate(sourceFile, resultFile, objectFile, dependencyFile string, debugLevel int, logger i18n.Logger) (bool, error) {
sourceFile = filepath.Clean(sourceFile)
resultFile = filepath.Clean(resultFile)
objectFile = filepath.Clean(objectFile)
dependencyFile = filepath.Clean(dependencyFile)

if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Checking build results for {0} (result = {1}, dep = {2})", sourceFile, objectFile, dependencyFile)
}

sourceFileStat, err := os.Stat(sourceFile)
if err != nil {
return false, i18n.WrapError(err)
}

objectFileStat, err := os.Stat(objectFile)
resultFileStat, err := os.Stat(resultFile)
if err != nil {
if os.IsNotExist(err) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", resultFile)
}
return false, nil
} else {
return false, i18n.WrapError(err)
Expand All @@ -183,16 +192,25 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err
dependencyFileStat, err := os.Stat(dependencyFile)
if err != nil {
if os.IsNotExist(err) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", dependencyFile)
}
return false, nil
} else {
return false, i18n.WrapError(err)
}
}

if sourceFileStat.ModTime().After(objectFileStat.ModTime()) {
if sourceFileStat.ModTime().After(resultFileStat.ModTime()) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, resultFile)
}
return false, nil
}
if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, dependencyFile)
}
return false, nil
}

Expand All @@ -212,10 +230,16 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err

firstRow := rows[0]
if !strings.HasSuffix(firstRow, ":") {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "No colon in first line of depfile")
}
return false, nil
}
objFileInDepFile := firstRow[:len(firstRow)-1]
if objFileInDepFile != objectFile {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Depfile is about different file: {0}", objFileInDepFile)
}
return false, nil
}

Expand All @@ -225,12 +249,22 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err
if err != nil && !os.IsNotExist(err) {
// There is probably a parsing error of the dep file
// Ignore the error and trigger a full rebuild anyway
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Failed to read: {0}", row)
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, i18n.WrapError(err).Error())
}
return false, nil
}
if os.IsNotExist(err) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", row)
}
return false, nil
}
if depStat.ModTime().After(objectFileStat.ModTime()) {
if depStat.ModTime().After(resultFileStat.ModTime()) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", row, resultFile)
}
return false, nil
}
}
Expand Down
1 change: 1 addition & 0 deletions src/arduino.cc/builder/constants/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ const BUILD_PROPERTIES_EXTRA_TIME_UTC = "extra.time.utc"
const BUILD_PROPERTIES_EXTRA_TIME_ZONE = "extra.time.zone"
const BUILD_PROPERTIES_INCLUDES = "includes"
const BUILD_PROPERTIES_OBJECT_FILE = "object_file"
const BUILD_PROPERTIES_DEP_FILE = "dep_file"
const BUILD_PROPERTIES_OBJECT_FILES = "object_files"
const BUILD_PROPERTIES_PATTERN = "pattern"
const BUILD_PROPERTIES_PID = "pid"
Expand Down
4 changes: 3 additions & 1 deletion src/arduino.cc/builder/container_add_prototypes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,10 @@ type ContainerAddPrototypes struct{}

func (s *ContainerAddPrototypes) Run(ctx *types.Context) error {
sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp")
depFile := sourceFile + ".d"
objFile := sourceFile + ".o"
commands := []types.Command{
&GCCPreprocRunner{SourceFilePath: sourceFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders},
&GCCPreprocRunner{SourceFilePath: sourceFile, ObjFilePath: objFile, DepFilePath: depFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders},
&ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE},
&FilterSketchSource{Source: &ctx.SourceGccMinusE},
&CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E},
Expand Down
12 changes: 7 additions & 5 deletions src/arduino.cc/builder/container_find_includes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,12 +289,14 @@ func writeCache(cache *includeCache, path string) error {

func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error {
sourcePath := sourceFile.SourcePath(ctx)
depPath := sourceFile.DepfilePath(ctx)
objPath := sourceFile.ObjectPath(ctx)
targetFilePath := utils.NULLFile()

// TODO: This should perhaps also compare against the
// include.cache file timestamp. Now, it only checks if the file
// changed after the object file was generated, but if it
// changed between generating the cache and the object file,
// changed after the dependency file was generated, but if it
// changed between generating the cache and the dependency file,
// this could show the file as unchanged when it really is
// changed. Changing files during a build isn't really
// supported, but any problems from it should at least be
Expand All @@ -303,7 +305,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
// TODO: This reads the dependency file, but the actual building
// does it again. Should the result be somehow cached? Perhaps
// remove the object file if it is found to be stale?
unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, sourceFile.ObjectPath(ctx), sourceFile.DepfilePath(ctx))
unchanged, err := builder_utils.BuildResultIsUpToDate(sourcePath, depPath, objPath, depPath, ctx.DebugLevel, ctx.GetLogger())
if err != nil {
return i18n.WrapError(err)
}
Expand All @@ -324,7 +326,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
}
} else {
commands := []types.Command{
&GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourcePath, TargetFilePath: targetFilePath, Includes: includes},
&GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourcePath, ObjFilePath: objPath, DepFilePath: depPath, TargetFilePath: targetFilePath, Includes: includes},
&IncludesFinderWithRegExp{Source: &ctx.SourceGccMinusE},
}
for _, command := range commands {
Expand All @@ -345,7 +347,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
library := ResolveLibrary(ctx, include)
if library == nil {
// Library could not be resolved, show error
err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: includes})
err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, ObjFilePath: objPath, DepFilePath: depPath, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: includes})
return i18n.WrapError(err)
}

Expand Down
16 changes: 13 additions & 3 deletions src/arduino.cc/builder/gcc_preproc_runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,14 @@ import (

type GCCPreprocRunner struct {
SourceFilePath string
ObjFilePath string
DepFilePath string
TargetFileName string
Includes []string
}

func (s *GCCPreprocRunner) Run(ctx *types.Context) error {
properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFileName, s.Includes)
properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.ObjFilePath, s.DepFilePath, s.TargetFileName, s.Includes)
if err != nil {
return i18n.WrapError(err)
}
Expand All @@ -72,12 +74,14 @@ func (s *GCCPreprocRunner) Run(ctx *types.Context) error {

type GCCPreprocRunnerForDiscoveringIncludes struct {
SourceFilePath string
ObjFilePath string
DepFilePath string
TargetFilePath string
Includes []string
}

func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(ctx *types.Context) error {
properties, _, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFilePath, s.Includes)
properties, _, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.ObjFilePath, s.DepFilePath, s.TargetFilePath, s.Includes)
if err != nil {
return i18n.WrapError(err)
}
Expand All @@ -100,7 +104,7 @@ func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(ctx *types.Context) error {
return nil
}

func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (properties.Map, string, error) {
func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, objFilePath string, depFilePath string, targetFilePath string, includes []string) (properties.Map, string, error) {
if targetFilePath != utils.NULLFile() {
preprocPath := ctx.PreprocPath
err := utils.EnsureFolderExists(preprocPath)
Expand All @@ -113,6 +117,12 @@ func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string
properties := ctx.BuildProperties.Clone()
properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = sourceFilePath
properties[constants.BUILD_PROPERTIES_PREPROCESSED_FILE_PATH] = targetFilePath
properties[constants.BUILD_PROPERTIES_DEP_FILE] = depFilePath
properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = objFilePath
err := utils.EnsureFolderExists(filepath.Dir(depFilePath))
if err != nil {
return nil, "", i18n.WrapError(err)
}

includes = utils.Map(includes, utils.WrapWithHyphenI)
properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE)
Expand Down
8 changes: 4 additions & 4 deletions src/arduino.cc/builder/phases/core_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func (s *CoreBuilder) Run(ctx *types.Context) error {
return i18n.WrapError(err)
}

archiveFile, objectFiles, err := compileCore(coreBuildPath, buildProperties, verbose, warningsLevel, logger)
archiveFile, objectFiles, err := compileCore(coreBuildPath, buildProperties, verbose, warningsLevel, ctx.DebugLevel, logger)
if err != nil {
return i18n.WrapError(err)
}
Expand All @@ -63,7 +63,7 @@ func (s *CoreBuilder) Run(ctx *types.Context) error {
return nil
}

func compileCore(buildPath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) {
func compileCore(buildPath string, buildProperties properties.Map, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) (string, []string, error) {
coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH]
variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH]

Expand All @@ -78,13 +78,13 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool,

variantObjectFiles := []string{}
if variantFolder != constants.EMPTY_STRING {
variantObjectFiles, err = builder_utils.CompileFiles(variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
variantObjectFiles, err = builder_utils.CompileFiles(variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger)
if err != nil {
return "", nil, i18n.WrapError(err)
}
}

coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger)
if err != nil {
return "", nil, i18n.WrapError(err)
}
Expand Down
Loading