diff --git a/.modules/collate_modules/_manifest.lua b/.modules/collate_modules/_manifest.lua new file mode 100644 index 0000000..8bb38f9 --- /dev/null +++ b/.modules/collate_modules/_manifest.lua @@ -0,0 +1,12 @@ +-- +-- Name: premake-ninja/.modules/collate_modules/_preload.lua +-- Purpose: Define the collate_modules action. +-- Author: Jan "GamesTrap" Schürkamp +-- Created: 2023/12/10 +-- Copyright: (c) 2023 Jan "GamesTrap" Schürkamp +-- + +return { + "_preload.lua", + "collate_modules.lua", +} diff --git a/.modules/collate_modules/_preload.lua b/.modules/collate_modules/_preload.lua new file mode 100644 index 0000000..f7c9dbd --- /dev/null +++ b/.modules/collate_modules/_preload.lua @@ -0,0 +1,53 @@ +-- +-- Name: premake-ninja/.modules/collate_modules/_preload.lua +-- Purpose: Define the collate_modules action. +-- Author: Jan "GamesTrap" Schürkamp +-- Created: 2023/12/10 +-- Copyright: (c) 2023 Jan "GamesTrap" Schürkamp +-- + +local p = premake + +newoption +{ + trigger = "modmapfmt", + description = "", + allowed = + { + { "clang", "Clang" }, + { "gcc", "GCC" }, + { "msvc", "MSVC" }, + } +} + +newoption +{ + trigger = "dd", + description = "", +} + +newoption +{ + trigger = "ddi", + description = "", +} + +newoption +{ + trigger = "deps", + description = "", +} + +newaction +{ + -- Metadata for the command line and help system + trigger = "collate_modules", + shortname = "collate_modules", + description = "collate_modules is a small utility to generate modmap files from ddi", + + execute = function() + p.modules.collate_modules.CollateModules() + end +} + +return p.modules.collate_modules diff --git a/.modules/collate_modules/collate_modules.lua b/.modules/collate_modules/collate_modules.lua new file mode 100644 index 0000000..18271e9 --- /dev/null +++ b/.modules/collate_modules/collate_modules.lua @@ -0,0 +1,983 @@ +-- +-- Name: premake-ninja/.modules/collate_modules/collate_modules.lua +-- Purpose: Define the collate_modules action. +-- Author: Jan "GamesTrap" Schürkamp +-- Created: 2023/12/10 +-- Copyright: (c) 2023 Jan "GamesTrap" Schürkamp +-- + +local p = premake + +premake.modules.collate_modules = {} +local collate_modules = p.modules.collate_modules + +local modmapfmt = _OPTIONS["modmapfmt"] +local dd = _OPTIONS["dd"] +local ddis = {} +local modDeps = {} + +local VersionStr = "version" +local RulesStr = "rules" +local WorkDirectoryStr = "work-directory" +local PrimaryOutputStr = "primary-output" +local OutputsStr = "outputs" +local ProvidesStr = "provides" +local LogicalNameStr = "logical-name" +local CompiledModulePathStr = "compiled-module-path" +local UniqueOnSourcePathStr = "unique-on-source-path" +local SourcePathStr = "source-path" +local IsInterfaceStr = "is-interface" +local RequiresStr = "requires" +local LookupMethodStr = "lookup-method" + +local function printError(msg) + term.setTextColor(term.errorColor) + print(msg) + term.setTextColor(nil) +end + +local function printDebug(msg) + term.setTextColor(term.magenta) + print(msg) + term.setTextColor(nil) +end + +local function validateInput() + if not _OPTIONS["ddi"] then + printError("collate_modules requires value for --ddi=") + return false + end + + if not dd then + printError("collate_modules requires value for --dd=") + return false + end + + return true +end + +LookupMethod = +{ + ByName = "by-name", + IncludeAngle = "include-angle", + IncludeQuote = "include-quote" +} + +-- SourceReqInfo = +-- { +-- LogicalName = "", +-- SourcePath = "", +-- CompiledModulePath = "", +-- UseSourcePath = false, +-- IsInterface = true, +-- Method = LookupMethod.ByName +-- } + +-- ScanDepInfo = +-- { +-- PrimaryOutput = "", +-- ExtraOutputs = {}, +-- Provides = {}, +-- Requires = {} +-- } + +-- ModuleReference = +-- { +-- Path = "", +-- Method = LookupMethod.ByName +-- } + +-- ModuleUsage = +-- { +-- Usage = {}, +-- Reference = {} +-- } + +-- AvailableModuleInfo = +-- { +-- BMIPath = "", +-- IsPrivate = false +-- } + +local function ScanDepFormatP1689Parse(ddiFilePath) + local scanDepInfo = + { + PrimaryOutput = "", + ExtraOutputs = {}, + Provides = {}, + Requires = {} + } + + --Load ddi JSON + local decodedDDI, error = json.decode(io.readfile(ddiFilePath)) + if not decodedDDI or error then + printError("Failed to parse \"" .. ddiFilePath .. "\" (" .. error .. ")") + return nil + end + + local version = iif(decodedDDI[VersionStr], decodedDDI[VersionStr], nil) + if version ~= nil and version > 1 then + printError("Failed to parse \"" .. ddiFilePath .. "\": version " .. tostring(version)) + return nil + end + + local rules = iif(decodedDDI[RulesStr], decodedDDI[RulesStr], nil) + if rules and type(rules) == "table" then + if #rules ~= 1 then + printError("Failed to parse \"" .. ddiFilePath .. "\": expected 1 source entry") + return nil + end + + for _, rule in pairs(rules) do + local workDir = rule[WorkDirectoryStr] + if workDir and type(workDir) ~= "string" then + printError("Failed to parse \"" .. ddiFilePath .. "\": work-directory is not a string") + return nil + end + + if rule[PrimaryOutputStr] then + scanDepInfo.PrimaryOutput = rule[PrimaryOutputStr] + if not scanDepInfo.PrimaryOutput then + printError("Failed to parse \"" .. ddiFilePath .. "\": invalid filename") + return nil + end + end + + if rule[OutputsStr] then + local outputs = rule[OutputsStr] + if outputs and type(outputs) == "table" then + for _1, output in pairs(outputs) do + if not output then + printError("Failed to parse \"" .. ddiFilePath .. "\": invalid filename") + return nil + end + table.insert(scanDepInfo.ExtraOutputs, output) + end + end + end + + if rule[ProvidesStr] then + local provides = rule[ProvidesStr] + if type(provides) ~= "table" then + printError("Failed to parse \"" .. ddiFilePath .. "\": provides is not an array") + return nil + end + + for _1, provide in pairs(provides) do + local provideInfo = + { + LogicalName = "", + SourcePath = "", + CompiledModulePath = "", + UseSourcePath = false, + IsInterface = true, + Method = LookupMethod.ByName + } + + provideInfo.LogicalName = provide[LogicalNameStr] + if not provideInfo.LogicalName then + printError("Failed to parse \"" .. ddiFilePath .. "\": invalid blob") + return nil + end + + if provide[CompiledModulePathStr] then + provideInfo.CompiledModulePath = provide[CompiledModulePathStr] + end + + if provide[UniqueOnSourcePathStr] then + local uniqueOnSourcePath = provide[UniqueOnSourcePathStr] + if type(uniqueOnSourcePath) ~= "boolean" then + printError("Failed to parse \"" .. ddiFilePath .. "\": unique-on-source-path is not a boolean") + return nil + end + provideInfo.UseSourcePath = uniqueOnSourcePath + else + provideInfo.UseSourcePath = false + end + + if provide[SourcePathStr] then + provideInfo.SourcePath = provide[SourcePathStr] + elseif provideInfo.UseSourcePath == true then + printError("Failed to parse \"" .. ddiFilePath .. "\": source-path is missing") + return nil + end + + if provide[IsInterfaceStr] then + local isInterface = provide[IsInterfaceStr] + if type(isInterface) ~= "boolean" then + printError("Failed to parse \"" .. ddiFilePath .. "\": is-interface is not a boolean") + return nil + end + provideInfo.IsInterface = isInterface + else + provideInfo.IsInterface = true + end + + table.insert(scanDepInfo.Provides, provideInfo) + end + end + + if rule[RequiresStr] then + local requires = rule[RequiresStr] + if type(requires) ~= "table" then + printError("Failed to parse \"" .. ddiFilePath .. "\": requires is not an array") + return nil + end + + for _1, require in pairs(requires) do + local requireInfo = + { + LogicalName = "", + SourcePath = "", + CompiledModulePath = "", + UseSourcePath = false, + IsInterface = true, + Method = LookupMethod.ByName + } + + requireInfo.LogicalName = require[LogicalNameStr] + if not requireInfo.LogicalName then + printError("Failed to parse \"" .. ddiFilePath .. "\": invalid blobl") + return nil + end + + if require[CompiledModulePathStr] then + requireInfo.CompiledModulePath = require[CompiledModulePathStr] + end + + if require[UniqueOnSourcePathStr] then + local uniqueOnSourcePath = require[UniqueOnSourcePathStr] + if type(uniqueOnSourcePath) ~= "boolean" then + printError("Failed to parse \"" .. ddiFilePath .. "\": unique-on-source-path is not a boolean") + return nil + end + requireInfo.UseSourcePath = uniqueOnSourcePath + else + requireInfo.UseSourcePath = false + end + + if require[SourcePathStr] then + requireInfo.SourcePath = require[SourcePathStr] + elseif requireInfo.UseSourcePath then + printError("Failed to parse \"" .. ddiFilePath .. "\": source-path is missing") + return nil + end + + if require[LookupMethodStr] then + local lookupMethod = require[LookupMethodStr] + if type(lookupMethod) ~= "string" then + printError("Failed to parse \"" .. ddiFilePath .. "\": lookup-method is not a string") + return nil + end + + if lookupMethod == "by-name" then + requireInfo.Method = LookupMethod.ByName + elseif lookupMethod == "include-angle" then + requireInfo.Method = LookupMethod.IncludeAngle + elseif lookupMethod == "include-quote" then + requireInfo.Method = LookupMethod.IncludeQuote + else + printError("Failed to parse \"" .. ddiFilePath .. "\": lookup-method is not valid: " .. lookupMethod) + return nil + end + elseif requireInfo.UseSourcePath then + requireInfo.Method = LookupMethod.ByName + end + + table.insert(scanDepInfo.Requires, requireInfo) + end + end + end + end + + return scanDepInfo +end + +local function getModuleMapExtension(modmapFormat) + if(modmapFormat == "clang") then + return ".pcm" + elseif (modmapFormat == "gcc") then + return ".gcm" + elseif(modmapFormat == "msvc") then + return ".ifc" + else + printError("collate_modules does not understand the " .. modmapFormat .. " module map format!") + os.exit(1) + end +end + +local function fileIsFullPath(name) + local nameLen = #name + + if os.target() == "windows" then + --On Windows, the name must be at least two characters long. + if nameLen < 2 then + return false + end + if name[1] == ":" then + return true + end + if name[2] == "\\" then + return true + end + else + --On UNIX, the name must be at least one character long. + if nameLen < 1 then + return false + end + end + if os.target ~= "windows" then + if name[1] == "~" then + return true + end + end + --On UNIX, the name must begin in a '/'. + --On Windows, if the name begins in a '/', then it is a full network path. + if name[1] == "/" then + return true + end + + return false +end + +local function ConvertToNinjaPath(path) + return path +end + +local function ModuleLocations_PathForGenerator(path) + return ConvertToNinjaPath(path) +end + +local function ModuleLocations_BMILocationForModule(modFiles, logicalName) + local m = modFiles[logicalName] + if m then + return m.BMIPath + end + + return nil +end + +local function ModuleLocations_BMIGeneratorPathForModule(modFiles, moduleLocations, logicalName) + local bmiLoc = ModuleLocations_BMILocationForModule(modFiles, logicalName) + if bmiLoc then + return ModuleLocations_PathForGenerator(bmiLoc) + end + + return bmiLoc +end + +local function Usages_AddReference(usages, logicalName, loc, lookupMethod) + if usages.Reference[logicalName] then + local r = usages.Reference[logicalName] + + if r.Path == loc and r.Method == lookupMethod then + return true + end + + printError("Disagreement of the location of the '" .. logicalName .. "' module. Location A: '" .. + r.Path .. "' via " .. r.Method .. "; Location B: '" .. loc .. "' via " .. lookupMethod .. ".") + return false + end + + usages.Reference[logicalName] = + { + Path = "", + Method = LookupMethod.ByName + } + local ref = usages.Reference[logicalName] + ref.Path = loc + ref.Method = lookupMethod +end + +local function moduleUsageSeed(modFiles, moduleLocations, objects, usages) + local internalUsages = {} + local unresolved = {} + + for _, object in pairs(objects) do + --Add references for each of the provided modules. + for _1, provide in pairs(object.Provides) do + local bmiLoc = ModuleLocations_BMIGeneratorPathForModule(modFiles, moduleLocations, provide.LogicalName) + if bmiLoc then + Usages_AddReference(usages, provide.LogicalName, bmiLoc, LookupMethod.ByName) + end + end + + --For each requires, pull in what is required. + for _1, require in pairs(object.Requires) do + --Find the required name in the current target. + local bmiLoc = ModuleLocations_BMIGeneratorPathForModule(modFiles, moduleLocations, require.LogicalName) + + --Find transitive usages. + local transitiveUsages = usages.Usage[require.LogicalName] + + for _2, provide in pairs(object.Provides) do + if not usages.Usage[provide.LogicalName] then + usages.Usage[provide.LogicalName] = {} + end + local thisUsages = usages.Usage[provide.LogicalName] + + --Add the direct usage. + thisUsages[require.LogicalName] = 1 + + if not transitiveUsages or internalUsages[require.LogicalName] then + --Mark that we need to update transitive usages later. + if bmiLoc then + if not internalUsages[provide.LogicalName] then + internalUsages[provide.LogicalName] = {} + end + internalUsages[provide.LogicalName][require.LogicalName] = 1 + end + else + --Add the transitive usage. + for tu, _3 in pairs(transitiveUsages) do + thisUsages[tu] = 1 + end + end + end + + if bmiLoc then + Usages_AddReference(usages, require.LogicalName, bmiLoc, require.Method) + end + end + end + + --While we have internal usages to manage. + while next(internalUsages) do + local startingSize = #internalUsages + + --For each internal usage. + for f, s in pairs(internalUsages) do + local thisUsages = usages.Usage[f] + + for f1, s2 in pairs(s) do + --Check if this required module uses other internal modules; defer if so. + if internalUsages[f1] then + goto continueUse + end + + local transitiveUsages = usages.Usage[f1] + if transitiveUsages then + for transitiveUsage, _ in pairs(transitiveUsages) do + thisUsages[transitiveUsage] = 1 + end + end + + s[f1] = nil + + ::continueUse:: + end + + --Erase the entry if it doesn't have any remaining usages. + if #s == 0 then + internalUsages[f] = nil + end + end + + --Check that at least one usage was resolved. + if startingSize == #internalUsages then + --Nothing could be resolved this loop; we have a cycle, so record the cycle and exit. + for f, s in pairs(internalUsages) do + if not table.contains(unresolved, f) then + table.insert(unresolved, f) + end + end + break + end + end + + return unresolved +end + +local function Ninja_WriteBuild(ninjaBuild) + local result = "" + + --Make sure there is a rule. + if ninjaBuild.Rule == "" then + printError("No rule for Ninja_WriteBuild! called with comment: " .. ninjaBuild.Comment) + os.exit(1) + end + + --Make sure there is at least one output file. + if #ninjaBuild.Outputs == 0 then + printError("No output files for Ninja_WriteBuild! called with comment: " .. ninjaBuild.Comment) + os.exit(1) + end + local buildStr = "" + + if ninjaBuild.Comment ~= "" then + result = result .. ninjaBuild.Comment .. "\n" + end + + --Write output files. + buildStr = buildStr .. "build" + + --Write explicit outputs + for _, output in pairs(ninjaBuild.Outputs) do + buildStr = buildStr .. " " .. output + -- if ComputingUnknownDependencies then + -- --TODO + -- end + end + + --Write implicit outputs + if #ninjaBuild.ImplicitOuts > 0 then + --Assume Ninja is new enough to support implicit outputs. + --Callers should not populate this field otherwise. + buildStr = buildStr .. " |" + for _, implicitOut in pairs(ninjaBuild.ImplicitOuts) do + buildStr = buildStr .. " " .. implicitOut + -- if ComputingUnknownDependencies then + -- --TODO + -- end + end + end + + --Repeat some outputs, but expressed as absolute paths. + --This helps Ninja handle absolute paths found in a depfile. + --FIXME: Unfortunately this causes Ninja to stat the file twice. + --We could avoid this if Ninja Issue #1251 were fixed. + if #ninjaBuild.WorkDirOuts > 0 then + if SupportsImplicitOuts() and #ninjaBuild.ImplicitOuts == 0 then + --Make them implicit outputs if supported by this version of Ninja. + buildStr = buildStr .. " |" + end + + for _, workDirOut in pairs(ninjaBuild.WorkDirOuts) do + buildStr = buildStr .. " " .. workDirOut + end + end + + --Write the rule. + buildStr = buildStr .. ": " .. ninjaBuild.Rule + + local arguments = "" + + --TODO: Better formatting for when there are multiple input/output files. + + --Write explicit dependencies. + for _, explicitDep in pairs(ninjaBuild.ExplicitDeps) do + arguments = arguments .. " " .. explicitDep + end + + --Write implicit dependencies. + if #ninjaBuild.ImplicitDeps > 0 then + arguments = arguments .. " |" + for _, implicitDep in pairs(ninjaBuild.ImplicitDeps) do + arguments = arguments .. " " .. implicitDep + end + end + + --Write oder-only dependencies. + if #ninjaBuild.OrderOnlyDeps > 0 then + arguments = arguments .. " ||" + for _, orderOnlyDep in pairs(ninjaBuild.OrderOnlyDeps) do + arguments = arguments .. " " .. orderOnlyDep + end + end + + arguments = arguments .. "\n" + + --Write the variables bound to this build statement. + local assignments = "" + for variable, value in pairs(ninjaBuild.Variables) do + assignments = assignments .. " " .. variable .. " = " .. value .. "\n" + end + + --Check if a response file rule should be used + local useResponseFile = false + --TODO + + return buildStr .. arguments .. assignments .. "\n" +end + +local function GetTransitiveUsages(modFiles, locs, required, usages) + local transitiveUsageDirects = {} + local transitiveUsageNames = {} + + local allUsages = {} + + for _, r in pairs(required) do + local bmiLoc = ModuleLocations_BMIGeneratorPathForModule(modFiles, locs, r.LogicalName) + if bmiLoc then + table.insert(allUsages, {LogicalName = r.LogicalName, Location = bmiLoc, Method = r.Method}) + if not table.contains(transitiveUsageDirects, r.LogicalName) then + table.insert(transitiveUsageDirects, r.LogicalName) + end + + --Insert transitive usages. + local transitiveUsages = usages.Usage[r.LogicalName] + if transitiveUsages then + for _1, tu in pairs(transitiveUsages) do + if not table.contains(transitiveUsageNames, tu) then + table.insert(transitiveUsageNames, tu) + end + end + end + end + end + + for _, transitiveName in pairs(transitiveUsageNames) do + if not table.contains(transitiveUsageDirects, transitiveName) then + local moduleRef = usages.Reference[transitiveName] + if moduleRef then + table.insert(allUsages, {LogicalName = transitiveName, Location = moduleRef.Path, Method = moduleRef.Method}) + end + end + end + + return allUsages +end + +local function ModuleMapContentClang(modFiles, locs, object, usages) + local mm = "" + + --Clang's command line only supports a single output. + --If more than one is expected, we cannot make a useful module map file. + if #object.Provides > 1 then + return "" + end + + --A series of flags which tell the compiler where to look for modules. + + for _, provide in pairs(object.Provides) do + local bmiLoc = ModuleLocations_BMIGeneratorPathForModule(modFiles, locs, provide.LogicalName) + if bmiLoc then + --Force the TU to be considered a C++ module source file regardless of extension. + mm = mm .. "-x c++-module\n" + + mm = mm .. "-fmodule-output=" .. bmiLoc .. "\n" + break + end + end + + local allUsages = GetTransitiveUsages(modFiles, locs, object.Requires, usages) + for _, usage in pairs(allUsages) do + mm = mm .. "-fmodule-file=" .. usage.LogicalName .. "=" .. usage.Location .. "\n" + end + + return mm +end + +local function ModuleMapContentGCC(modFiles, locs, object, usages) + local mm = "" + + --Documented in GCC's documentation. + --The format is a series of lines with a module name and the associated + --filename separated by spaces. The first line may use '$root' as the module + --name to specify a "repository root". + --That is used to anchor any relative paths present in the file + --(Premake-Ninja should never generate any). + + --Write the root directory to use for module paths. + mm = mm .. "$root " .. locs.RootDirectory .. "\n" + + for _, provide in pairs(object.Provides) do + local bmiLoc = ModuleLocations_BMIGeneratorPathForModule(modFiles, locs, provide.LogicalName) + if bmiLoc then + mm = mm .. provide.LogicalName .. " " .. bmiLoc .. "\n" + end + end + for _, require in pairs(object.Requires) do + local bmiLoc = ModuleLocations_BMIGeneratorPathForModule(modFiles, locs, require.LogicalName) + if bmiLoc then + mm = mm .. require.LogicalName .. " " .. bmiLoc .. "\n" + end + end + + return mm +end + +local function ModuleMapContentMSVC(modFiles, locs, object, usages) + local mm = "" + + --A response file of '-reference NAME=PATH' arguments. + + --MSVC's command line only supports a single output. + --If more than one is expected, we cannot make a useful module map file. + if #object.Provides > 1 then + return "" + end + + local function flagForMethod(method) + if method == LookupMethod.ByName then + return "-reference" + elseif method == LookupMethod.IncludeAngle then + return "-headerUnit:angle" + elseif method == LookupMethod.IncludeQuote then + return "-headerUnit:quote" + else + printError("Unsupported lookup method") + os.exit(1) + end + end + + for _, provide in pairs(object.Provides) do + if provide.IsInterface then + mm = mm .. "-interface\n" + else + mm = mm .. "-internalPartition\n" + end + + local bmiLoc = ModuleLocations_BMIGeneratorPathForModule(modFiles, locs, provide.LogicalName) + if bmiLoc then + mm = mm .. "-ifcOutput " .. bmiLoc .. "\n" + end + end + + local allUsages = GetTransitiveUsages(modFiles, locs, object.Requires, usages) + for _, usage in pairs(allUsages) do + local flag = flagForMethod(usage.Method) + + mm = mm .. flag .. " " .. usage.LogicalName .. "=" .. usage.Location .. "\n" + end + + return mm +end + +local function ModuleMapContent(modmapfmt, modFiles, locs, object, usages) + if modmapfmt == "clang" then + return ModuleMapContentClang(modFiles, locs, object, usages) + elseif modmapfmt == "gcc" then + return ModuleMapContentGCC(modFiles, locs, object, usages) + elseif modmapfmt == "msvc" then + return ModuleMapContentMSVC(modFiles, locs, object, usages) + end + + printError("Unknown modmapfmt: " .. modmapfmt) + os.exit(1) +end + +local function LoadModuleDependencies(modDeps, modFiles, usages) + if _OPTIONS["deps"] == nil or _OPTIONS["deps"] == "" then + return + end + + for i = 1, #modDeps do + local modDepFile = modDeps[i] + + local modDep = io.readfile(modDepFile) + if not modDep or modDep == "" then + printError("Failed to open \"" .. modDepFile .. "\" for module information") + os.exit(1) + end + + local modDepData, error = json.decode(modDep) + if not modDepData or error then + printError("Failed to parse \"" .. modDepFile .. "\" (" .. error .. ")") + os.exit(1) + end + + if modDepData == nil then + return + end + + local targetModules = modDepData["modules"] + if targetModules then + for moduleName, moduleData in pairs(targetModules) do + local bmiPath = moduleData["bmi"] + local isPrivate = moduleData["is-private"] + modFiles[moduleName] = { BMIPath = bmiPath, IsPrivate = isPrivate } + end + end + + local targetModulesReferences = modDepData["references"] + if targetModulesReferences then + for moduleName, moduleData in pairs(targetModulesReferences) do + local moduleReference = + { + Path = "", + Method = LookupMethod.ByName + } + + local referencePath = moduleData["path"] + if referencePath then + moduleReference.Path = referencePath + end + + local referenceMethod = moduleData["lookup-method"] + if referenceMethod then + if referenceMethod == "by-name" or referenceMethod == "include-angle" or referenceMethod == "include-quote" then + moduleReference.Method = referenceMethod + else + printError("Unknown lookup method \"" .. referenceMethod .. "\"") + os.exit(1) + end + end + + usages.Reference[moduleName] = moduleReference + end + end + + local targetModulesUsage = modDepData["usages"] + if targetModulesUsage then + for moduleName, modules in pairs(targetModulesUsage) do + for i = 1, #modules do + if not usages.Usage[moduleName] then + usages.Usage[moduleName] = {} + end + + table.insert(usages.Usage[moduleName], modules[i]) + end + end + end + end +end + +function collate_modules.CollateModules() + if not validateInput() then + os.exit(1) + end + + local moduleDir = path.getdirectory(dd) + + local ddis = iif(#_OPTIONS["ddi"] > 0, string.explode(_OPTIONS["ddi"], " "), {}) + + local modDeps = iif(#_OPTIONS["deps"] > 0, string.explode(_OPTIONS["deps"], " "), {}) + + local objects = {} + for _, ddiFilePath in pairs(ddis) do + local info = ScanDepFormatP1689Parse(ddiFilePath) + if not info then + printError("Failed to parse ddi file \"" .. ddiFilePath .. "\"") + os.exit(1) + end + + table.insert(objects, info) + end + + local usages = + { + Usage = {}, + Reference = {} + } + + local moduleExt = getModuleMapExtension(modmapfmt) + + local modFiles = {} + local targetModules = {} + + LoadModuleDependencies(modDeps, modFiles, usages) + + --Map from module name to module file path, if known. + for _, object in pairs(objects) do + for _1, provide in pairs(object.Provides) do + local mod = "" + + if provide.CompiledModulePath ~= "" then + --The scanner provided the path to the module file + mod = provide.CompiledModulePath + if not fileIsFullPath(mod) then + --Treat relative to work directory (top of build tree). + -- mod = CollapseFullPath(mod, dirTopBld) + --TODO (Use moduleDir to make path absolute?) + printError("Scanner provided compiled module relative paths are not supported!") + os.exit(1) + end + else + --Assume the module file path matches the logical module name. + local safeLogicalName = provide.LogicalName --TODO Needs fixing for header units + string.gsub(safeLogicalName, ":", "-") + mod = path.join(moduleDir, safeLogicalName) .. moduleExt + end + + modFiles[provide.LogicalName] = { BMIPath = mod, IsPrivate = false } --Always visible within our own target. + + targetModules[provide.LogicalName] = {} + local moduleInfo = targetModules[provide.LogicalName] + moduleInfo["bmi"] = mod + moduleInfo["is-private"] = false + end + end + + local moduleLocations = { RootDirectory = "." } + + --Insert information about the current target's modules. + if modmapfmt then + local cycleModules = moduleUsageSeed(modFiles, moduleLocations, objects, usages) + if #cycleModules ~= 0 then + printError("Circular dependency detected in the C++ module import graph. See modules named: \"" .. table.concat(cycleModules, "\", \"") .. "\"") + os.exit(1) + end + end + + --Create modmap and dyndep files + + local dynDepStr = "ninja_dyndep_version = 1.0\n" + + local ninjaBuild = + { + Comment = "", + Rule = "dyndep", + Outputs = {}, + ImplicitOuts = {}, + WorkDirOuts = {}, + ExplicitDeps = {}, + ImplicitDeps = {}, + OrderOnlyDeps = {}, + Variables = {}, + RspFile = "" + } + table.insert(ninjaBuild.Outputs, "") + for _, object in pairs(objects) do + ninjaBuild.Outputs[1] = object.PrimaryOutput + ninjaBuild.ImplicitOuts = {} + for _1, provide in pairs(object.Provides) do + local implicitOut = modFiles[provide.LogicalName].BMIPath + --Ignore the 'provides' when the BMI is the output. + if implicitOut ~= ninjaBuild.Outputs[1] then + table.insert(ninjaBuild.ImplicitOuts, implicitOut) + end + end + ninjaBuild.ImplicitDeps = {} + for _1, require in pairs(object.Requires) do + local mit = modFiles[require.LogicalName] + if mit then + table.insert(ninjaBuild.ImplicitDeps, mit.BMIPath) + end + end + ninjaBuild.Variables = {} + if #object.Provides > 0 then + ninjaBuild.Variables["restat"] = "1" + end + + if modmapfmt then + local mm = ModuleMapContent(modmapfmt, modFiles, moduleLocations, object, usages) + io.writefile(object.PrimaryOutput .. ".modmap", mm) + end + + dynDepStr = dynDepStr .. Ninja_WriteBuild(ninjaBuild) + end + + io.writefile(dd, dynDepStr) + + --Create CXXModules.json + + local targetModsFilepath = path.join(path.getdirectory(dd), "CXXModules.json") + local targetModuleInfo = {} + targetModuleInfo["modules"] = targetModules + + targetModuleInfo["usages"] = {} + local targetUsages = targetModuleInfo["usages"] + for moduleName, moduleUsages in pairs(usages.Usage) do + targetUsages[moduleName] = {} + local modUsage = targetUsages[moduleName] + for modules, _ in pairs(moduleUsages) do + table.insert(modUsage, modules) + end + end + + targetModuleInfo["references"] = {} + local targetReferences = targetModuleInfo["references"] + for moduleName, reference in pairs(usages.Reference) do + targetReferences[moduleName] = {} + local modRef = targetReferences[moduleName] + modRef["path"] = reference.Path + modRef["lookup-method"] = reference.Method + end + + io.writefile(targetModsFilepath, json.encode(targetModuleInfo)) +end + +include("_preload.lua") + +return collate_modules diff --git a/README.md b/README.md index e7f39b3..fe5c9fe 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ For each project - configuration pair we create separate .ninja file. For soluti Build.ninja file sets phony targets for configuration names so you can build them from command line. And default target is the first configuration name in your project (usually default). +### Experimental C++20 Modules support + +To enable the experimental C++ modules support you just need to provide the `--experimental-enable-cxx-modules` flag when generating the ninja build files. +By default only translation units with `.cxx`, `.cxxm`, `.ixx`, `.cppm`, `.c++m`, `.ccm`, `.mpp` file extensions are considered to be C++ modules. +To force scanning of translation units with file extensions like `.c`, `.cc`, `.cpp`, etc. provide the `--experimental-modules-scan-all` flag when generating the ninja build files. + ### Tested on [![ubuntu](https://github.com/GamesTrap/premake-ninja/actions/workflows/ubuntu.yml/badge.svg)](https://github.com/GamesTrap/premake-ninja/actions/workflows/ubuntu.yml) @@ -17,7 +23,3 @@ Build.ninja file sets phony targets for configuration names so you can build the ### Extra Tests Part of integration tests of several generators in https://github.com/Jarod42/premake-sample-projects - -### TODO - -- C++20 Modules support diff --git a/_preload.lua b/_preload.lua index a7cb070..0a8e6d6 100644 --- a/_preload.lua +++ b/_preload.lua @@ -8,6 +8,18 @@ local p = premake +newoption +{ + trigger = "experimental-enable-cxx-modules", + description = "Enable C++20 Modules support. This adds code scanning and collation step to the Ninja build." +} + +newoption +{ + trigger = "experimental-modules-scan-all", + description = "Enable scanning for all C++ translation units. By default only files ending with .cxx, .cxxm, .ixx, .cppm, .c++m, .ccm and .mpp are scanned." +} + newaction { -- Metadata for the command line and help system diff --git a/ninja.lua b/ninja.lua index e895ddc..cb48f6b 100644 --- a/ninja.lua +++ b/ninja.lua @@ -434,18 +434,33 @@ local function compilation_rules(cfg, toolset, pch) local all_resflags = getresflags(toolset, cfg, cfg) if toolset == p.tools.msc then + local force_include_pch = "" + if pch then + force_include_pch = " /Yu" .. ninja.shesc(path.getname(pch.input)) .. " /Fp" .. ninja.shesc(pch.pch) + p.outln("rule build_pch") + p.outln(" command = " .. iif(cfg.language == "C", cc .. all_cflags, cxx .. all_cxxflags) .. " /Yc" .. ninja.shesc(path.getname(pch.input)) .. " /Fp" .. ninja.shesc(pch.pch) .. " /nologo /showIncludes -c /Tp$in /Fo$out") + p.outln(" description = build_pch $out") + p.outln(" deps = msvc") + p.outln("") + end + p.outln("CFLAGS=" .. all_cflags) p.outln("rule cc") - p.outln(" command = " .. cc .. " $CFLAGS" .. " /nologo /showIncludes -c /Tc$in /Fo$out") + p.outln(" command = " .. cc .. " $CFLAGS" .. force_include_pch .. " /nologo /showIncludes -c /Tc$in /Fo$out") p.outln(" description = cc $out") p.outln(" deps = msvc") p.outln("") p.outln("CXXFLAGS=" .. all_cxxflags) p.outln("rule cxx") - p.outln(" command = " .. cxx .. " $CXXFLAGS" .. " /nologo /showIncludes -c /Tp$in /Fo$out") + p.outln(" command = " .. cxx .. " $CXXFLAGS" .. force_include_pch .. " /nologo /showIncludes -c /Tp$in /Fo$out") p.outln(" description = cxx $out") p.outln(" deps = msvc") p.outln("") + p.outln("rule cxx_module") + p.outln(" command = " .. cxx .. " $CXXFLAGS" .. force_include_pch .. " /nologo /showIncludes @$DYNDEP_MODULE_MAP_FILE /FS -c /Tp$in /Fo$out") + p.outln(" description = cxx_module $out") + p.outln(" deps = msvc") + p.outln("") p.outln("RESFLAGS = " .. all_resflags) p.outln("rule rc") p.outln(" command = " .. rc .. " /nologo /fo$out $in $RESFLAGS") @@ -486,6 +501,16 @@ local function compilation_rules(cfg, toolset, pch) p.outln(" depfile = $out.d") p.outln(" deps = gcc") p.outln("") + p.outln("rule cxx_module") + if toolset == p.tools.gcc then + p.outln(" command = " .. cxx .. " $CXXFLAGS " .. "-fmodules-ts -fmodule-mapper=$DYNDEP_MODULE_MAP_FILE -fdeps-format=p1689r5 -x c++ " .. force_include_pch .. " -MT $out -MF $out.d -c -o $out $in") + else + p.outln(" command = " .. cxx .. " $CXXFLAGS" .. force_include_pch .. " -MT $out -MF $out.d @$DYNDEP_MODULE_MAP_FILE -c -o $out $in") + end + p.outln(" description = cxx_module $out") + p.outln(" depfile = $out.d") + p.outln(" deps = gcc") + p.outln("") p.outln("RESFLAGS = " .. all_resflags) p.outln("rule rc") p.outln(" command = " .. rc .. " -i $in -o $out $RESFLAGS") @@ -513,6 +538,93 @@ local function custom_command_rule() p.outln("") end +local function get_module_scanner_name(toolset, toolsetVersion, cfg) + local scannerName = nil + + if toolset == p.tools.clang then + scannerName = "clang-scan-deps" + if toolsetVersion then + scannerName = scannerName .. "-" .. toolsetVersion + end + elseif toolset == p.tools.gcc or toolset == p.tools.msc then + scannerName = toolset.gettoolname(cfg, "cxx") + end + + return scannerName +end + +local function module_scan_rule(cfg, toolset) + local cmd = "" + + local scannerName = get_module_scanner_name(toolset, toolsetVersion, cfg) + + if toolset == p.tools.clang then + local _, toolsetVersion = p.tools.canonical(cfg.toolset) + local compilerName = toolset.gettoolname(cfg, "cxx") + + cmd = scannerName .. " -format=p1689 -- " .. compilerName .. " $CXXFLAGS -x c++ $in -c -o $OBJ_FILE -MT $DYNDEP_INTERMEDIATE_FILE -MD -MF $DEP_FILE > $DYNDEP_INTERMEDIATE_FILE.tmp && mv $DYNDEP_INTERMEDIATE_FILE.tmp $DYNDEP_INTERMEDIATE_FILE" + elseif toolset == p.tools.gcc then + cmd = scannerName .. " $CXXFLAGS -E -x c++ $in -MT $DYNDEP_INTERMEDIATE_FILE -MD -MF $DEP_FILE -fmodules-ts -fdeps-file=$DYNDEP_INTERMEDIATE_FILE -fdeps-target=$OBJ_FILE -fdeps-format=p1689r5 -o $PREPROCESSED_OUTPUT_FILE" + elseif toolset == p.tools.msc then + cmd = scannerName .. " $CXXFLAGS $in -nologo -TP -showIncludes -scanDependencies $DYNDEP_INTERMEDIATE_FILE -Fo$OBJ_FILE" + else + term.setTextColor(term.errorColor) + print("C++20 Modules are only supported with Clang, GCC and MSC!") + term.setTextColor(nil) + os.exit() + end + + p.outln("rule __module_scan") + if toolset == p.tools.msc then + p.outln(" deps = msvc") + else + p.outln(" depfile = $DEP_FILE") + end + p.outln(" command = " .. cmd) + p.outln(" description = Scanning $in for C++ dependencies") + p.outln("") +end + +local collateModuleScript = nil + +local function module_collate_rule(cfg, toolset) + if not collateModuleScript then + local collateModuleScripts = os.matchfiles(_MAIN_SCRIPT_DIR .. "/.modules/**/collate_modules/collate_modules.lua") + if collateModuleScripts == nil or collateModuleScripts[1] == nil then + term.setTextColor(term.errorColor) + print("Unable to find collate_modules.lua script!") + term.setTextColor(nil) + os.exit() + else + collateModuleScript = collateModuleScripts[1] + end + end + + local cmd = _PREMAKE_COMMAND .. " --file=" .. collateModuleScript .. " collate_modules " + + if toolset == p.tools.clang then + cmd = cmd .. "--modmapfmt=clang " + elseif toolset == p.tools.gcc then + cmd = cmd .. "--modmapfmt=gcc " + elseif toolset == p.tools.msc then + cmd = cmd .. "--modmapfmt=msvc " + else + term.setTextColor(term.errorColor) + print("C++20 Modules are only supported with Clang, GCC and MSC!") + term.setTextColor(nil) + os.exit() + end + + cmd = cmd .. "--dd=$out --ddi=\"$in\" --deps=$MODULE_DEPS @$out.rsp" + + p.outln("rule __module_collate") + p.outln(" command = " .. cmd) + p.outln(" description = Generating C++ dyndep file $out") + p.outln(" rspfile = $out.rsp") + p.outln(" rspfile_content = $in") + p.outln("") +end + local function collect_generated_files(prj, cfg) local generated_files = {} tree.traverse(project.getsourcetree(prj), { @@ -543,11 +655,36 @@ local function collect_generated_files(prj, cfg) return generated_files end -local function pch_build(cfg, pch) +local function is_module_file(file) + local fileEnding = path.getextension(file) + + if _OPTIONS["experimental-modules-scan-all"] and path.iscppfile(file) then + return true + end + + if fileEnding == ".cxx" or fileEnding == ".cxxm" or fileEnding == ".ixx" or + fileEnding == ".cppm" or fileEnding == ".c++m" or fileEnding == ".ccm" or + fileEnding == ".mpp" then + return true + end + + return false +end + +local function pch_build(cfg, pch, toolset) local pch_dependency = {} if pch then - pch_dependency = { pch.gch } - add_build(cfg, pch.gch, {}, "build_pch", {pch.input}, {}, {}, {}) + if toolset == p.tools.msc then + pch_dependency = { pch.pch } + + local obj_dir = project.getrelative(cfg.workspace, cfg.objdir) + local pchObj = obj_dir .. "/" .. path.getname(pch.inputSrc) .. (toolset.objectextension or ".o") + + add_build(cfg, pchObj, pch_dependency, "build_pch", {pch.inputSrc}, {}, {}, {}) + else + pch_dependency = { pch.gch } + add_build(cfg, pch.gch, {}, "build_pch", {pch.input}, {}, {}, {}) + end end return pch_dependency end @@ -589,13 +726,26 @@ local function compile_file_build(cfg, filecfg, toolset, pch_dependency, regular end add_build(cfg, objfilename, {}, "cc", {filepath}, pch_dependency, regular_file_dependencies, cflags) elseif shouldcompileascpp(filecfg) then - local objfilename = obj_dir .. "/" .. filecfg.objname .. (toolset.objectextension or ".o") + local objfilename = obj_dir .. "/" .. filecfg.objname .. path.getextension(filecfg.path) .. (toolset.objectextension or ".o") objfiles[#objfiles + 1] = objfilename local cxxflags = {} if has_custom_settings then cxxflags = {"CXXFLAGS = $CXXFLAGS " .. getcxxflags(toolset, cfg, filecfg)} end - add_build(cfg, objfilename, {}, "cxx", {filepath}, pch_dependency, regular_file_dependencies, cxxflags) + + local rule = "cxx" + local regFileDeps = table.arraycopy(regular_file_dependencies) + if _OPTIONS["experimental-enable-cxx-modules"] and is_module_file(filecfg.name) then + rule = "cxx_module" + local dynDepModMapFile = objfilename .. ".modmap" + local dynDepFile = path.join(obj_dir, "CXX.dd") + table.insert(cxxflags, "DYNDEP_MODULE_MAP_FILE = " .. dynDepModMapFile) + table.insert(cxxflags, "dyndep = " .. dynDepFile) + table.insert(regFileDeps, dynDepFile) + table.insert(regFileDeps, dynDepModMapFile) + end + + add_build(cfg, objfilename, {}, rule, {filepath}, pch_dependency, regFileDeps, cxxflags) elseif path.isresourcefile(filecfg.abspath) then local objfilename = obj_dir .. "/" .. filecfg.name .. ".res" objfiles[#objfiles + 1] = objfilename @@ -609,12 +759,22 @@ end local function files_build(prj, cfg, toolset, pch_dependency, regular_file_dependencies, file_dependencies) local objfiles = {} + local obj_dir = project.getrelative(cfg.workspace, cfg.objdir) + tree.traverse(project.getsourcetree(prj), { onleaf = function(node, depth) local filecfg = fileconfig.getconfig(node, cfg) if not filecfg or filecfg.flags.ExcludeFromBuild then return end + + -- Compiling PCH on MSVC is handled via build_pch build rule + if toolset == p.tools.msc and cfg.pchsource and cfg.pchsource == node.abspath then + local objfilename = obj_dir .. "/" .. path.getname(node.path) .. (toolset.objectextension or ".o") + objfiles[#objfiles + 1] = objfilename + return + end + local rule = p.global.getRuleForFile(node.name, prj.rules) local filepath = project.getrelative(cfg.workspace, node.abspath) @@ -639,6 +799,45 @@ local function files_build(prj, cfg, toolset, pch_dependency, regular_file_depen return objfiles end +local function scan_module_file_build(cfg, filecfg, toolset, modulefiles) + local obj_dir = project.getrelative(cfg.workspace, cfg.objdir) + local filepath = project.getrelative(cfg.workspace, filecfg.abspath) + local has_custom_settings = fileconfig.hasFileSettings(filecfg) + + local outputFilebase = obj_dir .. "/" .. filecfg.name + local dyndepfilename = outputFilebase .. toolset.objectextension .. ".ddi" + modulefiles[#modulefiles + 1] = dyndepfilename + + local vars = {} + table.insert(vars, "DEP_FILE = " .. outputFilebase .. toolset.objectextension .. ".ddi.d") + table.insert(vars, "DYNDEP_INTERMEDIATE_FILE = " .. dyndepfilename) + table.insert(vars, "OBJ_FILE = " .. outputFilebase .. toolset.objectextension) + table.insert(vars, "PREPROCESSED_OUTPUT_FILE = " .. outputFilebase .. toolset.objectextension .. ".ddi.i") + + add_build(cfg, dyndepfilename, {}, "__module_scan", {filepath}, {}, {}, vars) +end + +local function files_scan_modules(prj, cfg, toolset) + local modulefiles = {} + tree.traverse(project.getsourcetree(prj), { + onleaf = function(node, depth) + if not is_module_file(node.name) then + return + end + + local filecfg = fileconfig.getconfig(node, cfg) + if not filecfg or filecfg.flags.ExcludeFromBuild then + return + end + + scan_module_file_build(cfg, filecfg, toolset, modulefiles) + end, + }, false, 1) + p.outln("") + + return modulefiles +end + local function generated_files_build(cfg, generated_files, key) local final_dependency = {} if #generated_files > 0 then @@ -689,16 +888,15 @@ function ninja.generateProjectCfg(cfg) p.outln("") ---------------------------------------------------- figure out settings - local pch = nil - if toolset ~= p.tools.msc then - pch = p.tools.gcc.getpch(cfg) - if pch then - pch = { - input = project.getrelative(cfg.workspace, path.join(cfg.location, pch)), - placeholder = project.getrelative(cfg.workspace, path.join(cfg.objdir, path.getname(pch))), - gch = project.getrelative(cfg.workspace, path.join(cfg.objdir, path.getname(pch) .. ".gch")) - } - end + local pch = p.tools.gcc.getpch(cfg) + if pch then + pch = { + input = project.getrelative(cfg.workspace, path.join(cfg.location, pch)), + inputSrc = project.getrelative(cfg.workspace, path.join(cfg.location, cfg.pchsource)), + placeholder = project.getrelative(cfg.workspace, path.join(cfg.objdir, path.getname(pch))), + gch = project.getrelative(cfg.workspace, path.join(cfg.objdir, path.getname(pch) .. ".gch")), + pch = project.getrelative(cfg.workspace, path.join(cfg.objdir, path.getname(pch) .. ".pch")) + } end ---------------------------------------------------- write rules @@ -708,17 +906,54 @@ function ninja.generateProjectCfg(cfg) postbuild_rule(cfg) compilation_rules(cfg, toolset, pch) custom_command_rule() + if _OPTIONS["experimental-enable-cxx-modules"] then + module_scan_rule(cfg, toolset) + module_collate_rule(cfg, toolset) + end + + local modulefiles = nil + if _OPTIONS["experimental-enable-cxx-modules"] then + ---------------------------------------------------- scan all module files + p.outln("# scan modules") + modulefiles = files_scan_modules(prj, cfg, toolset) + end + + if _OPTIONS["experimental-enable-cxx-modules"] and modulefiles then + ---------------------------------------------------- collate all scanned module files + p.outln("# collate modules") + + local obj_dir = project.getrelative(cfg.workspace, cfg.objdir) + local outputFile = obj_dir .. "/CXX.dd" + + local implicitOutputs = {obj_dir .. "/CXXModules.json"} + for k,v in pairs(modulefiles) do + table.insert(implicitOutputs, path.replaceextension(v, "modmap")) + end + + local implicit_inputs = {} + local vars = {} + local dependencies = {} + for _, v in pairs(p.config.getlinks(cfg, "dependencies", "object")) do + local relDepObjDir = project.getrelative(cfg.workspace, v.objdir) + table.insert(dependencies, path.join(relDepObjDir, "CXXModules.json")) + end + + table.insert(vars, "MODULE_DEPS = \"" .. table.implode(dependencies, "", "", " ") .. "\"") + + add_build(cfg, outputFile, implicitOutputs, "__module_collate", modulefiles, implicit_inputs, dependencies, vars) + p.outln("") + end ---------------------------------------------------- build all files - p.outln("# build files") - local pch_dependency = pch_build(cfg, pch) + local pch_dependency = pch_build(cfg, pch, toolset) local generated_files = collect_generated_files(prj, cfg) local file_dependencies = getFileDependencies(cfg) local regular_file_dependencies = table.join(iif(#generated_files > 0, {"generated_files_" .. key}, {}), file_dependencies) local obj_dir = project.getrelative(cfg.workspace, cfg.objdir) + p.outln("# build files") local objfiles = files_build(prj, cfg, toolset, pch_dependency, regular_file_dependencies, file_dependencies) local final_dependency = generated_files_build(cfg, generated_files, key)