Improved target file format.
This commit is contained in:
parent
d6757c91b9
commit
4aa8bde693
|
|
@ -1,13 +1,12 @@
|
|||
import importlib
|
||||
import shutil
|
||||
import sys
|
||||
import os
|
||||
import random
|
||||
from types import ModuleType
|
||||
|
||||
import osdk.build as build
|
||||
import osdk.utils as utils
|
||||
import osdk.environments as environments
|
||||
import osdk.targets as targets
|
||||
|
||||
|
||||
CMDS = {}
|
||||
|
|
@ -37,20 +36,20 @@ def runCmd(opts: dict, args: list[str]) -> None:
|
|||
print(f"Usage: {args[0]} run <component>")
|
||||
sys.exit(1)
|
||||
|
||||
out = build.buildOne(opts.get('env', 'host-clang'), args[0])
|
||||
out = build.buildOne(opts.get('target', 'host-clang'), args[0])
|
||||
|
||||
print(f"{utils.Colors.BOLD}Running: {args[0]}{utils.Colors.RESET}")
|
||||
utils.runCmd(out, *args[1:])
|
||||
|
||||
|
||||
def buildCmd(opts: dict, args: list[str]) -> None:
|
||||
env = opts.get('env', 'host-clang')
|
||||
targetName = opts.get('target', 'host-clang')
|
||||
|
||||
if len(args) == 0:
|
||||
build.buildAll(env)
|
||||
build.buildAll(targetName)
|
||||
else:
|
||||
for component in args:
|
||||
build.buildOne(env, component)
|
||||
build.buildOne(targetName, component)
|
||||
|
||||
|
||||
def cleanCmd(opts: dict, args: list[str]) -> None:
|
||||
|
|
@ -81,17 +80,17 @@ def helpCmd(opts: dict, args: list[str]) -> None:
|
|||
print(" " + cmd + " - " + CMDS[cmd]["desc"])
|
||||
print("")
|
||||
|
||||
print("Enviroments:")
|
||||
availableToolchains = environments.available()
|
||||
if len(availableToolchains) == 0:
|
||||
print(" No environments available")
|
||||
print("Targets:")
|
||||
availableTargets = targets.available()
|
||||
if len(availableTargets) == 0:
|
||||
print(" No targets available")
|
||||
else:
|
||||
for env in environments.available():
|
||||
print(" " + env)
|
||||
for targetName in targets.available():
|
||||
print(" " + targetName)
|
||||
print("")
|
||||
|
||||
print("Variants:")
|
||||
for var in environments.VARIANTS:
|
||||
for var in targets.VARIANTS:
|
||||
print(" " + var)
|
||||
print("")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,36 +1,35 @@
|
|||
from os import environ
|
||||
from typing import TextIO, Tuple
|
||||
import json
|
||||
import copy
|
||||
|
||||
from . import ninja
|
||||
from . import manifests as m
|
||||
from . import environments as e
|
||||
from . import targets
|
||||
from . import utils
|
||||
|
||||
|
||||
def genNinja(out: TextIO, manifests: dict, env: dict) -> None:
|
||||
env = copy.deepcopy(env)
|
||||
|
||||
env["cflags"] += [m.cincludes(manifests)]
|
||||
env["cxxflags"] += [m.cincludes(manifests)]
|
||||
def genNinja(out: TextIO, manifests: dict, target: dict) -> None:
|
||||
target = copy.deepcopy(target)
|
||||
target = targets.patchToolArgs(target, "cc", [m.cincludes(manifests)])
|
||||
target = targets.patchToolArgs(target, "cxx", [m.cincludes(manifests)])
|
||||
|
||||
writer = ninja.Writer(out)
|
||||
|
||||
writer.comment("Generated by the meta build system")
|
||||
writer.newline()
|
||||
|
||||
writer.comment("Environment:")
|
||||
for key in env:
|
||||
if isinstance(env[key], list):
|
||||
writer.variable(key, " ".join(env[key]))
|
||||
else:
|
||||
writer.variable(key, env[key])
|
||||
writer.comment("Tools:")
|
||||
for key in target["tools"]:
|
||||
tool = target["tools"][key]
|
||||
writer.variable(key, tool["cmd"])
|
||||
writer.variable(key + "flags", " ".join(tool["args"]))
|
||||
writer.newline()
|
||||
|
||||
writer.newline()
|
||||
|
||||
writer.comment("Rules:")
|
||||
writer.rule(
|
||||
"cc", "$cc -c -o $out $in -MD -MF $out.d $cflags", depfile="$out.d")
|
||||
"cc", "$cc -c -o $out $in -MD -MF $out.d $ccflags", depfile="$out.d")
|
||||
writer.rule(
|
||||
"cxx", "$cxx -c -o $out $in -MD -MF $out.d $cxxflags", depfile="$out.d")
|
||||
writer.rule("ld", "$ld -o $out $in $ldflags")
|
||||
|
|
@ -71,46 +70,49 @@ def genNinja(out: TextIO, manifests: dict, env: dict) -> None:
|
|||
writer.build("all", "phony", all)
|
||||
|
||||
|
||||
def prepare(envName: str) -> Tuple[dict, dict]:
|
||||
env = e.load(envName)
|
||||
manifests = m.loadAll("./src", env)
|
||||
utils.mkdirP(env["dir"])
|
||||
genNinja(open(env["ninjafile"], "w"), manifests, env)
|
||||
def prepare(targetId: str) -> Tuple[dict, dict]:
|
||||
target = targets.load(targetId)
|
||||
manifests = m.loadAll("src", target)
|
||||
utils.mkdirP(target["dir"])
|
||||
genNinja(open(target["ninjafile"], "w"), manifests, target)
|
||||
|
||||
meta = {}
|
||||
meta["id"] = envName
|
||||
meta["id"] = target["key"]
|
||||
meta["type"] = "artifacts"
|
||||
meta["components"] = manifests
|
||||
meta["toolchain"] = env
|
||||
meta["target"] = target
|
||||
|
||||
with open(env["dir"] + "/manifest.json", "w") as f:
|
||||
with open(target["dir"] + "/manifest.json", "w") as f:
|
||||
json.dump(meta, f, indent=4)
|
||||
|
||||
return env, manifests
|
||||
with open(target["dir"] + "/_key", "w") as f:
|
||||
json.dump(target["key"], f, indent=4)
|
||||
|
||||
return target, manifests
|
||||
|
||||
|
||||
def buildAll(envName: str) -> None:
|
||||
environment, _ = prepare(envName)
|
||||
print(f"{utils.Colors.BOLD}Building all targets for {envName}{utils.Colors.RESET}")
|
||||
def buildAll(targetId: str) -> None:
|
||||
target, _ = prepare(targetId)
|
||||
print(f"{utils.Colors.BOLD}Building all components for target '{targetId}{utils.Colors.RESET}'")
|
||||
try:
|
||||
utils.runCmd("ninja", "-j", "1", "-f", environment["ninjafile"])
|
||||
utils.runCmd("ninja", "-j", "1", "-f", target["ninjafile"])
|
||||
except:
|
||||
raise utils.CliException(
|
||||
"Failed to build all for " + environment["key"])
|
||||
"Failed to build all for " + target["key"])
|
||||
|
||||
|
||||
def buildOne(envName: str, target: str) -> str:
|
||||
print(f"{utils.Colors.BOLD}Building {target} for {envName}{utils.Colors.RESET}")
|
||||
environment, manifests = prepare(envName)
|
||||
def buildOne(targetId: str, componentId: str) -> str:
|
||||
print(f"{utils.Colors.BOLD}Building {componentId} for target '{targetId}'{utils.Colors.RESET}")
|
||||
target, manifests = prepare(targetId)
|
||||
|
||||
if not target in manifests:
|
||||
raise utils.CliException("Unknown target: " + target)
|
||||
if not componentId in manifests:
|
||||
raise utils.CliException("Unknown component: " + componentId)
|
||||
|
||||
try:
|
||||
utils.runCmd("ninja", "-j", "1", "-f",
|
||||
environment["ninjafile"], manifests[target]["out"])
|
||||
target["ninjafile"], manifests[componentId]["out"])
|
||||
except:
|
||||
raise utils.CliException(
|
||||
f"Failed to build {target} for {environment['key']}")
|
||||
f"Failed to build {componentId} for target '{target['key']}'")
|
||||
|
||||
return manifests[target]["out"]
|
||||
return manifests[componentId]["out"]
|
||||
|
|
|
|||
|
|
@ -1,116 +0,0 @@
|
|||
|
||||
import copy
|
||||
import os
|
||||
|
||||
from . import utils
|
||||
|
||||
|
||||
PASSED_TO_BUILD = [
|
||||
"toolchain", "arch", "sub", "vendor", "sys", "abi", "encoding", "freestanding", "variant"]
|
||||
|
||||
|
||||
def enableCache(env: dict) -> dict:
|
||||
env = copy.deepcopy(env)
|
||||
env["cc"] = "ccache " + env["cc"]
|
||||
env["cxx"] = "ccache " + env["cxx"]
|
||||
return env
|
||||
|
||||
|
||||
def enableSan(env: dict) -> dict:
|
||||
if (env["freestanding"]):
|
||||
return env
|
||||
env = copy.deepcopy(env)
|
||||
env["cflags"] += ["-fsanitize=address", "-fsanitize=undefined"]
|
||||
env["cxxflags"] += ["-fsanitize=address", "-fsanitize=undefined"]
|
||||
env["ldflags"] += ["-fsanitize=address", "-fsanitize=undefined"]
|
||||
return env
|
||||
|
||||
|
||||
def enableColors(env: dict) -> dict:
|
||||
env = copy.deepcopy(env)
|
||||
if (env["toolchain"] == "clang"):
|
||||
env["cflags"] += ["-fcolor-diagnostics"]
|
||||
env["cxxflags"] += ["-fcolor-diagnostics"]
|
||||
elif (env["toolchain"] == "gcc"):
|
||||
env["cflags"] += ["-fdiagnostics-color=alaways"]
|
||||
env["cxxflags"] += ["-fdiagnostics-color=always"]
|
||||
|
||||
return env
|
||||
|
||||
|
||||
def enableOptimizer(env: dict, level: str) -> dict:
|
||||
env = copy.deepcopy(env)
|
||||
env["cflags"] += ["-O%s" % level]
|
||||
env["cxxflags"] += ["-O%s" % level]
|
||||
return env
|
||||
|
||||
|
||||
def available() -> list:
|
||||
return [file.removesuffix(".json") for file in utils.tryListDir("meta/toolchains")
|
||||
if file.endswith(".json")]
|
||||
|
||||
|
||||
VARIANTS = ["debug", "devel", "release", "sanatize"]
|
||||
|
||||
|
||||
def load(env: str) -> dict:
|
||||
variant = "devel"
|
||||
if ":" in env:
|
||||
env, variant = env.split(":")
|
||||
|
||||
if not env in available():
|
||||
raise utils.CliException(f"Environment '{env}' not available")
|
||||
|
||||
if not variant in VARIANTS:
|
||||
raise utils.CliException(f"Variant '{variant}' not available")
|
||||
|
||||
result = utils.loadJson(f"meta/toolchains/{env}.json")
|
||||
result["variant"] = variant
|
||||
|
||||
for key in PASSED_TO_BUILD:
|
||||
if isinstance(result[key], bool):
|
||||
if result[key]:
|
||||
result["cflags"] += [f"-D__osdk_{key}__"]
|
||||
result["cxxflags"] += [f"-D__osdk_{key}__"]
|
||||
else:
|
||||
result["cflags"] += [f"-D__osdk_{key}_{result[key]}__"]
|
||||
result["cxxflags"] += [f"-D__osdk_{key}_{result[key]}__"]
|
||||
|
||||
result["cflags"] += [
|
||||
"-std=gnu2x",
|
||||
"-Isrc",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror"
|
||||
]
|
||||
|
||||
result["cxxflags"] += [
|
||||
"-std=gnu++2b",
|
||||
"-Isrc",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
"-fno-exceptions",
|
||||
"-fno-rtti"
|
||||
]
|
||||
|
||||
result["hash"] = utils.objSha256(result, PASSED_TO_BUILD)
|
||||
result["key"] = utils.objKey(result, PASSED_TO_BUILD)
|
||||
result["dir"] = f".build/{result['hash'][:8]}"
|
||||
result["bindir"] = f"{result['dir']}/bin"
|
||||
result["objdir"] = f"{result['dir']}/obj"
|
||||
result["ninjafile"] = result["dir"] + "/build.ninja"
|
||||
|
||||
result = enableColors(result)
|
||||
|
||||
if variant == "debug":
|
||||
result = enableOptimizer(result, "g")
|
||||
elif variant == "devel":
|
||||
result = enableOptimizer(result, "2")
|
||||
elif variant == "release":
|
||||
result = enableOptimizer(result, "3")
|
||||
elif variant == "sanatize":
|
||||
result = enableOptimizer(result, "g")
|
||||
result = enableSan(result)
|
||||
|
||||
return result
|
||||
|
|
@ -23,7 +23,7 @@ def loadJsons(basedir: str) -> dict:
|
|||
return result
|
||||
|
||||
|
||||
def filter(manifests: dict, env: dict) -> dict:
|
||||
def filter(manifests: dict, target: dict) -> dict:
|
||||
result = {}
|
||||
for id in manifests:
|
||||
manifest = manifests[id]
|
||||
|
|
@ -31,7 +31,7 @@ def filter(manifests: dict, env: dict) -> dict:
|
|||
|
||||
if "requires" in manifest:
|
||||
for req in manifest["requires"]:
|
||||
if not env[req] in manifest["requires"][req]:
|
||||
if not target["props"][req] in manifest["requires"][req]:
|
||||
accepted = False
|
||||
break
|
||||
|
||||
|
|
@ -109,19 +109,19 @@ def prepareTests(manifests: dict) -> dict:
|
|||
return manifests
|
||||
|
||||
|
||||
def prepareInOut(manifests: dict, env: dict) -> dict:
|
||||
def prepareInOut(manifests: dict, target: dict) -> dict:
|
||||
manifests = copy.deepcopy(manifests)
|
||||
for key in manifests:
|
||||
item = manifests[key]
|
||||
basedir = os.path.dirname(item["dir"])
|
||||
|
||||
item["objs"] = [(x.replace(basedir, env["objdir"] + "/") + ".o", x)
|
||||
item["objs"] = [(x.replace(basedir, target["objdir"]) + ".o", x)
|
||||
for x in item["srcs"]]
|
||||
|
||||
if item["type"] == "lib":
|
||||
item["out"] = env["bindir"] + "/" + key + ".a"
|
||||
item["out"] = target["bindir"] + "/" + key + ".a"
|
||||
elif item["type"] == "exe":
|
||||
item["out"] = env["bindir"] + "/" + key
|
||||
item["out"] = target["bindir"] + "/" + key
|
||||
else:
|
||||
raise utils.CliException("Unknown type: " + item["type"])
|
||||
|
||||
|
|
@ -149,18 +149,18 @@ def cincludes(manifests: dict) -> str:
|
|||
cache: dict = {}
|
||||
|
||||
|
||||
def loadAll(basedir: str, env: dict) -> dict:
|
||||
cacheKey = basedir + ":" + env["id"]
|
||||
def loadAll(basedir: str, target: dict) -> dict:
|
||||
cacheKey = basedir + ":" + target["id"]
|
||||
if cacheKey in cache:
|
||||
return cache[cacheKey]
|
||||
|
||||
manifests = loadJsons(basedir)
|
||||
manifests = filter(manifests, env)
|
||||
manifests = filter(manifests, target)
|
||||
manifests = doInjects(manifests)
|
||||
manifests = resolveDeps(manifests)
|
||||
manifests = findFiles(manifests)
|
||||
manifests = prepareTests(manifests)
|
||||
manifests = prepareInOut(manifests, env)
|
||||
manifests = prepareInOut(manifests, target)
|
||||
|
||||
cache[cacheKey] = manifests
|
||||
return manifests
|
||||
|
|
|
|||
143
osdk/targets.py
Normal file
143
osdk/targets.py
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
|
||||
import copy
|
||||
import os
|
||||
|
||||
from . import utils
|
||||
|
||||
|
||||
def patchToolArgs(target, tool, args):
|
||||
target = copy.deepcopy(target)
|
||||
|
||||
target["tools"][tool]["args"] += args
|
||||
|
||||
return target
|
||||
|
||||
|
||||
def prefixToolCmd(target, tool, prefix):
|
||||
target = copy.deepcopy(target)
|
||||
|
||||
target["tools"][tool]["cmd"] = prefix + " " + target["tools"][tool]["cmd"]
|
||||
|
||||
return target
|
||||
|
||||
|
||||
def enableCache(target: dict) -> dict:
|
||||
target = copy.deepcopy(target)
|
||||
|
||||
target = prefixToolCmd(target, "cc", f"ccache")
|
||||
target = prefixToolCmd(target, "cxx", f"ccache")
|
||||
|
||||
return target
|
||||
|
||||
|
||||
def enableSan(target: dict) -> dict:
|
||||
if (target["freestanding"]):
|
||||
return target
|
||||
|
||||
target = copy.deepcopy(target)
|
||||
|
||||
target = patchToolArgs(
|
||||
target, "cc", ["-fsanitize=address", "-fsanitize=undefined"])
|
||||
target = patchToolArgs(
|
||||
target, "cxx", ["-fsanitize=address", "-fsanitize=undefined"])
|
||||
target = patchToolArgs(
|
||||
target, "ld", ["-fsanitize=address", "-fsanitize=undefined"])
|
||||
|
||||
return target
|
||||
|
||||
|
||||
def enableColors(target: dict) -> dict:
|
||||
target = copy.deepcopy(target)
|
||||
|
||||
if (target["props"]["toolchain"] == "clang"):
|
||||
target = patchToolArgs(target, "cc", ["-fcolor-diagnostics"])
|
||||
target = patchToolArgs(target, "cxx", ["-fcolor-diagnostics"])
|
||||
elif (target["props"]["toolchain"] == "gcc"):
|
||||
target = patchToolArgs(target, "cc", ["-fdiagnostics-color=alaways"])
|
||||
target = patchToolArgs(target, "cxx", ["-fdiagnostics-color=alaways"])
|
||||
|
||||
return target
|
||||
|
||||
|
||||
def enableOptimizer(target: dict, level: str) -> dict:
|
||||
target = copy.deepcopy(target)
|
||||
|
||||
target = patchToolArgs(target, "cc", ["-O" + level])
|
||||
target = patchToolArgs(target, "cxx", ["-O" + level])
|
||||
|
||||
return target
|
||||
|
||||
|
||||
def available() -> list:
|
||||
return [file.removesuffix(".json") for file in utils.tryListDir("meta/targets")
|
||||
if file.endswith(".json")]
|
||||
|
||||
|
||||
VARIANTS = ["debug", "devel", "release", "sanitize"]
|
||||
|
||||
|
||||
def load(targetId: str) -> dict:
|
||||
targetName = targetId
|
||||
targetVariant = "devel"
|
||||
if ":" in targetName:
|
||||
targetName, targetVariant = targetName.split(":")
|
||||
|
||||
if not targetName in available():
|
||||
raise utils.CliException(f"Target '{targetName}' not available")
|
||||
|
||||
if not targetVariant in VARIANTS:
|
||||
raise utils.CliException(f"Variant '{targetVariant}' not available")
|
||||
|
||||
target = utils.loadJson(f"meta/targets/{targetName}.json")
|
||||
target["props"]["variant"] = targetVariant
|
||||
|
||||
defines = []
|
||||
|
||||
for key in target["props"]:
|
||||
prop = target["props"][key]
|
||||
if isinstance(prop, bool):
|
||||
if prop:
|
||||
defines += [f"-D__osdk_{key}__"]
|
||||
else:
|
||||
defines += [f"-D__osdk_{key}_{prop}__"]
|
||||
|
||||
target = patchToolArgs(target, "cc", [
|
||||
"-std=gnu2x",
|
||||
"-Isrc",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
*defines
|
||||
])
|
||||
|
||||
target = patchToolArgs(target, "cxx", [
|
||||
"-std=gnu++2b",
|
||||
"-Isrc",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
"-fno-exceptions",
|
||||
"-fno-rtti",
|
||||
*defines
|
||||
])
|
||||
|
||||
target["hash"] = utils.objSha256(target, ["props", "tools"])
|
||||
target["key"] = utils.objKey(target["props"])
|
||||
target["dir"] = f".build/{target['hash'][:8]}"
|
||||
target["bindir"] = f"{target['dir']}/bin"
|
||||
target["objdir"] = f"{target['dir']}/obj"
|
||||
target["ninjafile"] = target["dir"] + "/build.ninja"
|
||||
|
||||
target = enableColors(target)
|
||||
|
||||
if targetVariant == "debug":
|
||||
target = enableOptimizer(target, "g")
|
||||
elif targetVariant == "devel":
|
||||
target = enableOptimizer(target, "2")
|
||||
elif targetVariant == "release":
|
||||
target = enableOptimizer(target, "3")
|
||||
elif targetVariant == "sanatize":
|
||||
target = enableOptimizer(target, "g")
|
||||
target = enableSan(target)
|
||||
|
||||
return target
|
||||
|
|
@ -77,6 +77,7 @@ def hashFile(filename: str) -> str:
|
|||
|
||||
def objSha256(obj: dict, keys: list[str] = []) -> str:
|
||||
toHash = {}
|
||||
|
||||
if len(keys) == 0:
|
||||
toHash = obj
|
||||
else:
|
||||
|
|
@ -88,15 +89,24 @@ def objSha256(obj: dict, keys: list[str] = []) -> str:
|
|||
return hashlib.sha256(data.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def objKey(obj: dict, keys: list[str]) -> str:
|
||||
def objKey(obj: dict, keys: list[str] = []) -> str:
|
||||
toKey = []
|
||||
for key in keys:
|
||||
if key in obj:
|
||||
|
||||
if len(keys) == 0:
|
||||
for key in obj:
|
||||
if isinstance(obj[key], bool):
|
||||
if obj[key]:
|
||||
toKey.append(key)
|
||||
else:
|
||||
toKey.append(obj[key])
|
||||
else:
|
||||
for key in keys:
|
||||
if key in obj:
|
||||
if isinstance(obj[key], bool):
|
||||
if obj[key]:
|
||||
toKey.append(key)
|
||||
else:
|
||||
toKey.append(obj[key])
|
||||
|
||||
return "-".join(toKey)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue