Implement debug command and apply mixins.

This patch also removes the old osdk code since we reached feature parity :>
This commit is contained in:
Sleepy Monax 2023-02-07 21:17:48 +01:00
parent 86a273d0b0
commit de27cb0365
7 changed files with 22 additions and 730 deletions

View file

@ -1,216 +0,0 @@
import importlib
import shutil
import sys
from types import ModuleType
import osdk.builder as builder
import osdk.utils as utils
import osdk.targets as targets
import osdk.manifests as manifests
__version__ = "0.3.2"
CMDS = {}
def parseOptions(args: list[str]) -> dict:
result = {
'opts': {},
'args': []
}
for arg in args:
if arg.startswith("--"):
if "=" in arg:
key, value = arg[2:].split("=", 1)
result['opts'][key] = value
else:
result['opts'][arg[2:]] = True
else:
result['args'].append(arg)
return result
def propsFromOptions(opt: dict) -> dict:
result = {}
for key in opt:
if key.startswith("prop:"):
result[key[5:]] = opt[key]
return result
def runCmd(opts: dict, args: list[str]) -> None:
props = propsFromOptions(opts)
if len(args) == 0:
print(f"Usage: osdk run <component>")
sys.exit(1)
out = builder.buildOne(opts.get('target', 'default'), args[0], props)
print()
print(f"{utils.Colors.BOLD}Running: {args[0]}{utils.Colors.RESET}")
utils.runCmd(out, *args[1:])
print()
print(f"{utils.Colors.GREEN}Process exited with success{utils.Colors.RESET}")
def debugCmd(opts: dict, args: list[str]) -> None:
props = propsFromOptions(opts)
if len(args) == 0:
print(f"Usage: osdk debug <component>")
sys.exit(1)
out = builder.buildOne(opts.get('target', 'default:debug'), args[0], props)
print()
print(f"{utils.Colors.BOLD}Debugging: {args[0]}{utils.Colors.RESET}")
utils.runCmd("/usr/bin/lldb", "-o", "run", out, *args[1:])
print()
print(f"{utils.Colors.GREEN}Process exited with success{utils.Colors.RESET}")
def buildCmd(opts: dict, args: list[str]) -> None:
props = propsFromOptions(opts)
allTargets = opts.get('all-targets', False)
targetName = opts.get('target', 'default')
if allTargets:
for target in targets.available():
if len(args) == 0:
builder.buildAll(target, props)
else:
for component in args:
builder.buildOne(target, component, props)
else:
if len(args) == 0:
builder.buildAll(targetName, props)
else:
for component in args:
builder.buildOne(targetName, component, props)
def listCmd(opts: dict, args: list[str]) -> None:
props = propsFromOptions(opts)
targetName = opts.get('target', 'default')
target = targets.load(targetName, props)
components = manifests.loadAll(["src"], target)
print(f"Available components for target '{targetName}':")
componentsNames = list(components.keys())
componentsNames.sort()
for component in componentsNames:
if components[component]["enabled"]:
print(" " + component)
print("")
def cleanCmd(opts: dict, args: list[str]) -> None:
shutil.rmtree(".osdk/build", ignore_errors=True)
def nukeCmd(opts: dict, args: list[str]) -> None:
shutil.rmtree(".osdk", ignore_errors=True)
def helpCmd(opts: dict, args: list[str]) -> None:
print(f"Usage: osdk <command> [options...] [<args...>]")
print("")
print("Description:")
print(" Operating System Development Kit.")
print("")
print("Commands:")
for cmd in CMDS:
print(" " + cmd + " - " + CMDS[cmd]["desc"])
print("")
print("Targets:")
availableTargets = targets.available()
if len(availableTargets) == 0:
print(" No targets available")
else:
for targetName in targets.available():
print(" " + targetName)
print("")
print("Variants:")
for var in targets.VARIANTS:
print(" " + var)
print("")
def versionCmd(opts: dict, args: list[str]) -> None:
print("OSDK v" + __version__)
CMDS = {
"run": {
"func": runCmd,
"desc": "Run a component on the host",
},
"debug": {
"func": debugCmd,
"desc": "Run a component on the host in debug mode",
},
"build": {
"func": buildCmd,
"desc": "Build one or more components",
},
"list": {
"func": listCmd,
"desc": "List available components",
},
"clean": {
"func": cleanCmd,
"desc": "Clean the build directory",
},
"nuke": {
"func": nukeCmd,
"desc": "Clean the build directory and cache",
},
"help": {
"func": helpCmd,
"desc": "Show this help message",
},
"version": {
"func": versionCmd,
"desc": "Show current version",
},
}
def loadPlugin(path: str) -> ModuleType:
"""Load a plugin from a path"""
spec = importlib.util.spec_from_file_location("plugin", path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
for files in utils.tryListDir("meta/plugins"):
if files.endswith(".py"):
plugin = loadPlugin(f"meta/plugins/{files}")
CMDS[plugin.__plugin__["name"]] = plugin.__plugin__
def main():
argv = sys.argv
try:
if len(argv) < 2:
helpCmd({}, [])
else:
o = parseOptions(argv[2:])
if not argv[1] in CMDS:
print(f"Unknown command: {argv[1]}")
print("")
print(f"Use '{argv[0]} help' for a list of commands")
return 1
CMDS[argv[1]]["func"](o['opts'], o['args'])
return 0
except utils.CliException as e:
print()
print(f"{utils.Colors.RED}{e.msg}{utils.Colors.RESET}")
return 1

View file

@ -1,5 +0,0 @@
import sys
from . import main
sys.exit(main())

View file

@ -1,163 +0,0 @@
from typing import TextIO, Tuple
import json
import copy
import os
from . import ninja
from . import manifests as m
from . import targets
from . import utils
def mergeToolsArgs(tool, layers):
args = []
for layer in layers:
if not layer.get("enabled", True):
continue
args.extend(layer
.get("tools", {})
.get(tool, {})
.get("args", []))
return args
def genNinja(out: TextIO, manifests: dict, target: dict) -> None:
target = copy.deepcopy(target)
target = targets.patchToolArgs(target, "cc", [m.cinc(manifests)])
target = targets.patchToolArgs(target, "cxx", [m.cinc(manifests)])
writer = ninja.Writer(out)
writer.comment("File generated by the build system, do not edit")
writer.newline()
writer.comment("Tools:")
for key in target["tools"]:
tool = target["tools"][key]
writer.variable(key, tool["cmd"])
writer.variable(
key + "flags", " ".join(mergeToolsArgs(key, [target] + list(manifests.values()))))
writer.newline()
writer.newline()
writer.comment("Rules:")
writer.rule(
"cc", "$cc -c -o $out $in -MD -MF $out.d $ccflags",
depfile="$out.d")
writer.rule(
"cxx", "$cxx -c -o $out $in -MD -MF $out.d $cxxflags",
depfile="$out.d")
writer.rule("ld", "$ld -o $out $in $ldflags")
writer.rule("ar", "$ar crs $out $in")
writer.rule("as", "$as -o $out $in $asflags")
writer.newline()
writer.comment("Build:")
all = []
for key in manifests:
item = manifests[key]
if not item["enabled"]:
continue
writer.comment("Project: " + item["id"])
for obj in item["objs"]:
if obj[1].endswith(".c"):
writer.build(
obj[0], "cc", obj[1], order_only=target["tools"]["cc"].get("files", ""))
elif obj[1].endswith(".cpp"):
writer.build(
obj[0], "cxx", obj[1], order_only=target["tools"]["cxx"].get("files", ""))
elif obj[1].endswith(".s") or obj[1].endswith(".asm"):
writer.build(
obj[0], "as", obj[1], order_only=target["tools"]["as"].get("files", ""))
writer.newline()
objs = [x[0] for x in item["objs"]]
if item["type"] == "lib":
writer.build(item["out"], "ar", objs,
order_only=target["tools"]["ar"].get("files", ""))
else:
objs = objs + item["libs"]
writer.build(item["out"], "ld", objs,
order_only=target["tools"]["ld"].get("files", ""))
all.append(item["out"])
writer.newline()
writer.comment("Phony:")
writer.build("all", "phony", all)
def prepare(targetId: str, props: dict) -> Tuple[dict, dict]:
target = targets.load(targetId, props)
includes = ["src"]
if os.path.exists("osdk.json"):
with open("osdk.json", "r") as f:
osdk = json.load(f)
includes = osdk["includes"]
print("includes: ", includes)
manifests = m.loadAll(includes, target)
utils.mkdirP(target["dir"])
genNinja(open(target["ninjafile"], "w"), manifests, target)
meta = {}
meta["id"] = target["key"]
meta["type"] = "artifacts"
meta["components"] = manifests
meta["target"] = target
with open(target["dir"] + "/manifest.json", "w") as f:
json.dump(meta, f, indent=4)
with open(target["dir"] + "/_key", "w") as f:
json.dump(target["key"], f, indent=4)
return target, manifests
def buildAll(targetId: str, props: dict = {}) -> None:
target, _ = prepare(targetId, props)
print(f"{utils.Colors.BOLD}Building all components for target '{targetId}{utils.Colors.RESET}'")
try:
utils.runCmd("ninja", "-v", "-f", target["ninjafile"])
except Exception as e:
raise utils.CliException(
"Failed to build all for " + target["key"] + ": " + e)
def buildOne(targetId: str, componentId: str, props: dict = {}) -> str:
print(f"{utils.Colors.BOLD}Building {componentId} for target '{targetId}'{utils.Colors.RESET}")
target, manifests = prepare(targetId, props)
if not componentId in manifests:
raise utils.CliException("Unknown component: " + componentId)
if not manifests[componentId]["enabled"]:
raise utils.CliException(
f"{componentId} is not enabled for the {targetId} target")
try:
utils.runCmd("ninja", "-v", "-f",
target["ninjafile"], manifests[componentId]["out"])
except Exception as e:
raise utils.CliException(
f"Failed to build {componentId} for target '{target['key']}': {e}")
return manifests[componentId]["out"]

View file

@ -1,191 +0,0 @@
import os
import copy
from pathlib import Path
from . import utils
def loadJsons(basedirs: list[str]) -> dict:
result = {}
for basedir in basedirs:
for root, dirs, files in os.walk(basedir):
for filename in files:
if filename == 'manifest.json':
filename = os.path.join(root, filename)
manifest = utils.loadJson(filename)
result[manifest["id"]] = manifest
return result
def filter(manifests: dict, target: dict) -> dict:
manifests = copy.deepcopy(manifests)
for id in manifests:
manifest = manifests[id]
accepted = True
if "requires" in manifest:
for req in manifest["requires"]:
if not req in target["props"] or \
not target["props"][req] in manifest["requires"][req]:
accepted = False
print(
f"Disabling {id} because it requires {req}: {manifest['requires'][req]}")
break
manifest["enabled"] = accepted
return manifests
def doInjects(manifests: dict) -> dict:
manifests = copy.deepcopy(manifests)
for key in manifests:
item = manifests[key]
if item["enabled"] and "inject" in item:
for inject in item["inject"]:
if inject in manifests:
manifests[inject]["deps"].append(key)
return manifests
def providersFor(key: str, manifests: dict) -> dict:
result = []
for k in manifests:
if manifests[k]["enabled"] and key in manifests[k].get("provide", []):
result.append(k)
return result
def resolveDeps(manifests: dict) -> dict:
manifests = copy.deepcopy(manifests)
def resolve(key: str, stack: list[str] = []) -> list[str]:
result: list[str] = []
if not key in manifests:
providers = providersFor(key, manifests)
if len(providers) == 0:
print("No providers for " + key)
return False, "", []
if len(providers) > 1:
raise utils.CliException(
f"Multiple providers for {key}: {providers}")
key = providers[0]
if key in stack:
raise utils.CliException("Circular dependency detected: " +
str(stack) + " -> " + key)
stack.append(key)
if "deps" in manifests[key]:
for dep in manifests[key]["deps"]:
keep, dep, res = resolve(dep, stack)
if not keep:
stack.pop()
print(f"Disabling {key} because we are missing a deps")
return False, "", []
result.append(dep)
result += res
stack.pop()
return True, key, result
for key in manifests:
keep, _, deps = resolve(key)
if not keep:
print(f"Disabling {key} because we are missing a deps")
manifests[key]["enabled"] = False
manifests[key]["deps"] = utils.stripDups(deps)
return manifests
def findFiles(manifests: dict) -> dict:
manifests = copy.deepcopy(manifests)
for key in manifests:
item = manifests[key]
path = manifests[key]["dir"]
testsPath = os.path.join(path, "tests")
assetsPath = os.path.join(path, "assets")
item["tests"] = utils.findFiles(testsPath, [".c", ".cpp"])
item["srcs"] = utils.findFiles(path, [".c", ".cpp", ".s", ".asm"])
item["assets"] = utils.findFiles(assetsPath)
return manifests
def prepareTests(manifests: dict) -> dict:
if not "tests" in manifests:
return manifests
manifests = copy.deepcopy(manifests)
tests = manifests["tests"]
for key in manifests:
item = manifests[key]
if "tests" in item and len(item["tests"]) > 0:
tests["deps"] += [item["id"]]
tests["srcs"] += item["tests"]
return manifests
def prepareInOut(manifests: dict, target: dict) -> dict:
manifests = copy.deepcopy(manifests)
for key in manifests:
item = manifests[key]
basedir = os.path.dirname(item["dir"])
item["objs"] = [(x.replace(basedir, target["objdir"]) + ".o", x)
for x in item["srcs"]]
if item["type"] == "lib":
item["out"] = target["bindir"] + "/" + key + ".a"
elif item["type"] == "exe":
item["out"] = target["bindir"] + "/" + key
else:
raise utils.CliException("Unknown type: " + item["type"])
for key in manifests:
item = manifests[key]
item["libs"] = [manifests[x]["out"]
for x in item["deps"] if manifests[x]["type"] == "lib"]
return manifests
def cinc(manifests: dict) -> str:
include_paths = []
for key in manifests:
item = manifests[key]
if item["enabled"]:
if "root-include" in item:
include_paths.append(item["dir"])
else:
include_paths.append(str(Path(item["dir"]).parent))
if len(include_paths) == 0:
return ""
# remove duplicates
include_paths = utils.stripDups(include_paths)
return " -I" + " -I".join(include_paths)
def loadAll(basedirs: list[str], target: dict) -> dict:
manifests = loadJsons(basedirs)
manifests = filter(manifests, target)
manifests = doInjects(manifests)
manifests = resolveDeps(manifests)
manifests = findFiles(manifests)
manifests = prepareTests(manifests)
manifests = prepareInOut(manifests, target)
return manifests

View file

@ -1,152 +0,0 @@
import copy
import os
from . import utils
def patchToolArgs(target, tool, args):
target = copy.deepcopy(target)
target["tools"][tool]["args"] += args
return target
def prefixToolCmd(target, tool, prefix):
target = copy.deepcopy(target)
target["tools"][tool]["cmd"] = prefix + " " + target["tools"][tool]["cmd"]
return target
def enableCache(target: dict) -> dict:
target = copy.deepcopy(target)
target = prefixToolCmd(target, "cc", f"ccache")
target = prefixToolCmd(target, "cxx", f"ccache")
return target
def enableSan(target: dict) -> dict:
if (target["props"]["freestanding"]):
print("Sanitization not supported for freestanding targets")
return target
target = copy.deepcopy(target)
target = patchToolArgs(
target, "cc", ["-fsanitize=address", "-fsanitize=undefined"])
target = patchToolArgs(
target, "cxx", ["-fsanitize=address", "-fsanitize=undefined"])
target = patchToolArgs(
target, "ld", ["-fsanitize=address", "-fsanitize=undefined"])
return target
def enableColors(target: dict) -> dict:
if (target["props"]["toolchain"] == "clang"):
target = patchToolArgs(target, "cc", ["-fcolor-diagnostics"])
target = patchToolArgs(target, "cxx", ["-fcolor-diagnostics"])
elif (target["props"]["toolchain"] == "gcc"):
target = patchToolArgs(target, "cc", ["-fdiagnostics-color=alaways"])
target = patchToolArgs(target, "cxx", ["-fdiagnostics-color=alaways"])
return target
def enableOptimizer(target: dict, level: str) -> dict:
target = patchToolArgs(target, "cc", ["-O" + level])
target = patchToolArgs(target, "cxx", ["-O" + level])
return target
def enableDebug(target: dict) -> dict:
target = patchToolArgs(target, "cc", ["-g"])
target = patchToolArgs(target, "cxx", ["-g"])
return target
def available() -> list:
return [file.removesuffix(".json") for file in utils.tryListDir("meta/targets")
if file.endswith(".json")]
VARIANTS = ["debug", "devel", "fast", "san"]
def load(targetId: str, props: dict) -> dict:
targetName = targetId
targetVariant = "devel"
if ":" in targetName:
targetName, targetVariant = targetName.split(":")
if not targetName in available():
raise utils.CliException(f"Target '{targetName}' not available")
if not targetVariant in VARIANTS:
raise utils.CliException(f"Variant '{targetVariant}' not available")
target = utils.loadJson(f"meta/targets/{targetName}.json")
target["props"]["variant"] = targetVariant
target["props"] = {**target["props"], **props}
defines = []
for key in target["props"]:
macroname = key.lower().replace("-", "_")
prop = target["props"][key]
macrovalue = str(prop).lower().replace(" ", "_").replace("-", "_")
if isinstance(prop, bool):
if prop:
defines += [f"-D__osdk_{macroname}__"]
else:
defines += [f"-D__osdk_{macroname}_{macrovalue}__"]
target = patchToolArgs(target, "cc", [
"-std=gnu2x",
"-Isrc",
"-Wall",
"-Wextra",
"-Werror",
*defines
])
target = patchToolArgs(target, "cxx", [
"-std=gnu++2b",
"-Isrc",
"-Wall",
"-Wextra",
"-Werror",
"-fno-exceptions",
"-fno-rtti",
*defines
])
target["hash"] = utils.objSha256(target, ["props", "tools"])
target["key"] = utils.objKey(target["props"])
target["dir"] = f".osdk/build/{target['hash'][:8]}"
target["bindir"] = f"{target['dir']}/bin"
target["objdir"] = f"{target['dir']}/obj"
target["ninjafile"] = target["dir"] + "/build.ninja"
target = enableColors(target)
if targetVariant == "debug":
target = enableDebug(target)
target = enableOptimizer(target, "0")
elif targetVariant == "devel":
target = enableOptimizer(target, "2")
elif targetVariant == "fast":
target = enableOptimizer(target, "3")
elif targetVariant == "san":
target = enableOptimizer(target, "g")
target = enableDebug(target)
target = enableSan(target)
return target

View file

@ -48,7 +48,17 @@ cmds += [Cmd("r", "run", "Run the target", runCmd)]
def debugCmd(args: Args): def debugCmd(args: Args):
pass targetSpec = cast(str, args.consumeOpt(
"target", "host-" + shell.uname().machine))
componentSpec = args.consumeArg()
if componentSpec is None:
raise Exception("Component not specified")
exe = builder.build(componentSpec, targetSpec)
shell.exec("/usr/bin/lldb", "-o", "run", exe)
cmds += [Cmd("d", "debug", "Debug the target", debugCmd)] cmds += [Cmd("d", "debug", "Debug the target", debugCmd)]

View file

@ -4,7 +4,7 @@ from pathlib import Path
from osdk.model import TargetManifest, ComponentManifest, Props, Type, Tool, Tools from osdk.model import TargetManifest, ComponentManifest, Props, Type, Tool, Tools
from osdk.logger import Logger from osdk.logger import Logger
from osdk import const, shell, jexpr, utils, rules from osdk import const, shell, jexpr, utils, rules, mixins
logger = Logger("context") logger = Logger("context")
@ -186,7 +186,12 @@ def instanciate(componentSpec: str, components: list[ComponentManifest], target:
def contextFor(targetSpec: str, props: Props) -> Context: def contextFor(targetSpec: str, props: Props) -> Context:
logger.log(f"Loading context for {targetSpec}") logger.log(f"Loading context for {targetSpec}")
target = loadTarget(targetSpec) targetEls = targetSpec.split(":")
if targetEls[0] == "":
targetEls[0] = "host-" + shell.uname().machine
target = loadTarget(targetEls[0])
components = loadAllComponents() components = loadAllComponents()
tools: Tools = {} tools: Tools = {}
@ -201,6 +206,10 @@ def contextFor(targetSpec: str, props: Props) -> Context:
tools[toolSpec].args += rules.rules[toolSpec].args tools[toolSpec].args += rules.rules[toolSpec].args
for m in targetEls[1:]:
mixin = mixins.byId(m)
tools = mixin(target, tools)
for component in components: for component in components:
for toolSpec in component.tools: for toolSpec in component.tools:
tool = component.tools[toolSpec] tool = component.tools[toolSpec]