From de27cb03659a05a8e6ab475fbfedf4c6c1563b95 Mon Sep 17 00:00:00 2001 From: VAN BOSSUYT Nicolas Date: Tue, 7 Feb 2023 21:17:48 +0100 Subject: [PATCH] Implement debug command and apply mixins. This patch also removes the old osdk code since we reached feature parity :> --- osdk-old/__init__.py | 216 ------------------------------------------ osdk-old/__main__.py | 5 - osdk-old/build.py | 163 ------------------------------- osdk-old/manifests.py | 191 ------------------------------------- osdk-old/targets.py | 152 ----------------------------- osdk/cmds.py | 12 ++- osdk/context.py | 13 ++- 7 files changed, 22 insertions(+), 730 deletions(-) delete mode 100644 osdk-old/__init__.py delete mode 100755 osdk-old/__main__.py delete mode 100644 osdk-old/build.py delete mode 100644 osdk-old/manifests.py delete mode 100644 osdk-old/targets.py diff --git a/osdk-old/__init__.py b/osdk-old/__init__.py deleted file mode 100644 index 2814525..0000000 --- a/osdk-old/__init__.py +++ /dev/null @@ -1,216 +0,0 @@ -import importlib -import shutil -import sys -from types import ModuleType - -import osdk.builder as builder -import osdk.utils as utils -import osdk.targets as targets -import osdk.manifests as manifests - -__version__ = "0.3.2" - -CMDS = {} - - -def parseOptions(args: list[str]) -> dict: - result = { - 'opts': {}, - 'args': [] - } - - for arg in args: - if arg.startswith("--"): - if "=" in arg: - key, value = arg[2:].split("=", 1) - result['opts'][key] = value - else: - result['opts'][arg[2:]] = True - else: - result['args'].append(arg) - - return result - - -def propsFromOptions(opt: dict) -> dict: - result = {} - for key in opt: - if key.startswith("prop:"): - result[key[5:]] = opt[key] - return result - - -def runCmd(opts: dict, args: list[str]) -> None: - props = propsFromOptions(opts) - - if len(args) == 0: - print(f"Usage: osdk run ") - sys.exit(1) - - out = builder.buildOne(opts.get('target', 'default'), args[0], props) - - print() - print(f"{utils.Colors.BOLD}Running: {args[0]}{utils.Colors.RESET}") - utils.runCmd(out, *args[1:]) - print() - print(f"{utils.Colors.GREEN}Process exited with success{utils.Colors.RESET}") - - -def debugCmd(opts: dict, args: list[str]) -> None: - props = propsFromOptions(opts) - if len(args) == 0: - print(f"Usage: osdk debug ") - sys.exit(1) - - out = builder.buildOne(opts.get('target', 'default:debug'), args[0], props) - - print() - print(f"{utils.Colors.BOLD}Debugging: {args[0]}{utils.Colors.RESET}") - utils.runCmd("/usr/bin/lldb", "-o", "run", out, *args[1:]) - print() - print(f"{utils.Colors.GREEN}Process exited with success{utils.Colors.RESET}") - - -def buildCmd(opts: dict, args: list[str]) -> None: - props = propsFromOptions(opts) - allTargets = opts.get('all-targets', False) - targetName = opts.get('target', 'default') - - if allTargets: - for target in targets.available(): - if len(args) == 0: - builder.buildAll(target, props) - else: - for component in args: - builder.buildOne(target, component, props) - else: - if len(args) == 0: - builder.buildAll(targetName, props) - else: - for component in args: - builder.buildOne(targetName, component, props) - - -def listCmd(opts: dict, args: list[str]) -> None: - props = propsFromOptions(opts) - targetName = opts.get('target', 'default') - target = targets.load(targetName, props) - components = manifests.loadAll(["src"], target) - - print(f"Available components for target '{targetName}':") - componentsNames = list(components.keys()) - componentsNames.sort() - for component in componentsNames: - if components[component]["enabled"]: - print(" " + component) - print("") - - -def cleanCmd(opts: dict, args: list[str]) -> None: - shutil.rmtree(".osdk/build", ignore_errors=True) - - -def nukeCmd(opts: dict, args: list[str]) -> None: - shutil.rmtree(".osdk", ignore_errors=True) - - -def helpCmd(opts: dict, args: list[str]) -> None: - print(f"Usage: osdk [options...] []") - print("") - - print("Description:") - print(" Operating System Development Kit.") - print("") - - print("Commands:") - for cmd in CMDS: - print(" " + cmd + " - " + CMDS[cmd]["desc"]) - print("") - - print("Targets:") - availableTargets = targets.available() - if len(availableTargets) == 0: - print(" No targets available") - else: - for targetName in targets.available(): - print(" " + targetName) - print("") - - print("Variants:") - for var in targets.VARIANTS: - print(" " + var) - print("") - - -def versionCmd(opts: dict, args: list[str]) -> None: - print("OSDK v" + __version__) - - -CMDS = { - "run": { - "func": runCmd, - "desc": "Run a component on the host", - }, - "debug": { - "func": debugCmd, - "desc": "Run a component on the host in debug mode", - }, - "build": { - "func": buildCmd, - "desc": "Build one or more components", - }, - "list": { - "func": listCmd, - "desc": "List available components", - }, - "clean": { - "func": cleanCmd, - "desc": "Clean the build directory", - }, - "nuke": { - "func": nukeCmd, - "desc": "Clean the build directory and cache", - }, - "help": { - "func": helpCmd, - "desc": "Show this help message", - }, - "version": { - "func": versionCmd, - "desc": "Show current version", - }, -} - - -def loadPlugin(path: str) -> ModuleType: - """Load a plugin from a path""" - spec = importlib.util.spec_from_file_location("plugin", path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -for files in utils.tryListDir("meta/plugins"): - if files.endswith(".py"): - plugin = loadPlugin(f"meta/plugins/{files}") - CMDS[plugin.__plugin__["name"]] = plugin.__plugin__ - - -def main(): - argv = sys.argv - try: - if len(argv) < 2: - helpCmd({}, []) - else: - o = parseOptions(argv[2:]) - if not argv[1] in CMDS: - print(f"Unknown command: {argv[1]}") - print("") - print(f"Use '{argv[0]} help' for a list of commands") - return 1 - CMDS[argv[1]]["func"](o['opts'], o['args']) - return 0 - except utils.CliException as e: - print() - print(f"{utils.Colors.RED}{e.msg}{utils.Colors.RESET}") - return 1 diff --git a/osdk-old/__main__.py b/osdk-old/__main__.py deleted file mode 100755 index 15170e5..0000000 --- a/osdk-old/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -import sys - -from . import main - -sys.exit(main()) diff --git a/osdk-old/build.py b/osdk-old/build.py deleted file mode 100644 index a29fffd..0000000 --- a/osdk-old/build.py +++ /dev/null @@ -1,163 +0,0 @@ -from typing import TextIO, Tuple -import json -import copy -import os - -from . import ninja -from . import manifests as m -from . import targets -from . import utils - - -def mergeToolsArgs(tool, layers): - args = [] - for layer in layers: - if not layer.get("enabled", True): - continue - - args.extend(layer - .get("tools", {}) - .get(tool, {}) - .get("args", [])) - return args - - -def genNinja(out: TextIO, manifests: dict, target: dict) -> None: - target = copy.deepcopy(target) - target = targets.patchToolArgs(target, "cc", [m.cinc(manifests)]) - target = targets.patchToolArgs(target, "cxx", [m.cinc(manifests)]) - - writer = ninja.Writer(out) - - writer.comment("File generated by the build system, do not edit") - writer.newline() - - writer.comment("Tools:") - for key in target["tools"]: - tool = target["tools"][key] - writer.variable(key, tool["cmd"]) - writer.variable( - key + "flags", " ".join(mergeToolsArgs(key, [target] + list(manifests.values())))) - writer.newline() - - writer.newline() - - writer.comment("Rules:") - writer.rule( - "cc", "$cc -c -o $out $in -MD -MF $out.d $ccflags", - depfile="$out.d") - - writer.rule( - "cxx", "$cxx -c -o $out $in -MD -MF $out.d $cxxflags", - depfile="$out.d") - - writer.rule("ld", "$ld -o $out $in $ldflags") - - writer.rule("ar", "$ar crs $out $in") - - writer.rule("as", "$as -o $out $in $asflags") - - writer.newline() - - writer.comment("Build:") - all = [] - for key in manifests: - item = manifests[key] - - if not item["enabled"]: - continue - - writer.comment("Project: " + item["id"]) - - for obj in item["objs"]: - if obj[1].endswith(".c"): - writer.build( - obj[0], "cc", obj[1], order_only=target["tools"]["cc"].get("files", "")) - elif obj[1].endswith(".cpp"): - writer.build( - obj[0], "cxx", obj[1], order_only=target["tools"]["cxx"].get("files", "")) - elif obj[1].endswith(".s") or obj[1].endswith(".asm"): - writer.build( - obj[0], "as", obj[1], order_only=target["tools"]["as"].get("files", "")) - - writer.newline() - - objs = [x[0] for x in item["objs"]] - - if item["type"] == "lib": - writer.build(item["out"], "ar", objs, - order_only=target["tools"]["ar"].get("files", "")) - else: - objs = objs + item["libs"] - writer.build(item["out"], "ld", objs, - order_only=target["tools"]["ld"].get("files", "")) - - all.append(item["out"]) - - writer.newline() - - writer.comment("Phony:") - writer.build("all", "phony", all) - - -def prepare(targetId: str, props: dict) -> Tuple[dict, dict]: - target = targets.load(targetId, props) - - includes = ["src"] - - if os.path.exists("osdk.json"): - with open("osdk.json", "r") as f: - osdk = json.load(f) - includes = osdk["includes"] - print("includes: ", includes) - - manifests = m.loadAll(includes, target) - - utils.mkdirP(target["dir"]) - genNinja(open(target["ninjafile"], "w"), manifests, target) - - meta = {} - meta["id"] = target["key"] - meta["type"] = "artifacts" - meta["components"] = manifests - meta["target"] = target - - with open(target["dir"] + "/manifest.json", "w") as f: - json.dump(meta, f, indent=4) - - with open(target["dir"] + "/_key", "w") as f: - json.dump(target["key"], f, indent=4) - - return target, manifests - - -def buildAll(targetId: str, props: dict = {}) -> None: - target, _ = prepare(targetId, props) - print(f"{utils.Colors.BOLD}Building all components for target '{targetId}{utils.Colors.RESET}'") - - try: - utils.runCmd("ninja", "-v", "-f", target["ninjafile"]) - except Exception as e: - raise utils.CliException( - "Failed to build all for " + target["key"] + ": " + e) - - -def buildOne(targetId: str, componentId: str, props: dict = {}) -> str: - print(f"{utils.Colors.BOLD}Building {componentId} for target '{targetId}'{utils.Colors.RESET}") - - target, manifests = prepare(targetId, props) - - if not componentId in manifests: - raise utils.CliException("Unknown component: " + componentId) - - if not manifests[componentId]["enabled"]: - raise utils.CliException( - f"{componentId} is not enabled for the {targetId} target") - - try: - utils.runCmd("ninja", "-v", "-f", - target["ninjafile"], manifests[componentId]["out"]) - except Exception as e: - raise utils.CliException( - f"Failed to build {componentId} for target '{target['key']}': {e}") - return manifests[componentId]["out"] diff --git a/osdk-old/manifests.py b/osdk-old/manifests.py deleted file mode 100644 index 18240b6..0000000 --- a/osdk-old/manifests.py +++ /dev/null @@ -1,191 +0,0 @@ -import os -import copy -from pathlib import Path -from . import utils - - -def loadJsons(basedirs: list[str]) -> dict: - result = {} - for basedir in basedirs: - for root, dirs, files in os.walk(basedir): - for filename in files: - if filename == 'manifest.json': - filename = os.path.join(root, filename) - manifest = utils.loadJson(filename) - result[manifest["id"]] = manifest - - return result - - -def filter(manifests: dict, target: dict) -> dict: - manifests = copy.deepcopy(manifests) - for id in manifests: - manifest = manifests[id] - accepted = True - - if "requires" in manifest: - for req in manifest["requires"]: - if not req in target["props"] or \ - not target["props"][req] in manifest["requires"][req]: - accepted = False - print( - f"Disabling {id} because it requires {req}: {manifest['requires'][req]}") - break - - manifest["enabled"] = accepted - - return manifests - - -def doInjects(manifests: dict) -> dict: - manifests = copy.deepcopy(manifests) - for key in manifests: - item = manifests[key] - if item["enabled"] and "inject" in item: - for inject in item["inject"]: - if inject in manifests: - manifests[inject]["deps"].append(key) - return manifests - - -def providersFor(key: str, manifests: dict) -> dict: - result = [] - for k in manifests: - if manifests[k]["enabled"] and key in manifests[k].get("provide", []): - result.append(k) - return result - - -def resolveDeps(manifests: dict) -> dict: - manifests = copy.deepcopy(manifests) - - def resolve(key: str, stack: list[str] = []) -> list[str]: - result: list[str] = [] - - if not key in manifests: - providers = providersFor(key, manifests) - - if len(providers) == 0: - print("No providers for " + key) - return False, "", [] - - if len(providers) > 1: - raise utils.CliException( - f"Multiple providers for {key}: {providers}") - - key = providers[0] - - if key in stack: - raise utils.CliException("Circular dependency detected: " + - str(stack) + " -> " + key) - - stack.append(key) - - if "deps" in manifests[key]: - for dep in manifests[key]["deps"]: - keep, dep, res = resolve(dep, stack) - if not keep: - stack.pop() - print(f"Disabling {key} because we are missing a deps") - return False, "", [] - result.append(dep) - result += res - - stack.pop() - return True, key, result - - for key in manifests: - keep, _, deps = resolve(key) - if not keep: - print(f"Disabling {key} because we are missing a deps") - manifests[key]["enabled"] = False - - manifests[key]["deps"] = utils.stripDups(deps) - - return manifests - - -def findFiles(manifests: dict) -> dict: - manifests = copy.deepcopy(manifests) - - for key in manifests: - item = manifests[key] - path = manifests[key]["dir"] - testsPath = os.path.join(path, "tests") - assetsPath = os.path.join(path, "assets") - - item["tests"] = utils.findFiles(testsPath, [".c", ".cpp"]) - item["srcs"] = utils.findFiles(path, [".c", ".cpp", ".s", ".asm"]) - item["assets"] = utils.findFiles(assetsPath) - - return manifests - - -def prepareTests(manifests: dict) -> dict: - if not "tests" in manifests: - return manifests - manifests = copy.deepcopy(manifests) - tests = manifests["tests"] - - for key in manifests: - item = manifests[key] - if "tests" in item and len(item["tests"]) > 0: - tests["deps"] += [item["id"]] - tests["srcs"] += item["tests"] - - return manifests - - -def prepareInOut(manifests: dict, target: dict) -> dict: - manifests = copy.deepcopy(manifests) - for key in manifests: - item = manifests[key] - basedir = os.path.dirname(item["dir"]) - - item["objs"] = [(x.replace(basedir, target["objdir"]) + ".o", x) - for x in item["srcs"]] - - if item["type"] == "lib": - item["out"] = target["bindir"] + "/" + key + ".a" - elif item["type"] == "exe": - item["out"] = target["bindir"] + "/" + key - else: - raise utils.CliException("Unknown type: " + item["type"]) - - for key in manifests: - item = manifests[key] - item["libs"] = [manifests[x]["out"] - for x in item["deps"] if manifests[x]["type"] == "lib"] - return manifests - - -def cinc(manifests: dict) -> str: - include_paths = [] - - for key in manifests: - item = manifests[key] - if item["enabled"]: - if "root-include" in item: - include_paths.append(item["dir"]) - else: - include_paths.append(str(Path(item["dir"]).parent)) - - if len(include_paths) == 0: - return "" - - # remove duplicates - include_paths = utils.stripDups(include_paths) - - return " -I" + " -I".join(include_paths) - - -def loadAll(basedirs: list[str], target: dict) -> dict: - manifests = loadJsons(basedirs) - manifests = filter(manifests, target) - manifests = doInjects(manifests) - manifests = resolveDeps(manifests) - manifests = findFiles(manifests) - manifests = prepareTests(manifests) - manifests = prepareInOut(manifests, target) - - return manifests diff --git a/osdk-old/targets.py b/osdk-old/targets.py deleted file mode 100644 index 8fe1bad..0000000 --- a/osdk-old/targets.py +++ /dev/null @@ -1,152 +0,0 @@ - -import copy -import os - -from . import utils - - -def patchToolArgs(target, tool, args): - target = copy.deepcopy(target) - - target["tools"][tool]["args"] += args - - return target - - -def prefixToolCmd(target, tool, prefix): - target = copy.deepcopy(target) - - target["tools"][tool]["cmd"] = prefix + " " + target["tools"][tool]["cmd"] - - return target - - -def enableCache(target: dict) -> dict: - target = copy.deepcopy(target) - - target = prefixToolCmd(target, "cc", f"ccache") - target = prefixToolCmd(target, "cxx", f"ccache") - - return target - - -def enableSan(target: dict) -> dict: - if (target["props"]["freestanding"]): - print("Sanitization not supported for freestanding targets") - return target - - target = copy.deepcopy(target) - - target = patchToolArgs( - target, "cc", ["-fsanitize=address", "-fsanitize=undefined"]) - target = patchToolArgs( - target, "cxx", ["-fsanitize=address", "-fsanitize=undefined"]) - target = patchToolArgs( - target, "ld", ["-fsanitize=address", "-fsanitize=undefined"]) - - return target - - -def enableColors(target: dict) -> dict: - if (target["props"]["toolchain"] == "clang"): - target = patchToolArgs(target, "cc", ["-fcolor-diagnostics"]) - target = patchToolArgs(target, "cxx", ["-fcolor-diagnostics"]) - elif (target["props"]["toolchain"] == "gcc"): - target = patchToolArgs(target, "cc", ["-fdiagnostics-color=alaways"]) - target = patchToolArgs(target, "cxx", ["-fdiagnostics-color=alaways"]) - - return target - - -def enableOptimizer(target: dict, level: str) -> dict: - target = patchToolArgs(target, "cc", ["-O" + level]) - target = patchToolArgs(target, "cxx", ["-O" + level]) - - return target - - -def enableDebug(target: dict) -> dict: - target = patchToolArgs(target, "cc", ["-g"]) - target = patchToolArgs(target, "cxx", ["-g"]) - - return target - - -def available() -> list: - return [file.removesuffix(".json") for file in utils.tryListDir("meta/targets") - if file.endswith(".json")] - - -VARIANTS = ["debug", "devel", "fast", "san"] - - -def load(targetId: str, props: dict) -> dict: - targetName = targetId - targetVariant = "devel" - if ":" in targetName: - targetName, targetVariant = targetName.split(":") - - if not targetName in available(): - raise utils.CliException(f"Target '{targetName}' not available") - - if not targetVariant in VARIANTS: - raise utils.CliException(f"Variant '{targetVariant}' not available") - - target = utils.loadJson(f"meta/targets/{targetName}.json") - target["props"]["variant"] = targetVariant - target["props"] = {**target["props"], **props} - - defines = [] - - for key in target["props"]: - macroname = key.lower().replace("-", "_") - prop = target["props"][key] - macrovalue = str(prop).lower().replace(" ", "_").replace("-", "_") - if isinstance(prop, bool): - if prop: - defines += [f"-D__osdk_{macroname}__"] - else: - defines += [f"-D__osdk_{macroname}_{macrovalue}__"] - - target = patchToolArgs(target, "cc", [ - "-std=gnu2x", - "-Isrc", - "-Wall", - "-Wextra", - "-Werror", - *defines - ]) - - target = patchToolArgs(target, "cxx", [ - "-std=gnu++2b", - "-Isrc", - "-Wall", - "-Wextra", - "-Werror", - "-fno-exceptions", - "-fno-rtti", - *defines - ]) - - target["hash"] = utils.objSha256(target, ["props", "tools"]) - target["key"] = utils.objKey(target["props"]) - target["dir"] = f".osdk/build/{target['hash'][:8]}" - target["bindir"] = f"{target['dir']}/bin" - target["objdir"] = f"{target['dir']}/obj" - target["ninjafile"] = target["dir"] + "/build.ninja" - - target = enableColors(target) - - if targetVariant == "debug": - target = enableDebug(target) - target = enableOptimizer(target, "0") - elif targetVariant == "devel": - target = enableOptimizer(target, "2") - elif targetVariant == "fast": - target = enableOptimizer(target, "3") - elif targetVariant == "san": - target = enableOptimizer(target, "g") - target = enableDebug(target) - target = enableSan(target) - - return target diff --git a/osdk/cmds.py b/osdk/cmds.py index 840981c..538dbba 100644 --- a/osdk/cmds.py +++ b/osdk/cmds.py @@ -48,7 +48,17 @@ cmds += [Cmd("r", "run", "Run the target", runCmd)] def debugCmd(args: Args): - pass + targetSpec = cast(str, args.consumeOpt( + "target", "host-" + shell.uname().machine)) + + componentSpec = args.consumeArg() + + if componentSpec is None: + raise Exception("Component not specified") + + exe = builder.build(componentSpec, targetSpec) + + shell.exec("/usr/bin/lldb", "-o", "run", exe) cmds += [Cmd("d", "debug", "Debug the target", debugCmd)] diff --git a/osdk/context.py b/osdk/context.py index 404942f..e92cb20 100644 --- a/osdk/context.py +++ b/osdk/context.py @@ -4,7 +4,7 @@ from pathlib import Path from osdk.model import TargetManifest, ComponentManifest, Props, Type, Tool, Tools from osdk.logger import Logger -from osdk import const, shell, jexpr, utils, rules +from osdk import const, shell, jexpr, utils, rules, mixins logger = Logger("context") @@ -186,7 +186,12 @@ def instanciate(componentSpec: str, components: list[ComponentManifest], target: def contextFor(targetSpec: str, props: Props) -> Context: logger.log(f"Loading context for {targetSpec}") - target = loadTarget(targetSpec) + targetEls = targetSpec.split(":") + + if targetEls[0] == "": + targetEls[0] = "host-" + shell.uname().machine + + target = loadTarget(targetEls[0]) components = loadAllComponents() tools: Tools = {} @@ -201,6 +206,10 @@ def contextFor(targetSpec: str, props: Props) -> Context: tools[toolSpec].args += rules.rules[toolSpec].args + for m in targetEls[1:]: + mixin = mixins.byId(m) + tools = mixin(target, tools) + for component in components: for toolSpec in component.tools: tool = component.tools[toolSpec]