Improved target file format.

osdk/__init__.py

@@ -1,13 +1,12 @@
 import importlib
 import shutil
 import sys
-import os
 import random
 from types import ModuleType
 
 import osdk.build as build
 import osdk.utils as utils
-import osdk.environments as environments
+import osdk.targets as targets
 
 
 CMDS = {}
@@ -37,20 +36,20 @@ def runCmd(opts: dict, args: list[str]) -> None:
         print(f"Usage: {args[0]} run <component>")
         sys.exit(1)
 
-    out = build.buildOne(opts.get('env', 'host-clang'), args[0])
+    out = build.buildOne(opts.get('target', 'host-clang'), args[0])
 
     print(f"{utils.Colors.BOLD}Running: {args[0]}{utils.Colors.RESET}")
     utils.runCmd(out, *args[1:])
 
 
 def buildCmd(opts: dict, args: list[str]) -> None:
-    env = opts.get('env', 'host-clang')
+    targetName = opts.get('target', 'host-clang')
 
     if len(args) == 0:
-        build.buildAll(env)
+        build.buildAll(targetName)
     else:
         for component in args:
-            build.buildOne(env, component)
+            build.buildOne(targetName, component)
 
 
 def cleanCmd(opts: dict, args: list[str]) -> None:
@@ -81,17 +80,17 @@ def helpCmd(opts: dict, args: list[str]) -> None:
         print("  " + cmd + " - " + CMDS[cmd]["desc"])
     print("")
 
-    print("Enviroments:")
+    print("Targets:")
-    availableToolchains = environments.available()
+    availableTargets = targets.available()
-    if len(availableToolchains) == 0:
+    if len(availableTargets) == 0:
-        print("   No environments available")
+        print("   No targets available")
     else:
-        for env in environments.available():
+        for targetName in targets.available():
-            print("  " + env)
+            print("  " + targetName)
     print("")
 
     print("Variants:")
-    for var in environments.VARIANTS:
+    for var in targets.VARIANTS:
         print("  " + var)
     print("")
 

osdk/build.py

@@ -1,36 +1,35 @@
-from os import environ
 from typing import TextIO, Tuple
 import json
 import copy
 
 from . import ninja
 from . import manifests as m
-from . import environments as e
+from . import targets
 from . import utils
 
 
-def genNinja(out: TextIO, manifests: dict, env: dict) -> None:
+def genNinja(out: TextIO, manifests: dict, target: dict) -> None:
-    env = copy.deepcopy(env)
+    target = copy.deepcopy(target)
-
+    target = targets.patchToolArgs(target, "cc", [m.cincludes(manifests)])
-    env["cflags"] += [m.cincludes(manifests)]
+    target = targets.patchToolArgs(target, "cxx", [m.cincludes(manifests)])
-    env["cxxflags"] += [m.cincludes(manifests)]
 
     writer = ninja.Writer(out)
 
     writer.comment("Generated by the meta build system")
     writer.newline()
 
-    writer.comment("Environment:")
+    writer.comment("Tools:")
-    for key in env:
+    for key in target["tools"]:
-        if isinstance(env[key], list):
+        tool = target["tools"][key]
-            writer.variable(key, " ".join(env[key]))
+        writer.variable(key, tool["cmd"])
-        else:
+        writer.variable(key + "flags", " ".join(tool["args"]))
-            writer.variable(key, env[key])
+        writer.newline()
+
     writer.newline()
 
     writer.comment("Rules:")
     writer.rule(
-        "cc", "$cc -c -o $out $in -MD -MF $out.d $cflags", depfile="$out.d")
+        "cc", "$cc -c -o $out $in -MD -MF $out.d $ccflags", depfile="$out.d")
     writer.rule(
         "cxx", "$cxx -c -o $out $in -MD -MF $out.d $cxxflags", depfile="$out.d")
     writer.rule("ld", "$ld -o $out $in $ldflags")
@@ -71,46 +70,49 @@ def genNinja(out: TextIO, manifests: dict, env: dict) -> None:
     writer.build("all", "phony", all)
 
 
-def prepare(envName: str) -> Tuple[dict, dict]:
+def prepare(targetId: str) -> Tuple[dict, dict]:
-    env = e.load(envName)
+    target = targets.load(targetId)
-    manifests = m.loadAll("./src", env)
+    manifests = m.loadAll("src", target)
-    utils.mkdirP(env["dir"])
+    utils.mkdirP(target["dir"])
-    genNinja(open(env["ninjafile"], "w"), manifests, env)
+    genNinja(open(target["ninjafile"], "w"), manifests, target)
 
     meta = {}
-    meta["id"] = envName
+    meta["id"] = target["key"]
     meta["type"] = "artifacts"
     meta["components"] = manifests
-    meta["toolchain"] = env
+    meta["target"] = target
 
-    with open(env["dir"] + "/manifest.json", "w") as f:
+    with open(target["dir"] + "/manifest.json", "w") as f:
         json.dump(meta, f, indent=4)
 
-    return env, manifests
+    with open(target["dir"] + "/_key",  "w") as f:
+        json.dump(target["key"], f, indent=4)
+
+    return target, manifests
 
 
-def buildAll(envName: str) -> None:
+def buildAll(targetId: str) -> None:
-    environment, _ = prepare(envName)
+    target, _ = prepare(targetId)
-    print(f"{utils.Colors.BOLD}Building all targets for {envName}{utils.Colors.RESET}")
+    print(f"{utils.Colors.BOLD}Building all components for target '{targetId}{utils.Colors.RESET}'")
     try:
-        utils.runCmd("ninja", "-j", "1", "-f", environment["ninjafile"])
+        utils.runCmd("ninja", "-j", "1", "-f", target["ninjafile"])
     except:
         raise utils.CliException(
-            "Failed to build all for " + environment["key"])
+            "Failed to build all for " + target["key"])
 
 
-def buildOne(envName: str, target: str) -> str:
+def buildOne(targetId: str, componentId: str) -> str:
-    print(f"{utils.Colors.BOLD}Building {target} for {envName}{utils.Colors.RESET}")
+    print(f"{utils.Colors.BOLD}Building {componentId} for target '{targetId}'{utils.Colors.RESET}")
-    environment, manifests = prepare(envName)
+    target, manifests = prepare(targetId)
 
-    if not target in manifests:
+    if not componentId in manifests:
-        raise utils.CliException("Unknown target: " + target)
+        raise utils.CliException("Unknown component: " + componentId)
 
     try:
         utils.runCmd("ninja", "-j", "1", "-f",
-                     environment["ninjafile"], manifests[target]["out"])
+                     target["ninjafile"], manifests[componentId]["out"])
     except:
         raise utils.CliException(
-            f"Failed to build {target} for {environment['key']}")
+            f"Failed to build {componentId} for target '{target['key']}'")
 
-    return manifests[target]["out"]
+    return manifests[componentId]["out"]

osdk/environments.py

@@ -1,116 +0,0 @@
-
-import copy
-import os
-
-from . import utils
-
-
-PASSED_TO_BUILD = [
-    "toolchain", "arch", "sub", "vendor", "sys", "abi", "encoding", "freestanding", "variant"]
-
-
-def enableCache(env: dict) -> dict:
-    env = copy.deepcopy(env)
-    env["cc"] = "ccache " + env["cc"]
-    env["cxx"] = "ccache " + env["cxx"]
-    return env
-
-
-def enableSan(env: dict) -> dict:
-    if (env["freestanding"]):
-        return env
-    env = copy.deepcopy(env)
-    env["cflags"] += ["-fsanitize=address", "-fsanitize=undefined"]
-    env["cxxflags"] += ["-fsanitize=address", "-fsanitize=undefined"]
-    env["ldflags"] += ["-fsanitize=address", "-fsanitize=undefined"]
-    return env
-
-
-def enableColors(env: dict) -> dict:
-    env = copy.deepcopy(env)
-    if (env["toolchain"] == "clang"):
-        env["cflags"] += ["-fcolor-diagnostics"]
-        env["cxxflags"] += ["-fcolor-diagnostics"]
-    elif (env["toolchain"] == "gcc"):
-        env["cflags"] += ["-fdiagnostics-color=alaways"]
-        env["cxxflags"] += ["-fdiagnostics-color=always"]
-
-    return env
-
-
-def enableOptimizer(env: dict, level: str) -> dict:
-    env = copy.deepcopy(env)
-    env["cflags"] += ["-O%s" % level]
-    env["cxxflags"] += ["-O%s" % level]
-    return env
-
-
-def available() -> list:
-    return [file.removesuffix(".json") for file in utils.tryListDir("meta/toolchains")
-            if file.endswith(".json")]
-
-
-VARIANTS = ["debug", "devel", "release", "sanatize"]
-
-
-def load(env: str) -> dict:
-    variant = "devel"
-    if ":" in env:
-        env, variant = env.split(":")
-
-    if not env in available():
-        raise utils.CliException(f"Environment '{env}' not available")
-
-    if not variant in VARIANTS:
-        raise utils.CliException(f"Variant '{variant}' not available")
-
-    result = utils.loadJson(f"meta/toolchains/{env}.json")
-    result["variant"] = variant
-
-    for key in PASSED_TO_BUILD:
-        if isinstance(result[key], bool):
-            if result[key]:
-                result["cflags"] += [f"-D__osdk_{key}__"]
-                result["cxxflags"] += [f"-D__osdk_{key}__"]
-        else:
-            result["cflags"] += [f"-D__osdk_{key}_{result[key]}__"]
-            result["cxxflags"] += [f"-D__osdk_{key}_{result[key]}__"]
-
-    result["cflags"] += [
-        "-std=gnu2x",
-        "-Isrc",
-        "-Wall",
-        "-Wextra",
-        "-Werror"
-    ]
-
-    result["cxxflags"] += [
-        "-std=gnu++2b",
-        "-Isrc",
-        "-Wall",
-        "-Wextra",
-        "-Werror",
-        "-fno-exceptions",
-        "-fno-rtti"
-    ]
-
-    result["hash"] = utils.objSha256(result, PASSED_TO_BUILD)
-    result["key"] = utils.objKey(result, PASSED_TO_BUILD)
-    result["dir"] = f".build/{result['hash'][:8]}"
-    result["bindir"] = f"{result['dir']}/bin"
-    result["objdir"] = f"{result['dir']}/obj"
-    result["ninjafile"] = result["dir"] + "/build.ninja"
-
-    result = enableColors(result)
-
-    if variant == "debug":
-        result = enableOptimizer(result, "g")
-    elif variant == "devel":
-        result = enableOptimizer(result, "2")
-    elif variant == "release":
-        result = enableOptimizer(result, "3")
-    elif variant == "sanatize":
-        result = enableOptimizer(result, "g")
-        result = enableSan(result)
-
-    return result

osdk/manifests.py

@@ -23,7 +23,7 @@ def loadJsons(basedir: str) -> dict:
     return result
 
 
-def filter(manifests: dict, env: dict) -> dict:
+def filter(manifests: dict, target: dict) -> dict:
     result = {}
     for id in manifests:
         manifest = manifests[id]
@@ -31,7 +31,7 @@ def filter(manifests: dict, env: dict) -> dict:
 
         if "requires" in manifest:
             for req in manifest["requires"]:
-                if not env[req] in manifest["requires"][req]:
+                if not target["props"][req] in manifest["requires"][req]:
                     accepted = False
                     break
 
@@ -109,19 +109,19 @@ def prepareTests(manifests: dict) -> dict:
     return manifests
 
 
-def prepareInOut(manifests: dict, env: dict) -> dict:
+def prepareInOut(manifests: dict, target: dict) -> dict:
     manifests = copy.deepcopy(manifests)
     for key in manifests:
         item = manifests[key]
         basedir = os.path.dirname(item["dir"])
 
-        item["objs"] = [(x.replace(basedir, env["objdir"] + "/") + ".o", x)
+        item["objs"] = [(x.replace(basedir, target["objdir"]) + ".o", x)
                         for x in item["srcs"]]
 
         if item["type"] == "lib":
-            item["out"] = env["bindir"] + "/" + key + ".a"
+            item["out"] = target["bindir"] + "/" + key + ".a"
         elif item["type"] == "exe":
-            item["out"] = env["bindir"] + "/" + key
+            item["out"] = target["bindir"] + "/" + key
         else:
             raise utils.CliException("Unknown type: " + item["type"])
 
@@ -149,18 +149,18 @@ def cincludes(manifests: dict) -> str:
 cache: dict = {}
 
 
-def loadAll(basedir: str, env: dict) -> dict:
+def loadAll(basedir: str, target: dict) -> dict:
-    cacheKey = basedir + ":" + env["id"]
+    cacheKey = basedir + ":" + target["id"]
     if cacheKey in cache:
         return cache[cacheKey]
 
     manifests = loadJsons(basedir)
-    manifests = filter(manifests, env)
+    manifests = filter(manifests, target)
     manifests = doInjects(manifests)
     manifests = resolveDeps(manifests)
     manifests = findFiles(manifests)
     manifests = prepareTests(manifests)
-    manifests = prepareInOut(manifests, env)
+    manifests = prepareInOut(manifests, target)
 
     cache[cacheKey] = manifests
     return manifests

osdk/targets.py

@@ -0,0 +1,143 @@
+
+import copy
+import os
+
+from . import utils
+
+
+def patchToolArgs(target, tool, args):
+    target = copy.deepcopy(target)
+
+    target["tools"][tool]["args"] += args
+
+    return target
+
+
+def prefixToolCmd(target, tool, prefix):
+    target = copy.deepcopy(target)
+
+    target["tools"][tool]["cmd"] = prefix + " " + target["tools"][tool]["cmd"]
+
+    return target
+
+
+def enableCache(target: dict) -> dict:
+    target = copy.deepcopy(target)
+
+    target = prefixToolCmd(target, "cc", f"ccache")
+    target = prefixToolCmd(target, "cxx", f"ccache")
+
+    return target
+
+
+def enableSan(target: dict) -> dict:
+    if (target["freestanding"]):
+        return target
+
+    target = copy.deepcopy(target)
+
+    target = patchToolArgs(
+        target, "cc", ["-fsanitize=address", "-fsanitize=undefined"])
+    target = patchToolArgs(
+        target, "cxx", ["-fsanitize=address", "-fsanitize=undefined"])
+    target = patchToolArgs(
+        target, "ld", ["-fsanitize=address", "-fsanitize=undefined"])
+
+    return target
+
+
+def enableColors(target: dict) -> dict:
+    target = copy.deepcopy(target)
+
+    if (target["props"]["toolchain"] == "clang"):
+        target = patchToolArgs(target, "cc", ["-fcolor-diagnostics"])
+        target = patchToolArgs(target, "cxx", ["-fcolor-diagnostics"])
+    elif (target["props"]["toolchain"] == "gcc"):
+        target = patchToolArgs(target, "cc", ["-fdiagnostics-color=alaways"])
+        target = patchToolArgs(target, "cxx", ["-fdiagnostics-color=alaways"])
+
+    return target
+
+
+def enableOptimizer(target: dict, level: str) -> dict:
+    target = copy.deepcopy(target)
+
+    target = patchToolArgs(target, "cc", ["-O" + level])
+    target = patchToolArgs(target, "cxx", ["-O" + level])
+
+    return target
+
+
+def available() -> list:
+    return [file.removesuffix(".json") for file in utils.tryListDir("meta/targets")
+            if file.endswith(".json")]
+
+
+VARIANTS = ["debug", "devel", "release", "sanitize"]
+
+
+def load(targetId: str) -> dict:
+    targetName = targetId
+    targetVariant = "devel"
+    if ":" in targetName:
+        targetName, targetVariant = targetName.split(":")
+
+    if not targetName in available():
+        raise utils.CliException(f"Target '{targetName}' not available")
+
+    if not targetVariant in VARIANTS:
+        raise utils.CliException(f"Variant '{targetVariant}' not available")
+
+    target = utils.loadJson(f"meta/targets/{targetName}.json")
+    target["props"]["variant"] = targetVariant
+
+    defines = []
+
+    for key in target["props"]:
+        prop = target["props"][key]
+        if isinstance(prop, bool):
+            if prop:
+                defines += [f"-D__osdk_{key}__"]
+        else:
+            defines += [f"-D__osdk_{key}_{prop}__"]
+
+    target = patchToolArgs(target, "cc", [
+        "-std=gnu2x",
+        "-Isrc",
+        "-Wall",
+        "-Wextra",
+        "-Werror",
+        *defines
+    ])
+
+    target = patchToolArgs(target, "cxx", [
+        "-std=gnu++2b",
+        "-Isrc",
+        "-Wall",
+        "-Wextra",
+        "-Werror",
+        "-fno-exceptions",
+        "-fno-rtti",
+        *defines
+    ])
+
+    target["hash"] = utils.objSha256(target, ["props", "tools"])
+    target["key"] = utils.objKey(target["props"])
+    target["dir"] = f".build/{target['hash'][:8]}"
+    target["bindir"] = f"{target['dir']}/bin"
+    target["objdir"] = f"{target['dir']}/obj"
+    target["ninjafile"] = target["dir"] + "/build.ninja"
+
+    target = enableColors(target)
+
+    if targetVariant == "debug":
+        target = enableOptimizer(target, "g")
+    elif targetVariant == "devel":
+        target = enableOptimizer(target, "2")
+    elif targetVariant == "release":
+        target = enableOptimizer(target, "3")
+    elif targetVariant == "sanatize":
+        target = enableOptimizer(target, "g")
+        target = enableSan(target)
+
+    return target

osdk/utils.py

@@ -77,6 +77,7 @@ def hashFile(filename: str) -> str:
 
 def objSha256(obj: dict, keys: list[str] = []) -> str:
     toHash = {}
+
     if len(keys) == 0:
         toHash = obj
     else:
@@ -88,15 +89,24 @@ def objSha256(obj: dict, keys: list[str] = []) -> str:
     return hashlib.sha256(data.encode("utf-8")).hexdigest()
 
 
-def objKey(obj: dict, keys: list[str]) -> str:
+def objKey(obj: dict, keys: list[str] = []) -> str:
     toKey = []
-    for key in keys:
+
-        if key in obj:
+    if len(keys) == 0:
+        for key in obj:
             if isinstance(obj[key], bool):
                 if obj[key]:
                     toKey.append(key)
             else:
                 toKey.append(obj[key])
+    else:
+        for key in keys:
+            if key in obj:
+                if isinstance(obj[key], bool):
+                    if obj[key]:
+                        toKey.append(key)
+                else:
+                    toKey.append(obj[key])
 
     return "-".join(toKey)