Allow custom props on a per build basis and move all temporary files to .osdk.

osdk/__init__.py

@@ -31,39 +31,53 @@ def parseOptions(args: list[str]) -> dict:
     return result
 
 
+def propsFromOptions(opt: dict) -> dict:
+    result = {}
+    for key in opt:
+        if key.startswith("prop:"):
+            result[key[5:]] = opt[key]
+    return result
+
+
 def runCmd(opts: dict, args: list[str]) -> None:
+    props = propsFromOptions(opts)
+
     if len(args) == 0:
         print(f"Usage: {args[0]} run <component>")
         sys.exit(1)
 
-    out = build.buildOne(opts.get('target', 'default'), args[0])
+    out = build.buildOne(opts.get('target', 'default'), args[0], props)
 
     print(f"{utils.Colors.BOLD}Running: {args[0]}{utils.Colors.RESET}")
     utils.runCmd(out, *args[1:])
+    print()
+    print(f"{utils.Colors.GREEN}Process exited with success{utils.Colors.RESET}")
 
 
 def buildCmd(opts: dict, args: list[str]) -> None:
+    props = propsFromOptions(opts)
     allTargets = opts.get('all-targets', False)
     targetName = opts.get('target', 'default')
 
     if allTargets:
         for target in targets.available():
             if len(args) == 0:
-                build.buildAll(target)
+                build.buildAll(target, props)
             else:
                 for component in args:
-                    build.buildOne(target, component)
+                    build.buildOne(target, component, props)
     else:
         if len(args) == 0:
-            build.buildAll(targetName)
+            build.buildAll(targetName, props)
         else:
             for component in args:
-                build.buildOne(targetName, component)
+                build.buildOne(targetName, component, props)
 
 
 def listCmd(opts: dict, args: list[str]) -> None:
+    props = propsFromOptions(opts)
     targetName = opts.get('target', 'default')
-    target = targets.load(targetName)
+    target = targets.load(targetName, props)
     components = manifests.loadAll("src", target)
 
     print(f"Available components for target '{targetName}':")
@@ -75,12 +89,11 @@ def listCmd(opts: dict, args: list[str]) -> None:
 
 
 def cleanCmd(opts: dict, args: list[str]) -> None:
-    shutil.rmtree(".build", ignore_errors=True)
+    shutil.rmtree(".osdk/build", ignore_errors=True)
 
 
 def nukeCmd(opts: dict, args: list[str]) -> None:
-    shutil.rmtree(".build", ignore_errors=True)
+    shutil.rmtree(".osdk", ignore_errors=True)
-    shutil.rmtree(".cache", ignore_errors=True)
 
 
 def helpCmd(opts: dict, args: list[str]) -> None:

osdk/build.py

@@ -8,6 +8,16 @@ from . import targets
 from . import utils
 
 
+def mergeToolsArgs(tool, layers):
+    args = []
+    for layer in layers:
+        args.extend(layer
+                    .get("tools", {})
+                    .get(tool, {})
+                    .get("args", []))
+    return args
+
+
 def genNinja(out: TextIO, manifests: dict, target: dict) -> None:
     target = copy.deepcopy(target)
     target = targets.patchToolArgs(target, "cc", [m.cincludes(manifests)])
@@ -15,14 +25,15 @@ def genNinja(out: TextIO, manifests: dict, target: dict) -> None:
 
     writer = ninja.Writer(out)
 
-    writer.comment("Generated by the meta build system")
+    writer.comment("File generated by the build system, do not edit")
     writer.newline()
 
     writer.comment("Tools:")
     for key in target["tools"]:
         tool = target["tools"][key]
         writer.variable(key, tool["cmd"])
-        writer.variable(key + "flags", " ".join(tool["args"]))
+        writer.variable(
+            key + "flags", " ".join(mergeToolsArgs(key, [target] + list(manifests.values()))))
         writer.newline()
 
     writer.newline()
@@ -82,8 +93,8 @@ def genNinja(out: TextIO, manifests: dict, target: dict) -> None:
     writer.build("all", "phony", all)
 
 
-def prepare(targetId: str) -> Tuple[dict, dict]:
+def prepare(targetId: str, props: dict) -> Tuple[dict, dict]:
-    target = targets.load(targetId)
+    target = targets.load(targetId, props)
     manifests = m.loadAll("src", target)
     utils.mkdirP(target["dir"])
     genNinja(open(target["ninjafile"], "w"), manifests, target)
@@ -103,9 +114,10 @@ def prepare(targetId: str) -> Tuple[dict, dict]:
     return target, manifests
 
 
-def buildAll(targetId: str) -> None:
+def buildAll(targetId: str, props: dict = {}) -> None:
-    target, _ = prepare(targetId)
+    target, _ = prepare(targetId, props)
     print(f"{utils.Colors.BOLD}Building all components for target '{targetId}{utils.Colors.RESET}'")
+
     try:
         utils.runCmd("ninja", "-v", "-j", "1", "-f",  target["ninjafile"])
     except:
@@ -113,9 +125,10 @@ def buildAll(targetId: str) -> None:
             "Failed to build all for " + target["key"])
 
 
-def buildOne(targetId: str, componentId: str) -> str:
+def buildOne(targetId: str, componentId: str, props: dict = {}) -> str:
     print(f"{utils.Colors.BOLD}Building {componentId} for target '{targetId}'{utils.Colors.RESET}")
-    target, manifests = prepare(targetId)
+
+    target, manifests = prepare(targetId, props)
 
     if not componentId in manifests:
         raise utils.CliException("Unknown component: " + componentId)

osdk/manifests.py

@@ -11,14 +11,8 @@ def loadJsons(basedir: str) -> dict:
         for filename in files:
             if filename == 'manifest.json':
                 filename = os.path.join(root, filename)
-                try:
+                manifest = utils.loadJson(filename)
-                    with open(filename) as f:
+                result[manifest["id"]] = manifest
-                        manifest = json.load(f)
-                        manifest["dir"] = os.path.dirname(filename)
-                        result[manifest["id"]] = manifest
-                except Exception as e:
-                    raise utils.CliException(
-                        f"Failed to load manifest {filename}: {e}")
 
     return result
 

osdk/targets.py

@@ -76,7 +76,7 @@ def available() -> list:
 VARIANTS = ["debug", "devel", "release", "sanitize"]
 
 
-def load(targetId: str) -> dict:
+def load(targetId: str, props: dict) -> dict:
     targetName = targetId
     targetVariant = "devel"
     if ":" in targetName:
@@ -90,16 +90,19 @@ def load(targetId: str) -> dict:
 
     target = utils.loadJson(f"meta/targets/{targetName}.json")
     target["props"]["variant"] = targetVariant
+    target["props"] = {**target["props"], **props}
 
     defines = []
 
     for key in target["props"]:
+        macroname = key.lower().replace("-", "_")
         prop = target["props"][key]
+        macrovalue = str(prop).lower().replace(" ", "_").replace("-", "_")
         if isinstance(prop, bool):
             if prop:
-                defines += [f"-D__osdk_{key}__"]
+                defines += [f"-D__osdk_{macroname}__"]
         else:
-            defines += [f"-D__osdk_{key}_{prop}__"]
+            defines += [f"-D__osdk_{macroname}_{macrovalue}__"]
 
     target = patchToolArgs(target, "cc", [
         "-std=gnu2x",
@@ -123,7 +126,7 @@ def load(targetId: str) -> dict:
 
     target["hash"] = utils.objSha256(target, ["props", "tools"])
     target["key"] = utils.objKey(target["props"])
-    target["dir"] = f".build/{target['hash'][:8]}"
+    target["dir"] = f".osdk/build/{target['hash'][:8]}"
     target["bindir"] = f"{target['dir']}/bin"
     target["objdir"] = f"{target['dir']}/obj"
     target["ninjafile"] = target["dir"] + "/build.ninja"

osdk/utils.py

@@ -89,24 +89,26 @@ def objSha256(obj: dict, keys: list[str] = []) -> str:
     return hashlib.sha256(data.encode("utf-8")).hexdigest()
 
 
+def toCamelCase(s: str) -> str:
+    s = ''.join(x for x in s.title() if x != '_' and x != '-')
+    s = s[0].lower() + s[1:]
+    return s
+
+
 def objKey(obj: dict, keys: list[str] = []) -> str:
     toKey = []
 
     if len(keys) == 0:
-        for key in obj:
+        keys = list(obj.keys())
+        keys.sort()
+
+    for key in keys:
+        if key in obj:
             if isinstance(obj[key], bool):
                 if obj[key]:
                     toKey.append(key)
             else:
-                toKey.append(obj[key])
+                toKey.append(f"{toCamelCase(key)}({obj[key]})")
-    else:
-        for key in keys:
-            if key in obj:
-                if isinstance(obj[key], bool):
-                    if obj[key]:
-                        toKey.append(key)
-                else:
-                    toKey.append(obj[key])
 
     return "-".join(toKey)
 
@@ -123,7 +125,7 @@ def mkdirP(path: str) -> str:
 
 
 def downloadFile(url: str) -> str:
-    dest = ".cache/remote/" + hashlib.sha256(url.encode('utf-8')).hexdigest()
+    dest = ".osdk/cache/" + hashlib.sha256(url.encode('utf-8')).hexdigest()
     tmp = dest + ".tmp"
 
     if os.path.isfile(dest):
@@ -162,48 +164,74 @@ def runCmd(*args: str) -> bool:
     return True
 
 
+def getCmdOutput(*args: str) -> str:
+    try:
+        proc = subprocess.run(args, stdout=subprocess.PIPE)
+    except FileNotFoundError:
+        raise CliException(f"Failed to run {args[0]}: command not found")
+
+    if proc.returncode == -signal.SIGSEGV:
+        raise CliException("Segmentation fault")
+
+    if proc.returncode != 0:
+        raise CliException(
+            f"Failed to run {' '.join(args)}: process exited with code {proc.returncode}")
+
+    return proc.stdout.decode('utf-8')
+
+
 CACHE = {}
 
 
 MACROS = {
-    "uname": lambda what: getattr(os.uname(), what),
+    "uname": lambda what: getattr(os.uname(), what).lower(),
     "include": lambda *path: loadJson(''.join(path)),
-    "merge": lambda lhs, rhs: {**lhs, **rhs},
+    "join": lambda lhs, rhs: {**lhs, **rhs} if isinstance(lhs, dict) else lhs + rhs,
+    "concat": lambda *args: ''.join(args),
+    "exec": lambda *args: getCmdOutput(*args).splitlines(),
 }
 
 
-def processJson(e: any) -> any:
+def isJexpr(jexpr: list) -> bool:
+    return isinstance(jexpr, list) and len(jexpr) > 0 and isinstance(jexpr[0], str) and jexpr[0].startswith("@")
+
+
+def jsonEval(jexpr: list) -> any:
+    macro = jexpr[0][1:]
+    if not macro in MACROS:
+        raise CliException(f"Unknown macro {macro}")
+    return MACROS[macro](*list(map((lambda x: jsonWalk(x)), jexpr[1:])))
+
+
+def jsonWalk(e: any) -> any:
     if isinstance(e, dict):
         for k in e:
-            e[processJson(k)] = processJson(e[k])
+            e[jsonWalk(k)] = jsonWalk(e[k])
-    elif isinstance(e, list) and len(e) > 0 and isinstance(e[0], str) and e[0].startswith("@"):
+    elif isJexpr(e):
-        macro = e[0][1:]
+        return jsonEval(e)
-
-        if not macro in MACROS:
-            raise CliException(f"Unknown macro {macro}")
-
-        return MACROS[macro](*list(map((lambda x: processJson(x)), e[1:])))
     elif isinstance(e, list):
         for i in range(len(e)):
-            e[i] = processJson(e[i])
+            e[i] = jsonWalk(e[i])
 
     return e
 
 
 def loadJson(filename: str) -> dict:
-    result = {}
+    try:
-    if filename in CACHE:
+        result = {}
-        result = CACHE[filename]
+        if filename in CACHE:
-    else:
+            result = CACHE[filename]
-        with open(filename) as f:
+        else:
-            result = processJson(json.load(f))
+            with open(filename) as f:
+                result = jsonWalk(json.load(f))
+                result["dir"] = os.path.dirname(filename)
+                result["json"] = filename
+                CACHE[filename] = result
 
-            result["dir"] = os.path.dirname(filename)
+        result = copy.deepcopy(result)
-            result["json"] = filename
+        return result
-            CACHE[filename] = result
+    except Exception as e:
-
+        raise CliException(f"Failed to load json {filename}: {e}")
-    result = copy.deepcopy(result)
-    return result
 
 
 def tryListDir(path: str) -> list[str]: