Continue work on ninja file generation.

osdk-old/utils.py

@@ -1,281 +0,0 @@
-from copy import copy
-import errno
-import os
-import hashlib
-import signal
-import requests
-import subprocess
-import json
-import copy
-import re
-
-
-class Colors:
-    BLACK = "\033[0;30m"
-    RED = "\033[0;31m"
-    GREEN = "\033[0;32m"
-    BROWN = "\033[0;33m"
-    BLUE = "\033[0;34m"
-    PURPLE = "\033[0;35m"
-    CYAN = "\033[0;36m"
-    LIGHT_GRAY = "\033[0;37m"
-    DARK_GRAY = "\033[1;30m"
-    LIGHT_RED = "\033[1;31m"
-    LIGHT_GREEN = "\033[1;32m"
-    YELLOW = "\033[1;33m"
-    LIGHT_BLUE = "\033[1;34m"
-    LIGHT_PURPLE = "\033[1;35m"
-    LIGHT_CYAN = "\033[1;36m"
-    LIGHT_WHITE = "\033[1;37m"
-    BOLD = "\033[1m"
-    FAINT = "\033[2m"
-    ITALIC = "\033[3m"
-    UNDERLINE = "\033[4m"
-    BLINK = "\033[5m"
-    NEGATIVE = "\033[7m"
-    CROSSED = "\033[9m"
-    RESET = "\033[0m"
-
-
-class CliException(Exception):
-    def __init__(self, msg: str):
-        self.msg = msg
-
-
-def stripDups(l: list[str]) -> list[str]:
-    # Remove duplicates from a list
-    # by keeping only the last occurence
-    result: list[str] = []
-    for item in l:
-        if item in result:
-            result.remove(item)
-        result.append(item)
-    return result
-
-
-def findFiles(dir: str, exts: list[str] = []) -> list[str]:
-    if not os.path.isdir(dir):
-        return []
-
-    result: list[str] = []
-
-    for f in os.listdir(dir):
-        if len(exts) == 0:
-            result.append(f)
-        else:
-            for ext in exts:
-                if f.endswith(ext):
-                    result.append(os.path.join(dir, f))
-                    break
-
-    return result
-
-
-def hashFile(filename: str) -> str:
-    with open(filename, "rb") as f:
-        return hashlib.sha256(f.read()).hexdigest()
-
-
-def objSha256(obj: dict, keys: list[str] = []) -> str:
-    toHash = {}
-
-    if len(keys) == 0:
-        toHash = obj
-    else:
-        for key in keys:
-            if key in obj:
-                toHash[key] = obj[key]
-
-    data = json.dumps(toHash, sort_keys=True)
-    return hashlib.sha256(data.encode("utf-8")).hexdigest()
-
-
-def toCamelCase(s: str) -> str:
-    s = ''.join(x for x in s.title() if x != '_' and x != '-')
-    s = s[0].lower() + s[1:]
-    return s
-
-
-def objKey(obj: dict, keys: list[str] = []) -> str:
-    toKey = []
-
-    if len(keys) == 0:
-        keys = list(obj.keys())
-        keys.sort()
-
-    for key in keys:
-        if key in obj:
-            if isinstance(obj[key], bool):
-                if obj[key]:
-                    toKey.append(key)
-            else:
-                toKey.append(f"{toCamelCase(key)}({obj[key]})")
-
-    return "-".join(toKey)
-
-
-def mkdirP(path: str) -> str:
-    try:
-        os.makedirs(path)
-    except OSError as exc:
-        if exc.errno == errno.EEXIST and os.path.isdir(path):
-            pass
-        else:
-            raise
-    return path
-
-
-def downloadFile(url: str) -> str:
-    dest = ".osdk/cache/" + hashlib.sha256(url.encode('utf-8')).hexdigest()
-    tmp = dest + ".tmp"
-
-    if os.path.isfile(dest):
-        return dest
-
-    print(f"Downloading {url} to {dest}")
-
-    try:
-        r = requests.get(url, stream=True)
-        r.raise_for_status()
-        mkdirP(os.path.dirname(dest))
-        with open(tmp, 'wb') as f:
-            for chunk in r.iter_content(chunk_size=8192):
-                if chunk:
-                    f.write(chunk)
-
-        os.rename(tmp, dest)
-        return dest
-    except requests.exceptions.RequestException as e:
-        raise CliException(f"Failed to download {url}: {e}")
-
-
-def runCmd(*args: str) -> bool:
-    try:
-        proc = subprocess.run(args)
-    except FileNotFoundError:
-        raise CliException(f"Failed to run {args[0]}: command not found")
-    except KeyboardInterrupt:
-        raise CliException("Interrupted")
-
-    if proc.returncode == -signal.SIGSEGV:
-        raise CliException("Segmentation fault")
-
-    if proc.returncode != 0:
-        raise CliException(
-            f"Failed to run {' '.join(args)}: process exited with code {proc.returncode}")
-
-    return True
-
-
-def getCmdOutput(*args: str) -> str:
-    try:
-        proc = subprocess.run(args, stdout=subprocess.PIPE)
-    except FileNotFoundError:
-        raise CliException(f"Failed to run {args[0]}: command not found")
-
-    if proc.returncode == -signal.SIGSEGV:
-        raise CliException("Segmentation fault")
-
-    if proc.returncode != 0:
-        raise CliException(
-            f"Failed to run {' '.join(args)}: process exited with code {proc.returncode}")
-
-    return proc.stdout.decode('utf-8')
-
-
-def sanitizedUname():
-    un = os.uname()
-    if un.machine == "aarch64":
-        un.machine = "arm64"
-    return un
-
-
-def findLatest(command) -> str:
-    """
-    Find the latest version of a command
-
-    Exemples
-    clang -> clang-15
-    clang++ -> clang++-15
-    gcc -> gcc10
-    """
-    print("Searching for latest version of " + command)
-
-    regex = re.compile(r"^" + re.escape(command) + r"(-.[0-9]+)?$")
-
-    versions = []
-    for path in os.environ["PATH"].split(os.pathsep):
-        if os.path.isdir(path):
-            for f in os.listdir(path):
-                if regex.match(f):
-                    versions.append(f)
-
-    if len(versions) == 0:
-        raise CliException(f"Failed to find {command}")
-
-    versions.sort()
-    chosen = versions[-1]
-
-    print(f"Using {chosen} as {command}")
-    return chosen
-
-
-CACHE = {}
-
-MACROS = {
-    "uname": lambda what: getattr(sanitizedUname(), what).lower(),
-    "include": lambda *path: loadJson(''.join(path)),
-    "join": lambda lhs, rhs: {**lhs, **rhs} if isinstance(lhs, dict) else lhs + rhs,
-    "concat": lambda *args: ''.join(args),
-    "exec": lambda *args: getCmdOutput(*args).splitlines(),
-    "latest": findLatest,
-}
-
-
-def isJexpr(jexpr: list) -> bool:
-    return isinstance(jexpr, list) and len(jexpr) > 0 and isinstance(jexpr[0], str) and jexpr[0].startswith("@")
-
-
-def jsonEval(jexpr: list) -> any:
-    macro = jexpr[0][1:]
-    if not macro in MACROS:
-        raise CliException(f"Unknown macro {macro}")
-    return MACROS[macro](*list(map((lambda x: jsonWalk(x)), jexpr[1:])))
-
-
-def jsonWalk(e: any) -> any:
-    if isinstance(e, dict):
-        for k in e:
-            e[jsonWalk(k)] = jsonWalk(e[k])
-    elif isJexpr(e):
-        return jsonEval(e)
-    elif isinstance(e, list):
-        for i in range(len(e)):
-            e[i] = jsonWalk(e[i])
-
-    return e
-
-
-def loadJson(filename: str) -> dict:
-    try:
-        result = {}
-        if filename in CACHE:
-            result = CACHE[filename]
-        else:
-            with open(filename) as f:
-                result = jsonWalk(json.load(f))
-                result["dir"] = os.path.dirname(filename)
-                result["json"] = filename
-                CACHE[filename] = result
-
-        result = copy.deepcopy(result)
-        return result
-    except Exception as e:
-        raise CliException(f"Failed to load json {filename}: {e}")
-
-
-def tryListDir(path: str) -> list[str]:
-    try:
-        return os.listdir(path)
-    except FileNotFoundError:
-        return []

osdk/args.py

@@ -17,6 +17,13 @@ class Args:
                 del self.opts[key]
         return result
 
+    def consumeOpt(self, key: str, default: Value) -> Value:
+        if key in self.opts:
+            result = self.opts[key]
+            del self.opts[key]
+            return result
+        return default
+
     def consumeArg(self) -> str | None:
         if len(self.args) == 0:
             return None

osdk/builder.py

@@ -1,4 +1,4 @@
-from typing import Any, TextIO
+from typing import TextIO
 
 from osdk.model import ComponentManifest, TargetManifest, Props
 from osdk.ninja import Writer
@@ -30,7 +30,7 @@ def gen(out: TextIO, context: Context):
         tool = target.tools[i]
         rule = rules.rules[i]
         writer.rule(
-            i, f"{tool.cmd} {rule.rule.replace('$flags',f'${i}flags')}")
+            i, f"{tool.cmd} {rule.rule.replace('$flags',f'${i}flags')}", deps=rule.deps)
         writer.newline()
 
     writer.separator("Components")
@@ -38,14 +38,14 @@ def gen(out: TextIO, context: Context):
     for instance in context.instances:
         objects = instance.objsfiles()
         writer.comment(f"Component: {instance.manifest.id}")
-
+        writer.comment(f"Resolved: {', '.join(instance.resolved)}")
-        writer.newline()
 
         for obj in objects:
             r = rules.byFileIn(obj[0])
             if r is None:
                 raise Exception(f"Unknown rule for file {obj[0]}")
-            writer.build(obj[1], r.id,  obj[0])
+            t = target.tools[r.id]
+            writer.build(obj[1], r.id,  obj[0], order_only=t.files)
 
         writer.newline()
 
@@ -53,13 +53,26 @@ def gen(out: TextIO, context: Context):
             writer.build(instance.libfile(), "ar",
                          list(map(lambda o: o[1], objects)))
         else:
+            libraries: list[str] = []
+
+            for req in instance.resolved:
+                reqInstance = context.componentByName(req)
+
+                if reqInstance is None:
+                    raise Exception(f"Component {req} not found")
+
+                if not reqInstance.isLib():
+                    raise Exception(f"Component {req} is not a library")
+
+                libraries.append(reqInstance.outfile())
+
             writer.build(instance.binfile(), "ld",
-                         list(map(lambda o: o[1], objects)))
+                         list(map(lambda o: o[1], objects)) + libraries)
 
         writer.newline()
 
 
-def build(componentSpec: str, targetSpec: str = "default",  props: Props = {}) -> str:
+def build(componentSpec: str, targetSpec: str,  props: Props = {}) -> str:
     context = contextFor(targetSpec, props)
     target = context.target
 
@@ -69,6 +82,26 @@ def build(componentSpec: str, targetSpec: str = "default",  props: Props = {}) -
     with open(ninjaPath, "w") as f:
         gen(f, context)
 
-    raise NotImplementedError()
+    component = context.componentByName(componentSpec)
 
-    return ""
+    if component is None:
+        raise Exception(f"Component {componentSpec} not found")
+
+    shell.exec(f"ninja", "-v", "-f", ninjaPath, component.outfile())
+
+    return component.outfile()
+
+
+def buildAll(targetSpec: str, props: Props = {}) -> str:
+    context = contextFor(targetSpec, props)
+    target = context.target
+
+    shell.mkdir(target.builddir())
+    ninjaPath = f"{target.builddir()}/build.ninja"
+
+    with open(ninjaPath, "w") as f:
+        gen(f, context)
+
+    shell.exec(f"ninja", "-v", "-f", ninjaPath)
+
+    return target.builddir()

osdk/cmds.py

@@ -1,7 +1,7 @@
-from typing import Callable
+from typing import Callable, cast
 
 from osdk.args import Args
-from osdk import context, shell, const, vt100, model
+from osdk import context, shell, const, vt100, builder
 
 Callback = Callable[[Args], None]
 
@@ -30,7 +30,15 @@ def append(cmd: Cmd):
 
 
 def runCmd(args: Args):
-    pass
+    targetSpec = cast(str, args.consumeOpt(
+        "target", "host-" + shell.uname().machine))
+
+    componentSpec = args.consumeArg()
+
+    if componentSpec is None:
+        raise Exception("Component not specified")
+
+    builder.build(componentSpec, targetSpec)
 
 
 cmds += [Cmd("r", "run", "Run the target", runCmd)]
@@ -44,7 +52,15 @@ cmds += [Cmd("d", "debug", "Debug the target", debugCmd)]
 
 
 def buildCmd(args: Args):
-    pass
+    targetSpec = cast(str, args.consumeOpt(
+        "target", "host-" + shell.uname().machine))
+
+    componentSpec = args.consumeArg()
+
+    if componentSpec is None:
+        raise Exception("Component not specified")
+
+    builder.build(componentSpec, targetSpec)
 
 
 cmds += [Cmd("b", "build", "Build the target", buildCmd)]

osdk/context.py

@@ -1,6 +1,6 @@
 from typing import cast
 
-from osdk.model import TargetManifest, ComponentManifest, Props
+from osdk.model import TargetManifest, ComponentManifest, Props, Type
 from osdk.logger import Logger
 from osdk import const, shell, jexpr
 
@@ -25,7 +25,7 @@ class ComponentInstance:
         self.resolved = resolved
 
     def isLib(self):
-        return self.manifest.type == "lib"
+        return self.manifest.type == Type.LIB
 
     def binfile(self) -> str:
         return f"{self.target.builddir()}/bin/{self.manifest.id}.out"
@@ -43,6 +43,11 @@ class ComponentInstance:
     def libfile(self) -> str:
         return f"{self.target.builddir()}/lib/{self.manifest.id}.a"
 
+    def outfile(self) -> str:
+        if self.isLib():
+            return self.libfile()
+        return self.binfile()
+
 
 class Context:
     target: TargetManifest
@@ -52,6 +57,12 @@ class Context:
         self.target = target
         self.instances = instances
 
+    def componentByName(self, name: str) -> ComponentInstance | None:
+        result = list(filter(lambda x: x.manifest.id == name, self.instances))
+        if len(result) == 0:
+            return None
+        return result[0]
+
 
 def loadAllTargets() -> list[TargetManifest]:
     files = shell.find(const.TARGETS_DIR, ["*.json"])

osdk/ninja.py

@@ -49,7 +49,7 @@ class Writer(object):
                                   break_on_hyphens=False):
             self.output.write('# ' + line + '\n')
 
-    def separator(self, text) -> None:
+    def separator(self, text : str) -> None:
         self.output.write(f"# --- {text} ---" + '-' *
                           (self.width - 10 - len(text)) + " #\n\n")
 

osdk/rules.py

@@ -3,17 +3,19 @@ class Rule:
     fileIn: list[str]
     fileOut: list[str]
     rule: str
+    deps: str | None = None
 
-    def __init__(self, id: str, fileIn: list[str], fileOut: list[str], rule: str):
+    def __init__(self, id: str, fileIn: list[str], fileOut: list[str], rule: str, deps: str | None = None):
         self.id = id
         self.fileIn = fileIn
         self.fileOut = fileOut
         self.rule = rule
+        self.deps = deps
 
 
 rules: dict[str, Rule] = {
-    "cc": Rule("cc", ["c"], ["o"], "-c -o $out $in -MD -MF $out.d $flags"),
+    "cc": Rule("cc", ["c"], ["o"], "-c -o $out $in -MD -MF $out.d $flags", "$out.d"),
-    "cxx": Rule("cxx", ["cpp", "cc", "cxx"], ["o"], "-c -o $out $in -MD -MF $out.d $flags"),
+    "cxx": Rule("cxx", ["cpp", "cc", "cxx"], ["o"], "-c -o $out $in -MD -MF $out.d $flags", "$out.d"),
     "as": Rule("as", ["s", "asm", "S"], ["o"], "-o $out $in $flags"),
     "ar": Rule("ar", ["o"], ["a"], "$flags $out $in"),
     "ld": Rule("ld", ["o", "a"], ["out"], "$flags $out $in"),