Continue work on ninja file generation.
This commit is contained in:
parent
e7b93db95f
commit
3dbcf7b9cc
7 changed files with 88 additions and 300 deletions
|
|
@ -1,281 +0,0 @@
|
|||
from copy import copy
|
||||
import errno
|
||||
import os
|
||||
import hashlib
|
||||
import signal
|
||||
import requests
|
||||
import subprocess
|
||||
import json
|
||||
import copy
|
||||
import re
|
||||
|
||||
|
||||
class Colors:
|
||||
BLACK = "\033[0;30m"
|
||||
RED = "\033[0;31m"
|
||||
GREEN = "\033[0;32m"
|
||||
BROWN = "\033[0;33m"
|
||||
BLUE = "\033[0;34m"
|
||||
PURPLE = "\033[0;35m"
|
||||
CYAN = "\033[0;36m"
|
||||
LIGHT_GRAY = "\033[0;37m"
|
||||
DARK_GRAY = "\033[1;30m"
|
||||
LIGHT_RED = "\033[1;31m"
|
||||
LIGHT_GREEN = "\033[1;32m"
|
||||
YELLOW = "\033[1;33m"
|
||||
LIGHT_BLUE = "\033[1;34m"
|
||||
LIGHT_PURPLE = "\033[1;35m"
|
||||
LIGHT_CYAN = "\033[1;36m"
|
||||
LIGHT_WHITE = "\033[1;37m"
|
||||
BOLD = "\033[1m"
|
||||
FAINT = "\033[2m"
|
||||
ITALIC = "\033[3m"
|
||||
UNDERLINE = "\033[4m"
|
||||
BLINK = "\033[5m"
|
||||
NEGATIVE = "\033[7m"
|
||||
CROSSED = "\033[9m"
|
||||
RESET = "\033[0m"
|
||||
|
||||
|
||||
class CliException(Exception):
|
||||
def __init__(self, msg: str):
|
||||
self.msg = msg
|
||||
|
||||
|
||||
def stripDups(l: list[str]) -> list[str]:
|
||||
# Remove duplicates from a list
|
||||
# by keeping only the last occurence
|
||||
result: list[str] = []
|
||||
for item in l:
|
||||
if item in result:
|
||||
result.remove(item)
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def findFiles(dir: str, exts: list[str] = []) -> list[str]:
|
||||
if not os.path.isdir(dir):
|
||||
return []
|
||||
|
||||
result: list[str] = []
|
||||
|
||||
for f in os.listdir(dir):
|
||||
if len(exts) == 0:
|
||||
result.append(f)
|
||||
else:
|
||||
for ext in exts:
|
||||
if f.endswith(ext):
|
||||
result.append(os.path.join(dir, f))
|
||||
break
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def hashFile(filename: str) -> str:
|
||||
with open(filename, "rb") as f:
|
||||
return hashlib.sha256(f.read()).hexdigest()
|
||||
|
||||
|
||||
def objSha256(obj: dict, keys: list[str] = []) -> str:
|
||||
toHash = {}
|
||||
|
||||
if len(keys) == 0:
|
||||
toHash = obj
|
||||
else:
|
||||
for key in keys:
|
||||
if key in obj:
|
||||
toHash[key] = obj[key]
|
||||
|
||||
data = json.dumps(toHash, sort_keys=True)
|
||||
return hashlib.sha256(data.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def toCamelCase(s: str) -> str:
|
||||
s = ''.join(x for x in s.title() if x != '_' and x != '-')
|
||||
s = s[0].lower() + s[1:]
|
||||
return s
|
||||
|
||||
|
||||
def objKey(obj: dict, keys: list[str] = []) -> str:
|
||||
toKey = []
|
||||
|
||||
if len(keys) == 0:
|
||||
keys = list(obj.keys())
|
||||
keys.sort()
|
||||
|
||||
for key in keys:
|
||||
if key in obj:
|
||||
if isinstance(obj[key], bool):
|
||||
if obj[key]:
|
||||
toKey.append(key)
|
||||
else:
|
||||
toKey.append(f"{toCamelCase(key)}({obj[key]})")
|
||||
|
||||
return "-".join(toKey)
|
||||
|
||||
|
||||
def mkdirP(path: str) -> str:
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EEXIST and os.path.isdir(path):
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
return path
|
||||
|
||||
|
||||
def downloadFile(url: str) -> str:
|
||||
dest = ".osdk/cache/" + hashlib.sha256(url.encode('utf-8')).hexdigest()
|
||||
tmp = dest + ".tmp"
|
||||
|
||||
if os.path.isfile(dest):
|
||||
return dest
|
||||
|
||||
print(f"Downloading {url} to {dest}")
|
||||
|
||||
try:
|
||||
r = requests.get(url, stream=True)
|
||||
r.raise_for_status()
|
||||
mkdirP(os.path.dirname(dest))
|
||||
with open(tmp, 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
os.rename(tmp, dest)
|
||||
return dest
|
||||
except requests.exceptions.RequestException as e:
|
||||
raise CliException(f"Failed to download {url}: {e}")
|
||||
|
||||
|
||||
def runCmd(*args: str) -> bool:
|
||||
try:
|
||||
proc = subprocess.run(args)
|
||||
except FileNotFoundError:
|
||||
raise CliException(f"Failed to run {args[0]}: command not found")
|
||||
except KeyboardInterrupt:
|
||||
raise CliException("Interrupted")
|
||||
|
||||
if proc.returncode == -signal.SIGSEGV:
|
||||
raise CliException("Segmentation fault")
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise CliException(
|
||||
f"Failed to run {' '.join(args)}: process exited with code {proc.returncode}")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def getCmdOutput(*args: str) -> str:
|
||||
try:
|
||||
proc = subprocess.run(args, stdout=subprocess.PIPE)
|
||||
except FileNotFoundError:
|
||||
raise CliException(f"Failed to run {args[0]}: command not found")
|
||||
|
||||
if proc.returncode == -signal.SIGSEGV:
|
||||
raise CliException("Segmentation fault")
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise CliException(
|
||||
f"Failed to run {' '.join(args)}: process exited with code {proc.returncode}")
|
||||
|
||||
return proc.stdout.decode('utf-8')
|
||||
|
||||
|
||||
def sanitizedUname():
|
||||
un = os.uname()
|
||||
if un.machine == "aarch64":
|
||||
un.machine = "arm64"
|
||||
return un
|
||||
|
||||
|
||||
def findLatest(command) -> str:
|
||||
"""
|
||||
Find the latest version of a command
|
||||
|
||||
Exemples
|
||||
clang -> clang-15
|
||||
clang++ -> clang++-15
|
||||
gcc -> gcc10
|
||||
"""
|
||||
print("Searching for latest version of " + command)
|
||||
|
||||
regex = re.compile(r"^" + re.escape(command) + r"(-.[0-9]+)?$")
|
||||
|
||||
versions = []
|
||||
for path in os.environ["PATH"].split(os.pathsep):
|
||||
if os.path.isdir(path):
|
||||
for f in os.listdir(path):
|
||||
if regex.match(f):
|
||||
versions.append(f)
|
||||
|
||||
if len(versions) == 0:
|
||||
raise CliException(f"Failed to find {command}")
|
||||
|
||||
versions.sort()
|
||||
chosen = versions[-1]
|
||||
|
||||
print(f"Using {chosen} as {command}")
|
||||
return chosen
|
||||
|
||||
|
||||
CACHE = {}
|
||||
|
||||
MACROS = {
|
||||
"uname": lambda what: getattr(sanitizedUname(), what).lower(),
|
||||
"include": lambda *path: loadJson(''.join(path)),
|
||||
"join": lambda lhs, rhs: {**lhs, **rhs} if isinstance(lhs, dict) else lhs + rhs,
|
||||
"concat": lambda *args: ''.join(args),
|
||||
"exec": lambda *args: getCmdOutput(*args).splitlines(),
|
||||
"latest": findLatest,
|
||||
}
|
||||
|
||||
|
||||
def isJexpr(jexpr: list) -> bool:
|
||||
return isinstance(jexpr, list) and len(jexpr) > 0 and isinstance(jexpr[0], str) and jexpr[0].startswith("@")
|
||||
|
||||
|
||||
def jsonEval(jexpr: list) -> any:
|
||||
macro = jexpr[0][1:]
|
||||
if not macro in MACROS:
|
||||
raise CliException(f"Unknown macro {macro}")
|
||||
return MACROS[macro](*list(map((lambda x: jsonWalk(x)), jexpr[1:])))
|
||||
|
||||
|
||||
def jsonWalk(e: any) -> any:
|
||||
if isinstance(e, dict):
|
||||
for k in e:
|
||||
e[jsonWalk(k)] = jsonWalk(e[k])
|
||||
elif isJexpr(e):
|
||||
return jsonEval(e)
|
||||
elif isinstance(e, list):
|
||||
for i in range(len(e)):
|
||||
e[i] = jsonWalk(e[i])
|
||||
|
||||
return e
|
||||
|
||||
|
||||
def loadJson(filename: str) -> dict:
|
||||
try:
|
||||
result = {}
|
||||
if filename in CACHE:
|
||||
result = CACHE[filename]
|
||||
else:
|
||||
with open(filename) as f:
|
||||
result = jsonWalk(json.load(f))
|
||||
result["dir"] = os.path.dirname(filename)
|
||||
result["json"] = filename
|
||||
CACHE[filename] = result
|
||||
|
||||
result = copy.deepcopy(result)
|
||||
return result
|
||||
except Exception as e:
|
||||
raise CliException(f"Failed to load json {filename}: {e}")
|
||||
|
||||
|
||||
def tryListDir(path: str) -> list[str]:
|
||||
try:
|
||||
return os.listdir(path)
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
|
|
@ -17,6 +17,13 @@ class Args:
|
|||
del self.opts[key]
|
||||
return result
|
||||
|
||||
def consumeOpt(self, key: str, default: Value) -> Value:
|
||||
if key in self.opts:
|
||||
result = self.opts[key]
|
||||
del self.opts[key]
|
||||
return result
|
||||
return default
|
||||
|
||||
def consumeArg(self) -> str | None:
|
||||
if len(self.args) == 0:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from typing import Any, TextIO
|
||||
from typing import TextIO
|
||||
|
||||
from osdk.model import ComponentManifest, TargetManifest, Props
|
||||
from osdk.ninja import Writer
|
||||
|
|
@ -30,7 +30,7 @@ def gen(out: TextIO, context: Context):
|
|||
tool = target.tools[i]
|
||||
rule = rules.rules[i]
|
||||
writer.rule(
|
||||
i, f"{tool.cmd} {rule.rule.replace('$flags',f'${i}flags')}")
|
||||
i, f"{tool.cmd} {rule.rule.replace('$flags',f'${i}flags')}", deps=rule.deps)
|
||||
writer.newline()
|
||||
|
||||
writer.separator("Components")
|
||||
|
|
@ -38,14 +38,14 @@ def gen(out: TextIO, context: Context):
|
|||
for instance in context.instances:
|
||||
objects = instance.objsfiles()
|
||||
writer.comment(f"Component: {instance.manifest.id}")
|
||||
|
||||
writer.newline()
|
||||
writer.comment(f"Resolved: {', '.join(instance.resolved)}")
|
||||
|
||||
for obj in objects:
|
||||
r = rules.byFileIn(obj[0])
|
||||
if r is None:
|
||||
raise Exception(f"Unknown rule for file {obj[0]}")
|
||||
writer.build(obj[1], r.id, obj[0])
|
||||
t = target.tools[r.id]
|
||||
writer.build(obj[1], r.id, obj[0], order_only=t.files)
|
||||
|
||||
writer.newline()
|
||||
|
||||
|
|
@ -53,13 +53,26 @@ def gen(out: TextIO, context: Context):
|
|||
writer.build(instance.libfile(), "ar",
|
||||
list(map(lambda o: o[1], objects)))
|
||||
else:
|
||||
libraries: list[str] = []
|
||||
|
||||
for req in instance.resolved:
|
||||
reqInstance = context.componentByName(req)
|
||||
|
||||
if reqInstance is None:
|
||||
raise Exception(f"Component {req} not found")
|
||||
|
||||
if not reqInstance.isLib():
|
||||
raise Exception(f"Component {req} is not a library")
|
||||
|
||||
libraries.append(reqInstance.outfile())
|
||||
|
||||
writer.build(instance.binfile(), "ld",
|
||||
list(map(lambda o: o[1], objects)))
|
||||
list(map(lambda o: o[1], objects)) + libraries)
|
||||
|
||||
writer.newline()
|
||||
|
||||
|
||||
def build(componentSpec: str, targetSpec: str = "default", props: Props = {}) -> str:
|
||||
def build(componentSpec: str, targetSpec: str, props: Props = {}) -> str:
|
||||
context = contextFor(targetSpec, props)
|
||||
target = context.target
|
||||
|
||||
|
|
@ -69,6 +82,26 @@ def build(componentSpec: str, targetSpec: str = "default", props: Props = {}) -
|
|||
with open(ninjaPath, "w") as f:
|
||||
gen(f, context)
|
||||
|
||||
raise NotImplementedError()
|
||||
component = context.componentByName(componentSpec)
|
||||
|
||||
return ""
|
||||
if component is None:
|
||||
raise Exception(f"Component {componentSpec} not found")
|
||||
|
||||
shell.exec(f"ninja", "-v", "-f", ninjaPath, component.outfile())
|
||||
|
||||
return component.outfile()
|
||||
|
||||
|
||||
def buildAll(targetSpec: str, props: Props = {}) -> str:
|
||||
context = contextFor(targetSpec, props)
|
||||
target = context.target
|
||||
|
||||
shell.mkdir(target.builddir())
|
||||
ninjaPath = f"{target.builddir()}/build.ninja"
|
||||
|
||||
with open(ninjaPath, "w") as f:
|
||||
gen(f, context)
|
||||
|
||||
shell.exec(f"ninja", "-v", "-f", ninjaPath)
|
||||
|
||||
return target.builddir()
|
||||
|
|
|
|||
24
osdk/cmds.py
24
osdk/cmds.py
|
|
@ -1,7 +1,7 @@
|
|||
from typing import Callable
|
||||
from typing import Callable, cast
|
||||
|
||||
from osdk.args import Args
|
||||
from osdk import context, shell, const, vt100, model
|
||||
from osdk import context, shell, const, vt100, builder
|
||||
|
||||
Callback = Callable[[Args], None]
|
||||
|
||||
|
|
@ -30,7 +30,15 @@ def append(cmd: Cmd):
|
|||
|
||||
|
||||
def runCmd(args: Args):
|
||||
pass
|
||||
targetSpec = cast(str, args.consumeOpt(
|
||||
"target", "host-" + shell.uname().machine))
|
||||
|
||||
componentSpec = args.consumeArg()
|
||||
|
||||
if componentSpec is None:
|
||||
raise Exception("Component not specified")
|
||||
|
||||
builder.build(componentSpec, targetSpec)
|
||||
|
||||
|
||||
cmds += [Cmd("r", "run", "Run the target", runCmd)]
|
||||
|
|
@ -44,7 +52,15 @@ cmds += [Cmd("d", "debug", "Debug the target", debugCmd)]
|
|||
|
||||
|
||||
def buildCmd(args: Args):
|
||||
pass
|
||||
targetSpec = cast(str, args.consumeOpt(
|
||||
"target", "host-" + shell.uname().machine))
|
||||
|
||||
componentSpec = args.consumeArg()
|
||||
|
||||
if componentSpec is None:
|
||||
raise Exception("Component not specified")
|
||||
|
||||
builder.build(componentSpec, targetSpec)
|
||||
|
||||
|
||||
cmds += [Cmd("b", "build", "Build the target", buildCmd)]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from typing import cast
|
||||
|
||||
from osdk.model import TargetManifest, ComponentManifest, Props
|
||||
from osdk.model import TargetManifest, ComponentManifest, Props, Type
|
||||
from osdk.logger import Logger
|
||||
from osdk import const, shell, jexpr
|
||||
|
||||
|
|
@ -25,7 +25,7 @@ class ComponentInstance:
|
|||
self.resolved = resolved
|
||||
|
||||
def isLib(self):
|
||||
return self.manifest.type == "lib"
|
||||
return self.manifest.type == Type.LIB
|
||||
|
||||
def binfile(self) -> str:
|
||||
return f"{self.target.builddir()}/bin/{self.manifest.id}.out"
|
||||
|
|
@ -43,6 +43,11 @@ class ComponentInstance:
|
|||
def libfile(self) -> str:
|
||||
return f"{self.target.builddir()}/lib/{self.manifest.id}.a"
|
||||
|
||||
def outfile(self) -> str:
|
||||
if self.isLib():
|
||||
return self.libfile()
|
||||
return self.binfile()
|
||||
|
||||
|
||||
class Context:
|
||||
target: TargetManifest
|
||||
|
|
@ -52,6 +57,12 @@ class Context:
|
|||
self.target = target
|
||||
self.instances = instances
|
||||
|
||||
def componentByName(self, name: str) -> ComponentInstance | None:
|
||||
result = list(filter(lambda x: x.manifest.id == name, self.instances))
|
||||
if len(result) == 0:
|
||||
return None
|
||||
return result[0]
|
||||
|
||||
|
||||
def loadAllTargets() -> list[TargetManifest]:
|
||||
files = shell.find(const.TARGETS_DIR, ["*.json"])
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ class Writer(object):
|
|||
break_on_hyphens=False):
|
||||
self.output.write('# ' + line + '\n')
|
||||
|
||||
def separator(self, text) -> None:
|
||||
def separator(self, text : str) -> None:
|
||||
self.output.write(f"# --- {text} ---" + '-' *
|
||||
(self.width - 10 - len(text)) + " #\n\n")
|
||||
|
||||
|
|
|
|||
|
|
@ -3,17 +3,19 @@ class Rule:
|
|||
fileIn: list[str]
|
||||
fileOut: list[str]
|
||||
rule: str
|
||||
deps: str | None = None
|
||||
|
||||
def __init__(self, id: str, fileIn: list[str], fileOut: list[str], rule: str):
|
||||
def __init__(self, id: str, fileIn: list[str], fileOut: list[str], rule: str, deps: str | None = None):
|
||||
self.id = id
|
||||
self.fileIn = fileIn
|
||||
self.fileOut = fileOut
|
||||
self.rule = rule
|
||||
self.deps = deps
|
||||
|
||||
|
||||
rules: dict[str, Rule] = {
|
||||
"cc": Rule("cc", ["c"], ["o"], "-c -o $out $in -MD -MF $out.d $flags"),
|
||||
"cxx": Rule("cxx", ["cpp", "cc", "cxx"], ["o"], "-c -o $out $in -MD -MF $out.d $flags"),
|
||||
"cc": Rule("cc", ["c"], ["o"], "-c -o $out $in -MD -MF $out.d $flags", "$out.d"),
|
||||
"cxx": Rule("cxx", ["cpp", "cc", "cxx"], ["o"], "-c -o $out $in -MD -MF $out.d $flags", "$out.d"),
|
||||
"as": Rule("as", ["s", "asm", "S"], ["o"], "-o $out $in $flags"),
|
||||
"ar": Rule("ar", ["o"], ["a"], "$flags $out $in"),
|
||||
"ld": Rule("ld", ["o", "a"], ["out"], "$flags $out $in"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue