diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 096e366..c719b31 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,6 +22,7 @@ jobs: - name: Install dependencies run: | + sudo apt install -y ruff python -m pip install --upgrade pip python -m pip install -r requirements.txt python -m pip install mypy pytest @@ -33,3 +34,7 @@ jobs: - name: Run PyTest run: | python -m pytest + + - name: Run Ruff + run: | + ruff check cutekit diff --git a/cutekit/__init__.py b/cutekit/__init__.py index c49ac6f..41a815c 100644 --- a/cutekit/__init__.py +++ b/cutekit/__init__.py @@ -36,15 +36,15 @@ def ensure(version: tuple[int, int, int]): def setupLogger(verbose: bool): if verbose: logging.basicConfig( - level=logging.INFO, + level=logging.DEBUG, format=f"{vt100.CYAN}%(asctime)s{vt100.RESET} {vt100.YELLOW}%(levelname)s{vt100.RESET} %(name)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S", ) else: - projectRoot = model.Project.root() + projectRoot = model.Project.topmost() logFile = const.GLOBAL_LOG_FILE if projectRoot is not None: - logFile = os.path.join(projectRoot, const.PROJECT_LOG_FILE) + logFile = os.path.join(projectRoot.dirname(), const.PROJECT_LOG_FILE) # create the directory if it doesn't exist logDir = os.path.dirname(logFile) diff --git a/cutekit/builder.py b/cutekit/builder.py index 938ae95..a21c01f 100644 --- a/cutekit/builder.py +++ b/cutekit/builder.py @@ -1,7 +1,8 @@ import os import logging +import dataclasses as dt + from pathlib import Path -from dataclasses import dataclass from typing import TextIO, Union from . import shell, rules, model, ninja, const, cli @@ -187,7 +188,7 @@ def gen(out: TextIO, target: model.Target, registry: model.Registry): all(w, registry, target) -@dataclass +@dt.dataclass class Product: path: Path target: model.Target diff --git a/cutekit/cli.py b/cutekit/cli.py index dac3870..60a82e0 100644 --- a/cutekit/cli.py +++ b/cutekit/cli.py @@ -1,10 +1,10 @@ import inspect import logging import sys +import dataclasses as dt from pathlib import Path from typing import Optional, Union, Callable -from dataclasses import dataclass from . import const, vt100 @@ -72,7 +72,7 @@ def parse(args: list[str]) -> Args: Callback = Callable[[Args], None] -@dataclass +@dt.dataclass class Command: shortName: Optional[str] longName: str diff --git a/cutekit/jexpr.py b/cutekit/jexpr.py index 4293954..aa8ee52 100644 --- a/cutekit/jexpr.py +++ b/cutekit/jexpr.py @@ -1,17 +1,20 @@ import os import json +import re +import tomllib + from pathlib import Path -from typing import Any, cast, Callable, Final -from . import shell, compat +from typing import Any, Optional, cast, Callable, Final +from . import shell Json = Any Builtin = Callable[..., Json] BUILTINS: Final[dict[str, Builtin]] = { "uname": lambda arg, ctx: getattr(shell.uname(), arg).lower(), - "include": lambda arg, ctx: evalRead(arg), - "evalRead": lambda arg, ctx: evalRead(arg), + "include": lambda arg, ctx: evalRead(Path(arg)), + "evalRead": lambda arg, ctx: evalRead(Path(arg)), "join": lambda lhs, rhs, ctx: cast( Json, {**lhs, **rhs} if isinstance(lhs, dict) else lhs + rhs ), @@ -50,12 +53,26 @@ def eval(jexpr: Json, filePath: Path) -> Json: return jexpr +def extraSchema(toml: str) -> Optional[str]: + schemaRegex = re.compile(r"#:schema\s+(.*)") + schema = schemaRegex.search(toml) + return schema.group(1) if schema else None + + def read(path: Path) -> Json: try: with open(path, "r") as f: - return json.load(f) - except: - raise RuntimeError(f"Failed to read {path}") + if path.suffix == ".toml": + tomlStr = f.read() + toml = tomllib.loads(tomlStr) + schema = extraSchema(tomlStr) + if schema: + toml["$schema"] = schema + return toml + else: + return json.load(f) + except Exception as e: + raise RuntimeError(f"Failed to read {path}: {e}") def evalRead(path: Path) -> Json: diff --git a/cutekit/model.py b/cutekit/model.py index bae5de1..6cd5a3f 100644 --- a/cutekit/model.py +++ b/cutekit/model.py @@ -1,13 +1,12 @@ import os import logging +import dataclasses as dt from enum import Enum from typing import Any, Generator, Optional, Type, cast from pathlib import Path from dataclasses_json import DataClassJsonMixin -import dataclasses -from dataclasses import dataclass, field from cutekit import const, shell @@ -29,11 +28,13 @@ class Kind(Enum): # --- Manifest --------------------------------------------------------------- # -@dataclass +@dt.dataclass class Manifest(DataClassJsonMixin): id: str - type: Kind = field(default=Kind.UNKNOWN) - path: str = field(default="") + type: Kind = dt.field(default=Kind.UNKNOWN) + path: str = dt.field(default="") + SUFFIXES = [".json", ".toml"] + SUFFIXES_GLOBS = ["*.json", "*.toml"] @staticmethod def parse(path: Path, data: dict[str, Any]) -> "Manifest": @@ -47,12 +48,24 @@ class Manifest(DataClassJsonMixin): obj.path = str(path) return obj + @staticmethod + def tryLoad(path: Path) -> Optional["Manifest"]: + for suffix in Manifest.SUFFIXES: + pathWithSuffix = path.with_suffix(suffix) + if pathWithSuffix.exists(): + _logger.debug(f"Loading manifest from '{pathWithSuffix}'") + return Manifest.parse(pathWithSuffix, jexpr.evalRead(pathWithSuffix)) + return None + @staticmethod def load(path: Path) -> "Manifest": """ Load a manifest from a given path """ - return Manifest.parse(path, jexpr.evalRead(path)) + manifest = Manifest.tryLoad(path) + if manifest is None: + raise RuntimeError(f"Could not find manifest at '{path}'") + return manifest def dirname(self) -> str: """ @@ -79,16 +92,16 @@ class Manifest(DataClassJsonMixin): _project: Optional["Project"] = None -@dataclass +@dt.dataclass class Extern(DataClassJsonMixin): git: str tag: str -@dataclass +@dt.dataclass class Project(Manifest): - description: str = field(default="(No description)") - extern: dict[str, Extern] = field(default_factory=dict) + description: str = dt.field(default="(No description)") + extern: dict[str, Extern] = dt.field(default_factory=dict) @property def externDirs(self) -> list[str]: @@ -96,47 +109,37 @@ class Project(Manifest): return list(res) @staticmethod - def root() -> Optional[str]: - """ - Find the root of the project by looking for a project.json - """ + def topmost() -> Optional["Project"]: cwd = Path.cwd() + topmost: Optional["Project"] = None while str(cwd) != cwd.root: - if (cwd / "project.json").is_file(): - return str(cwd) + projectManifest = Manifest.tryLoad(cwd / "project") + if projectManifest is not None: + topmost = projectManifest.ensureType(Project) cwd = cwd.parent - return None - - @staticmethod - def chdir() -> None: - """ - Change the current working directory to the root of the project - """ - path = Project.root() - if path is None: - raise RuntimeError( - "No project.json found in this directory or any parent directory" - ) - os.chdir(path) - - @staticmethod - def at(path: str) -> Optional["Project"]: - path = os.path.join(path, "project.json") - if not os.path.exists(path): - return None - return Manifest.load(Path(path)).ensureType(Project) + return topmost @staticmethod def ensure() -> "Project": - root = Project.root() - if root is None: + project = Project.topmost() + if project is None: raise RuntimeError( - "No project.json found in this directory or any parent directory" + "No project found in this directory or any parent directory" ) - os.chdir(root) - return Manifest.load(Path(os.path.join(root, "project.json"))).ensureType( - Project - ) + return project + + def chdir(self): + """ + Change the current working directory to the root of the project + """ + os.chdir(self.dirname()) + + @staticmethod + def at(path: Path) -> Optional["Project"]: + projectManifest = Manifest.tryLoad(path / "project") + if projectManifest is None: + return None + return projectManifest.ensureType(Project) @staticmethod def fetchs(extern: dict[str, Extern]): @@ -159,7 +162,7 @@ class Project(Manifest): ext.git, extPath, ) - project = Project.at(extPath) + project = Project.at(Path(extPath)) if project is not None: Project.fetchs(project.extern) @@ -188,6 +191,7 @@ def initCmd(args: cli.Args): name = args.consumeArg() _logger.info("Fetching registry...") + r = requests.get(f"https://raw.githubusercontent.com/{repo}/main/registry.json") if r.status_code != 200: @@ -235,11 +239,11 @@ def initCmd(args: cli.Args): # --- Target ----------------------------------------------------------------- # -@dataclass +@dt.dataclass class Tool(DataClassJsonMixin): - cmd: str = field(default="") - args: list[str] = field(default_factory=list) - files: list[str] = field(default_factory=list) + cmd: str = dt.field(default="") + args: list[str] = dt.field(default_factory=list) + files: list[str] = dt.field(default_factory=list) Tools = dict[str, Tool] @@ -249,11 +253,11 @@ DEFAULT_TOOLS: Tools = { } -@dataclass +@dt.dataclass class Target(Manifest): - props: Props = field(default_factory=dict) - tools: Tools = field(default_factory=dict) - routing: dict[str, str] = field(default_factory=dict) + props: Props = dt.field(default_factory=dict) + tools: Tools = dt.field(default_factory=dict) + routing: dict[str, str] = dt.field(default_factory=dict) @property def hashid(self) -> str: @@ -283,27 +287,27 @@ class Target(Manifest): # --- Component -------------------------------------------------------------- # -@dataclass +@dt.dataclass class Resolved: reason: Optional[str] = None - resolved: list[str] = field(default_factory=list) + resolved: list[str] = dt.field(default_factory=list) @property def enabled(self) -> bool: return self.reason is None -@dataclass +@dt.dataclass class Component(Manifest): - decription: str = field(default="(No description)") - props: Props = field(default_factory=dict) - tools: Tools = field(default_factory=dict) - enableIf: dict[str, list[Any]] = field(default_factory=dict) - requires: list[str] = field(default_factory=list) - provides: list[str] = field(default_factory=list) - subdirs: list[str] = field(default_factory=list) - injects: list[str] = field(default_factory=list) - resolved: dict[str, Resolved] = field(default_factory=dict) + decription: str = dt.field(default="(No description)") + props: Props = dt.field(default_factory=dict) + tools: Tools = dt.field(default_factory=dict) + enableIf: dict[str, list[Any]] = dt.field(default_factory=dict) + requires: list[str] = dt.field(default_factory=list) + provides: list[str] = dt.field(default_factory=list) + subdirs: list[str] = dt.field(default_factory=list) + injects: list[str] = dt.field(default_factory=list) + resolved: dict[str, Resolved] = dt.field(default_factory=dict) def isEnabled(self, target: Target) -> tuple[bool, str]: for k, v in self.enableIf.items(): @@ -334,12 +338,12 @@ KINDS: dict[Kind, Type[Manifest]] = { # --- Dependency resolution -------------------------------------------------- # -@dataclass +@dt.dataclass class Resolver: _registry: "Registry" _target: Target - _mappings: dict[str, list[Component]] = field(default_factory=dict) - _cache: dict[str, Resolved] = field(default_factory=dict) + _mappings: dict[str, list[Component]] = dt.field(default_factory=dict) + _cache: dict[str, Resolved] = dt.field(default_factory=dict) _baked = False def _bake(self): @@ -446,21 +450,25 @@ class Resolver: _registry: Optional["Registry"] = None -@dataclass +@dt.dataclass class Registry(DataClassJsonMixin): project: Project - manifests: dict[str, Manifest] = field(default_factory=dict) + manifests: dict[str, Manifest] = dt.field(default_factory=dict) - def _append(self, m: Manifest): + def _append(self, m: Optional[Manifest]) -> Optional[Manifest]: """ Append a manifest to the model """ + if m is None: + return m + if m.id in self.manifests: raise RuntimeError( f"Duplicated manifest '{m.id}' at '{m.path}' already loaded from '{self.manifests[m.id].path}'" ) self.manifests[m.id] = m + return m def iter(self, type: Type[utils.T]) -> Generator[utils.T, None, None]: """ @@ -525,62 +533,55 @@ class Registry(DataClassJsonMixin): @staticmethod def load(project: Project, mixins: list[str], props: Props) -> "Registry": - registry = Registry(project) - registry._append(project) + r = Registry(project) + r._append(project) # Lookup and load all extern projects for externDir in project.externDirs: - projectPath = os.path.join(externDir, "project.json") - manifestPath = os.path.join(externDir, "manifest.json") + extern = r._append( + Manifest.tryLoad(Path(externDir) / "project") + or Manifest.tryLoad(Path(externDir) / "manifest") + ) - if os.path.exists(projectPath): - registry._append(Manifest.load(Path(projectPath)).ensureType(Project)) - elif os.path.exists(manifestPath): - # For simple library allow to have a manifest.json instead of a project.json - registry._append( - Manifest.load(Path(manifestPath)).ensureType(Component) - ) - else: - _logger.warn( - "Extern project does not have a project.json or manifest.json" - ) + if extern is not None: + _logger.warn("Extern project does not have a project or manifest") # Load all manifests from projects - for project in list(registry.iter(Project)): + for project in list(r.iter(Project)): targetDir = os.path.join(project.dirname(), const.TARGETS_DIR) - targetFiles = shell.find(targetDir, ["*.json"]) + targetFiles = shell.find(targetDir, Manifest.SUFFIXES_GLOBS) for targetFile in targetFiles: - registry._append(Manifest.load(Path(targetFile)).ensureType(Target)) + r._append(Manifest.load(Path(targetFile)).ensureType(Target)) - componentDir = os.path.join(project.dirname(), const.SRC_DIR) - rootComponent = os.path.join(project.dirname(), "manifest.json") - componentFiles = shell.find(componentDir, ["manifest.json"]) + componentFiles = shell.find( + os.path.join(project.dirname(), const.SRC_DIR), + ["manifest" + s for s in Manifest.SUFFIXES], + ) - if os.path.exists(rootComponent): - componentFiles += [rootComponent] + rootComponent = Manifest.tryLoad(Path(project.dirname()) / "manifest") + if rootComponent is not None: + r._append(rootComponent) for componentFile in componentFiles: - registry._append( - Manifest.load(Path(componentFile)).ensureType(Component) - ) + r._append(Manifest.load(Path(componentFile)).ensureType(Component)) # Resolve all dependencies for all targets - for target in registry.iter(Target): + for target in r.iter(Target): target.props |= props - resolver = Resolver(registry, target) + resolver = Resolver(r, target) # Apply injects - for c in registry.iter(Component): + for c in r.iter(Component): if c.isEnabled(target)[0]: for inject in c.injects: - victim = registry.lookup(inject, Component) + victim = r.lookup(inject, Component) if not victim: raise RuntimeError(f"Cannot find component '{inject}'") victim.requires += [c.id] # Resolve all components - for c in registry.iter(Component): + for c in r.iter(Component): resolved = resolver.resolve(c.id) if resolved.reason: _logger.info(f"Component '{c.id}' disabled: {resolved.reason}") @@ -592,7 +593,7 @@ class Registry(DataClassJsonMixin): # Merge in default tools for k, v in DEFAULT_TOOLS.items(): if k not in tools: - tools[k] = dataclasses.replace(v) + tools[k] = dt.replace(v) from . import mixins as mxs @@ -601,12 +602,12 @@ class Registry(DataClassJsonMixin): tools = mixin(target, tools) # Apply tooling from components - for c in registry.iter(Component): + for c in r.iter(Component): if c.resolved[target.id].enabled: for k, v in c.tools.items(): tools[k].args += v.args - return registry + return r @cli.command("l", "list", "List all components and targets") diff --git a/cutekit/plugins.py b/cutekit/plugins.py index cbd7c15..a56b4f0 100644 --- a/cutekit/plugins.py +++ b/cutekit/plugins.py @@ -23,19 +23,16 @@ def load(path: str): def loadAll(): _logger.info("Loading plugins...") - root = model.Project.root() - - if root is None: + project = model.Project.topmost() + if project is None: _logger.info("Not in project, skipping plugin loading") return - - project = model.Project.at(root) paths = list( map(lambda e: os.path.join(const.EXTERN_DIR, e), project.extern.keys()) ) + ["."] for dirname in paths: - pluginDir = os.path.join(root, dirname, const.META_DIR, "plugins") + pluginDir = os.path.join(project.dirname(), dirname, const.META_DIR, "plugins") for files in shell.readdir(pluginDir): if files.endswith(".py"): diff --git a/cutekit/utils.py b/cutekit/utils.py index 560e2b3..ef9cfee 100644 --- a/cutekit/utils.py +++ b/cutekit/utils.py @@ -6,9 +6,9 @@ import hashlib T = TypeVar("T") -def uniq(l: list[T]) -> list[T]: +def uniq(lst: list[T]) -> list[T]: result: list[T] = [] - for i in l: + for i in lst: if i in result: result.remove(i) result.append(i)