Massive ackbuilder refactor --- cleaner and more expressive. Lists

are automatically flattened (leading to better build files), and the
list and filename functions are vastly more orthogonal.

first/ackbuilder.lua

@@ -18,27 +18,51 @@ local cwd = "."
 local vars = {}
 local parente = {}
 
+-- Forward references
+local loadtarget
+
 local function print(...)
-	for _, s in ipairs({...}) do
+	local function print_no_nl(list)
-		if (type(s) ~= "string") then
+		for _, s in ipairs(list) do
-			s = tostring(s)
+			if (type(s) == "table") then
+				io.stderr:write("{")
+				for k, v in pairs(s) do
+					print_no_nl({k})
+					io.stderr:write("=")
+					print_no_nl({v})
+					io.stderr:write(" ")
 				end
-		io.stderr:write(s)
+				io.stderr:write("}")
+			else
+				io.stderr:write(tostring(s))
 			end
+		end
+	end
+
+	print_no_nl({...})
 	io.stderr:write("\n")
 end
 
+local function assertString(s, i)
+	if (type(s) ~= "string") then
+		error(string.format("parameter %d must be a string", i))
+	end
+end
+
 local function concat(...)
 	local r = {}
-	for _, t in ipairs({...}) do
+
+	local function process(list)
+		for _, t in ipairs(list) do
 			if (type(t) == "table") and not t.is then
-			for _, v in ipairs(t) do
+				process(t)
-				r[#r+1] = v
-			end
 			else
 				r[#r+1] = t
 			end
 		end
+	end
+	process({...})
+
 	return r
 end
 
@@ -90,22 +114,50 @@ local function concatpath(...)
 	return (p:gsub("/+", "/"):gsub("^%./", ""):gsub("/%./", "/"))
 end
 
-local function filenamesof(targets, pattern)
+-- Turns a list of strings or targets into a list of targets, expanding
-	local f = {}
+-- recursive lists and wildcards.
-	if targets then
+local function targetsof(...)
-		if targets.is then
+	local o = {}
-			targets = {targets}
+
+	local function process(items)
+		for _, item in ipairs(items) do
+			if (type(item) == "table") then
+				if item.is then
+					-- This is a target.
+					o[#o+1] = item
+				else
+					-- This is a list.
+					process(item)
+				end
+			elseif (type(item) == "string") then
+				-- Filename!
+				if item:find("^%+") then
+					item = cwd..item
+				elseif item:find("^%./") then
+					item = concatpath(cwd, item)
+				end
+				o[#o+1] = loadtarget(item)
+			else
+				error(string.format("member of target list is not a string or a target"))
+			end
+		end
 	end
 
-		for _, r in pairs(targets) do
+	process({...})
+	return o
+end
+
+local function filenamesof(...)
+	local targets = targetsof(...)
+
+	local f = {}
+	for _, r in ipairs(targets) do
 		if (type(r) == "table") and r.is then
 			if r.outs then
-					for _, o in pairs(r.outs) do
+				for _, o in ipairs(r.outs) do
-						if not pattern or o:find(pattern) then
 					f[#f+1] = o
 				end
 			end
-				end
 		elseif (type(r) == "string") then
 			f[#f+1] = r
 		else
@@ -113,17 +165,13 @@ local function filenamesof(targets, pattern)
 				type(r)))
 		end
 	end
-	end
 	return f
 end
 
-local function targetnamesof(targets)
+local function targetnamesof(...)
-	local f = {}
+	local targets = targetsof(...)
-	if targets then
-		if targets.is then
-			targets = {targets}
-		end
 
+	local f
 	for _, r in pairs(targets) do
 		if (type(r) == "table") and r.is then
 			f[#f+1] = r.fullname
@@ -134,26 +182,22 @@ local function targetnamesof(targets)
 				type(r)))
 		end
 	end
-	end
 	return f
 end
 
-local function dotocollection(collection, callback)
+local function dotocollection(files, callback)
-	if (type(collection) == "string") then
+	if (#files == 1) and (type(files[1]) == "string") then
-		return callback(collection)
-	elseif collection.is then
-		local files = filenamesof(collection.outs)
-		if (#files ~= 1) then
-			error("inputs with more than one output need to be in a list")
-		end
 		return callback(files[1])
 	end
 
 	local o = {}
-	for _, s in pairs(collection) do
+	local function process(files)
+		for _, s in ipairs(files) do
+			if (type(s) == "table") then
 				if s.is then
-			for _, b in pairs(dotocollection(filenamesof(s), callback)) do
+					error("passed target to a filename manipulation function")
-				o[#o+1] = b
+				else
+					process(s)
 				end
 			else
 				local b = callback(s)
@@ -162,12 +206,16 @@ local function dotocollection(collection, callback)
 				end
 			end
 		end
+	end
+	process(files)
 	return o
 end
 
-local function abspath(collection)
+
-	return dotocollection(collection,
+local function abspath(...)
+	return dotocollection({...},
 		function(filename)
+			assertString(filename, 1)
 			if not filename:find("^[/$]") then
 				filename = concatpath(posix.getcwd(), filename)
 			end
@@ -176,9 +224,11 @@ local function abspath(collection)
 	)
 end
 
-local function basename(collection)
+
-	return dotocollection(collection,
+local function basename(...)
+	return dotocollection({...},
 		function(filename)
+			assertString(filename, 1)
 			local _, _, b = filename:find("^.*/([^/]*)$")
 			if not b then
 				return filename
@@ -188,9 +238,11 @@ local function basename(collection)
 	)
 end
 
-local function dirname(collection)
+
-	return dotocollection(collection,
+local function dirname(...)
+	return dotocollection({...},
 		function(filename)
+			assertString(filename, 1)
 			local _, _, b = filename:find("^(.*)/[^/]*$")
 			if not b then
 				return ""
@@ -200,25 +252,34 @@ local function dirname(collection)
 	)
 end
 
-local function replace(collection, pattern, repl)
+local function replace(files, pattern, repl)
-	return dotocollection(collection,
+	return dotocollection(files,
 		function(filename)
 			return filename:gsub(pattern, repl)
 		end
 	)
 end
 
-local function fpairs(collection)
+local function fpairs(...)
-	if (type(collection) == "string") or collection.is then
+	return ipairs(filenamesof(...))
-		return fpairs({collection})
 end
 
-	return pairs(filenamesof(collection))
+local function matching(collection, pattern)
+	local o = {}
+	dotocollection(collection,
+		function(filename)
+			if filename:find(pattern) then
+				o[#o+1] = filename
+			end
+		end
+	)
+	return o
 end
 		
 -- Selects all targets containing at least one output file that matches
 -- the pattern (or all, if the pattern is nil).
 local function selectof(targets, pattern)
+	local targets = targetsof(targets)
 	local o = {}
 	for k, v in pairs(targets) do
 		if v.is and v.outs then
@@ -237,16 +298,16 @@ local function selectof(targets, pattern)
 	return o
 end
 
-local function uniquify(collection)
+local function uniquify(...)
 	local s = {}
-	local o = {}
+	return dotocollection({...},
-	for _, v in pairs(collection) do
+		function(filename)
-		if not s[v] then
+			if not s[filename] then
-			s[v] = true
+				s[filename] = true
-			o[#o+1] = v
+				return filename
 			end
 		end
-	return o
+	)
 end
 
 local function startswith(needle, haystack)
@@ -274,13 +335,13 @@ local function templateexpand(list, vars)
 		o[#o+1] = s:gsub("%%%b{}",
 			function(expr)
 				expr = expr:sub(3, -2)
-				local chunk, e = loadstring("return "..expr, expr, "text", vars)
+				local chunk, e = load("return ("..expr..")", expr, "text", vars)
 				if e then
 					error(string.format("error evaluating expression: %s", e))
 				end
 				local value = chunk()
 				if (value == nil) then
-					error(string.format("template expression expands to nil (probably an undefined variable)"))
+					error(string.format("template expression '%s' expands to nil (probably an undefined variable)", expr))
 				end
 				return asstring(value)
 			end
@@ -302,7 +363,7 @@ local function loadbuildfile(filename)
 			if not e then
 				local thisglobals = {_G = thisglobals}
 				setmetatable(thisglobals, {__index = globals})
-				chunk, e = loadstring(data, filename, "text", thisglobals)
+				chunk, e = load(data, "@"..filename, "text", thisglobals)
 			end
 		end
 		if e then
@@ -316,7 +377,7 @@ local function loadbuildfile(filename)
 	end
 end
 
-local function loadtarget(targetname)
+loadtarget = function(targetname)
 	if targets[targetname] then
 		return targets[targetname]
 	end
@@ -368,24 +429,7 @@ local typeconverters = {
 		elseif (type(i) ~= "table") then
 			error(string.format("property '%s' must be a target list", propname))
 		end
-
+		return targetsof(i)
-		local o = {}
-		for k, s in pairs(i) do
-			if (type(s) == "table") and s.is then
-				o[k] = s
-			elseif (type(s) == "string") then
-				if s:find("^%+") then
-					s = cwd..s
-				elseif s:find("^%./") then
-					s = concatpath(cwd, s)
-				end
-				o[k] = loadtarget(s)
-			else
-				error(string.format("member of target list '%s' is not a string or a target",
-					propname))
-			end
-		end
-		return o
 	end,
 
 	strings = function(propname, i)
@@ -394,7 +438,7 @@ local typeconverters = {
 		elseif (type(i) ~= "table") then
 			error(string.format("property '%s' must be a string list", propname))
 		end
-		return i
+		return concat(i)
 	end,
 
 	boolean = function(propname, i)
@@ -547,7 +591,7 @@ local function install_ninja_emitter()
 	emit("\n")
 
 	local function unmake(collection)
-		return dotocollection(collection,
+		return dotocollection({collection},
 			function(s)
 				return s:gsub("%$%b()",
 					function(expr)
@@ -589,14 +633,12 @@ definerule("simplerule",
 		vars = { type="table", default={} },
 	},
 	function (e)
-		emitter:rule(e.fullname,
+		emitter:rule(e.fullname, filenamesof(e.ins, e.deps), e.outs)
-			concat(filenamesof(e.ins), filenamesof(e.deps)),
-			e.outs)
 		emitter:label(e.fullname, " ", e.label or "")
 
 		local vars = inherit(e.vars, {
-			ins = e.ins,
+			ins = filenamesof(e.ins),
-			outs = e.outs
+			outs = filenamesof(e.outs)
 		})
 
 		emitter:exec(templateexpand(e.commands, vars))
@@ -610,7 +652,7 @@ definerule("simplerule",
 
 definerule("installable",
 	{
-		map = { type="targets", default={} },
+		map = { type="table", default={} },
 	},
 	function (e)
 		local deps = {}
@@ -618,6 +660,12 @@ definerule("installable",
 		local srcs = {}
 		local dests = {}
 		for dest, src in pairs(e.map) do
+			src = targetsof(src)
+			if (#src ~= 1) then
+				error("installable can only cope with one target at a time")
+			end
+			src = src[1]
+
 			if src.is.installable then
 				if (type(dest) ~= "number") then
 					error("can't specify a destination filename when installing an installable")
@@ -631,7 +679,7 @@ definerule("installable",
 					error("installable can only cope with targets emitting single files")
 				end
 
-				deps[#deps+1] = src
+				deps[#deps+1] = src.fullname
 				dests[#dests+1] = dest
 				commands[#commands+1] = "cp "..f[1].." "..dest
 			end
@@ -717,9 +765,9 @@ globals = {
 	inherit = inherit,
 	print = print,
 	replace = replace,
+	matching = matching,
 	selectof = selectof,
 	startswith = startswith,
-	targetnamesof = targetnamesof,
 	uniquify = uniquify,
 	vars = vars,
 }

first/build.lua

@@ -14,8 +14,8 @@ definerule("normalrule",
 	function (e)
 		local dir = e.objdir or objdir(e)
 		local realouts = {}
-		for k, v in pairs(e.outleaves) do
+		for _, v in pairs(e.outleaves) do
-			realouts[k] = concatpath(dir, v)
+			realouts[#realouts+1] = concatpath(dir, v)
 		end
 
 		local vars = inherit(e.vars, {
@@ -185,7 +185,7 @@ definerule("clibrary",
 			cwd = e.cwd,
 			ins = ins,
 			deps = concat(e.hdrs, e.deps),
-			outleaves = { e.name..".a", unpack(basename(hdrs)) },
+			outleaves = concat(e.name..".a", basename(hdrs)),
 			label = e.label,
 			commands = commands,
 			vars = {
@@ -207,19 +207,21 @@ definerule("cprogram",
 		}
 	},
 	function (e)
-		local libs = filenamesof(e.deps, "%.a$")
+		local libs = matching(filenamesof(e.deps), "%.a$")
 		if (#e.srcs > 0) then
-			for _, f in pairs(filenamesof(
+			for _, f in pairs(
-				{
+				matching(
+					filenamesof(
 						clibrary {
 							name = e.name .. "/main",
 							cwd = e.cwd,
 							srcs = e.srcs,
 							deps = e.deps,
 						}
-				},
+					),
 					"%.a$"
-			)) do
+				)
+			) do
 				libs[#libs+1] = f
 			end
 		end

lang/cem/cemcom.ansi/build.lua

@@ -1,4 +1,3 @@
-local posix = require("posix")
 include("util/LLgen/build.lua")
 
 normalrule {
@@ -13,7 +12,7 @@ normalrule {
 	}
 }
 
-local str_files = basename(posix.glob(cwd().."/*.str"))
+local str_files = basename(filenamesof("./*.str"))
 local str_targets = {}
 
 for _, f in ipairs(str_files) do
@@ -46,7 +45,7 @@ clibrary {
 	hdrs = str_targets,
 	deps = {
 		"+parameters",
-		unpack(str_targets)
+		str_targets
 	}
 }
 

lang/cem/cpp.ansi/build.lua

@@ -9,9 +9,9 @@ local allocd_header = definerule(null,
 			name = e.name,
 			ins = {
 				"./make.allocd",
-				unpack(e.srcs)
+				e.srcs
 			},
-			outleaves = replace(basename(e.srcs), "%.str$", ".h"),
+			outleaves = basename(filenamesof(e.srcs)[1]):gsub("%.str$", ".h"),
 			commands = {
 				"%{ins[1]} < %{ins[2]} > %{outs}"
 			}
@@ -82,7 +82,7 @@ cprogram {
 	name = "cpp",
 	srcs = concat(
 		"./*.c",
-		filenamesof(llgen, "%.c$"),
+		matching(filenamesof(llgen), "%.c$"),
 		"+next_c",
 		"+symbol2str_c",
 		"+tabgen_c"

mach/proto/as/build.lua

@@ -32,7 +32,7 @@ definerule("build_as",
 			name = e.name,
 			srcs = concat(
 				"mach/proto/as/*.c",
-				filenamesof(yaccfiles, "%.c$")
+				matching(filenamesof(yaccfiles), "%.c$")
 			),
 			deps = {
 				"h+emheaders",

mach/proto/ncg/build.lua

@@ -35,7 +35,6 @@ definerule("build_ncg",
 				"modules/src/flt_arith+lib",
 				"modules/src/object+lib",
 				"util/data+em_data",
-				archlib, -- for .h file
 				headers,
 				tables, -- for .h file
 			}

plat/build.lua

@@ -16,7 +16,7 @@ definerule("ackfile",
 				"plat/"..plat.."+tools",
 				"util/ack+pkg",
 				"lang/cem/cpp.ansi+pkg",
-				unpack(e.deps)
+				e.deps
 			},
 			commands = {
 				"ACKDIR=$(INSDIR) $(INSDIR)/bin/ack -m%{plat} -c -o %{outs} %{ins} %{hdrpaths} %{ackcflags}"
@@ -38,10 +38,11 @@ definerule("acklibrary",
 			hdrs = e.hdrs,
 			deps = {
 				"util/arch+pkg",
-				unpack(e.deps)
+				e.deps
 			},
 			_cfile = ackfile,
 			commands = {
+				"rm -f %{outs[1]}",
 				"ACKDIR=$(INSDIR) $(INSDIR)/bin/aal q %{outs[1]} %{ins}"
 			}
 		}

util/LLgen/build.lua

@@ -21,7 +21,7 @@ definerule("llgen",
 	function(e)
 		-- Remember this is executed from the caller's directory; local
 		-- target names will resolve there
-		local fs = replace(basename(e.srcs), "%.g$", "")
+		local fs = replace(basename(filenamesof(e.srcs)), "%.g$", "")
 
 		return normalrule {
 			name = e.name,
@@ -29,11 +29,11 @@ definerule("llgen",
 			outleaves = {
 				"Lpars.c",
 				"Lpars.h",
-				unpack(replace(fs, "$", ".c"))
+				replace(fs, "$", ".c")
 			},
 			ins = {
 				"util/LLgen+llgen",
-				unpack(e.srcs),
+				e.srcs,
 			},
 			commands = {
 				"cd %{dir} && %{abspath(ins)}"

util/cmisc/build.lua

@@ -8,13 +8,13 @@ definerule("tabgen",
 		srcs = { type="targets" },
 	},
 	function(e)
-		local symname = replace(basename(e.srcs[1]), "%.tab$", "")
+		local symname = basename(filenamesof(e.srcs)[1]):gsub("%.tab$", "")
 
 		return normalrule {
 			name = e.name,
 			ins = {
 				"util/cmisc+tabgen",
-				unpack(e.srcs)
+				e.srcs
 			},
 			outleaves = { symname..".c" },
 			commands = {

util/data/build.lua

@@ -24,7 +24,7 @@ clibrary {
 	name = "em_data",
 	srcs = concat(
 		"./em_ptyp.c",
-		filenamesof(generated, "%.c$")
+		matching(filenamesof(generated), "%.c$")
 	),
 	hdrs = {
 		"+generated" -- so we export the H files

util/ncgg/build.lua

@@ -12,11 +12,11 @@ local cgglexer = flex {
 
 normalrule {
 	name = "keywords",
-	ins = concat(
+	ins = {
 		"./cvtkeywords",
 		"./keywords",
-		filenamesof({cggparser}, "%.h$")
+		matching(filenamesof(cggparser), "%.h$")
-	),
+	},
 	outleaves = { "enterkeyw.c" },
 	commands = {
 		"%{ins[1]} %{ins[2]} %{ins[3]} %{outs[1]}"
@@ -27,8 +27,8 @@ cprogram {
 	name = "ncgg",
 	srcs = concat(
 		"./*.c",
-		filenamesof({cggparser}, "%.c$"),
+		matching(filenamesof(cggparser), "%.c$"),
-		filenamesof({cgglexer}, "%.c$"),
+		matching(filenamesof(cgglexer), "%.c$"),
 		"+keywords"
 	),
 	deps = {