xbmk: make all projects multi-tree

we currently distinguish, by virtue of logic, multi-
and single-tree projects. this introduces a lot of
complexity, so the following patch has removed this
distinction. all single-tree projects are now multi
tree, but with only one tree.

other design changes:

config/git removed. rev/tree now handled with mkhelper
and target configs. this allows per-tree remote handling,
so we could (for example) add the mrchromebox coreboot
fork easily, for specific chromebook models.

config/project is now config/module/project. this
cleans up the main config/ directory.

cfgname variable: .config no longer hardcoded when
doing configs in projects. it is configurable with
this variable. if unset, the previous .cfg is used

noconfig variable: set this in target/mkhelper
configs, and config/module/project/tree/config/ is
unused; instead, make is called without config copying.
a "noconfig" placeholder is used. this enables cleaner
usage of previously single-tree projects in the new
multi-tree-only structure.

cfgvars: variables reinitialised in project config are
cleaned more reliably. in particular, several variables
e.g. autogenargs were not being reset between projects,
although the autogenargs variable is currently unused.

general cleanup has been conducted across the build
system, as part of this redesign. several parts of the
build system more over-engineered as a result of the
previous design, and the new design enabled a severe
reduction in sloccount (75 lines reduced in lbmk).

yes, this change *adds features* but there are fewer
lines of code than before. this is the best kind of
change. way better than the 3-line-diff. yes.

rev= and tree= are set more centrally in configs. the
mkhelper config can be used to set url/url_bkup, but
individual trees can override them.

configs have been cleaned up, with several variables
now much more heavily centralised. config/git/ was
used before to first download the bare repository, and
then the real clone was made with patches. this allows
git repositories to be cached for future re-builds.

with config/git/ gone, the download logic for projects
is more centrally implemented. a separate call to ./mk -f
is no longer performed, and in fact ./mk -f never needs
to be used anymore as a result. ./mk -f now is basically
the same thing as -b, just without any action performed.

this change will allow rapid expansion of the libreboot
project. i want to add hundreds of chromebooks to the
project, using mrchromebox, but the previous design did
not permit this. this change means that i can easily have
unlimited upstreams per project. coreboot will, in some
follow-up changes, have the following upstreams:

coreboot/default: coreboot.org
coreboot/fam15h: 15h.org fork
coreboot/chromebook: mrchromebox fork

this can now be done freely without massive changes to
the build system. under the previous design, i worked on
a now-abandoned patch to do N remotes per project, reading
repo links from a file. this meant that i'd have coreboot.org
first (main and backup), then mrchromebox. this meant that
lbmk would failover through to mrchromebox, wasting time
for the user.

this new design is much cleaner, and will enable the
project to be maintained much more easily.

Signed-off-by: Leah Rowe <leah@libreboot.org>

7 files changed, 86 insertions, 61 deletions

diff --git a/include/env/get.sh b/include/env/get.sh
index 09126297..0ed9d6cc 100644
--- a/include/env/get.sh
+++ b/include/env/get.sh
@@ -3,52 +3,83 @@
 # Copyright (c) 2020-2021,2023-2026 Leah Rowe <leah@libreboot.org>
 # Copyright (c) 2022 Caleb La Grange <thonkpeasant@protonmail.com>
 
-eval "`newvar url bkup_url depend loc subcurl subcurl_bkup subgit subgit_bkup \
-    subhash`"
+eval "`newvar url bkup_url fetch_depend loc subcurl subcurl_bkup subgit \
+    subgit_bkup subhash badhash badtghash`"
 
 tmpgit="$xbtmp/gitclone"
 tmpgitcache="$xbtmp/tmpgit"
 
-multi_tree_fetch()
+# delete a source tree and corresponding builds if metadata changed
+# inside config/ - this is done by hashing a concatenation of hashes
+# from all metadata, checked against a previously calculated hash
+
+delete_old_project_files()
 {
-	[ -d "src/$project/$tree" ] && return 0
+	# delete an entire source tree along with its builds:
+	same_pjhash mhash "$tree" badhash "$datadir" \
+	    "$configdir/$tree" "$mdir" || \
+		x_ rm -Rf "src/$project/$tree" "elf/$project/$tree"
 
-	git_prep "$url" "$bkup_url" "$xbmkpwd/$configdir/$tree/patches" \
-	    "src/$project/$tree" "submod"
+	x_ cp "$xbtmp/new.hash" "$XBMK_CACHE/mhash/$project$tree"
+
+	if [ -n "$target" ] && [ "$target" != "$tree" ]; then
+		# delete only target builds, if only the target changed
+		same_pjhash mtghash "$target" badtghash "$configdir/$target" \
+		    || x_ rm -Rf "elf/$project/$tree/$target"
+
+		x_ cp "$xbtmp/new.hash" "$XBMK_CACHE/mtghash/$project$target"
+	fi
 }
 
-single_tree_fetch()
+same_pjhash()
 {
-	xgcctree=""
+	eval "`newvar hash old_hash`"
+
+	hashdir="$1"
+	hashname="$2"
+	badhashvar="$3"
 
-	. "config/git/$project/pkg.cfg" || \
-	    err "Can't read config 'config/git/$project/pkg.cfg'" \
-	    "single_tree_fetch" "@"
+	shift 3
 
-	if [ -z "$url" ] || [ -z "$bkup_url" ]; then
-		err "url/bkup_url not both set 'config/git/$project/pkg.cfg'" \
-		    "single_tree_fetch" "$@"
+	x_ mkdir -p "$XBMK_CACHE/$hashdir"
+
+	if [ -f "$XBMK_CACHE/$hashdir/$project$hashname" ]; then
+		read -r old_hash < "$XBMK_CACHE/$hashdir/$project$hashname" \
+		    || err \
+		    "$hashdir: err '$XBMK_CACHE/$hashdir/$project$hashname'" \
+		    same_pjhash "$hashdir" "$hashname" "$badhashvar" "$@"
 	fi
 
-	[ -n "$xgcctree" ] && \
-		x_ "$mk" -f coreboot "$xgcctree"
+	eval "$build_sbase"
+	fx_ "x_ util/sbase/sha512sum" find "$@" -type f -not -path \
+	    "*/.git*/*" | awk '{print $1}' > "$xbtmp/tmp.hash" || \
+	    err "!h $project $hashdir" \
+	    "same_pjhash" "$hashdir" "$hashname" "$badhashvar" "$@"
+
+	hash="$(x_ "$sha512sum" "$xbtmp/tmp.hash" | awk '{print $1}' || \
+	    err)" || err "$hashname: Can't read sha512 of '$xbtmp/tmp.hash'" \
+	    "same_pjhash" "$hashdir" "$hashname" "$badhashvar" "$@"
+
+	if [ "$hash" != "$old_hash" ] || \
+	    [ ! -f "$XBMK_CACHE/$hashdir/$project$hashname" ]; then
+		eval "$badhashvar=\"y\""
+	fi
 
-	[ -n "$depend" ] && \
-		for d in $depend ; do
-			x_ "$mk" -f $d
-		done
+	printf "%s\n" "$hash" > "$xbtmp/new.hash" || \
+	    err "!mkhash $xbtmp/new.hash ($hashdir $hashname $badhashvar)" \
+	    "same_pjhash" "$hashdir" "$hashname" "$badhashvar" "$@"
 
-	$if_single_tree \
-		clone_project; :
+	eval "[ \"\$$badhashvar\" = \"y\" ] && return 1"; :
 }
 
-clone_project()
+fetch_trees()
 {
-	loc="src/$project"
-	[ -d "$loc" ] && return 0
+	[ -d "src/$project/$tree" ] && return 0
 
-	remkdir "${tmpgit%/*}"
-	git_prep "$url" "$bkup_url" "$xbmkpwd/config/$project/patches" "$loc"
+	mk_dependencies -f $fetch_depend
+
+	git_prep "$url" "$bkup_url" "$xbmkpwd/$configdir/$tree/patches" \
+	    "src/$project/$tree"
 }
 
 git_prep()
@@ -59,12 +90,10 @@ git_prep()
 	_loc="$4" # $1 and $2 are gitrepo and gitrepo_backup
 
 	[ -z "$rev" ] && \
-		err "$project/$tree: rev not set" "git_prep" "$@"
+		err "$project/$tree: rev unset" "git_prep" "$@"
 
 	xbget git "$1" "$2" "$tmpgit" "$rev" "$_patchdir"
-	if singletree "$project" || [ $# -gt 4 ]; then
-		dx_ fetch_submodule "$mdir/module.list"
-	fi
+	dx_ fetch_submodule "$mdir/module.list"
 
 	[ "$_loc" != "${_loc%/*}" ] && \
 		x_ mkdir -p "${_loc%/*}"
diff --git a/include/fw/inject.sh b/include/fw/inject.sh
index 98502d3c..2d4c99e6 100644
--- a/include/fw/inject.sh
+++ b/include/fw/inject.sh
@@ -4,7 +4,7 @@
 # Copyright (c) 2022 Ferass El Hafidi <vitali64pmemail@protonmail.com>
 # Copyright (c) 2023-2026 Leah Rowe <leah@libreboot.org>
 
-cbcfgsdir="config/coreboot"
+cbcfgsdir="config/module/coreboot"
 tmpromdel="$XBMK_CACHE/DO_NOT_FLASH"
 nvmutil="util/nvmutil/nvmutil"
 ifdtool="elf/coreboot/default/ifdtool"
diff --git a/include/fw/mrc.sh b/include/fw/mrc.sh
index 10d7e437..19003af7 100644
--- a/include/fw/mrc.sh
+++ b/include/fw/mrc.sh
@@ -24,7 +24,7 @@ extract_refcode()
 
 	# enable the Intel GbE device, if told by offset MRC_refcode_gbe
 	[ -n "$MRC_refcode_gbe" ] && \
-		x_ dd if="config/ifd/hp820g2/1.bin" of="$appdir/ref" bs=1 \
+		x_ dd if="config/snippet/1" of="$appdir/ref" bs=1 \
 		    seek=$MRC_refcode_gbe count=1 conv=notrunc; :
 
 	x_ mv "$appdir/ref" "$_pre_dest"
diff --git a/include/fw/rom.sh b/include/fw/rom.sh
index 4189ac6b..20ac2604 100644
--- a/include/fw/rom.sh
+++ b/include/fw/rom.sh
@@ -26,7 +26,8 @@ buildser()
 copyps1bios()
 {
 	remkdir "bin/playstation"
-	x_ cp src/pcsx-redux/src/mips/openbios/openbios.bin bin/playstation
+	x_ cp src/pcsx-redux/default/src/mips/openbios/openbios.bin \
+	    bin/playstation
 
 	printf "MIT License\n\nCopyright (c) 2019-2025 PCSX-Redux authors\n\n" \
 	    > bin/playstation/COPYING.txt || \
@@ -59,7 +60,7 @@ corebootpremake()
 	[ "$XBMK_RELEASE" = "y" ] && [ "$release" = "n" ] && \
 		return 0
 
-	fx_ check_coreboot_util printf "cbfstool\nifdtool\n"
+	forx 'check_coreboot_util %s' cbfstool ifdtool
 
 	printf "%s\n" "${version%%-*}" > "$srcdir/.coreboot-version" || \
 	    err "!mk $srcdir .coreboot-version" "corebootpremake" "$@"
@@ -70,8 +71,6 @@ corebootpremake()
 
 check_coreboot_util()
 {
-	[ "$badhash" = "y" ] && \
-		x_ rm -f "elf/coreboot/$tree/$1"
 	e "elf/coreboot/$tree/$1" f && \
 		return 0
 
@@ -111,10 +110,8 @@ mkcorebootbin()
 	[ "$XBMK_RELEASE" = "y" ] && [ "$release" = "n" ] && \
 		return 0
 
-	forx 'check_coreboot_util %s' cbfstool ifdtool
-
-	for y in "$target_dir/config"/*; do
-		defconfig="$y"
+	for y in "$defscan"/*; do
+		buildcfg="$y"
 		mkcorebootbin_real
 	done
 
@@ -127,7 +124,7 @@ mkcorebootbin_real()
 
 	tmprom="$xbtmp/coreboot.rom"
 
-	initmode="${defconfig##*/}"
+	initmode="${buildcfg##*/}"
 	displaymode="${initmode##*_}"
 	if [ "$displaymode" = "$initmode" ]; then
 		# blank it for "normal" or "fspgop" configs:
@@ -138,7 +135,7 @@ mkcorebootbin_real()
 	cbfstool="elf/coreboot/$tree/cbfstool"
 
 	# cbfstool option backends, if they exist
-	cbfscfg="config/coreboot/$target/cbfs.cfg"
+	cbfscfg="config/module/coreboot/$target/cbfs.cfg"
 
 	elfrom="elf/coreboot/$tree/$target/$initmode"
 	[ -n "$displaymode" ] && \
@@ -183,7 +180,7 @@ mkcorebootbin_real()
 	[ -f "$cbfscfg" ] && \
 		dx_ add_cbfs_option "$cbfscfg"
 
-	if grep "CONFIG_PAYLOAD_NONE=y" "$defconfig"; then
+	if grep "CONFIG_PAYLOAD_NONE=y" "$buildcfg"; then
 		if [ "$payload_seabios" = "y" ]; then
 			pname="seabios"
 			add_seabios
@@ -252,7 +249,9 @@ add_seabios()
 	fi
 
 	[ "$payload_memtest" = "y" ] && \
-		cbfs "$tmprom" "elf/memtest86plus/memtest.bin" img/memtest
+		cbfs "$tmprom" \
+		    "elf/memtest86plus/default/default/noconfig/memtest.bin" \
+		    img/memtest
 
 	[ "$payload_grub" = "y" ] && \
 		add_grub
@@ -377,6 +376,9 @@ cbfs()
 		lzma="-c lzma -l 0x1110000 -e 0x1110000"
 	fi
 
+	if [ $# -lt 2 ]; then
+		echo TEST
+	fi
 	x_ "$cbfstool" "$1" $ccmd -f "$2" -n "$3" $lzma
 }
 
diff --git a/include/fw/vendor.sh b/include/fw/vendor.sh
index 95cebb66..74946ac0 100644
--- a/include/fw/vendor.sh
+++ b/include/fw/vendor.sh
@@ -13,12 +13,12 @@
 # with inject.sh concatenated onto vendor.sh; they are inexorably intertwined.
 # The main "mk" script sources vendor.sh first, and then inject.sh, in lbmk.
 
-e6400_unpack="$xbmkpwd/src/bios_extract/dell_inspiron_1100_unpacker.py"
+e6400_unpack="$xbmkpwd/src/bios_extract/default/dell_inspiron_1100_unpacker.py"
 me7updateparser="$xbmkpwd/util/me7_update_parser/me7_update_parser.py"
-pfs_extract="$xbmkpwd/src/biosutilities/Dell_PFS_Extract.py"
-uefiextract="$xbmkpwd/elf/uefitool/uefiextract"
-bsdtar="$xbmkpwd/elf/libarchive/bsdtar"
-bsdunzip="$xbmkpwd/elf/libarchive/bsdunzip"
+pfs_extract="$xbmkpwd/src/biosutilities/default/Dell_PFS_Extract.py"
+uefiextract="$xbmkpwd/elf/uefitool/default/default/noconfig/uefiextract"
+bsdtar="$xbmkpwd/elf/libarchive/default/default/noconfig/bsdtar"
+bsdunzip="$xbmkpwd/elf/libarchive/default/default/noconfig/bsdunzip"
 vendir="vendorfiles"
 appdir="$vendir/app"
 vfix="DO_NOT_FLASH_YET._FIRST,_INJECT_FILES_VIA_INSTRUCTIONS_ON_LIBREBOOT.ORG_"
@@ -260,7 +260,7 @@ find_me()
 
 apply_deguard_hack()
 {
-	x_ cd src/deguard
+	x_ cd src/deguard/default
 
 	x_ ./finalimage.py --delta "data/delta/$ME11delta" \
 	     --version "$ME11version" --pch "$ME11pch" --sku "$ME11sku" \
@@ -414,7 +414,7 @@ bootstrap()
 
 	mecleaner="$xbmkpwd/$cbdir/util/me_cleaner/me_cleaner.py"
 	[ "$XBMKmecleaner" = "y" ] && \
-		mecleaner="$xbmkpwd/src/me_cleaner/me_cleaner.py"
+		mecleaner="$xbmkpwd/src/me_cleaner/default/me_cleaner.py"
 
 	x_ "$mk" -f coreboot "${cbdir##*/}"
 	x_ "$mk" -f me_cleaner
diff --git a/include/lib.sh b/include/lib.sh
index b410a173..dde99065 100644
--- a/include/lib.sh
+++ b/include/lib.sh
@@ -82,18 +82,12 @@ e()
 	printf "%s %s\n" "$1" "$es2" 1>&2
 }
 
-singletree()
-{
-	( fx_ "eval exit 1 && err" find "config/$1/"*/ -type f \
-	    -name "target.cfg" ) || return 1; :
-}
-
 pad_one_byte()
 {
 	paddedfile="`mktemp || err "mktemp pad_one_byte"`" || \
 	    err "can't make tmp file" "pad_one_byte" "$@"
 
-	x_ cat "$1" config/data/coreboot/0 > "$paddedfile" || \
+	x_ cat "$1" "config/snippet/0" > "$paddedfile" || \
 	    err "could not pad file '$paddedfile'" "pad_one_byte" "$1"; :
 
 	x_ mv "$paddedfile" "$1"
diff --git a/include/mk/release.sh b/include/mk/release.sh
index 9063a4d4..f9f185c3 100644
--- a/include/mk/release.sh
+++ b/include/mk/release.sh
@@ -55,13 +55,13 @@ prep_release_src()
 
 	forx 'fx_ \"x_ rm -Rf\" x_ find . -name \".%s\"' git gitmodules
 
-	( fx_ nuke x_ find config -type f -name "nuke.list" ) || \
+	( fx_ nuke x_ find config/module -type f -name "nuke.list" ) || \
 	    err "can't prune project files" "prep_release_src" "$@"; :
 }
 
 nuke()
 {
-	r="$rsrc/src/${1#config/}"
+	r="$rsrc/src/${1#config/module/}"
 
 	if [ -d "${r%/*}" ]; then
 		x_ cd "${r%/*}"