From 076c87ccf68f7fe5a3998e8cd8edd2033a2f2ef0 Mon Sep 17 00:00:00 2001 From: Leah Rowe Date: Fri, 18 Sep 2026 21:04:00 +0100 Subject: xbmk: make all projects multi-tree we currently distinguish, by virtue of logic, multi- and single-tree projects. this introduces a lot of complexity, so the following patch has removed this distinction. all single-tree projects are now multi tree, but with only one tree. other design changes: config/git removed. rev/tree now handled with mkhelper and target configs. this allows per-tree remote handling, so we could (for example) add the mrchromebox coreboot fork easily, for specific chromebook models. config/project is now config/module/project. this cleans up the main config/ directory. cfgname variable: .config no longer hardcoded when doing configs in projects. it is configurable with this variable. if unset, the previous .cfg is used noconfig variable: set this in target/mkhelper configs, and config/module/project/tree/config/ is unused; instead, make is called without config copying. a "noconfig" placeholder is used. this enables cleaner usage of previously single-tree projects in the new multi-tree-only structure. cfgvars: variables reinitialised in project config are cleaned more reliably. in particular, several variables e.g. autogenargs were not being reset between projects, although the autogenargs variable is currently unused. general cleanup has been conducted across the build system, as part of this redesign. several parts of the build system more over-engineered as a result of the previous design, and the new design enabled a severe reduction in sloccount (75 lines reduced in lbmk). yes, this change *adds features* but there are fewer lines of code than before. this is the best kind of change. way better than the 3-line-diff. yes. rev= and tree= are set more centrally in configs. the mkhelper config can be used to set url/url_bkup, but individual trees can override them. configs have been cleaned up, with several variables now much more heavily centralised. config/git/ was used before to first download the bare repository, and then the real clone was made with patches. this allows git repositories to be cached for future re-builds. with config/git/ gone, the download logic for projects is more centrally implemented. a separate call to ./mk -f is no longer performed, and in fact ./mk -f never needs to be used anymore as a result. ./mk -f now is basically the same thing as -b, just without any action performed. this change will allow rapid expansion of the libreboot project. i want to add hundreds of chromebooks to the project, using mrchromebox, but the previous design did not permit this. this change means that i can easily have unlimited upstreams per project. coreboot will, in some follow-up changes, have the following upstreams: coreboot/default: coreboot.org coreboot/fam15h: 15h.org fork coreboot/chromebook: mrchromebox fork this can now be done freely without massive changes to the build system. under the previous design, i worked on a now-abandoned patch to do N remotes per project, reading repo links from a file. this meant that i'd have coreboot.org first (main and backup), then mrchromebox. this meant that lbmk would failover through to mrchromebox, wasting time for the user. this new design is much cleaner, and will enable the project to be maintained much more easily. Signed-off-by: Leah Rowe --- include/env/get.sh | 91 +++++++++++++++++++++++++++++++++------------------ include/fw/inject.sh | 2 +- include/fw/mrc.sh | 2 +- include/fw/rom.sh | 26 ++++++++------- include/fw/vendor.sh | 14 ++++---- include/lib.sh | 8 +---- include/mk/release.sh | 4 +-- 7 files changed, 86 insertions(+), 61 deletions(-) (limited to 'include') diff --git a/include/env/get.sh b/include/env/get.sh index 09126297..0ed9d6cc 100644 --- a/include/env/get.sh +++ b/include/env/get.sh @@ -3,52 +3,83 @@ # Copyright (c) 2020-2021,2023-2026 Leah Rowe # Copyright (c) 2022 Caleb La Grange -eval "`newvar url bkup_url depend loc subcurl subcurl_bkup subgit subgit_bkup \ - subhash`" +eval "`newvar url bkup_url fetch_depend loc subcurl subcurl_bkup subgit \ + subgit_bkup subhash badhash badtghash`" tmpgit="$xbtmp/gitclone" tmpgitcache="$xbtmp/tmpgit" -multi_tree_fetch() +# delete a source tree and corresponding builds if metadata changed +# inside config/ - this is done by hashing a concatenation of hashes +# from all metadata, checked against a previously calculated hash + +delete_old_project_files() { - [ -d "src/$project/$tree" ] && return 0 + # delete an entire source tree along with its builds: + same_pjhash mhash "$tree" badhash "$datadir" \ + "$configdir/$tree" "$mdir" || \ + x_ rm -Rf "src/$project/$tree" "elf/$project/$tree" - git_prep "$url" "$bkup_url" "$xbmkpwd/$configdir/$tree/patches" \ - "src/$project/$tree" "submod" + x_ cp "$xbtmp/new.hash" "$XBMK_CACHE/mhash/$project$tree" + + if [ -n "$target" ] && [ "$target" != "$tree" ]; then + # delete only target builds, if only the target changed + same_pjhash mtghash "$target" badtghash "$configdir/$target" \ + || x_ rm -Rf "elf/$project/$tree/$target" + + x_ cp "$xbtmp/new.hash" "$XBMK_CACHE/mtghash/$project$target" + fi } -single_tree_fetch() +same_pjhash() { - xgcctree="" + eval "`newvar hash old_hash`" + + hashdir="$1" + hashname="$2" + badhashvar="$3" - . "config/git/$project/pkg.cfg" || \ - err "Can't read config 'config/git/$project/pkg.cfg'" \ - "single_tree_fetch" "@" + shift 3 - if [ -z "$url" ] || [ -z "$bkup_url" ]; then - err "url/bkup_url not both set 'config/git/$project/pkg.cfg'" \ - "single_tree_fetch" "$@" + x_ mkdir -p "$XBMK_CACHE/$hashdir" + + if [ -f "$XBMK_CACHE/$hashdir/$project$hashname" ]; then + read -r old_hash < "$XBMK_CACHE/$hashdir/$project$hashname" \ + || err \ + "$hashdir: err '$XBMK_CACHE/$hashdir/$project$hashname'" \ + same_pjhash "$hashdir" "$hashname" "$badhashvar" "$@" fi - [ -n "$xgcctree" ] && \ - x_ "$mk" -f coreboot "$xgcctree" + eval "$build_sbase" + fx_ "x_ util/sbase/sha512sum" find "$@" -type f -not -path \ + "*/.git*/*" | awk '{print $1}' > "$xbtmp/tmp.hash" || \ + err "!h $project $hashdir" \ + "same_pjhash" "$hashdir" "$hashname" "$badhashvar" "$@" + + hash="$(x_ "$sha512sum" "$xbtmp/tmp.hash" | awk '{print $1}' || \ + err)" || err "$hashname: Can't read sha512 of '$xbtmp/tmp.hash'" \ + "same_pjhash" "$hashdir" "$hashname" "$badhashvar" "$@" + + if [ "$hash" != "$old_hash" ] || \ + [ ! -f "$XBMK_CACHE/$hashdir/$project$hashname" ]; then + eval "$badhashvar=\"y\"" + fi - [ -n "$depend" ] && \ - for d in $depend ; do - x_ "$mk" -f $d - done + printf "%s\n" "$hash" > "$xbtmp/new.hash" || \ + err "!mkhash $xbtmp/new.hash ($hashdir $hashname $badhashvar)" \ + "same_pjhash" "$hashdir" "$hashname" "$badhashvar" "$@" - $if_single_tree \ - clone_project; : + eval "[ \"\$$badhashvar\" = \"y\" ] && return 1"; : } -clone_project() +fetch_trees() { - loc="src/$project" - [ -d "$loc" ] && return 0 + [ -d "src/$project/$tree" ] && return 0 - remkdir "${tmpgit%/*}" - git_prep "$url" "$bkup_url" "$xbmkpwd/config/$project/patches" "$loc" + mk_dependencies -f $fetch_depend + + git_prep "$url" "$bkup_url" "$xbmkpwd/$configdir/$tree/patches" \ + "src/$project/$tree" } git_prep() @@ -59,12 +90,10 @@ git_prep() _loc="$4" # $1 and $2 are gitrepo and gitrepo_backup [ -z "$rev" ] && \ - err "$project/$tree: rev not set" "git_prep" "$@" + err "$project/$tree: rev unset" "git_prep" "$@" xbget git "$1" "$2" "$tmpgit" "$rev" "$_patchdir" - if singletree "$project" || [ $# -gt 4 ]; then - dx_ fetch_submodule "$mdir/module.list" - fi + dx_ fetch_submodule "$mdir/module.list" [ "$_loc" != "${_loc%/*}" ] && \ x_ mkdir -p "${_loc%/*}" diff --git a/include/fw/inject.sh b/include/fw/inject.sh index 98502d3c..2d4c99e6 100644 --- a/include/fw/inject.sh +++ b/include/fw/inject.sh @@ -4,7 +4,7 @@ # Copyright (c) 2022 Ferass El Hafidi # Copyright (c) 2023-2026 Leah Rowe -cbcfgsdir="config/coreboot" +cbcfgsdir="config/module/coreboot" tmpromdel="$XBMK_CACHE/DO_NOT_FLASH" nvmutil="util/nvmutil/nvmutil" ifdtool="elf/coreboot/default/ifdtool" diff --git a/include/fw/mrc.sh b/include/fw/mrc.sh index 10d7e437..19003af7 100644 --- a/include/fw/mrc.sh +++ b/include/fw/mrc.sh @@ -24,7 +24,7 @@ extract_refcode() # enable the Intel GbE device, if told by offset MRC_refcode_gbe [ -n "$MRC_refcode_gbe" ] && \ - x_ dd if="config/ifd/hp820g2/1.bin" of="$appdir/ref" bs=1 \ + x_ dd if="config/snippet/1" of="$appdir/ref" bs=1 \ seek=$MRC_refcode_gbe count=1 conv=notrunc; : x_ mv "$appdir/ref" "$_pre_dest" diff --git a/include/fw/rom.sh b/include/fw/rom.sh index 4189ac6b..20ac2604 100644 --- a/include/fw/rom.sh +++ b/include/fw/rom.sh @@ -26,7 +26,8 @@ buildser() copyps1bios() { remkdir "bin/playstation" - x_ cp src/pcsx-redux/src/mips/openbios/openbios.bin bin/playstation + x_ cp src/pcsx-redux/default/src/mips/openbios/openbios.bin \ + bin/playstation printf "MIT License\n\nCopyright (c) 2019-2025 PCSX-Redux authors\n\n" \ > bin/playstation/COPYING.txt || \ @@ -59,7 +60,7 @@ corebootpremake() [ "$XBMK_RELEASE" = "y" ] && [ "$release" = "n" ] && \ return 0 - fx_ check_coreboot_util printf "cbfstool\nifdtool\n" + forx 'check_coreboot_util %s' cbfstool ifdtool printf "%s\n" "${version%%-*}" > "$srcdir/.coreboot-version" || \ err "!mk $srcdir .coreboot-version" "corebootpremake" "$@" @@ -70,8 +71,6 @@ corebootpremake() check_coreboot_util() { - [ "$badhash" = "y" ] && \ - x_ rm -f "elf/coreboot/$tree/$1" e "elf/coreboot/$tree/$1" f && \ return 0 @@ -111,10 +110,8 @@ mkcorebootbin() [ "$XBMK_RELEASE" = "y" ] && [ "$release" = "n" ] && \ return 0 - forx 'check_coreboot_util %s' cbfstool ifdtool - - for y in "$target_dir/config"/*; do - defconfig="$y" + for y in "$defscan"/*; do + buildcfg="$y" mkcorebootbin_real done @@ -127,7 +124,7 @@ mkcorebootbin_real() tmprom="$xbtmp/coreboot.rom" - initmode="${defconfig##*/}" + initmode="${buildcfg##*/}" displaymode="${initmode##*_}" if [ "$displaymode" = "$initmode" ]; then # blank it for "normal" or "fspgop" configs: @@ -138,7 +135,7 @@ mkcorebootbin_real() cbfstool="elf/coreboot/$tree/cbfstool" # cbfstool option backends, if they exist - cbfscfg="config/coreboot/$target/cbfs.cfg" + cbfscfg="config/module/coreboot/$target/cbfs.cfg" elfrom="elf/coreboot/$tree/$target/$initmode" [ -n "$displaymode" ] && \ @@ -183,7 +180,7 @@ mkcorebootbin_real() [ -f "$cbfscfg" ] && \ dx_ add_cbfs_option "$cbfscfg" - if grep "CONFIG_PAYLOAD_NONE=y" "$defconfig"; then + if grep "CONFIG_PAYLOAD_NONE=y" "$buildcfg"; then if [ "$payload_seabios" = "y" ]; then pname="seabios" add_seabios @@ -252,7 +249,9 @@ add_seabios() fi [ "$payload_memtest" = "y" ] && \ - cbfs "$tmprom" "elf/memtest86plus/memtest.bin" img/memtest + cbfs "$tmprom" \ + "elf/memtest86plus/default/default/noconfig/memtest.bin" \ + img/memtest [ "$payload_grub" = "y" ] && \ add_grub @@ -377,6 +376,9 @@ cbfs() lzma="-c lzma -l 0x1110000 -e 0x1110000" fi + if [ $# -lt 2 ]; then + echo TEST + fi x_ "$cbfstool" "$1" $ccmd -f "$2" -n "$3" $lzma } diff --git a/include/fw/vendor.sh b/include/fw/vendor.sh index 95cebb66..74946ac0 100644 --- a/include/fw/vendor.sh +++ b/include/fw/vendor.sh @@ -13,12 +13,12 @@ # with inject.sh concatenated onto vendor.sh; they are inexorably intertwined. # The main "mk" script sources vendor.sh first, and then inject.sh, in lbmk. -e6400_unpack="$xbmkpwd/src/bios_extract/dell_inspiron_1100_unpacker.py" +e6400_unpack="$xbmkpwd/src/bios_extract/default/dell_inspiron_1100_unpacker.py" me7updateparser="$xbmkpwd/util/me7_update_parser/me7_update_parser.py" -pfs_extract="$xbmkpwd/src/biosutilities/Dell_PFS_Extract.py" -uefiextract="$xbmkpwd/elf/uefitool/uefiextract" -bsdtar="$xbmkpwd/elf/libarchive/bsdtar" -bsdunzip="$xbmkpwd/elf/libarchive/bsdunzip" +pfs_extract="$xbmkpwd/src/biosutilities/default/Dell_PFS_Extract.py" +uefiextract="$xbmkpwd/elf/uefitool/default/default/noconfig/uefiextract" +bsdtar="$xbmkpwd/elf/libarchive/default/default/noconfig/bsdtar" +bsdunzip="$xbmkpwd/elf/libarchive/default/default/noconfig/bsdunzip" vendir="vendorfiles" appdir="$vendir/app" vfix="DO_NOT_FLASH_YET._FIRST,_INJECT_FILES_VIA_INSTRUCTIONS_ON_LIBREBOOT.ORG_" @@ -260,7 +260,7 @@ find_me() apply_deguard_hack() { - x_ cd src/deguard + x_ cd src/deguard/default x_ ./finalimage.py --delta "data/delta/$ME11delta" \ --version "$ME11version" --pch "$ME11pch" --sku "$ME11sku" \ @@ -414,7 +414,7 @@ bootstrap() mecleaner="$xbmkpwd/$cbdir/util/me_cleaner/me_cleaner.py" [ "$XBMKmecleaner" = "y" ] && \ - mecleaner="$xbmkpwd/src/me_cleaner/me_cleaner.py" + mecleaner="$xbmkpwd/src/me_cleaner/default/me_cleaner.py" x_ "$mk" -f coreboot "${cbdir##*/}" x_ "$mk" -f me_cleaner diff --git a/include/lib.sh b/include/lib.sh index b410a173..dde99065 100644 --- a/include/lib.sh +++ b/include/lib.sh @@ -82,18 +82,12 @@ e() printf "%s %s\n" "$1" "$es2" 1>&2 } -singletree() -{ - ( fx_ "eval exit 1 && err" find "config/$1/"*/ -type f \ - -name "target.cfg" ) || return 1; : -} - pad_one_byte() { paddedfile="`mktemp || err "mktemp pad_one_byte"`" || \ err "can't make tmp file" "pad_one_byte" "$@" - x_ cat "$1" config/data/coreboot/0 > "$paddedfile" || \ + x_ cat "$1" "config/snippet/0" > "$paddedfile" || \ err "could not pad file '$paddedfile'" "pad_one_byte" "$1"; : x_ mv "$paddedfile" "$1" diff --git a/include/mk/release.sh b/include/mk/release.sh index 9063a4d4..f9f185c3 100644 --- a/include/mk/release.sh +++ b/include/mk/release.sh @@ -55,13 +55,13 @@ prep_release_src() forx 'fx_ \"x_ rm -Rf\" x_ find . -name \".%s\"' git gitmodules - ( fx_ nuke x_ find config -type f -name "nuke.list" ) || \ + ( fx_ nuke x_ find config/module -type f -name "nuke.list" ) || \ err "can't prune project files" "prep_release_src" "$@"; : } nuke() { - r="$rsrc/src/${1#config/}" + r="$rsrc/src/${1#config/module/}" if [ -d "${r%/*}" ]; then x_ cd "${r%/*}" -- cgit v1.2.1