handle project downloads in main lbmk script

this means the unified /tmp handling is now provided for
in both the former "fetch" and "fetch_trees" script, which
are now (respectively):

./update project repo
./update project trees

if the fetch scripts weren't cleaning /tmp before, they
now are, because lbmk handles it

Signed-off-by: Leah Rowe <leah@libreboot.org>

13 files changed, 339 insertions, 15 deletions

diff --git a/script/build/coreboot/utils b/script/build/coreboot/utils
index bf0c47a7..f7af680b 100755
--- a/script/build/coreboot/utils
+++ b/script/build/coreboot/utils
@@ -55,7 +55,7 @@ build_for_mainboard() {
 buildutils() {
 	tree="${1}"
 	[ -d "coreboot/${tree}/" ] || \
-		./fetch_trees coreboot $tree || \
+		./update project trees coreboot $tree || \
 		    err "buildutils: cannot fetch ${tree}"
 	for util in cbfstool ifdtool; do
 		[ -f "cbutils/${tree}/${util}" ] && continue
diff --git a/script/build/grub/payload b/script/build/grub/payload
index f8a49b64..359fc1f7 100755
--- a/script/build/grub/payload
+++ b/script/build/grub/payload
@@ -72,7 +72,7 @@ main()
 handle_dependencies()
 {
 	[ -d "grub/" ] || \
-		./fetch grub || \
+		./update project repo grub || \
 		    err "handle_dependencies: cannot fetch grub"
 	[ -f "grub/grub-mkstandalone" ] || \
 		./build grub utils || \
diff --git a/script/build/grub/utils b/script/build/grub/utils
index 3a282f2e..cb80d15d 100755
--- a/script/build/grub/utils
+++ b/script/build/grub/utils
@@ -27,7 +27,7 @@ set -u -e
 main()
 {
 	printf "Building GRUB\n"
-	[ -d "grub/" ] || ./fetch grub || err "cannot fetch grub"
+	[ -d "grub/" ] || ./update project repo grub || err "cannot fetch grub"
 	build_grub
 }
 
diff --git a/script/build/release/roms b/script/build/release/roms
index fa27c031..ad789477 100755
--- a/script/build/release/roms
+++ b/script/build/release/roms
@@ -145,7 +145,7 @@ strip_archive()
 	romdir=${1}
 
 	[ -d "coreboot/${tree}" ] || \
-		./fetch_trees coreboot ${tree} || \
+		./update project trees coreboot ${tree} || \
 		    err "strip_archive: coreboot/${tree}: can't fetch source"
 	./build coreboot utils ${tree} || \
 	    err "strip_archive: coreboot/${tree}: can't build utils"
diff --git a/script/build/release/src b/script/build/release/src
index d72347c3..3067440b 100755
--- a/script/build/release/src
+++ b/script/build/release/src
@@ -85,11 +85,11 @@ create_release_directory()
 download_modules()
 {
 	for modname in ${trees_fetch_list}; do
-		[ -d "${modname}" ] || ./fetch_trees ${modname} || \
+		[ -d "${modname}" ] || ./update project trees ${modname} || \
 		    err "download_modules: couldn't download ${modname} trees"
 	done
 	for modname in ${simple_fetch_list}; do
-		[ -d "${modname}/" ] || ./fetch ${modname} || \
+		[ -d "${modname}/" ] || ./update project repo ${modname} || \
 		    err "download_modules: couldn't download ${modname} repo"
 	done
 }
diff --git a/script/handle/make/config b/script/handle/make/config
index 5af7e9e7..e7016c85 100755
--- a/script/handle/make/config
+++ b/script/handle/make/config
@@ -139,7 +139,7 @@ handle_dependencies()
 
 	codedir="${project}/${tree}"
 	[ -d "${codedir}" ] || \
-		./fetch_trees "${project}" "${target}" || \
+		./update project trees "${project}" "${target}" || \
 		    fail "handle_dependencies: can't fetch ${project}/${target}"
 
 	# u-boot and coreboot are both compiled with coreboot's crossgcc
@@ -166,7 +166,7 @@ check_cross_compiler()
 	# only true if not building coreboot:
 	ctarget="${cbdir#coreboot/}"
 	[ -d "${cbdir}" ] || \
-		./fetch_trees coreboot ${ctarget} || \
+		./update project trees coreboot ${ctarget} || \
 		    fail "check_cross_compiler: can't fetch coreboot/${ctarget}"
 
 	if [ "${arch}" = "x86_32" ] || [ "${arch}" = "x86_64" ]; then
diff --git a/script/handle/make/file b/script/handle/make/file
index 855ea1c4..dcb2fc6b 100755
--- a/script/handle/make/file
+++ b/script/handle/make/file
@@ -52,7 +52,7 @@ main()
 
 handle_dependencies()
 {
-	[ -d "${project}" ] || ./fetch "${project%/*}" || \
+	[ -d "${project}" ] || ./update project repo "${project%/*}" || \
 	    err "handle_dependencies: can't fetch ${project%/*}"
 	[ -d "${project}" ] || \
 		err "handle_dependencies: ${project%/*} not downloaded"
diff --git a/script/update/blobs/download b/script/update/blobs/download
index 1d18cf7e..f49ab79f 100755
--- a/script/update/blobs/download
+++ b/script/update/blobs/download
@@ -157,11 +157,11 @@ EOF
 build_dependencies()
 {
 	[ -d ${cbdir} ] || \
-		./fetch_trees coreboot ${cbdir##*/} || \
+		./update project trees coreboot ${cbdir##*/} || \
 		    err "build_dependencies: can't fetch ${cbdir}"
 	for d in uefitool biosutilities bios_extract me_cleaner; do
 		[ -d "${d}" ] && continue
-		./fetch "${d}" || \
+		./update project repo "${d}" || \
 		    err "build_dependencies: can't fetch ${d}"
 	done
 	[ -f uefitool/uefiextract ] || \
diff --git a/script/update/blobs/extract b/script/update/blobs/extract
index c62700f2..ad77abd4 100755
--- a/script/update/blobs/extract
+++ b/script/update/blobs/extract
@@ -56,10 +56,10 @@ check_board()
 build_dependencies()
 {
 	if [ ! -d me_cleaner ]; then
-		./fetch me_cleaner || \
+		./update project repo me_cleaner || \
 		    err "build_dependencies: can't fetch me_cleaner"
 	elif [ ! -d "${cbdir}" ]; then
-		./fetch_trees coreboot default || \
+		./update project trees coreboot default || \
 		    err "build_dependencies: can't fetch coreboot"
 	elif [ ! -f "${ifdtool}" ]; then
 		make -C "${ifdtool%/ifdtool}" || \
diff --git a/script/update/blobs/inject b/script/update/blobs/inject
index 08bbf41a..7efab2a9 100755
--- a/script/update/blobs/inject
+++ b/script/update/blobs/inject
@@ -123,7 +123,8 @@ detect_board()
 
 build_dependencies()
 {
-	[ -d "${cbdir}" ] || ./fetch_trees coreboot default
+	[ -d "${cbdir}" ] || ./update project trees coreboot default || \
+	    err "build_dependencies: could not download coreboot/default"
 	./build coreboot utils default || \
 	    err "build_dependencies: could not build cbutils"
 	./update blobs download ${board} || \
diff --git a/script/update/blobs/mrc b/script/update/blobs/mrc
index c069e678..420fa42f 100755
--- a/script/update/blobs/mrc
+++ b/script/update/blobs/mrc
@@ -71,7 +71,7 @@ check_existing()
 
 build_dependencies()
 {
-	[ -d "${cbdir}/" ] || ./fetch_trees coreboot default || \
+	[ -d "${cbdir}/" ] || ./update project trees coreboot default || \
 	    err "build_dependencies: cannot fetch coreboot/default"
 	./build coreboot utils default || \
 	    err "build_dependencies: cannot build cbutils/default"
diff --git a/script/update/project/repo b/script/update/project/repo
new file mode 100755
index 00000000..270d6cb9
--- /dev/null
+++ b/script/update/project/repo
@@ -0,0 +1,124 @@
+#!/usr/bin/env sh
+
+# SPDX-FileCopyrightText: 2022 Caleb La Grange <thonkpeasant@protonmail.com>
+# SPDX-FileCopyrightText: 2022 Ferass El Hafidi <vitali64pmemail@protonmail.com>
+# SPDX-FileCopyrightText: 2023 Leah Rowe <leah@libreboot.org>
+# SPDX-License-Identifier: GPL-3.0-only
+
+. "include/err.sh"
+
+name=""
+revision=""
+location=""
+url=""
+bkup_url=""
+tmp_dir=""
+depend=""
+
+main()
+{
+	id -u 1>/dev/null 2>/dev/null || \
+	    fail "cannot ascertain user id"
+	if [ "$(id -u)" = "0" ]; then
+		fail "running lbmk as root is not permitted"
+	fi
+	./checkgit || err "Please read: https://libreboot.org/docs/build/"
+
+	[ $# -gt 0 ] || fail "no argument given"
+
+	[ -z "${1+x}" ] && fail 'main(): name not set'
+	name=${1}
+
+	read_config
+	verify_config
+
+	clone_project
+	[ "${depend}" = "" ] || ./fetch ${depend} || \
+	    fail "Cannot fetch dependency, ${depend}, for project, ${name}"
+
+	rm -Rf "${tmp_dir}" || fail "cannot remove tmpdir, ${tmp_dir}"
+}
+
+read_config()
+{
+	awkstr=" /\{.*${name}.*}{/ {flag=1;next} /\}/{flag=0} flag { print }"
+	while read -r line ; do
+		set ${line} || fail "read_config: set line"
+		case ${line} in
+		rev:*)
+			revision=${2} ;;
+		loc:*)
+			location=${2} ;;
+		url:*)
+			url=${2} ;;
+		bkup_url:*)
+			bkup_url=${2} ;;
+		depend:*)
+			depend=${2} ;;
+		esac
+	done << EOF
+	$(eval "awk '${awkstr}' resources/git/revisions")
+EOF
+}
+
+verify_config()
+{
+	[ -z "${revision+x}" ] && fail 'verify_config: revision not set'
+	[ -z "${location+x}" ] && fail 'verify_config: location not set'
+	[ -z "${url+x}" ] && fail 'verify_config: url not set'
+}
+
+clone_project()
+{
+	tmp_dir=$(mktemp -dt "${name}_XXXXX")
+
+	git clone ${url} "${tmp_dir}" || git clone ${bkup_url} "${tmp_dir}" || \
+	    fail "clone_project: could not download ${name}"
+	(
+	cd "${tmp_dir}" || fail "clone_project: tmpdir not created"
+	git reset --hard ${revision} || \
+	    fail "clone_project: Cannot reset revision"
+	)
+	patch_project
+
+	[ ! -d "${location}" ] || \
+		rm -Rf "${location}" || \
+		    fail "clone_project: Can't remove directory '${location}'"
+	mv "${tmp_dir}" "${location}" || \
+	    fail "clone_project: could not copy temp file to destination"
+}
+
+patch_project()
+{
+	patchdir="resources/${name}/patches"
+
+	for patchfile in "${PWD}/${patchdir}"/*.patch ; do
+		[ -f "${patchfile}" ] || continue
+		(
+		cd "${tmp_dir}" || fail "patch_project: tmpdir unavailable"
+		git am "${patchfile}" || \
+		    fail "patch_project: Cannot patch project: $name"
+		)
+	done
+}
+
+fail()
+{
+	for x in "${location}" "${tmp_dir}"; do
+		[ -z "${x}" ] || [ ! -d "${x}" ] || rm -Rf "${location}" || :
+	done
+	usage
+	err "${1}"
+}
+
+usage()
+{
+	cat <<- EOF
+	Usage: ./fetch [name]
+
+	Options:
+	name: Module name as specified in resources/git/revisions
+	EOF
+}
+
+main $@
diff --git a/script/update/project/trees b/script/update/project/trees
new file mode 100755
index 00000000..20bc1587
--- /dev/null
+++ b/script/update/project/trees
@@ -0,0 +1,199 @@
+#!/usr/bin/env sh
+
+#  helper script: create code trees based on git revision, with patches
+#  (currently used for downloading coreboot, seabios and u-boot)
+#
+#	Copyright (C) 2014-2016,2020,2021,2023 Leah Rowe <info@minifree.org>
+#	Copyright (C) 2022 Alper Nebi Yasak <alpernebiyasak@gmail.com>
+#	Copyright (C) 2022 Ferass El Hafidi <vitali64pmemail@protonmail.com>
+#
+#	This program is free software: you can redistribute it and/or modify
+#	it under the terms of the GNU General Public License as published by
+#	the Free Software Foundation, either version 3 of the License, or
+#	(at your option) any later version.
+#
+#	This program is distributed in the hope that it will be useful,
+#	but WITHOUT ANY WARRANTY; without even the implied warranty of
+#	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#	GNU General Public License for more details.
+#
+#	You should have received a copy of the GNU General Public License
+#	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+[ "x${DEBUG+set}" = 'xset' ] && set -v
+set -u -e
+
+. "include/err.sh"
+
+_target=""
+tree=""
+rev=""
+project=""
+cfgsdir=""
+
+main()
+{
+	id -u 1>/dev/null 2>/dev/null || \
+	    err "cannot ascertain user id"
+	if [ "$(id -u)" = "0" ]; then
+		err "running lbmk as root is not permitted"
+	fi
+	./checkgit || err "Please read: https://libreboot.org/docs/build/"
+
+	rm -f "${cfgsdir}"/*/seen || err_rm_seen "main 1"
+
+	printf "Downloading %s and (if available) applying patches\n" \
+	    ${project}
+
+	[ -z "${1}" ] && err "project name not specified"
+	project="${1}"
+	cfgsdir="resources/${project}"
+	[ -d "${cfgsdir}" ] || err "unsupported project name"
+	shift 1
+
+	targets=$(./build command options "${cfgsdir}")
+	[ $# -gt 0 ] && targets=$@
+	[ -z "${targets}" ] && \
+	    err "No targets available for project: ${project}"
+
+	for x in ${targets}; do
+		rm -f "${cfgsdir}"/*/seen || err_rm_seen "main 2"
+		download_for_target "${x}" || \
+		    err "${project}/${target}: cannot download source tree"
+	done
+
+	rm -f "${cfgsdir}"/*/seen || err_rm_seen "main 3"
+}
+
+download_for_target()
+{
+	_target="${1}"
+	tree="undefined"
+	rev="undefined"
+
+	fetch_config "${_target}" || \
+	    err "download_for_target: ${project}/${_target}: bad target.cfg"
+
+	rm -f "${cfgsdir}"/*/seen || err_rm_seen "download_for_target"
+
+	if [ -d "${project}/${tree}" ]; then
+		printf "REMARK: download/%s %s: exists. Skipping.\n" \
+		    "${project}" "${tree}" 1>&2
+		[ "${tree}" != "${_target}" ] && \
+			printf "(for target: '%s}')\n" "${_target}" 1>&2
+		return 0
+	fi
+
+	fetch_from_upstream || \
+	    err "download_for_target: cannot fetch: ${project}"
+	prepare_new_tree "${_target}" "${tree}" "${rev}" || \
+	    err "download_for_target: cannot create tree: ${project}/${tree}"
+}
+
+fetch_config()
+{
+	_target=${1}
+
+	while true; do
+		rev="undefined"
+		tree="undefined"
+
+		check_config_for_target "${_target}" || return 1
+
+		# This is to override $rev and $tree
+		. "${cfgsdir}/${_target}/target.cfg" || \
+		    err "fetch_config: no \"${cfgsdir}/${_target}/target.cfg\""
+
+		if [ "${_target}" != "${tree}" ]; then
+			_target="${tree}"
+			continue
+		elif [ "${tree}" = "undefined" ]; then
+			printf "ERROR (fetch_config): download/%s:" 1>&2
+			printf " tree name undefined for '%s\n'" \
+			    "${project}" "${_target}" 1>&2
+			return 1
+		elif [ "${rev}" = "undefined" ]; then
+			printf "ERROR (fetch_config): download/%s:" 1>&2
+			printf " commit ID undefined for '%s'\n" \
+			    "${project}" "${_target}" 1>&2
+			return 1
+		else
+			break
+		fi
+	done
+}
+
+check_config_for_target()
+{
+	_target=${1}
+
+	if [ ! -f "${cfgsdir}/${_target}/target.cfg" ]; then
+		printf "ERROR: download/%s: target.cfg does not" \
+		    "${project}" 1>&2
+		printf " exist for '%s'\n" "${_target}" 1>&2
+		return 1
+	elif [ -f "${cfgsdir}/${_target}/seen" ]; then
+		printf "ERROR: download/%s: logical loop:" "${project}" 1>&2
+		printf " '%s' target.cfg refers to another tree," "${_target}" \
+		    1>&2
+		printf " which ultimately refers back to '%s'." "${_target}" \
+		    1>&2
+		return 1
+	fi
+	touch "${cfgsdir}/${_target}/seen" || \
+	    err "${project}/${_target}: touch \"${cfgsdir}/${_target}/seen\""
+}
+
+fetch_from_upstream()
+{
+	[ -d "${project}" ] || mkdir -p "${project}" || return 1
+	[ -d "${project}" ] || return 1
+	[ -d "${project}/${project}" ] && return 0
+
+	./fetch ${project} || return 1
+}
+
+prepare_new_tree()
+{
+	target=${1}
+	tree=${2}
+	rev=${3}
+
+	printf "Preparing %s tree: %s\n" ${project} ${tree}
+	[ "${tree}" != "${target}" ] && \
+		printf "(for target, %s)\n" "${target}"
+
+	cp -R "${project}/${project}" "${project}/${tree}" || \
+	    err "${project}/${tree}: cannot copy source tree"
+	(
+	cd "${project}/${tree}" || err "cannot cd to ${project}/${tree}"
+	git reset --hard ${rev} || \
+	    err "cannot reset ${project} revision for tree, ${tree}"
+	git submodule update --init --checkout || \
+	    err "cannot update ${project} submodules for tree, ${tree}"
+
+	for patch in "../../${cfgsdir}/${tree}/patches/"*.patch; do
+		[ -f "${patch}" ] || continue
+		if ! git am "${patch}"; then
+			git am --abort || \
+			    err "${project}/${tree}: FAILED: git am --abort"
+			err "cannot patch: ${project}/${tree}"
+		fi
+	done
+
+	# extra.sh can be used for anything
+	# but should *only* be a last resort
+	if [ -f "../../${cfgsdir}/${tree}/extra.sh" ]; then
+		"../../${cfgsdir}/${tree}/extra.sh" || \
+		    err "prepare_new_tree ${project}/${tree}: extra.sh: error"
+	fi
+	)
+}
+
+err_rm_seen()
+{
+	err "${1}: ${project}/${target}: cannot rm: \"${cfgsdir}/*/seen\""
+}
+
+main $@