From c9696e233389f1f896dc70076cfc03f14f8a940a Mon Sep 17 00:00:00 2001
From: Leah Rowe <leah@libreboot.org>
Date: Mon, 12 May 2025 16:30:05 +0100
Subject: lib.sh: move xbmkget() to git.sh

in cbmk, it's only used from there.

in lbmk, it's also used from vendor.sh.

however, i plan to further expand git.sh at
some point, tidying it up so that git cloning
is also done from xbmkget, with dlop=git and
git.sh would then be renamed to get.sh

Signed-off-by: Leah Rowe <leah@libreboot.org>
---
 include/git.sh | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

(limited to 'include/git.sh')

diff --git a/include/git.sh b/include/git.sh
index bbc5140e..b65bd196 100644
--- a/include/git.sh
+++ b/include/git.sh
@@ -87,6 +87,56 @@ fetch_submodule()
 	    "$mdir/${1##*/}/patches"
 }
 
+# can grab from the internet, or copy locally.
+# if copying locally, it can only copy a file.
+xbmkget()
+{
+	_dlop="curl" && [ $# -gt 4 ] && _dlop="$5"
+	x_ mkdir -p "${3%/*}" "$XBMK_CACHE/file"
+	for url in "$1" "$2"; do
+		[ -n "$url" ] && try_file "$url" "$_dlop" "$@" && return 0
+	done && err "$1 $2 $3 $4: not downloaded"; :
+}
+
+try_file()
+{
+	cached="$XBMK_CACHE/file/$6"
+	dl_fail="n" # 1 url, 2 url backup, 3 destination, 4 checksum
+	bad_checksum "$6" "$cached" 2>/dev/null && dl_fail="y"
+	[ "$dl_fail" = "n" ] && e "$5" f && return 0
+
+	x_ rm -f "$cached"
+	[ "$2" = "curl" ] || [ "$2" = "copy" ] || \
+	    err "$3 $4 $5 $6: Unsupported dlop type: '$2'"
+
+	try_$2 "$cached" "$@" || return 1
+
+	bad_checksum "$6" "$cached" && return 1
+	[ "$cached" = "$5" ] || x_ cp "$cached" "$5"; :
+}
+
+try_curl()
+{
+	_ua="Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0"
+	curl --location --retry 3 -A "$_ua" "$2" -o "$1" || \
+	    wget --tries 3 -U "$_ua" "$2" -O "$1" || return 1; :
+}
+
+try_copy()
+{
+	[ -L "$2" ] && printf "dl %s %s %s %s: '%s' is a symlink\n" \
+	    "$4" "$5" "$6" "$7" "$2" 1>&2 && return 1
+	[ ! -f "$2" ] && printf "dl %s %s %s %s: '%s' not a file\n" \
+	    "$4" "$5" "$6" "$7" "$2" 1>&2 && return 1
+	cp "$2" "$1" || return 1; :
+}
+
+bad_checksum()
+{
+	[ "$(sha512sum "$2" | awk '{print $1}')" != "$1" ] || return 1
+	printf "Bad checksum for file: %s\n" "$2" 1>&2; rm -f "$2" || :; :
+}
+
 tmpclone()
 {
 	[ -d "$3" ] && return 0
-- 
cgit v1.2.1


From 92954eeb38f69ebc0cfef7b9629e96a1599667be Mon Sep 17 00:00:00 2001
From: Leah Rowe <leah@libreboot.org>
Date: Mon, 12 May 2025 19:20:50 +0100
Subject: lib.sh: remove rmgit()

We don't need to call it from git.sh, because it's
only being done when building a release anyway,
and we already run rmgit when doing a release.

The function itself is only two simple fx_ calls,
so we can just do that from build_release().

Signed-off-by: Leah Rowe <leah@libreboot.org>
---
 include/git.sh | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'include/git.sh')

diff --git a/include/git.sh b/include/git.sh
index b65bd196..0091c832 100644
--- a/include/git.sh
+++ b/include/git.sh
@@ -57,9 +57,6 @@ git_prep()
 		dx_ fetch_submodule "$mdir/module.list"
 	fi
 
-	[ "$_loc" != "$XBMK_CACHE/repo/$project" ] && \
-	    [ "$XBMK_RELEASE" = "y" ] && rmgit "$tmpgit"
-
 	[ "$_loc" = "${_loc%/*}" ] || x_ mkdir -p "${_loc%/*}"
 	x_ mv "$tmpgit" "$_loc"
 }
-- 
cgit v1.2.1


From 1ca26c5d238d2acb85e491f949ff7cc369959bd9 Mon Sep 17 00:00:00 2001
From: Leah Rowe <leah@libreboot.org>
Date: Tue, 13 May 2025 18:41:48 +0100
Subject: git.sh: Re-implement redundant git downloads

And this time it works.

I'm now calling xbmkget() which in turn calls tmpclone(),
instead of me calling tmpclone() directly.

The git-pull is done on both remotes, regardless of whether
the first succeeds. This way, if I forgot to update a mirror,
downloads would probably still work.

This also fixes an issue people were having, for example where
the gnulib repository of GRUB was always being downloaded
every time.

I'm using a new directory, XBMK_CACHE/clone, instead
of XBMK_CACHE/repo (which I used before), in case people
still have the old caches from before.

Signed-off-by: Leah Rowe <leah@libreboot.org>
---
 include/git.sh | 56 +++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 37 insertions(+), 19 deletions(-)

(limited to 'include/git.sh')

diff --git a/include/git.sh b/include/git.sh
index 0091c832..af0e8f2a 100644
--- a/include/git.sh
+++ b/include/git.sh
@@ -12,7 +12,7 @@ fetch_targets()
 	e "src/$project/$tree" d && return 0
 
 	printf "Creating %s tree %s\n" "$project" "$tree"
-	git_prep "$loc" "$loc" "$xbmkpwd/$configdir/$tree/patches" \
+	git_prep "$url" "$bkup_url" "$xbmkpwd/$configdir/$tree/patches" \
 	    "src/$project/$tree" with_submodules
 	nuke "$project/$tree" "$project/$tree"
 }
@@ -42,6 +42,7 @@ clone_project()
 	    loc="src/$project"
 	printf "Downloading project '%s' to '%s'\n" "$project" "$loc"
 
+	singletree "$project" || x_ rm -Rf "$loc"
 	e "$loc" d missing && remkdir "${tmpgit%/*}" && git_prep \
 	    "$url" "$bkup_url" "$xbmkpwd/config/$project/patches" "$loc"; :
 }
@@ -52,7 +53,7 @@ git_prep()
 	_loc="$4" # $1 and $2 are gitrepo and gitrepo_backup
 
 	chkvars rev
-	tmpclone "$1" "$2" "$tmpgit" "$rev" "$_patchdir"
+	xbmkget "$1" "$2" "$tmpgit" "$rev" git "$_patchdir"
 	if singletree "$project" || [ $# -gt 4 ]; then
 		dx_ fetch_submodule "$mdir/module.list"
 	fi
@@ -80,7 +81,7 @@ fetch_submodule()
 	[ "$st" = "file" ] && xbmkget "$subfile" "$subfile_bkup" \
 	    "$tmpgit/$1" "$subhash" && return 0
 	x_ rm -Rf "$tmpgit/$1"
-	tmpclone "$subrepo" "$subrepo_bkup" "$tmpgit/$1" "$subhash" \
+	xbmkget "$subrepo" "$subrepo_bkup" "$tmpgit/$1" "$subhash" git \
 	    "$mdir/${1##*/}/patches"
 }
 
@@ -89,7 +90,9 @@ fetch_submodule()
 xbmkget()
 {
 	_dlop="curl" && [ $# -gt 4 ] && _dlop="$5"
-	x_ mkdir -p "${3%/*}" "$XBMK_CACHE/file"
+	[ "$_dlop" = "curl" ] || [ "$_dlop" = "copy" ] || \
+	    [ "$_dlop" = "git" ] || err "$1 $2 $3 $4: Bad dlop type: '$_dlop'"
+
 	for url in "$1" "$2"; do
 		[ -n "$url" ] && try_file "$url" "$_dlop" "$@" && return 0
 	done && err "$1 $2 $3 $4: not downloaded"; :
@@ -97,19 +100,25 @@ xbmkget()
 
 try_file()
 {
-	cached="$XBMK_CACHE/file/$6"
-	dl_fail="n" # 1 url, 2 url backup, 3 destination, 4 checksum
-	bad_checksum "$6" "$cached" 2>/dev/null && dl_fail="y"
-	[ "$dl_fail" = "n" ] && e "$5" f && return 0
+	cached="file/$6" && [ "$2" = "git" ] && cached="clone/${3##*/}" && \
+	    cached="${cached%.git}"
+	cached="$XBMK_CACHE/$cached"
+	x_ mkdir -p "${5%/*}" "${cached%/*}"
+
+	echk="d" && dl_fail="n" && [ "$2" != "git" ] && echk="f" && \
+	    bad_checksum "$6" "$cached" 2>/dev/null && dl_fail="y"
+	[ "$dl_fail" = "n" ] && e "$5" $echk && return 0
 
-	x_ rm -f "$cached"
-	[ "$2" = "curl" ] || [ "$2" = "copy" ] || \
-	    err "$3 $4 $5 $6: Unsupported dlop type: '$2'"
+	[ "$2" != "git" ] && x_ rm -f "$cached"
 
 	try_$2 "$cached" "$@" || return 1
 
-	bad_checksum "$6" "$cached" && return 1
-	[ "$cached" = "$5" ] || x_ cp "$cached" "$5"; :
+	if [ "$2" = "git" ]; then
+		tmpclone "$cached" "$5" "$6" "$8" || return 1
+	else
+		bad_checksum "$6" "$cached" && return 1
+		[ "$cached" != "$5" ] && x_ cp "$cached" "$5"
+	fi
 }
 
 try_curl()
@@ -128,6 +137,16 @@ try_copy()
 	cp "$2" "$1" || return 1; :
 }
 
+try_git()
+{
+	[ -d "$1" ] || git clone "$2" "$1" || return 1
+	git -C "$1" remote add main "$4" 2>/dev/null || :
+	git -C "$1" remote add backup "$5" 2>/dev/null || :
+	git -C "$1" fetch --all || :
+	git -C "$1" pull --all || :
+	git -C "$1" pull --all || :
+}
+
 bad_checksum()
 {
 	[ "$(sha512sum "$2" | awk '{print $1}')" != "$1" ] || return 1
@@ -136,12 +155,11 @@ bad_checksum()
 
 tmpclone()
 {
-	[ -d "$3" ] && return 0
-	printf "Creating git clone '%s' from '%s', '%s'\n" "$3" "$1" "$2"
-	git clone "$1" "$3" || x_ rm -Rf "$3"
-	[ -d "$3" ] || x_ git clone "$2" "$3"
-	x_ git -C "$3" reset --hard "$4"
-	fx_ "eval x_ git -C \"$3\" am" find "$5" -type f
+	[ -d "$2" ] && return 0
+	printf "Creating git clone '%s' from '%s'\n" "$2" "$1"
+	( x_ git clone "$1" "$2" ) || return 1
+	( x_ git -C "$2" reset --hard "$3" ) || return 1
+	( fx_ "eval x_ git -C \"$2\" am" find "$4" -type f ) || return 1; :
 }
 
 nuke()
-- 
cgit v1.2.1


From ed8a33d6fb1c380e70fec881e6a0308cea99333e Mon Sep 17 00:00:00 2001
From: Leah Rowe <leah@libreboot.org>
Date: Tue, 13 May 2025 20:47:41 +0100
Subject: git.sh: cleanup

Signed-off-by: Leah Rowe <leah@libreboot.org>
---
 include/git.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'include/git.sh')

diff --git a/include/git.sh b/include/git.sh
index af0e8f2a..cc9c2f2a 100644
--- a/include/git.sh
+++ b/include/git.sh
@@ -38,11 +38,10 @@ fetch_project()
 
 clone_project()
 {
-	loc="$XBMK_CACHE/repo/$project" && singletree "$project" && \
+	loc="$XBMK_CACHE/clone/$project" && singletree "$project" && \
 	    loc="src/$project"
 	printf "Downloading project '%s' to '%s'\n" "$project" "$loc"
 
-	singletree "$project" || x_ rm -Rf "$loc"
 	e "$loc" d missing && remkdir "${tmpgit%/*}" && git_prep \
 	    "$url" "$bkup_url" "$xbmkpwd/config/$project/patches" "$loc"; :
 }
@@ -144,7 +143,6 @@ try_git()
 	git -C "$1" remote add backup "$5" 2>/dev/null || :
 	git -C "$1" fetch --all || :
 	git -C "$1" pull --all || :
-	git -C "$1" pull --all || :
 }
 
 bad_checksum()
-- 
cgit v1.2.1


From 3c23ff4fa1873e9170326141e31eb588d71a7d3a Mon Sep 17 00:00:00 2001
From: Leah Rowe <leah@libreboot.org>
Date: Tue, 13 May 2025 21:57:34 +0100
Subject: git.sh: Only create destination repo on success

Don't leave a broken cache laying around, which would
otherwise break lbmk for the user.

Signed-off-by: Leah Rowe <leah@libreboot.org>
---
 include/git.sh | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

(limited to 'include/git.sh')

diff --git a/include/git.sh b/include/git.sh
index cc9c2f2a..9405828e 100644
--- a/include/git.sh
+++ b/include/git.sh
@@ -6,6 +6,7 @@ eval "`setvars "" loc url bkup_url subfile subhash subrepo subrepo_bkup \
     depend subfile_bkup repofail`"
 
 tmpgit="$xbmklocal/gitclone"
+tmpgitcache="$XBMK_CACHE/tmpgit"
 
 fetch_targets()
 {
@@ -138,11 +139,17 @@ try_copy()
 
 try_git()
 {
-	[ -d "$1" ] || git clone "$2" "$1" || return 1
-	git -C "$1" remote add main "$4" 2>/dev/null || :
-	git -C "$1" remote add backup "$5" 2>/dev/null || :
-	git -C "$1" fetch --all || :
-	git -C "$1" pull --all || :
+	gitdest="`findpath "$1"`" || err "Can't get readpath for '$1'"
+	x_ rm -Rf "$tmpgitcache"
+
+	[ -d "$gitdest" ] || git clone "$2" "$tmpgitcache" || return 1
+	[ -d "$gitdest" ] || x_ mkdir -p "${gitdest##*/}"
+	[ -d "$gitdest" ] || x_ mv "$tmpgitcache" "$gitdest"
+
+	git -C "$gitdest" remote add main "$4" 2>/dev/null || :
+	git -C "$gitdest" remote add backup "$5" 2>/dev/null || :
+	git -C "$gitdest" fetch --all || :
+	git -C "$gitdest" pull --all || :
 }
 
 bad_checksum()
-- 
cgit v1.2.1


From b04c86e5740f92cbea708f52a444df3aa1a2061f Mon Sep 17 00:00:00 2001
From: Leah Rowe <leah@libreboot.org>
Date: Tue, 13 May 2025 22:00:28 +0100
Subject: git.sh: rename to get.sh

it now handles more than just git, and i forsee
it handling even more in the future, e.g. rsync,
ftp, bittorrent.

Signed-off-by: Leah Rowe <leah@libreboot.org>
---
 include/git.sh | 176 ---------------------------------------------------------
 1 file changed, 176 deletions(-)
 delete mode 100644 include/git.sh

(limited to 'include/git.sh')

diff --git a/include/git.sh b/include/git.sh
deleted file mode 100644
index 9405828e..00000000
--- a/include/git.sh
+++ /dev/null
@@ -1,176 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-# Copyright (c) 2020-2021,2023-2025 Leah Rowe <leah@libreboot.org>
-# Copyright (c) 2022 Caleb La Grange <thonkpeasant@protonmail.com>
-
-eval "`setvars "" loc url bkup_url subfile subhash subrepo subrepo_bkup \
-    depend subfile_bkup repofail`"
-
-tmpgit="$xbmklocal/gitclone"
-tmpgitcache="$XBMK_CACHE/tmpgit"
-
-fetch_targets()
-{
-	e "src/$project/$tree" d && return 0
-
-	printf "Creating %s tree %s\n" "$project" "$tree"
-	git_prep "$url" "$bkup_url" "$xbmkpwd/$configdir/$tree/patches" \
-	    "src/$project/$tree" with_submodules
-	nuke "$project/$tree" "$project/$tree"
-}
-
-fetch_project()
-{
-	eval "`setvars "" xtree`"
-	eval "`setcfg "config/git/$project/pkg.cfg"`"
-
-	chkvars url
-
-	[ -n "$xtree" ] && x_ ./mk -f coreboot "$xtree"
-	[ -z "$depend" ] || for d in $depend ; do
-		printf "'%s' needs '%s'; grabbing '%s'\n" "$project" "$d" "$d"
-		x_ ./mk -f $d
-	done
-	clone_project
-
-	for x in config/git/*; do
-		[ -d "$x" ] && nuke "${x##*/}" "src/${x##*/}" 2>/dev/null; :
-	done; :
-}
-
-clone_project()
-{
-	loc="$XBMK_CACHE/clone/$project" && singletree "$project" && \
-	    loc="src/$project"
-	printf "Downloading project '%s' to '%s'\n" "$project" "$loc"
-
-	e "$loc" d missing && remkdir "${tmpgit%/*}" && git_prep \
-	    "$url" "$bkup_url" "$xbmkpwd/config/$project/patches" "$loc"; :
-}
-
-git_prep()
-{
-	_patchdir="$3"
-	_loc="$4" # $1 and $2 are gitrepo and gitrepo_backup
-
-	chkvars rev
-	xbmkget "$1" "$2" "$tmpgit" "$rev" git "$_patchdir"
-	if singletree "$project" || [ $# -gt 4 ]; then
-		dx_ fetch_submodule "$mdir/module.list"
-	fi
-
-	[ "$_loc" = "${_loc%/*}" ] || x_ mkdir -p "${_loc%/*}"
-	x_ mv "$tmpgit" "$_loc"
-}
-
-fetch_submodule()
-{
-	mcfgdir="$mdir/${1##*/}"; eval \
-	    "`setvars "" subhash subrepo subrepo_bkup subfile subfile_bkup st`"
-	[ ! -f "$mcfgdir/module.cfg" ] || . "$mcfgdir/module.cfg" || \
-	    err "! . $mcfgdir/module.cfg"
-
-	for xt in repo file; do
-		_seval="if [ -n \"\$sub$xt\" ] || [ -n \"\$sub${xt}_bkup\" ]"
-		eval "$_seval; then st=\"\$st \$xt\"; fi"
-	done
-	st="${st# }" && [ "$st" = "repo file" ] && err "$mdir: repo+file"
-
-	[ -z "$st" ] && return 0 # subrepo/subfile not defined
-	chkvars "sub${st}" "sub${st}_bkup" "subhash"
-
-	[ "$st" = "file" ] && xbmkget "$subfile" "$subfile_bkup" \
-	    "$tmpgit/$1" "$subhash" && return 0
-	x_ rm -Rf "$tmpgit/$1"
-	xbmkget "$subrepo" "$subrepo_bkup" "$tmpgit/$1" "$subhash" git \
-	    "$mdir/${1##*/}/patches"
-}
-
-# can grab from the internet, or copy locally.
-# if copying locally, it can only copy a file.
-xbmkget()
-{
-	_dlop="curl" && [ $# -gt 4 ] && _dlop="$5"
-	[ "$_dlop" = "curl" ] || [ "$_dlop" = "copy" ] || \
-	    [ "$_dlop" = "git" ] || err "$1 $2 $3 $4: Bad dlop type: '$_dlop'"
-
-	for url in "$1" "$2"; do
-		[ -n "$url" ] && try_file "$url" "$_dlop" "$@" && return 0
-	done && err "$1 $2 $3 $4: not downloaded"; :
-}
-
-try_file()
-{
-	cached="file/$6" && [ "$2" = "git" ] && cached="clone/${3##*/}" && \
-	    cached="${cached%.git}"
-	cached="$XBMK_CACHE/$cached"
-	x_ mkdir -p "${5%/*}" "${cached%/*}"
-
-	echk="d" && dl_fail="n" && [ "$2" != "git" ] && echk="f" && \
-	    bad_checksum "$6" "$cached" 2>/dev/null && dl_fail="y"
-	[ "$dl_fail" = "n" ] && e "$5" $echk && return 0
-
-	[ "$2" != "git" ] && x_ rm -f "$cached"
-
-	try_$2 "$cached" "$@" || return 1
-
-	if [ "$2" = "git" ]; then
-		tmpclone "$cached" "$5" "$6" "$8" || return 1
-	else
-		bad_checksum "$6" "$cached" && return 1
-		[ "$cached" != "$5" ] && x_ cp "$cached" "$5"
-	fi
-}
-
-try_curl()
-{
-	_ua="Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0"
-	curl --location --retry 3 -A "$_ua" "$2" -o "$1" || \
-	    wget --tries 3 -U "$_ua" "$2" -O "$1" || return 1; :
-}
-
-try_copy()
-{
-	[ -L "$2" ] && printf "dl %s %s %s %s: '%s' is a symlink\n" \
-	    "$4" "$5" "$6" "$7" "$2" 1>&2 && return 1
-	[ ! -f "$2" ] && printf "dl %s %s %s %s: '%s' not a file\n" \
-	    "$4" "$5" "$6" "$7" "$2" 1>&2 && return 1
-	cp "$2" "$1" || return 1; :
-}
-
-try_git()
-{
-	gitdest="`findpath "$1"`" || err "Can't get readpath for '$1'"
-	x_ rm -Rf "$tmpgitcache"
-
-	[ -d "$gitdest" ] || git clone "$2" "$tmpgitcache" || return 1
-	[ -d "$gitdest" ] || x_ mkdir -p "${gitdest##*/}"
-	[ -d "$gitdest" ] || x_ mv "$tmpgitcache" "$gitdest"
-
-	git -C "$gitdest" remote add main "$4" 2>/dev/null || :
-	git -C "$gitdest" remote add backup "$5" 2>/dev/null || :
-	git -C "$gitdest" fetch --all || :
-	git -C "$gitdest" pull --all || :
-}
-
-bad_checksum()
-{
-	[ "$(sha512sum "$2" | awk '{print $1}')" != "$1" ] || return 1
-	printf "Bad checksum for file: %s\n" "$2" 1>&2; rm -f "$2" || :; :
-}
-
-tmpclone()
-{
-	[ -d "$2" ] && return 0
-	printf "Creating git clone '%s' from '%s'\n" "$2" "$1"
-	( x_ git clone "$1" "$2" ) || return 1
-	( x_ git -C "$2" reset --hard "$3" ) || return 1
-	( fx_ "eval x_ git -C \"$2\" am" find "$4" -type f ) || return 1; :
-}
-
-nuke()
-{
-	e "config/${1%/}/nuke.list" f missing || while read -r nukefile; do
-		rmf="src/${2%/}/$nukefile" && [ -L "$rmf" ] && continue
-		e "$rmf" e missing || x_ rm -Rf "$rmf"
-	done < "config/${1%/}/nuke.list"; :
-}
-- 
cgit v1.2.1