get.sh: More reliable git remote caching

Don't do one repository for all remotes. Do one *clone* per
remote.

This also means that users no longer download information twice,
in practice, because the backup repository will only be downloaded
if the main one didn't work.

Theoretically, this change is makes the process less efficient, but
in practise it's more reliable now.

We do now use --mirror on the git clone command for caches, but we
already did git pull --all before.

This just ensures that we absolutely have all local code.

NOTE:

The new code isn't used by default. To use it, you must do:

export XBMK_CACHE_MIRROR="y"

Otherwise, the old behaviour will continue to be used. This is
because the new code, while correct, puts more strain on upstream
servers (more code being downloaded), and can result in higher amounts
of disk space being used. The old behaviour wasn't broken, so we'll
also support that method.

TODO: perhaps also have a check in place to re-use both caches,
where available, regardless of XBMK_CACHE_MIRROR?

Signed-off-by: Leah Rowe <leah@libreboot.org>

2 files changed, 64 insertions, 12 deletions

diff --git a/include/get.sh b/include/get.sh
index 4731c694..35cf2dbf 100644
--- a/include/get.sh
+++ b/include/get.sh
@@ -52,7 +52,9 @@ fetch_project()
 
 clone_project()
 {
-	loc="$XBMK_CACHE/clone/$project"
+	# if loc is blank, don't create a target
+	# directory; just update the caches
+	loc=""
 	if singletree "$project"; then
 		loc="src/$project"
 	fi
@@ -83,7 +85,17 @@ git_prep()
 	if [ "$_loc" != "${_loc%/*}" ]; then
 		x_ xbmkdir "${_loc%/*}"
 	fi
-	x_ mv "$tmpgit" "$_loc"
+
+	if [ -z "$_loc" ]; then
+		# we only used git_prep to update caches, on
+		# a multi-tree project. tmpgit is useless now.
+
+		x_ rm -Rf "$tmpgit"
+	else
+		# actual downloaded
+
+		x_ mv "$tmpgit" "$_loc"
+	fi
 }
 
 fetch_submodule()
@@ -191,11 +203,19 @@ try_fetch()
 
 try_fetch_git()
 {
-	# always the main repo as basis for naming,
-	# in case the backup has another name
+	if [ "$XBMK_CACHE_MIRROR" = "y" ]; then
+		# 1st argument $1 is the current git remote being tried,
+		# let's say it was https://foo.example.com/repo, then cached
+		# directories becomes cache/mirror/foo.example.com/repo
 
-	cached="clone/${3##*/}"
-	cached="${cached%.git}"
+		cached="mirror/${1#*://}"
+	else
+		# always the main repo as basis for naming,
+		# in case the backup has another name
+
+		cached="clone/${3##*/}"
+		cached="${cached%.git}"
+	fi
 	cached="$XBMK_CACHE/$cached"
 
 	x_ xbmkdir "${5%/*}" "${cached%/*}"
@@ -282,7 +302,12 @@ try_git()
 	x_ rm -Rf "$tmpgitcache"
 
 	if [ ! -d "$gitdest" ]; then
-		( x_ git clone "$2" "$tmpgitcache" ) || return 1
+		if [ "$XBMK_CACHE_MIRROR" = "y" ]; then
+			( x_ git clone --mirror "$2" "$tmpgitcache" ) || \
+			    return 1
+		else
+			( x_ git clone "$2" "$tmpgitcache" ) || return 1
+		fi
 
 		x_ xbmkdir "${gitdest%/*}"
 		x_ mv "$tmpgitcache" "$gitdest"
@@ -298,13 +323,18 @@ try_git()
 		return 0
 	fi
 
-	( x_ git -C "$gitdest" remote remove main ) || :
-	( x_ git -C "$gitdest" remote remove backup ) || :
+	if [ "$XBMK_CACHE_MIRROR" = "y" ]; then
+		( x_ git -C "$gitdest" fetch ) || :; :
+		( x_ git -C "$gitdest" update-server-info ) || :; :
+	else
+		( x_ git -C "$gitdest" remote remove main ) || :
+		( x_ git -C "$gitdest" remote remove backup ) || :
 
-	x_ git -C "$gitdest" remote add main "$4"
-	x_ git -C "$gitdest" remote add backup "$5"
+		x_ git -C "$gitdest" remote add main "$4"
+		x_ git -C "$gitdest" remote add backup "$5"
 
-	( x_ git -C "$gitdest" pull --all ) || :; :
+		( x_ git -C "$gitdest" pull --all ) || :; :
+	fi
 }
 
 bad_checksum()
diff --git a/include/init.sh b/include/init.sh
index 96247908..9904d780 100644
--- a/include/init.sh
+++ b/include/init.sh
@@ -176,6 +176,9 @@ xbmk_child_set_env()
 	if [ -z "${XBMK_THREADS+x}" ]; then
 		xbmk_set_threads; :
 	fi
+	if [ -z "${XBMK_CACHE_MIRROR+x}" ]; then
+		xbmk_set_mirror
+	fi
 }
 
 xbmk_child_set_tmp()
@@ -220,6 +223,7 @@ xbmk_parent_set_env()
 	remkdir "$xbtmp" "$xbtmp/gnupath" "$xbtmp/xbmkpath"
 
 	xbmk_set_pyver
+	xbmk_set_mirror
 }
 
 xbmk_parent_check_tmp()
@@ -409,6 +413,24 @@ pybin()
 	fi
 }
 
+xbmk_set_mirror()
+{
+	# defines whether cache/clone/ (regular clones)
+	# or cache/mirror (--mirror clones) are used, per project
+
+	# to use cache/mirror/ do: export XBMK_CACHE_MIRROR="y"
+	# mirror/ stores a separate directory per repository, even per backup.
+	# it's slower, and uses more disk space, and some upstreams might not
+	# appreciate it, so it should only be used for development or archival
+
+	if [ -z "${XBMK_CACHE_MIRROR+x}" ]; then
+		export XBMK_CACHE_MIRROR="n"
+	fi
+	if [ "$XBMK_CACHE_MIRROR" != "y" ]; then
+		export XBMK_CACHE_MIRROR="n"
+	fi
+}
+
 xbmk_git_init()
 {
 	for gitarg in "--global user.name" "--global user.email"; do