diff options
Diffstat (limited to 'include/get.sh')
| -rw-r--r-- | include/get.sh | 387 |
1 files changed, 301 insertions, 86 deletions
diff --git a/include/get.sh b/include/get.sh index 6f1b5f1d..c463bbc5 100644 --- a/include/get.sh +++ b/include/get.sh @@ -1,168 +1,383 @@ # SPDX-License-Identifier: GPL-3.0-or-later + # Copyright (c) 2020-2021,2023-2025 Leah Rowe <leah@libreboot.org> # Copyright (c) 2022 Caleb La Grange <thonkpeasant@protonmail.com> -eval "`setvars "" loc url bkup_url subcurl subhash subgit subgit_bkup \ - depend subcurl_bkup repofail`" +depend="" +loc="" +url="" +bkup_url="" +subgit="" +subgit_bkup="" +subcurl="" +subcurl_bkup="" +subhash="" -tmpgit="$xbmklocal/gitclone" -tmpgitcache="$XBMK_CACHE/tmpgit" +tmpgit="$xbtmp/gitclone" +tmpgitcache="$xbtmp/tmpgit" fetch_targets() { - e "src/$project/$tree" d && return 0 - - printf "Creating %s tree %s\n" "$project" "$tree" - git_prep "$url" "$bkup_url" "$xbmkpwd/$configdir/$tree/patches" \ - "src/$project/$tree" with_submodules + if [ ! -d "src/$project/$tree" ]; then + git_prep "$url" "$bkup_url" \ + "$xbmkpwd/$configdir/$tree/patches" \ + "src/$project/$tree" "submod" + fi } fetch_project() { - eval "`setvars "" xtree`" - eval "`setcfg "config/git/$project/pkg.cfg"`" + xgcctree="" - chkvars url bkup_url + . "config/git/$project/pkg.cfg" || \ + err "Can't read config 'config/git/$project/pkg.cfg'" \ + "fetch_project" "@" + + if [ -z "$url" ] || [ -z "$bkup_url" ]; then + err "url/bkup_url not both set 'config/git/$project/pkg.cfg'" \ + "fetch_project" "$@" + fi + + if [ -n "$xgcctree" ]; then + x_ ./mk -f coreboot "$xgcctree" + fi + if [ -n "$depend" ]; then + for d in $depend ; do + x_ ./mk -f $d + done + fi - [ -n "$xtree" ] && x_ ./mk -f coreboot "$xtree" - [ -z "$depend" ] || for d in $depend ; do - printf "'%s' needs '%s'; grabbing '%s'\n" "$project" "$d" "$d" - x_ ./mk -f $d - done clone_project } clone_project() { - loc="$XBMK_CACHE/clone/$project" && singletree "$project" && \ - loc="src/$project" - printf "Downloading project '%s' to '%s'\n" "$project" "$loc" + # if loc is blank, don't create a target + # directory; just update the caches + loc="" + if singletree "$project"; then + loc="src/$project" + fi - e "$loc" d missing && remkdir "${tmpgit%/*}" && git_prep \ - "$url" "$bkup_url" "$xbmkpwd/config/$project/patches" "$loc"; : + if e "$loc" d missing; then + remkdir "${tmpgit%/*}" + git_prep "$url" "$bkup_url" \ + "$xbmkpwd/config/$project/patches" "$loc" + fi } git_prep() { + printf "Creating code directory, src/%s/%s\n" "$project" "$tree" + _patchdir="$3" _loc="$4" # $1 and $2 are gitrepo and gitrepo_backup - chkvars rev - xbmkget git "$1" "$2" "$tmpgit" "$rev" "$_patchdir" + if [ -z "$rev" ]; then + err "$project/$tree: rev not set" "git_prep" "$@" + fi + + xbget git "$1" "$2" "$tmpgit" "$rev" "$_patchdir" if singletree "$project" || [ $# -gt 4 ]; then dx_ fetch_submodule "$mdir/module.list" fi - [ "$_loc" = "${_loc%/*}" ] || x_ mkdir -p "${_loc%/*}" - x_ mv "$tmpgit" "$_loc" + if [ "$_loc" != "${_loc%/*}" ]; then + x_ xbmkdir "${_loc%/*}" + fi + + if [ -z "$_loc" ]; then + # we only used git_prep to update caches, on + # a multi-tree project. tmpgit is useless now. + + x_ rm -Rf "$tmpgit" + else + # actual downloaded + + x_ mv "$tmpgit" "$_loc" + fi } fetch_submodule() { - mcfgdir="$mdir/${1##*/}"; eval \ - "`setvars "" subhash subgit subgit_bkup subcurl subcurl_bkup st`" - eval "`setcfg "$mcfgdir/module.cfg" 0`" + mcfgdir="$mdir/${1##*/}" - for xt in git curl; do - _seval="if [ -n \"\$sub$xt\" ] || [ -n \"\$sub${xt}_bkup\" ]" - eval "$_seval; then st=\"\$st \$xt\"; fi" - done + subhash="" + subgit="" + subgit_bkup="" + subcurl="" + subcurl_bkup="" + st="" - st="${st# }" && [ "$st" = "git curl" ] && err "$mdir: git+curl defined" - [ -z "$st" ] && return 0 # subgit/subcurl not defined - chkvars "sub${st}" "sub${st}_bkup" "subhash" + if e "$mcfgdir/module.cfg" f missing; then + return 0 + fi + . "$mcfgdir/module.cfg" || \ + err "Can't read '$mcfgdir/module.cfg'" "fetch_submodules" "$@" - [ "$st" = "git" ] && x_ rm -Rf "$tmpgit/$1" - eval xbmkget "$st" "\$sub$st" "\$sub${st}_bkup" "$tmpgit/$1" \ - "$subhash" "$mdir/${1##*/}/patches" + if [ -n "$subgit" ] || [ -n "$subgit_bkup" ]; then + st="$st git" + fi + if [ -n "$subcurl" ] || [ -n "$subcurl_bkup" ]; then + st="$st curl" + fi + + st="${st# }" + if [ "$st" = "git curl" ]; then + err "$mdir: git+curl defined" "fetch_submodule" "$@" + fi + + if [ -z "$st" ]; then + return 0 + fi + + if [ "$st" = "curl" ]; then + if [ -z "$subcurl" ] || [ -z "$subcurl_bkup" ]; then + err "subcurl/subcurl_bkup not both set" \ + "fetch_submodule" "$@" + fi + elif [ -z "$subgit" ] || [ -z "$subgit_bkup" ]; then + err "subgit/subgit_bkup not both set" "fetch_submodule" "$@" + elif [ -z "$subhash" ]; then + err "subhash not set" "fetch_submodule" "$@" + fi + + if [ "$st" = "git" ]; then + x_ rm -Rf "$tmpgit/$1" + xbget "$st" "$subgit" "$subgit_bkup" "$tmpgit/$1" \ + "$subhash" "$mdir/${1##*/}/patches" + else + xbget "$st" "$subcurl" "$subcurl_bkup" "$tmpgit/$1" \ + "$subhash" "$mdir/${1##*/}/patches" + fi } -xbmkget() +# TODO: in the following functions, argument numbers are used +# which is hard to understand. the code should be modified +# so that variable names are used instead, for easy reading + +xbget() { - [ "$1" = "curl" ] || [ "$1" = "copy" ] || [ "$1" = "git" ] || \ - err "Bad dlop (arg 1): xbmkget $*" + if [ "$1" != "curl" ] && [ "$1" != "copy" ] && [ "$1" != "git" ]; then + err "Bad dlop (arg 1)" "xbget" "$@" + fi - echk="f" && [ "$1" = "git" ] && echk="d" + for url in "$2" "$3" + do + if [ -z "$url" ]; then + err "empty URL given in" "xbget" "$@" + elif ! try_fetch "$url" "$@"; then + continue + fi - for url in "$2" "$3"; do - [ -n "$url" ] || err "empty URL given in: xbmkget $*" - try_file "$url" "$@" || continue - eval "[ -$echk \"$4\" ] || continue" + case "$1" in + git) + if [ ! -d "$4" ]; then + continue + fi + ;; + *) + if [ ! -f "$4" ]; then + continue + fi + ;; + esac return 0 # successful download/copy done - err "$1 $2 $3 $4: not downloaded"; : + + err "failed to download file/repository" "xbget" "$@"; : } -try_file() +try_fetch() { - cached="file/$6" && [ "$2" = "git" ] && cached="clone/${3##*/}" && \ - cached="${cached%.git}" # always the main repo as basis for naming, - # in case the backup has another name + if [ "$2" = "git" ]; then + if ! try_fetch_git "$@"; then + return 1 + fi + else + if ! try_fetch_file "$@"; then + return 1 + fi + fi +} + +try_fetch_git() +{ + if [ "$XBMK_CACHE_MIRROR" = "y" ]; then + # 1st argument $1 is the current git remote being tried, + # let's say it was https://foo.example.com/repo, then cached + # directories becomes cache/mirror/foo.example.com/repo + + cached="mirror/${1#*://}" + else + # always the main repo as basis for naming, + # in case the backup has another name + + cached="clone/${3##*/}" + cached="${cached%.git}" + fi cached="$XBMK_CACHE/$cached" - x_ mkdir -p "${5%/*}" "${cached%/*}" - echk="d" && [ "$2" != "git" ] && echk="f" && \ - bad_checksum "$6" "$cached" 2>/dev/null && x_ rm -f "$cached" + x_ xbmkdir "${5%/*}" "${cached%/*}" - evalchk="[ -$echk \"$cached\" ] || " && [ "$2" = "git" ] && evalchk="" - eval "${evalchk}try_$2 \"\$cached\" \"\$@\" || return 1" - [ "$2" != "git" ] && [ -f "$5" ] && \ - bad_checksum "$6" "$5" 2>/dev/null && x_ cp "$cached" "$5" - eval "[ -$echk \"$cached\" ] || return 1" + if ! try_$2 "$cached" "$@"; then + return 1 + elif [ ! -d "$cached" ]; then + return 1 + fi - if [ "$2" = "git" ]; then - [ -d "$5" ] || tmpclone "$cached" "$5" "$6" "$7" || return 1 - else - bad_checksum "$6" "$cached" && x_ rm -f "$cached" && return 1 - [ "$cached" != "$5" ] && x_ cp "$cached" "$5" - bad_checksum "$6" "$5" && x_ rm -f "$5" && return 1; : + if [ ! -d "$5" ]; then + tmpclone "$cached" "$5" "$6" "$7" || \ + err "Can't clone final repo" "try_fetch" "$@"; : fi - eval "[ -$echk \"$5\" ] || return 1" + if [ ! -d "$5" ]; then + return 1 + fi +} + +try_fetch_file() +{ + cached="file/$6" + cached="$XBMK_CACHE/$cached" + + x_ xbmkdir "${5%/*}" "${cached%/*}" + + if bad_checksum "$6" "$cached" 2>/dev/null; then + x_ rm -f "$cached" + fi + + if [ ! -f "$cached" ]; then + if ! try_$2 "$cached" "$@"; then + return 1 + fi + fi + + if [ -f "$5" ]; then + if bad_checksum "$6" "$5" 2>/dev/null; then + x_ cp "$cached" "$5" + fi + fi + + if [ ! -f "$cached" ]; then + return 1 + elif bad_checksum "$6" "$cached"; then + x_ rm -f "$cached" + + return 1 + fi + + if [ "$cached" != "$5" ]; then + x_ cp "$cached" "$5" + fi + + if bad_checksum "$6" "$5"; then + x_ rm -f "$5" + + return 1 + elif [ ! -f "$5" ]; then + return 1 + fi } try_curl() { _ua="Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0" - ( x_ curl --location --retry 3 -A "$_ua" "$2" -o "$1" ) || \ - ( x_ wget --tries 3 -U "$_ua" "$2" -O "$1" ) || return 1; : + + ( x_ curl --location --retry 3 -A "$_ua" "$2" -o "$1" ) \ + || ( x_ wget --tries 3 -U "$_ua" "$2" -O "$1" ) \ + || return 1; : } try_copy() { - [ -L "$2" ] && printf "symlink %s (trycp %s)\n" "$2" "$*" && return 1 - [ ! -f "$2" ] && printf "%s missing (trycp %s)\n" "$2" "$*" && return 1 ( x_ cp "$2" "$1" ) || return 1; : } try_git() { - gitdest="`findpath "$1"`" || err "Can't get readpath for '$1'" + gitdest="`findpath "$1" || err "Can't get findpath for '$1'"`" || \ + err "failed findpath for '$1'" try_get "$@" + x_ rm -Rf "$tmpgitcache" - [ -d "$gitdest" ] || ( x_ git clone "$2" "$tmpgitcache" ) || return 1 - [ -d "$gitdest" ] || x_ mkdir -p "${gitdest%/*}" - [ -d "$gitdest" ] || x_ mv "$tmpgitcache" "$gitdest" + if [ ! -d "$gitdest" ]; then + if [ "$XBMK_CACHE_MIRROR" = "y" ]; then + ( x_ git clone --mirror "$2" "$tmpgitcache" ) || \ + return 1 + else + ( x_ git clone "$2" "$tmpgitcache" ) || return 1 + fi + + x_ xbmkdir "${gitdest%/*}" + x_ mv "$tmpgitcache" "$gitdest" + fi + + if git -C "$gitdest" show "$7" 1>/dev/null 2>/dev/null && \ + [ "$forcepull" != "y" ]; then + # don't try to pull the latest changes if the given target + # revision already exists locally. this saves a lot of time + # during release builds, and reduces the chance that we will + # interact with grub.git or gnulib.git overall during runtime + + return 0 + fi + + if [ "$XBMK_CACHE_MIRROR" = "y" ]; then + ( x_ git -C "$gitdest" fetch ) || :; : + ( x_ git -C "$gitdest" update-server-info ) || :; : + else + ( x_ git -C "$gitdest" remote remove main ) || : + ( x_ git -C "$gitdest" remote remove backup ) || : + + x_ git -C "$gitdest" remote add main "$4" + x_ git -C "$gitdest" remote add backup "$5" - ( x_ git -C "$gitdest" remote remove main ) || : - ( x_ git -C "$gitdest" remote remove backup ) || : - x_ git -C "$gitdest" remote add main "$4" - x_ git -C "$gitdest" remote add backup "$5" - ( x_ git -C "$gitdest" fetch --all ) || : - ( x_ git -C "$gitdest" pull --all ) || :; : + ( x_ git -C "$gitdest" pull --all ) || :; : + fi } bad_checksum() { - [ ! -f "$2" ] && printf "'%s' missing, %s\n" "$2" "$1" 1>&2 && return 0 - csum="$(x_ sha512sum "$2" | awk '{print $1}')" || err "!sha512 '$2' $1" - [ "$csum" != "$1" ] && printf "BAD SHA512, '%s'; need '%s', got %s\n" \ - "$2" "$1" "$csum" 1>&2 && x_ rm -f "$2" && return 0; return 1 + if e "$2" f missing; then + return 0 + fi + + build_sbase + csum="$(x_ "$sha512sum" "$2" | awk '{print $1}')" || \ + err "!sha512 '$2' $1" bad_checksum "$@" + + if [ "$csum" = "$1" ]; then + return 1 + else + x_ rm -f "$2" + printf "BAD SHA512 %s, '%s'; need %s\n" "$csum" "$2" "$1" 1>&2 + fi } tmpclone() { ( x_ git clone "$1" "$2" ) || return 1 ( x_ git -C "$2" reset --hard "$3" ) || return 1 - ( fx_ "eval x_ git -C \"$2\" am" find "$4" -type f ) || return 1; : + + if [ ! -d "$4" ]; then + return 0 + fi + + tmpclone_patchlist="`mktemp || err "Can't create tmp patch list"`" || \ + err "Can't create tmp patch list" "tmpclone" "$@" + + x_ find "$4" -type f | sort > "$tmpclone_patchlist" || \ + err "Can't write patch names to '$tmpclone_patchlist'" \ + "tmpclone" "$@" + + while read -r tmpclone_patch; do + + ( x_ git -C "$2" am --keep-cr "$tmpclone_patch" ) || \ + err "Can't apply '$tmpclone_patch'" "tmpclone" "$@"; : + + done < "$tmpclone_patchlist" || \ + err "Can't read '$tmpclone_patchlist'" "tmpclone" "$@" + + x_ rm -f "$tmpclone_patchlist" } |