diff options
Diffstat (limited to 'include/env')
| -rw-r--r-- | include/env/get.sh | 304 | ||||
| -rw-r--r-- | include/env/init.sh | 388 |
2 files changed, 692 insertions, 0 deletions
diff --git a/include/env/get.sh b/include/env/get.sh new file mode 100644 index 00000000..0ed9d6cc --- /dev/null +++ b/include/env/get.sh @@ -0,0 +1,304 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +# Copyright (c) 2020-2021,2023-2026 Leah Rowe <leah@libreboot.org> +# Copyright (c) 2022 Caleb La Grange <thonkpeasant@protonmail.com> + +eval "`newvar url bkup_url fetch_depend loc subcurl subcurl_bkup subgit \ + subgit_bkup subhash badhash badtghash`" + +tmpgit="$xbtmp/gitclone" +tmpgitcache="$xbtmp/tmpgit" + +# delete a source tree and corresponding builds if metadata changed +# inside config/ - this is done by hashing a concatenation of hashes +# from all metadata, checked against a previously calculated hash + +delete_old_project_files() +{ + # delete an entire source tree along with its builds: + same_pjhash mhash "$tree" badhash "$datadir" \ + "$configdir/$tree" "$mdir" || \ + x_ rm -Rf "src/$project/$tree" "elf/$project/$tree" + + x_ cp "$xbtmp/new.hash" "$XBMK_CACHE/mhash/$project$tree" + + if [ -n "$target" ] && [ "$target" != "$tree" ]; then + # delete only target builds, if only the target changed + same_pjhash mtghash "$target" badtghash "$configdir/$target" \ + || x_ rm -Rf "elf/$project/$tree/$target" + + x_ cp "$xbtmp/new.hash" "$XBMK_CACHE/mtghash/$project$target" + fi +} + +same_pjhash() +{ + eval "`newvar hash old_hash`" + + hashdir="$1" + hashname="$2" + badhashvar="$3" + + shift 3 + + x_ mkdir -p "$XBMK_CACHE/$hashdir" + + if [ -f "$XBMK_CACHE/$hashdir/$project$hashname" ]; then + read -r old_hash < "$XBMK_CACHE/$hashdir/$project$hashname" \ + || err \ + "$hashdir: err '$XBMK_CACHE/$hashdir/$project$hashname'" \ + same_pjhash "$hashdir" "$hashname" "$badhashvar" "$@" + fi + + eval "$build_sbase" + fx_ "x_ util/sbase/sha512sum" find "$@" -type f -not -path \ + "*/.git*/*" | awk '{print $1}' > "$xbtmp/tmp.hash" || \ + err "!h $project $hashdir" \ + "same_pjhash" "$hashdir" "$hashname" "$badhashvar" "$@" + + hash="$(x_ "$sha512sum" "$xbtmp/tmp.hash" | awk '{print $1}' || \ + err)" || err "$hashname: Can't read sha512 of '$xbtmp/tmp.hash'" \ + "same_pjhash" "$hashdir" "$hashname" "$badhashvar" "$@" + + if [ "$hash" != "$old_hash" ] || \ + [ ! -f "$XBMK_CACHE/$hashdir/$project$hashname" ]; then + eval "$badhashvar=\"y\"" + fi + + printf "%s\n" "$hash" > "$xbtmp/new.hash" || \ + err "!mkhash $xbtmp/new.hash ($hashdir $hashname $badhashvar)" \ + "same_pjhash" "$hashdir" "$hashname" "$badhashvar" "$@" + + eval "[ \"\$$badhashvar\" = \"y\" ] && return 1"; : +} + +fetch_trees() +{ + [ -d "src/$project/$tree" ] && return 0 + + mk_dependencies -f $fetch_depend + + git_prep "$url" "$bkup_url" "$xbmkpwd/$configdir/$tree/patches" \ + "src/$project/$tree" +} + +git_prep() +{ + printf "Creating code directory, src/%s/%s\n" "$project" "$tree" + + _patchdir="$3" + _loc="$4" # $1 and $2 are gitrepo and gitrepo_backup + + [ -z "$rev" ] && \ + err "$project/$tree: rev unset" "git_prep" "$@" + + xbget git "$1" "$2" "$tmpgit" "$rev" "$_patchdir" + dx_ fetch_submodule "$mdir/module.list" + + [ "$_loc" != "${_loc%/*}" ] && \ + x_ mkdir -p "${_loc%/*}" + + x_ mv "$tmpgit" "$_loc" +} + +fetch_submodule() +{ + mcfgdir="$mdir/${1##*/}" + eval "`newvar st subcurl subcurl_bkup subgit subgit_bkup subhash`" + + e "$mcfgdir/module.cfg" f missing && return 0 + . "$mcfgdir/module.cfg" || \ + err "Can't read '$mcfgdir/module.cfg'" "fetch_submodules" "$@" + + [ -n "$subgit" ] && [ -n "$subgit_bkup" ] && \ + st="git" + [ -n "$subcurl" ] && [ -n "$subcurl_bkup" ] && \ + st="curl$st" + [ "$st" = "curlgit" ] && \ + err "$mdir: git+curl defined" "fetch_submodule" "$@" + [ -z "$st" ] && \ + return 0 + [ -z "$subhash" ] && \ + err "subhash not set" "fetch_submodule" "$@" + + [ "$st" = "git" ] && \ + x_ rm -Rf "$tmpgit/$1" + + eval "xbget \"\$st\" \"\$sub$st\" \"\$sub${st}_bkup\" \"\$tmpgit/\$1\" \ + \"\$subhash\" \"\$mdir/\${1##*/}/patches\"" +} + +# TODO: in the following functions, argument numbers are used +# which is hard to understand. the code should be modified +# so that variable names are used instead, for easy reading + +xbget() +{ + [ "$1" != "curl" ] && [ "$1" != "copy" ] && [ "$1" != "git" ] && \ + err "Bad dlop (arg 1)" "xbget" "$@" + + for url in "$2" "$3"; do + [ -z "$url" ] && \ + err "empty URL given in" "xbget" "$@" + try_fetch_$1 "$url" "$@" || \ + continue + [ -e "$4" ] && return 0; : # successful download/copy + done + + err "failed to download file/repository" "xbget" "$@"; : +} + +try_fetch_git() +{ + # 1st argument $1 is the current git remote being tried, + # let's say it was https://foo.example.com/repo, then cached + # directories becomes cache/mirror/foo.example.com/repo + + cached="clone" + [ "$XBMK_CACHE_MIRROR" = "y" ] && \ + cached="mirror" + cached="$cached/${1#*://}" + cached="$XBMK_CACHE/$cached" + + x_ mkdir -p "${5%/*}" "${cached%/*}" + + try_$2 "$cached" "$@" || \ + return 1 + [ ! -d "$cached" ] && \ + return 1 + + if [ ! -d "$5" ]; then + tmpclone "$cached" "$5" "$6" "$7" || \ + err "Can't clone final repo" "try_fetch" "$@"; : + fi + + [ ! -d "$5" ] && \ + return 1; : +} + +try_fetch_copy() +{ + try_fetch_curl "$@" || return 1; : +} + +try_fetch_curl() +{ + cached="file/$6" + cached="$XBMK_CACHE/$cached" + + x_ mkdir -p "${5%/*}" "${cached%/*}" + + if bad_checksum "$6" "$cached" 2>/dev/null; then + x_ rm -f "$cached" + fi + + [ -f "$cached" ] || \ + try_$2 "$cached" "$@" || \ + return 1 + + [ -f "$5" ] && bad_checksum "$6" "$5" 2>/dev/null && \ + x_ cp "$cached" "$5" + + if [ ! -f "$cached" ]; then + return 1 + elif bad_checksum "$6" "$cached"; then + x_ rm -f "$cached" + return 1 + fi + + [ "$cached" != "$5" ] && \ + x_ cp "$cached" "$5" + + if bad_checksum "$6" "$5"; then + x_ rm -f "$5" + return 1 + elif [ ! -f "$5" ]; then + return 1 + fi +} + +try_curl() +{ + ua="Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0" + + ( x_ curl --location --retry 3 -A "$ua" "$2" -o "$1" ) \ + || ( x_ wget --tries 3 -U "$ua" "$2" -O "$1" ) || return 1; : +} + +try_copy() +{ + ( x_ cp "$2" "$1" ) || return 1; : +} + +try_git() +{ + gitdest="`findpath "$1" || err "Can't get findpath for '$1'"`" || \ + err "failed findpath for '$1'" try_get "$@" + + x_ rm -Rf "$tmpgitcache" + + if [ ! -d "$gitdest" ]; then + if [ "$XBMK_CACHE_MIRROR" = "y" ]; then + ( x_ git clone --mirror "$2" "$tmpgitcache" ) || \ + return 1 + else + ( x_ git clone "$2" "$tmpgitcache" ) || return 1 + fi + + x_ mkdir -p "${gitdest%/*}" + x_ mv "$tmpgitcache" "$gitdest" + fi + + git -C "$gitdest" show "$7" 1>/dev/null 2>/dev/null && \ + [ "$forcepull" != "y" ] && \ + return 0 # don't pull changes if target rev exists locally + + if [ "$XBMK_CACHE_MIRROR" = "y" ]; then + ( x_ git -C "$gitdest" fetch ) || :; : + ( x_ git -C "$gitdest" update-server-info ) || :; : + else + ( x_ git -C "$gitdest" pull --all ) || :; : + fi +} + +bad_checksum() +{ + e "$2" f missing && \ + return 0 + + eval "$build_sbase" + csum="$(x_ "$sha512sum" "$2" | awk '{print $1}')" || \ + err "!sha512 '$2' $1" bad_checksum "$@" + + [ "$csum" = "$1" ] && \ + return 1 + + x_ rm -f "$2" + printf "BAD SHA512 %s, '%s'; need %s\n" "$csum" "$2" "$1" 1>&2 +} + +tmpclone() +{ + ( x_ git clone "$1" "$2" ) || return 1 + ( x_ git -C "$2" reset --hard "$3" ) || return 1 + + [ ! -d "$4" ] && \ + return 0 + + tmpclone_patchlist="`mktemp || err "Can't create tmp patch list"`" || \ + err "Can't create tmp patch list" "tmpclone" "$@" + + x_ find "$4" -type f | sort > "$tmpclone_patchlist" || \ + err "Can't write patch names to '$tmpclone_patchlist'" \ + "tmpclone" "$@" + + while read -r tmpclone_patch; do + + ( x_ git -C "$2" am -3 --keep-cr "$tmpclone_patch" ) || \ + err "Can't apply '$tmpclone_patch'" "tmpclone" "$@"; : + + done < "$tmpclone_patchlist" || \ + err "Can't read '$tmpclone_patchlist'" "tmpclone" "$@" + + x_ rm -f "$tmpclone_patchlist" +} diff --git a/include/env/init.sh b/include/env/init.sh new file mode 100644 index 00000000..d04474e3 --- /dev/null +++ b/include/env/init.sh @@ -0,0 +1,388 @@ +# SPDX-License-Identifier: GPL-3.0-only + +# Copyright (c) 2022 Caleb La Grange <thonkpeasant@protonmail.com> +# Copyright (c) 2022 Ferass El Hafidi <vitali64pmemail@protonmail.com> +# Copyright (c) 2020-2026 Leah Rowe <leah@libreboot.org> +# Copyright (c) 2025 Alper Nebi Yasak <alpernebiyasak@gmail.com> + +projectname="libreboot" +projectsite="https://libreboot.org/" + +eval "`newvar aur_notice basetmp board checkvarschk checkvarsxbmk configdir \ + datadir is_child python pyver reinstall relname version versiondate \ + xbmklock xbmkpath xbtmp sha512sum`" + +xbmk_init() +{ + xbmklock="$xbmkpwd/lock" + basetmp="$xbmkpwd/xbmkwd" + sha512sum="$xbmkpwd/util/sbase/sha512sum" + + if [ $# -gt 0 ] && [ "$1" = "dependencies" ]; then + x_ xbmkpkg "$@" + exit 0 + fi + + id -u 1>/dev/null 2>/dev/null || \ + err "suid check failed" "xbmk_init" "$@" + + [ "$(id -u)" = "0" ] && \ + err "this command as root is not permitted" "xbmk_init" "$@" + + export PWD="$xbmkpwd" + x_ mkdir -p "$basetmp" "cache" + + eval "$(printf "get_ver set_env set_threads git_init xchild\n" | awk \ + '{ for (i=1; i<=NF; i++) $i = $i " \"$@\" || return 0;"; print }')" +} + +xbmkpkg() +{ + xchk xbmkpkg "$@" + + [ $# -gt 2 ] && \ + reinstall="$3" + + . "config/dependencies/$2" || \ + err "Can't read 'config/dependencies/$2'" "xbmkpkg" "$@" + + if [ -z "$pkg_add" ] || [ -z "$pkglist" ]; then + err "pkg_add/pkglist not both set" "xbmkpkg" "$@" + fi + + if [ "$2" == "gentoo" ]; then + + if [ -z "$pkg_use" ] || [ -z "$pkg_use_file" ]; then + err "pkg_use/pkg_use_file not both set" "xbmkpkg" "$@" + fi + + printf "Updating package use...\n" + printf "Writing into %s" "$pkg_use_file..." + + x_ mkdir -p "`dirname $pkg_use_file`" + + printf "%s\n" "$pkg_use" >> "$pkg_use_file" + fi + + x_ $pkg_add $pkglist + + [ -n "$aur_notice" ] && \ + printf "You need AUR packages: %s\n" "$aur_notice" 1>&2; : +} + +get_ver() +{ + [ ! -f ".version" ] || \ + read -r version < ".version" || \ + err "can't read version file" "get_ver" "$@" + [ ! -f ".versiondate" ] || \ + read -r versiondate < ".versiondate" || \ + err "can't read versiondate" get_ver "$@" + + [ -f ".version" ] && [ -z "$version" ] && \ + err "version not set" "get_ver" "$@" + [ -f ".versiondate" ] && [ -z "$versiondate" ] && \ + err "versiondate not set" "get_ver" "$@" + + [ ! -e ".git" ] && [ ! -f ".version" ] && \ + version="unknown" + [ ! -e ".git" ] && [ ! -f ".versiondate" ] && \ + versiondate="1716415872" + + xbmk_sanitize_version + + [ -n "$version" ] && \ + relname="$projectname-$version"; : +} + +# a parent instance will cause this function to return 0. +# a child instance will return 1, skipping further initialisation +# after this function is called. +set_env() +{ + is_child="n" + xbmkpath="$PATH" + + # unify all temporary files/directories in a single TMPDIR + [ -n "${TMPDIR+x}" ] && [ "${TMPDIR%_*}" != "$basetmp/xbmk" ] && \ + unset TMPDIR + if [ -n "${TMPDIR+x}" ]; then + export TMPDIR="$TMPDIR" + xbtmp="$TMPDIR" + fi + [ -n "${TMPDIR+x}" ] && \ + is_child="y" + + if [ "$is_child" = "y" ]; then + xbmk_child_set_env + return 1 # child instance. discontinue initialisation. + fi + xbmk_parent_set_env +} + +xbmk_child_set_env() +{ + xbmk_child_set_tmp + + [ -z "${XBMK_CACHE+x}" ] && \ + err "XBMK_CACHE unset on child" "set_env" "$@" + [ -z "${XBMK_THREADS+x}" ] && \ + set_threads + [ -z "${XBMK_CACHE_MIRROR+x}" ] && \ + xbmk_set_mirror; : +} + +xbmk_child_set_tmp() +{ + eval "`newvar badtmp locktmp xbtmpchk`" + + xbtmpchk="`findpath "$TMPDIR" || err "!findpath $TMPDIR"`" || \ + err "!findpath '$TMPDIR'" "xbmk_child_set_tmp" "$@" + + read -r locktmp < "$xbmklock" || \ + err "can't read '$xbmklock'" "xbmk_child_set_tmp" "$@" + + if [ "$locktmp" != "$xbtmpchk" ]; then + badtmp="TMPDIR '$xbtmpchk' changed; was '$locktmp'" + printf "bad TMPDIR init, '%s': %s\n" "$TMPDIR" "$badtmp" 1>&2 + err "'$xbmklock' present with bad tmpdir. is a build running?" + fi + + xbtmp="$xbtmpchk" + export TMPDIR="$xbtmpchk" +} + +xbmk_parent_set_env() +{ + xbmk_parent_check_tmp + + printf "%s\n" "$xbtmp" > "$xbmklock" || \ + err "cannot create '$xbmklock'" set_env "$@"; : + + # not really critical for security, but it's a barrier + # against the user to make them think twice before deleting it + # in case an actual instance of xbmk is already running: + + x_ chmod -w "$xbmklock" + + forx 'xbmk_%s' parent_set_export set_version + + remkdir "$xbtmp" "$xbtmp/xbmkpath" + + forx 'xbmk_set_%s' pyver mirror +} + +xbmk_parent_check_tmp() +{ + export TMPDIR="$basetmp" + + xbmklist="`mktemp || err "can't make tmplist"`" || \ + err "can't make tmplist" xbmk_parent_check_tmp "$@" + + x_ rm -f "$xbmklist" + x_ touch "$xbmklist" + + for xtmpdir in "$basetmp"/xbmk_*; do + [ ! -e "$xtmpdir" ] || \ + printf "%s\n" "$xtmpdir" >> "$xbmklist" || \ + err "can't write '$xtmpdir' to '$xbmklist'" \ + "xbmk_parent_check_tmp" "$@"; : + done + + # set up a unified temporary directory, for common deletion later: + export TMPDIR="`x_ mktemp -d -t xbmk_XXXXXXXX`" || \ + err "can't export TMPDIR" "xbmk_parent_check_tmp" "$@" + xbtmp="$TMPDIR" + + while read -r xtmpdir; do + [ "$xtmpdir" = "$xbtmp" ] && \ + err "pre-existing $xbtmp" xbmk_parent_check_tmp "$@"; : + done < "$xbmklist" || \ + err "Can't read xbmklist: '$xbmklist'" "xbmk_parent_check_tmp" "$@" + + x_ rm -f "$xbmklist" +} + +xbmk_parent_set_export() +{ + export XBMK_CACHE="$xbmkpwd/cache" + + [ -e "$XBMK_CACHE" ] && [ ! -d "$XBMK_CACHE" ] && \ + err "cachedir '$XBMK_CACHE' is a file" \ + "xbmk_parent_set_export" "$@" + + export PATH="$xbtmp/xbmkpath:$PATH" + xbmkpath="$PATH" + + # if "y": a coreboot target won't be built if target.cfg says release=n + # (this is used to exclude certain build targets from releases) + + [ -z "${XBMK_RELEASE+x}" ] && \ + export XBMK_RELEASE="n" + [ "$XBMK_RELEASE" = "Y" ] && \ + export XBMK_RELEASE="y" + [ "$XBMK_RELEASE" != "y" ] && \ + export XBMK_RELEASE="n"; : +} + +set_threads() +{ + [ -z "${XBMK_THREADS+x}" ] && \ + export XBMK_THREADS=1 + expr "X$XBMK_THREADS" : "X-\{0,1\}[0123456789][0123456789]*$" \ + 1>/dev/null 2>/dev/null || \ + export XBMK_THREADS=1; : +} + +xbmk_set_version() +{ + version_="$version" + [ ! -e ".git" ] || \ + version="$(git describe --tags HEAD 2>&1)" || \ + version="git-$(git rev-parse HEAD 2>&1)" || \ + version="$version_" + + versiondate_="$versiondate" + [ ! -e ".git" ] || \ + versiondate="$(git show --no-patch --no-notes \ + --pretty='%ct' HEAD)" || versiondate="$versiondate_" + + update_xbmkver "." + + relname="$projectname-$version" + export LOCALVERSION="-$projectname-${version%%-*}" +} + +xbmk_set_pyver() +{ + python="python3" + pyver="2" + pyv="import sys; print(sys.version_info[:])" + + pybin python3 1>/dev/null || \ + python="python" + [ "$python" = "python3" ] && \ + pyver="3" + pybin "$python" 1>/dev/null || \ + pyver="" + + [ -z "$pyver" ] || \ + "`x_ pybin "$python"`" -c "$pyv" 1>/dev/null \ + 2>/dev/null || \ + err "Can't detect Python version." "xbmk_set_pyver" "$@" + if [ -n "$pyver" ]; then + pyver="$("$(pybin "$python")" -c "$pyv" | awk '{print $1}')" + pyver="${pyver#(}" + pyver="${pyver%,}" + fi + [ "${pyver%%.*}" != "3" ] && \ + err "Bad python version (must by 3.x)" "xbmk_set_pyver" "$@" + + # set up python in PATH (environmental variable): + ( + x_ cd "$xbtmp/xbmkpath" + x_ ln -s "`x_ pybin "$python"`" python || \ + err "can't make symlink" "xbmk_set_pyver" "$@" + + ) || err "Can't link Python in $xbtmp/xbmkpath" xbmk_set_pyver "$@"; : +} + +# Use direct path, to prevent a hang if Python is using a virtual environment, +# not command -v, to prevent a hang when checking python's version +# See: https://docs.python.org/3/library/venv.html#how-venvs-work +pybin() +{ + py="import sys; quit(1) if sys.prefix == sys.base_prefix else quit(0)" + + venv=1 + command -v "$1" 1>/dev/null 2>/dev/null || \ + venv=0 + + [ $venv -lt 1 ] || \ + "$1" -c "$py" 1>/dev/null 2>/dev/null || \ + venv=0 + + # try the direct path from PATH first. the binary specified by venv + # can be called directly, which then removes use of the venv + if [ $venv -gt 0 ] && [ -L "`command -v "$1" 2>/dev/null`" ]; then + pypath="$(findpath \ + "$(command -v "$1" 2>/dev/null)" 2>/dev/null || :)" + if [ -e "$pypath" ] && [ ! -d "$pypath" ] && \ + [ -x "$pypath" ]; then + printf "%s\n" "$pypath" + return 0 + fi + fi + + # if python venv: fall back to common PATH directories for checking: + [ $venv -gt 0 ] && for pypath in "/usr/local/bin" "/usr/bin"; do + if [ -e "$pypath/$1" ] && [ ! -d "$pypath/$1" ] && \ + [ -x "$pypath/$1" ]; then + printf "%s/%s\n" "$pypath" "$1" + return 0 + fi + done && return 1 + + # Defer to normal command -v if not a venv + command -v "$1" 2>/dev/null || \ + return 1; : +} + +xbmk_set_mirror() +{ + # defines whether cache/clone/ (regular clones) + # or cache/mirror (--mirror clones) are used, per project + + # to use cache/mirror/ do: export XBMK_CACHE_MIRROR="y" + # mirror/ stores a separate directory per repository, even per backup. + # it's slower, and uses more disk space, and some upstreams might not + # appreciate it, so it should only be used for development or archival + + [ -z "${XBMK_CACHE_MIRROR+x}" ] && \ + export XBMK_CACHE_MIRROR="n" + [ "$XBMK_CACHE_MIRROR" != "y" ] && \ + export XBMK_CACHE_MIRROR="n"; : +} + +git_init() +{ + # the git identity check is only needed for: "$mk" -b coreboot [args] + [ "${2-}" = "-b" ] && [ "${3-}" = "coreboot" ] && \ + for gitarg in "user.name" "user.email"; do + git config --includes $gitarg 1>/dev/null 2>/dev/null || \ + err "Run this first: $gitcmd \"your ${gitcmd##*.}\"" \ + "git_init" "$@"; : + done + + [ -L ".git" ] && \ + err "'$xbmkpwd/.git' is a symlink" "git_init" "$@" + [ -e ".git" ] && \ + return 0 + + ( + export GIT_COMMITTER_DATE="@$versiondate +0000" + export GIT_AUTHOR_DATE="@$versiondate +0000" + + x_ git init 1>/dev/null 2>/dev/null + x_ git add -A . 1>/dev/null 2>/dev/null + x_ git -c user.name="xbmk" -c user.email="xbmk@example.com" \ + commit -m "$projectname $version" \ + --author="xbmk <xbmk@example.com>" 1>/dev/null 2>/dev/null + x_ git -c user.name="xbmk" -c user.email="xbmk@example.com" \ + tag -a "$version" -m "$projectname $version" \ + 1>/dev/null 2>/dev/null; : + + ) || err "Can't initialise workdir" git_init "$@"; : +} + +xchild() +{ + xbmk_rval=0 + + ( x_ "$mk" "$@" ) || xbmk_rval=1 + + ( x_ rm -Rf "$xbtmp" ) || xbmk_rval=1 + ( x_ rm -f "$xbmklock" ) || xbmk_rval=1 + + exit $xbmk_rval +} |
