From d1ba94ea0842c2e464f2c9177529de5319c8a33f Mon Sep 17 00:00:00 2001 From: Leah Rowe Date: Sun, 8 Oct 2023 02:34:50 +0100 Subject: update/release/*: merge to update/project/release The logic has been re-written, where source archives are concerned. This clones the current repository, and starts a new build from scratch. A custom release directory is possible, by passing -d This eliminates a step during build-testing, saving hours of time, because it builds the release archive *inside* the release archive, with git files removed, thus replicating the same setup that the user would have. This also makes everything a bit more consistent, because it's guaranteed that a release archive will always have the same files; previously, the release build script would only copy what was already built, without building anything. Now, this script builds everything itself. The script also builds serprog images, not just coreboot. Usage: ./update project release If -d is not passed, release/ is used inside lbmk. Otherwise, you could do: ./update project release -d /path/to/directory If the directory exists, this script will exit (error). Other minor fixes: build/fw/coreboot: make version in coreboot-version (file) not contain hyphens, to work around a quirk in coreboot's build system when not building on regular libreboot releases. this quirk only appears when lbmk is not being compiled under git. The other main benefit of this change is that the new script will probably require a lot less maintenance. Signed-off-by: Leah Rowe --- script/update/project/release | 299 ++++++++++++++++++++++++++++++++++++++++++ script/update/project/trees | 6 +- script/update/release/roms | 175 ------------------------ script/update/release/src | 111 ---------------- 4 files changed, 302 insertions(+), 289 deletions(-) create mode 100755 script/update/project/release delete mode 100755 script/update/release/roms delete mode 100755 script/update/release/src diff --git a/script/update/project/release b/script/update/project/release new file mode 100755 index 00000000..8843f2d1 --- /dev/null +++ b/script/update/project/release @@ -0,0 +1,299 @@ +#!/usr/bin/env sh +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2020-2023 Leah Rowe + +[ "x${DEBUG+set}" = 'xset' ] && set -v +set -u -e + +. "include/err.sh" + +eval "$(setvars "" vdir basename src_dirname srcdir _nogit _xm target romdir \ + microcode_required CONFIG_HAVE_MRC CONFIG_HAVE_ME_BIN \ + CONFIG_KBC1126_FIRMWARE CONFIG_INCLUDE_SMSC_SCH5545_EC_FW)" + +_f="coreboot u-boot seabios flashrom grub memtest86plus me_cleaner uefitool" +_f="${_f} bios_extract biosutilities pico-serprog stm32-vserprog" +_f="${_f} rpi-pico-serprog" + +ifdtool="cbutils/default/ifdtool" +cbfstool="cbutils/default/cbfstool" + +main() +{ + vdir="release" + while getopts d: option + do + case "${1}" in + -d) vdir="${2}" ;; + *) err "Invalid option" ;; + esac + done + [ -z "${vdir}" ] && err "Empty directory names not allowed" + + vdir="${vdir}/${version}" + basename="${projectname}-${version}" + src_dirname="${basename}_src" + srcdir="${vdir}/${src_dirname}" + + [ -e "${vdir}" ] && err "already exists: \"${vdir}\"" + + mkvdir + build_release +} + +mkvdir() +{ + mkdir -p "${vdir}" || err "mkvdir: !mkdir -p \"${vdir}\"" + + git_init || err "mkvdir: !git_init \"${vdir}\"" + git clone . "${srcdir}" || err "mkdir: !gitclone \"${srcdir}\"" + [ -z "${_nogit}" ] || x_ rm -Rf ".git" + + rm -Rf "${srcdir}/.git" + insert_version_files "${srcdir}" || err "mkvdir ${srcdir}: versionfile" +} + +# release archives contain .gitignore, but not .git. +# this script can be run from lbmk.git, or an archive. +git_init() +{ + [ -L ".git" ] && err "Reference .git is a symlink" + [ -e ".git" ] && return 0 + eval "$(setvars "$(date -Rd @${versiondate})" cdate _nogit)" + + git init || return 1 + git add -A . || return 1 + git commit -m "${projectname} ${version}" --date "${cdate}" || return 1 + git tag -a "${version}" -m "${projectname} ${version}" || return 1 +} + +build_release() +{ + _xm="build_release ${vdir}" + ( + cd "${srcdir}" || err "${_xm}: !cd \"${srcdir}\"" + + fetch_trees + mkrom_images + clean_release + ) + ( + cd "${srcdir%/*}" || err "${_xm}: mktarball \"${srcdir}\"" + mktarball "${srcdir##*/}" "${srcdir##*/}.tar.xz" || err "${_xm}: mksrc" + ) + rm -Rf "${srcdir}" || err "${_xm}: !rm -Rf \"${srcdir}\"" +} + +fetch_trees() +{ + for x in ${_f}; do + ./update project trees -f "${x}" || err "${_xm}: fetch ${x}" + done + + x_ rm -Rf src/coreboot/coreboot src/u-boot/u-boot src/seabios/seabios + + # by doing this first, we build-test under the same conditions as + # the user, when they extract and build from the src tarball. this + # saves time on build-testing, because we build once, not twice. thus: + + rm -Rf */.git* */*/.git* */*/*/.git* */*/*/*/.git* */*/*/*/*/.git* \ + */*/*/*/*/*/.git* */*/*/*/*/*/*/.git* */*/*/*/*/*/*/*/.git* || \ + err "${_xm}: rm-dotgit" +} + +mkrom_images() +{ + ./build fw coreboot all || err "${_xm}: roms-all" + ./build fw serprog rp2040 || err "${_xm}: rp2040" + ./build fw serprog stm32 || err "${_xm}: stm32" + + for rombuild in bin/*; do + [ -d "${rombuild}" ] || continue + handle_rom_archive "${rombuild}" + done + + mv "release/${version}/roms/" ../roms || err "${_xm}: copy roms/" +} + +handle_rom_archive() +{ + builddir="${1}" + romdir="tmp/romdir" + x_ rm -Rf "${romdir}" + target="${builddir##*/}" + + if [ ! -f "config/coreboot/${target}/target.cfg" ]; then + # No config, just make a tarball + tarball="release/${version}/roms/${basename}_${target}.tar.xz" + insert_copying_files "${builddir}" + mktarball "${builddir}" "${tarball}" + return 0 + fi + + romdir="${romdir}/bin/${target}" + x_ mkdir -p "${romdir}" + x_ cp "${builddir}/"* "${romdir}" + + handle_vendorfiles + + printf "Generating release/%s/roms/%s-%s_%s.tar.xz\n" \ + "${version}" "${projectname}" "${version}" "${target##*/}" + insert_version_files "${romdir}" || \ + err "mkrom_tarball ${romdir}: versionfile" + + insert_copying_files "${romdir}" + mkrom_tarball +} + +handle_vendorfiles() +{ + microcode_required="y" + . "config/coreboot/${target}/target.cfg" + if [ "${microcode_required}" != "y" ] && \ + [ "${microcode_required}" != "n" ]; then microcode_required="y"; fi + + for x in CONFIG_HAVE_MRC CONFIG_HAVE_ME_BIN CONFIG_KBC1126_FIRMWARE \ + CONFIG_INCLUDE_SMSC_SCH5545_EC_FW; do + eval "${x}=\"y\"" + grep "${x}=y" "config/coreboot/${target}/config/"* || \ + eval "${x}=\"n\"" + done + + # remove ME/MRC/EC firmware from ROM images + if [ "${CONFIG_HAVE_ME_BIN}" = "y" ] || \ + [ "${target}" = "e6400_4mb" ] || \ + [ "${microcode_required}" = "n" ]; then + strip_archive + fi +} + +strip_archive() +{ + if [ "${microcode_required}" = "n" ]; then + for romfile in "${romdir}"/*.rom; do + [ -f "${romfile}" ] || continue + strip_ucode "${romfile}" + done + for romfile in "${romdir}"/*.tmprom; do + [ -f "${romfile}" ] || continue + x_ mv "${romfile}" "${romfile%.tmprom}.rom" + done + fi + + # Hash the rom before removing vendor files + x_ rm -f "${romdir}/vendorhashes" + x_ touch "${romdir}/vendorhashes" + ( + x_ cd "${romdir}" + x_ sha512sum *.rom >> vendorhashes + ) + for romfile in "${romdir}"/*.rom; do + [ -f "${romfile}" ] || continue + strip_rom_image "${romfile}" + done +} + +strip_ucode() +{ + romfile=${1} + _newrom_b="${romfile%.rom}_nomicrocode.tmprom" + x_ cp "${romfile}" "${_newrom_b}" + microcode_present="y" + "${cbfstool}" "${_newrom_b}" remove -n \ + cpu_microcode_blob.bin 2>/dev/null || microcode_present="n" + [ "${microcode_present}" = "n" ] || return 0 + printf "REMARK: '%s' already lacks microcode\n" "${romfile}" 1>&2 + printf "Renaming default ROM file instead.\n" 1>&2 + x_ mv "${romfile}" "${_newrom_b}" +} + +strip_rom_image() +{ + [ -f "${1}" ] || return 0 + [ "${CONFIG_HAVE_ME_BIN}" != "y" ] || \ + x_ "${ifdtool}" --nuke me "${1}" -O "${1}" + [ "${CONFIG_HAVE_MRC}" != "y" ] || \ + x_ "${cbfstool}" "${1}" remove -n mrc.bin + [ "${CONFIG_INCLUDE_SMSC_SCH5545_EC_FW}" != "y" ] || \ + x_ "${cbfstool}" "${1}" remove -n sch5545_ecfw.bin + if [ "${CONFIG_KBC1126_FIRMWARE}" = "y" ]; then + x_ "${cbfstool}" "${1}" remove -n ecfw1.bin + x_ "${cbfstool}" "${1}" remove -n ecfw2.bin + fi + # TODO: replace this board-specific hack + [ "${target}" = "e6400_4mb" ] || return 0 + x_ "${cbfstool}" "${1}" remove -n "pci10de,06eb.rom" +} + +insert_copying_files() +{ + x_ rm -Rf "${1}/licenses" + x_ mkdir -p "${1}/licenses" + l="${1}/licenses" + # copy licenses to rom image archive, for completion + x_ cp "src/grub/COPYING" "${l}/COPYING.grub" + x_ cp "src/coreboot/default/COPYING" "${l}/COPYING.coreboot" + x_ cp -R "src/coreboot/default/LICENSES" "${l}/LICENSES.coreboot" + x_ cp "src/seabios/default/COPYING" "${l}/COPYING.coreboot" + x_ cp "src/seabios/default/COPYING.LESSER" "${l}/COPYING.LESSER.seabios" + x_ cp -R "src/u-boot/default/Licenses" "${l}/COPYING.u-boot" +} + +mkrom_tarball() +{ + archivename="${basename}_${target##*/}" + f="release/${version}/roms/${archivename}" + x_ mkdir -p "${f%/*}" + ( + x_ cd "${romdir%/bin/${target}}" + mktarball "bin/${target}" "${archivename}.tar.xz" + ) + x_ mv "${romdir%/bin/${target}}/${archivename}.tar.xz"* "${f%/*}" + + printf "Created ROM archive: ${f%/*}/${archivename}.tar.xz" +} + +clean_release() +{ + for x in "src/coreboot/default/util/kbc1126" util/*; do + [ ! -f "${x}/Makefile" ] && continue + x_ make clean -C "${x}" + done + + for x in ${_f}; do + ./update project trees -c "${x}" || err "${_xm}: pclean ${x}" + done + ./update project trees -x coreboot || err "${_xm}: clean-crossgcc" + ./update project trees -c "stm32-vserprog/libopencm3" + + # make absolutely sure crossgcc was purged. for some reason, + # crossgcc-clean isn't always reliable on every coreboot tree + for xgcc in "build-" "binutils-" "gcc-" "gmp-" "mpc-" "mpfr-" \ + "llvm-" "clang-tools-" "cfe-" "compiler-rt-" "acpica-" \ + "getopt" "xgcc"; do + x_ rm -Rf src/coreboot/*/util/crossgcc/${xgcc}* + done + + rm -Rf vendor mrc ec pciroms || err "${_xm}: vendor files" + rm -Rf elf tmp cbutils src/pico-serprog/build release bin || \ + err "${_xm}: elf, tmp, cbutils, pico-serprog/build" +} + +insert_version_files() +{ + printf "%s\n" "${version}" > "${1}/version" || return 1 + printf "%s\n" "${versiondate}" > "${1}/versiondate" || return 1 + printf "%s\n" "${projectname}" > "${1}/projectname" || return 1 +} + +mktarball() +{ + [ "${2%/*}" = "${2}" ] || x_ mkdir -p "${2%/*}" + x_ tar -c "${1}" | xz -T0 -9e > "${2}" + ( + [ "${2%/*}" != "${2}" ] && x_ cd "${2%/*}" + x_ sha512sum "${2##*/}" > "${2##*/}.sha512" + ) +} + +main $@ diff --git a/script/update/project/trees b/script/update/project/trees index 1055d500..7df3c8bf 100755 --- a/script/update/project/trees +++ b/script/update/project/trees @@ -11,7 +11,7 @@ set -u -e . "include/option.sh" . "include/git.sh" -export LOCALVERSION="-${projectname}-${version}" +export LOCALVERSION="-${projectname}-${version%%-*}" eval "$(setvars "" arch cfgsdir codedir config config_name crossgcc_ada mode \ elfdir listfile project romtype target target_dir targets tree cbfstool _f \ @@ -271,9 +271,9 @@ run_make_command() [ -f "${codedir}/Makefile" ] || [ -f "${codedir}/makefile" ] || \ [ -f "${codedir}/GNUmakefile" ] || return 1 [ "${project}" = "coreboot" ] && [ -z "${mode}" ] && \ - x_ printf "%s\n" "${version}" >"${codedir}/.coreboot-version" + x_ printf "%s\n" "${version%%-*}" >"${codedir}/.coreboot-version" - x_ make -C "${codedir}" -j$(nproc) ${mode} + x_ make ${mode} -j$(nproc) -C "${codedir}" [ "${mode}" != "clean" ] && return 0 make -C "${codedir}" distclean 2>/dev/null || : } diff --git a/script/update/release/roms b/script/update/release/roms deleted file mode 100755 index cec93951..00000000 --- a/script/update/release/roms +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env sh -# SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2020,2021,2022,2023 Leah Rowe - -[ "x${DEBUG+set}" = 'xset' ] && set -v -set -u -e - -. "include/err.sh" - -tree="default" -eval "$(setvars "" target CONFIG_HAVE_MRC CONFIG_HAVE_ME_BIN \ - CONFIG_KBC1126_FIRMWARE CONFIG_INCLUDE_SMSC_SCH5545_EC_FW)" -ifdtool="cbutils/${tree}/ifdtool" -cbfstool="cbutils/${tree}/cbfstool" -microcode_required="y" - -main() -{ - printf "Building ROM image archives for version %s\n" "${version}" - - init_check - for romdir in bin/*; do - [ -d "${romdir}" ] || continue - make_archive "${romdir}" - done - printf "\nROM archives available at release/%s/roms/\n\n" "${version}" -} - -init_check() -{ - [ ! -d "bin/" ] && \ - err "init_check: no ROMs built yet (error)" - [ -d "release/" ] || \ - x_ mkdir -p release/ - [ -d "release/${version}/" ] || \ - x_ mkdir -p "release/${version}/" - [ ! -d "release/${version}/roms/" ] || \ - x_ rm -Rf "release/${version}/roms/" - [ -d "release/${version}/roms/" ] && return 0 - x_ mkdir -p "release/${version}/roms/" -} - -make_archive() -{ - builddir="${1}" - romdir="tmp/romdir" - x_ rm -Rf "${romdir}" - target="${builddir##*/}" - - if [ ! -f "config/coreboot/${target}/target.cfg" ]; then - # No config, just make a tarball - tarball=release/${version}/roms/${target}_${version}.tar.xz - x_ tar -c "${builddir}" | xz -T0 -6 > ${tarball} || \ - x_ rm -f ${tarball} - return 0 - fi - - romdir="${romdir}/bin/${target}" - x_ mkdir -p "${romdir}" - x_ cp "${builddir}"/* "${romdir}" - - printf "%s\n" "${target}" - - microcode_required="y" - . "config/coreboot/${target}/target.cfg" - if [ "${microcode_required}" != "y" ] && \ - [ "${microcode_required}" != "n" ]; then microcode_required="y"; fi - - for x in CONFIG_HAVE_MRC CONFIG_HAVE_ME_BIN CONFIG_KBC1126_FIRMWARE \ - CONFIG_INCLUDE_SMSC_SCH5545_EC_FW; do - eval "${x}=\"y\"" - grep "${x}=y" "config/coreboot/${target}/config/"* || \ - eval "${x}=\"n\"" - done - - # remove ME/MRC/EC firmware from ROM images - if [ "${CONFIG_HAVE_ME_BIN}" = "y" ] || \ - [ "${target}" = "e6400_4mb" ] || \ - [ "${microcode_required}" = "n" ]; then - strip_archive "${romdir}" - fi - - printf "Generating release/%s/roms/%s-%s_%s.tar.xz\n" \ - "${version}" "${projectname}" "${version}" "${target##*/}" - x_ printf "%s\n" "${version}" > "${romdir}/version" - x_ printf "%s\n" "${versiondate}" > "${romdir}/versiondate" - x_ printf "%s\n" "${projectname}" > "${romdir}/projectname" - - for x in u-boot seabios coreboot; do - [ -d "src/${x}/default" ] || \ - x_ ./update project trees -f ${x} default - done - [ -d src/grub ] || x_ ./update project trees -f grub - - # copy licenses to rom image archive, for completion - x_ cp "src/grub/COPYING" "${romdir}/COPYING.grub" - x_ cp "src/coreboot/default/COPYING" "${romdir}/COPYING.coreboot" - x_ cp "src/seabios/default/COPYING" "${romdir}/COPYING.coreboot" - x_ cp -R "src/u-boot/default/Licenses" "${romdir}/COPYING.u-boot" - - archivename="${projectname}-${version}_${target##*/}" - f="release/${version}/roms/${archivename}" - ( - x_ cd "${romdir%/bin/${target}}" - x_ tar -c "bin/${target}/" | xz -T0 -9e > "${archivename}.tar.xz" - ) - x_ mv "${romdir%/bin/${target}}/${archivename}.tar.xz" "${f}.tar.xz" - x_ rm -Rf "${romdir%/bin/${target}}" -} - -strip_archive() -{ - romdir=${1} - [ -d "src/coreboot/${tree}" ] || \ - x_ ./update project trees -f coreboot ${tree} - - x_ ./update project trees -b coreboot utils ${tree} - - if [ "${microcode_required}" = "n" ]; then - for romfile in "${romdir}"/*.rom; do - [ -f "${romfile}" ] || continue - strip_ucode "${romfile}" - done - for romfile in "${romdir}"/*.tmprom; do - [ -f "${romfile}" ] || continue - x_ mv "${romfile}" "${romfile%.tmprom}.rom" - done - fi - - # Hash the rom before removing vendor files - x_ rm -f "${romdir}/vendorhashes" - x_ touch "${romdir}/vendorhashes" - ( - x_ cd "${romdir}" - x_ sha512sum *.rom >> vendorhashes - ) - for romfile in "${romdir}"/*.rom; do - [ -f "${romfile}" ] || continue - strip_rom_image "${romfile}" - done -} - -strip_ucode() -{ - romfile=${1} - _newrom_b="${romfile%.rom}_nomicrocode.tmprom" - x_ cp "${romfile}" "${_newrom_b}" - microcode_present="y" - "${cbfstool}" "${_newrom_b}" remove -n \ - cpu_microcode_blob.bin 2>/dev/null || microcode_present="n" - [ "${microcode_present}" = "n" ] || return 0 - printf "REMARK: '%s' already lacks microcode\n" "${romfile}" 1>&2 - printf "Renaming default ROM file instead.\n" 1>&2 - x_ mv "${romfile}" "${_newrom_b}" -} - -strip_rom_image() -{ - [ -f "${1}" ] || return 0 - [ "${CONFIG_HAVE_ME_BIN}" != "y" ] || \ - x_ "${ifdtool}" --nuke me "${1}" -O "${1}" - [ "${CONFIG_HAVE_MRC}" != "y" ] || \ - x_ "${cbfstool}" "${1}" remove -n mrc.bin - [ "${CONFIG_INCLUDE_SMSC_SCH5545_EC_FW}" != "y" ] || \ - x_ "${cbfstool}" "${1}" remove -n sch5545_ecfw.bin - if [ "${CONFIG_KBC1126_FIRMWARE}" = "y" ]; then - x_ "${cbfstool}" "${1}" remove -n ecfw1.bin - x_ "${cbfstool}" "${1}" remove -n ecfw2.bin - fi - # TODO: replace this board-specific hack - [ "${target}" = "e6400_4mb" ] || return 0 - x_ "${cbfstool}" "${1}" remove -n "pci10de,06eb.rom" -} - -main $@ diff --git a/script/update/release/src b/script/update/release/src deleted file mode 100755 index d75ec170..00000000 --- a/script/update/release/src +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env sh -# SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2020,2021,2023 Leah Rowe - -[ "x${DEBUG+set}" = 'xset' ] && set -v -set -u -e - -. "include/err.sh" - -trees_fetch_list="coreboot u-boot seabios" -simple_fetch_list="flashrom grub memtest86plus me_cleaner uefitool" -simple_fetch_list="${simple_fetch_list} bios_extract biosutilities" -download_only_list="pico-serprog stm32-vserprog" - -dirlist="config util script include" -linklist="update" # symlinks in main directory, to script: lbmk -filelist="build README.md COPYING version versiondate projectname .gitignore" - -eval "$(setvars "" reldir dirname srcdir)" - -printf "Building source code archive, version %s\n" "${version}" - -main() -{ - create_release_directory - download_modules - copy_files - purge_files - - create_release_archive - printf "Source code archive available at %s.tar.xz\n\n" "${srcdir}" -} - -create_release_directory() -{ - reldir="release/${version}" - dirname="${projectname}-${version}_src" - srcdir="${reldir}/${dirname}" - - xx_ mkdir -p "${reldir}" - xx_ rm -Rf "${srcdir}" "${srcdir}.tar.xz" - xx_ mkdir -p "${srcdir}" -} - -download_modules() -{ - for modname in ${trees_fetch_list} ${simple_fetch_list} \ - ${download_only_list}; do - [ -d "src/${modname}/" ] || \ - xx_ ./update project trees -f ${modname} - done -} - -copy_files() -{ - xx_ cp -R "src" "${srcdir}/src" - for dir in ${trees_fetch_list}; do - xx_ rm -Rf "${srcdir}/src/${dir}/${dir}" - done - for dir in ${dirlist}; do - xx_ cp -R "${dir}/" "${srcdir}/" - done - - for i in ${filelist}; do - [ -f "${i}" ] || fail "copy_files: '${i}' does not exist" - xx_ cp "${i}" "${srcdir}/" - done - ( - xx_ cd "${srcdir}/" - for i in ${linklist}; do - xx_ ln -s build "${i}" - done - ) -} - -purge_files() -{ - ( - xx_ cd "${srcdir}" - [ ! -d "src/coreboot/default/util/kbc1126" ] || \ - xx_ ./update project trees -c "src/coreboot/default/util/kbc1126" - xx_ ./update project trees -x coreboot - for p in u-boot seabios coreboot; do - xx_ ./update project trees -c "${p}" - done - xx_ ./update project trees -c bios_extract flashrom grub uefitool \ - stm32-vserprog stm32-vserprog/libopencm3 util/* memtest86plus - - xx_ rm -Rf .git */.git* */*/.git* */*/*/.git* */*/*/*/.git* \ - */*/*/*/*/.git* */*/*/*/*/*/.git* */*/*/*/*/*/*/.git* \ - */*/*/*/*/*/*/*/.git* - xx_ rm -Rf cbutils elf src/pico-serprog/build - ) -} - -create_release_archive() -{ - ( - xx_ cd "${reldir}/" - xx_ tar -c "${dirname}/" | xz -T0 -9e >"${dirname}.tar.xz" - xx_ rm -Rf "${dirname}/" - ) -} - -fail() -{ - [ -z "${srcdir}" ] || rm -Rf "${srcdir}" 1>/dev/null 2>/dev/null || : - fail "${1}" -} - -main $@ -- cgit v1.2.1