From 99510d5479ae893889e81fd37d63eaa97b25c5e1 Mon Sep 17 00:00:00 2001 From: Yuri Bushmelev Date: Sat, 5 Mar 2011 16:55:24 +0300 Subject: contrib/testing/testbuilder: update to 0.4 --- contrib/testing/testbuilder/README | 15 ++- contrib/testing/testbuilder/setup-testbuilder | 12 +- contrib/testing/testbuilder/testbuilder | 127 +++++++++++++-------- .../testing/testbuilder/testbuilder.conf.sample | 65 +---------- .../testbuilder/testbuilder.d/master.conf.sample | 13 +++ .../testbuilder/testbuilder.d/off/my.conf.sample | 10 ++ .../testbuilder/testbuilder.d/testing.conf.sample | 37 ++++++ 7 files changed, 159 insertions(+), 120 deletions(-) create mode 100644 contrib/testing/testbuilder/testbuilder.d/master.conf.sample create mode 100644 contrib/testing/testbuilder/testbuilder.d/off/my.conf.sample create mode 100644 contrib/testing/testbuilder/testbuilder.d/testing.conf.sample diff --git a/contrib/testing/testbuilder/README b/contrib/testing/testbuilder/README index fe7810567f..af4aa313dc 100644 --- a/contrib/testing/testbuilder/README +++ b/contrib/testing/testbuilder/README @@ -9,7 +9,7 @@ I assume below that you wish to install TestBuilder into `~/testbuilder` directo 2. Run the `setup-testbuilder` script with an OE branch and BitBake branch you wish to use. E.g.: - ./setup-testbuilder "testing-next" "1.10" + ./setup-testbuilder "testing-next" "1.12" It will clone the OE and BitBake trees and prepare the file `build/conf/local.conf` for you. You can review `local.conf` afterward and add some settings there. @@ -17,15 +17,18 @@ I assume below that you wish to install TestBuilder into `~/testbuilder` directo 4. Create `testbuilder.conf` according to your needs. You can use `testbuilder.conf.sample` as reference. -5. Run `./testbuilder` and wait for the build results. Check the `logs` subdir for log +5. Create your build configs in `testbuilder.d` directory. You can use provided + sample config files as reference. + +6. Run `./testbuilder` and wait for the build results. Check the `logs` subdir for log files (`tail logs/B.*` is useful) You can run a single configured build (e.g. "testing"). - ./testbuilder -B testing + ./testbuilder -B ./testbuilder.d/testing.conf -Or you can use TestBuilder to build a specific distro/machine/image -(e.g. angstrom-2008.1/qemuarm/console-image). +Or you can use TestBuilder to build a specific distro/machine/image just like +bitbake (e.g. angstrom-2008.1/qemuarm/console-image). ./testbuilder -D angstrom-2008.1 -M qemuarm console-image @@ -34,6 +37,8 @@ use the option `-S`. ./testbuilder -D angstrom-2008.1 -M qemuarm -S +Look ./testbuilder -h for full list of possible options. + -- Yury 'Jay7' Bushmelev email: jay4mail at gmail com diff --git a/contrib/testing/testbuilder/setup-testbuilder b/contrib/testing/testbuilder/setup-testbuilder index 240e99b3b9..a403c9b385 100755 --- a/contrib/testing/testbuilder/setup-testbuilder +++ b/contrib/testing/testbuilder/setup-testbuilder @@ -72,12 +72,12 @@ TMPDIR = "${TMP_DIR}" BBFILES ?= "${OE_DIR}/recipes/*/*.bb" # ENABLE_BINARY_LOCALE_GENERATION = "0" -# GLIBC_GENERATE_LOCALES = "en_US.UTF-8 en_GB.UTF-8 de_DE.UTF-8 fr_FR.UTF-8 ru_RU.UTF-8" -# IMAGE_LINGUAS="en-us en-gb de-de fr-fr ru-ru" +# GLIBC_GENERATE_LOCALES = "en_US.UTF-8 en_GB.UTF-8 de_DE.UTF-8" +# IMAGE_LINGUAS="en-us en-gb" # jffs2, tar(.gz|bz2), cpio(.gz), cramfs, ext2(.gz), ext3(.gz) # squashfs, squashfs-lzma -IMAGE_FSTYPES = "jffs2 tar.gz" +IMAGE_FSTYPES ?= "tar.gz" # CCACHE = "${@bb.which(bb.data.getVar('PATH', d, 1), 'ccache') and 'ccache '}" PARALLEL_MAKE = "-j ${MAKE_NUMBER_THREADS}" @@ -90,9 +90,9 @@ INHERIT += "rm_work" BBINCLUDELOGS = "yes" # OE stats client (make troubleshooting easier) -INHERIT += "oestats-client" -OESTATS_BUILDER = "${TB_BUILDER}" -OESTATS_SERVER = "tinderbox.openembedded.org" +#INHERIT += "oestats-client" +#OESTATS_BUILDER = "${TB_BUILDER}" +#OESTATS_SERVER = "tinderbox.openembedded.org" EOF diff --git a/contrib/testing/testbuilder/testbuilder b/contrib/testing/testbuilder/testbuilder index b86c8303d9..755ac5e73b 100755 --- a/contrib/testing/testbuilder/testbuilder +++ b/contrib/testing/testbuilder/testbuilder @@ -1,7 +1,7 @@ #!/bin/sh # TestBuilder for OpenEmbedded -# Copyright (c) 2010 Yuri Bushmelev +# Copyright (c) 2010-2011 Yuri Bushmelev # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,26 +12,27 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -VERSION="0.3.1" +VERSION="0.4" PACKAGE="TestBuilder" TB_DIR=`dirname $0` TB_NAME=`basename $0` cd "${TB_DIR}" +CFG_FILE="${TB_DIR}/testbuilder.conf" +CFG_DIR="${TB_DIR}/testbuilder.d" +. "${CFG_FILE}" + tb_usage() { echo "${PACKAGE} v${VERSION}" # Run only specified build echo "Usage: ${TB_NAME} -Vh" - echo " or ${TB_NAME} [-B build [-D distro] [-M machine] [-S]]" + echo " or ${TB_NAME} [-B ${CFG_DIR}/build.conf [-D distro] [-M machine] [-S]]" # Behave like DISTRO=distro MACHINE=machine bitbake images # with -S testbuilder will only setup env and run interactive sub-shell echo " or ${TB_NAME} -D distro -M machine [-T /full/path/to/tmpdir] [-S | images]" } -CFG_FILE="${TB_DIR}/testbuilder.conf" -. "${CFG_FILE}" - # Sanity cleanup ORIG_PATH=${PATH} ORIG_LD_LIBRARY_PATH=${LD_LIBRARY_PATH} @@ -54,11 +55,15 @@ TB_LOG="${LOG_DIR}/tb.log" #export LD_LIBRARY_PATH LANG PATH export LD_LIBRARY_PATH LANG -BB_ENV_EXTRAWHITE="MACHINE DISTRO TB_BUILDER OE_DIR TMP_DIR DL_DIR MAKE_NUMBER_THREADS BB_NUMBER_THREADS" +BB_ENV_EXTRAWHITE="MACHINE DISTRO TB_BUILDER OE_DIR TMP_DIR DL_DIR MAKE_NUMBER_THREADS BB_NUMBER_THREADS IMAGE_FSTYPES" export BB_ENV_EXTRAWHITE BBPATH TB_BUILDER OE_DIR TMP_DIR DL_DIR MAKE_NUMBER_THREADS BB_NUMBER_THREADS +BB_CMD=bitbake + mkdir -p ${LOG_DIR} +[ -f "${TB_LOG}" ] && mv "${TB_LOG}" "${TB_LOG}.0" + ### Functions # log message @@ -123,10 +128,29 @@ tb_git_branch() { cd "${_cwd}" } +# detect build status by log +tb_get_build_status() { + local _log=$1 + local _line + + if [ ! -f ${_log} ]; then + echo "UNKNOWN" + return + fi + + _line=$(tail -n1 ${_log}) + tb_log " ++ ${_line}" + if [ "${_line}" != "${_line#ERROR}" ]; then + echo "FAIL" + else + echo "OK" + fi +} # Do build of image-set $bbimages for machine-set $MLIST -# tb_build_machines "$MLIST" $bbimages +# tb_build_machines "$btype" "$MLIST" $bbimages tb_build_machines() { + local btype=$1; shift local _MLIST=$1; shift local _bbimages=$@ local _machine @@ -149,29 +173,30 @@ tb_build_machines() { tb_interactive_shell else - tb_log "${_machine} build started, images: ${_bbimages}" + tb_log " ++ ${_machine} build started" case "${SEPARATE_BUILD}" in [Yy][Ee][Ss]) # Build images separately for _image in $_bbimages; do - tb_log "${_image} build started" - _log="${LOG_DIR}/B.${build}.${DISTRO}.${_machine}.${_image}.log" + tb_log " ++ ${_image} build started" + _log="${LOG_DIR}/B.${build}.${DISTRO}.${_machine}.${_image}.${btype}.log" [ -f "${_log}" ] && mv "${_log}" "${_log}.0" - bitbake ${BB_OPTS} ${_image} 2>&1 | tee "${_log}" + ${BB_CMD} ${BB_OPTS} ${_image} 2>&1 | tee "${_log}" _rc=$? - tb_log "${_image} build finished. Exit code: ${_rc}." + tb_log " ++ ${_image} build finished. Exit code: ${_rc}. Build status: $(tb_get_build_status ${_log})" done ;; *) # Build all images with one bitbake invocation - _log="${LOG_DIR}/B.${build}.${DISTRO}.${_machine}.log" + _log="${LOG_DIR}/B.${build}.${DISTRO}.${_machine}.${btype}.log" [ -f "${_log}" ] && mv "${_log}" "${_log}.0" - bitbake ${BB_OPTS} ${_bbimages} 2>&1 | tee "${_log}" + ${BB_CMD} ${BB_OPTS} ${_bbimages} 2>&1 | tee "${_log}" _rc=$? + tb_log " ++ Build finished. Exit code: ${_rc}. Build status: $(tb_get_build_status ${_log})" ;; esac - tb_log "${_machine} build finished. Exit code:${_rc}." + tb_log " + ${_machine} build finished." fi done @@ -257,32 +282,44 @@ fi [ -z "${DO_RUN_BUILDS}" ] && exit 0 # Use build from cmdline if any -[ -n "$OBUILD" ] && BUILDS=$OBUILD +if [ -n "$OBUILD" ]; then + BUILDS=$OBUILD +else + BUILDS="$CFG_DIR"/*.conf +fi # Auto-building mode tb_log "${PACKAGE} v${VERSION}" -for build in $BUILDS; do - [ "${build}" != "${build#!}" ] && continue +for buildconf in ${BUILDS}; do + # Cleanup env + unset DISABLED SETS DISTROS DISTRO TASKS OE_BRANCH BB_BRANCH BUILD IMAGE_FSTYPES + for bset in $SETS; do + unset MACHINES_${bset} + done + + . ${buildconf} + + [ "$DISABLED" == "yes" ] && continue + + b=$(basename ${buildconf}) + build=${b%.conf} - eval BUILD="\$BUILD_${build}" - tb_log "= Started build '$build'" - [ -n "$BUILD" ] && tb_log "= $BUILD" + tb_log "" + tb_log "=== Started build '$build'" + [ -n "$BUILD" ] && tb_log "=== $BUILD" # change BB branch - eval BB_BRANCH="\$BB_BRANCH_${build}" if [ -n "${BB_BRANCH}" ]; then tb_git_branch "$BB_DIR" "$BB_BRANCH" fi # change OE branch - eval OE_BRANCH="\$OE_BRANCH_${build}" if [ -n "${OE_BRANCH}" ]; then tb_git_branch "$OE_DIR" "$OE_BRANCH" fi # parse tasks - eval TASKS="\$TASKS_${build}" unset DO_BUILD_CLEAN DO_BUILD_INCREMENTAL for task in $TASKS; do [ "${task}" != "${task#!}" ] && continue @@ -297,46 +334,40 @@ for build in $BUILDS; do done # distros - eval DISTROS="\$DISTROS_${build}" for distro in $DISTROS; do [ "${distro}" != "${distro#!}" ] && continue [ -n "${ODISTRO}" -a "${distro#!}" != "${ODISTRO}" ] && continue export DISTRO=$distro - # Get build sets - eval BSLIST="\$BUILD_SETS_${build}" - - # Empty $BUILD_SETS is exception - we should use $MACHINES + # Empty $SETS is exception - we should use $MACHINES # to build every machine in separate TMPDIRs - if [ -z "${BSLIST}" ]; then + if [ -z "${SETS}" ]; then # Use every machine as separate build set - eval MACHINES="\$MACHINES_${build}" for machine in $MACHINES; do [ "${machine}" != "${machine#!}" ] && continue eval "MACHINE_${machine}=${machine}" - BSLIST="${BSLIST} ${machine}" + SETS="${SETS} ${machine}" done fi - tb_log "== Processing distro ${DISTRO}, build set: ${BSLIST}" + tb_log "|" + tb_log "++= DISTRO: ${DISTRO}" - for bset in $BSLIST; do + for bset in $SETS; do [ "${bset}" != "${bset#!}" ] && continue - eval MLIST="\$MACHINES_${build}_${bset}" - [ -z "$MLIST" ] && eval MLIST="\$MACHINES_${build}" + eval MLIST="\$MACHINES_${bset}" + [ -z "$MLIST" ] && MLIST=$MACHINES - eval ILIST="\$IMAGES_${build}" - - tb_log "=== Processing set ${bset}, machines: ${MLIST}" + tb_log " += Processing set ${bset}" INC_TMP_DIR="${INC_DIR}/${build}/${distro}/${bset}" # collect images to build in $bbimages bbimages='' - for image in $ILIST; do + for image in $IMAGES; do [ "${image}" != "${image#!}" ] && continue bbimages="${bbimages} ${image}" done @@ -348,7 +379,7 @@ for build in $BUILDS; do -a "${DO_BUILD_CLEAN}" = 'y' \ -a ! -d "${INC_TMP_DIR}/work" \ ]; then - tb_log "Will do incremental build instead of clean to populate TMPDIR" + tb_log " + Will do incremental build instead of clean to populate TMPDIR" DO_BUILD_CLEAN='skip-once' fi @@ -358,14 +389,14 @@ for build in $BUILDS; do export TMP_DIR="${CLN_DIR}" mkdir -p "${TMP_DIR}" - tb_log "=== Cleaning ${TMP_DIR}" + tb_log " + Cleaning ${TMP_DIR}" #rm -rf ${TMP_DIR}/* find ${TMP_DIR}/* -delete - tb_log "=== Starting clean builds for machines {$MLIST}" - tb_build_machines "$MLIST" $bbimages + tb_log " + Starting clean builds for machines {$MLIST}" + tb_build_machines c "$MLIST" $bbimages - tb_log "=== Cleaning ${TMP_DIR} again" + tb_log " + Cleaning ${TMP_DIR} again" find ${TMP_DIR}/* -delete ;; 'skip-once') @@ -381,8 +412,8 @@ for build in $BUILDS; do export TMP_DIR="${INC_TMP_DIR}" mkdir -p "${TMP_DIR}" - tb_log "=== Starting incremental builds for machine-set {$MLIST}" - tb_build_machines "$MLIST" $bbimages + tb_log " + Starting incremental builds for machines {$MLIST}" + tb_build_machines i "$MLIST" $bbimages ;; esac done diff --git a/contrib/testing/testbuilder/testbuilder.conf.sample b/contrib/testing/testbuilder/testbuilder.conf.sample index cf795c2265..3e2f90b5e9 100644 --- a/contrib/testing/testbuilder/testbuilder.conf.sample +++ b/contrib/testing/testbuilder/testbuilder.conf.sample @@ -2,66 +2,9 @@ ### NOTES # 0. For first time setup look into end of file for "Misc dirs" -# 1. All names in BUILDS and BUILD_SETS variables should -# consists only of alphanumeric characters and underscores, -# and beginning with an alphabetic character or an underscore. -# i.e. [a-zA-Z_][a-zA-Z0-9_]* -# My recommendation is not to use underscore as well -# 2. You can prefix any word in following 'list' variables +# 1. You can prefix any word in following 'list' variables # with '!' do disable it. -# 3. All lists items will be processed in order they appears in list - -### BUILDS: Define here your build profiles -# -BUILDS="testing master my" - -## BUILD: Just some text comment to show in build log -BUILD_testing="Regular build of testing branch" -## OE_BRANCH: OE branch to use for this build -OE_BRANCH_testing="testing-next" -## BB_BRANCH: bitbake branch to use for this build -BB_BRANCH_testing="1.10" -## DISTROS: OE distros -DISTROS_testing="angstrom-2008.1 angstrom-2010.x" -## IMAGES: OE images -IMAGES_testing="console-image x11-image opie-image" -## BUILD_SETS: build all machines in set within same TMPDIR -BUILD_SETS_testing="armv5 mips32 mips64 ppc sh4 i686" -## MACHINES: OE machines per build set -MACHINES_testing_armv5="qemuarm" -MACHINES_testing_mips32="qemumips qemumipsel" -MACHINES_testing_mips64="qemumips64" -MACHINES_testing_ppc="qemuppc" -MACHINES_testing_sh4="qemush4" -MACHINES_testing_i686="qemux86" -## TASKS: What to do exactly -# build_clean - do clean builds -# build_incremental - do incremental builds -TASKS_testing="!build_clean build_incremental" - -# Other sample build profile -BUILD_master="Build from master branch" -OE_BRANCH_master="master" -BB_BRANCH_master="master" -# Note that angstrom-2010.x is disabled below ('!' before) -DISTROS_master="!angstrom-2010.x minimal" -IMAGES_master="console-image" -BUILD_SETS_master="zauruses qemux86" -MACHINES_master_zauruses="collie tosa akita" -MACHINES_master_qemu="qemux86" -TASKS_master="build_clean !build_incremental" - -# Sample build from local branch -BUILD_my="Build my local branch" -OE_BRANCH_my="my/work" -BB_BRANCH_my="master" -DISTROS_my="minimal" -IMAGES_my="console-image !my-own-image" -# Note BUILD_SETS_my absence -# Every machine will be built in own TMPDIR -MACHINES_my="tosa akita spitz" -TASKS_my="build_clean !build_incremental" - +# 2. All lists items will be processed in order they appears in list ### Common options @@ -100,8 +43,8 @@ TB_SHELL='/bin/bash --norc' # Use $SHELL variable or fallback to /bin/sh #TB_SHELL=${SHELL:-'/bin/sh'} -# Name for oestats-client -TB_BUILDER="You_should_change_this" +# Name for oestats-client (empty to disable) +TB_BUILDER="" # Misc dirs TB_BASE=`pwd` diff --git a/contrib/testing/testbuilder/testbuilder.d/master.conf.sample b/contrib/testing/testbuilder/testbuilder.d/master.conf.sample new file mode 100644 index 0000000000..2e48d4b07e --- /dev/null +++ b/contrib/testing/testbuilder/testbuilder.d/master.conf.sample @@ -0,0 +1,13 @@ +# Other sample build profile +DISABLED=yes # This build is disabled +BUILD="Build from master branch" +OE_BRANCH="master" +BB_BRANCH="master" +# Note that angstrom-2010.x is disabled below ('!' before) +DISTROS="!angstrom-2010.x minimal" +IMAGES="console-image" +SETS="zauruses qemux86" +MACHINES_zauruses="collie tosa akita" +MACHINES_qemu="qemux86" +TASKS="build_clean !build_incremental" + diff --git a/contrib/testing/testbuilder/testbuilder.d/off/my.conf.sample b/contrib/testing/testbuilder/testbuilder.d/off/my.conf.sample new file mode 100644 index 0000000000..fecfd75bec --- /dev/null +++ b/contrib/testing/testbuilder/testbuilder.d/off/my.conf.sample @@ -0,0 +1,10 @@ +# Sample build from local branch +BUILD="Build my local branch" +OE_BRANCH="my/work" +BB_BRANCH="master" +DISTROS="minimal" +IMAGES="console-image !my-own-image" +# Note SETS absence +# Every machine will be built in own TMPDIR +MACHINES="tosa akita spitz" +TASKS="build_clean !build_incremental" diff --git a/contrib/testing/testbuilder/testbuilder.d/testing.conf.sample b/contrib/testing/testbuilder/testbuilder.d/testing.conf.sample new file mode 100644 index 0000000000..47d9b683bc --- /dev/null +++ b/contrib/testing/testbuilder/testbuilder.d/testing.conf.sample @@ -0,0 +1,37 @@ +### NOTES +# 1. All names in SETS variables should consists only of alphanumeric +# characters and underscores, and beginning with an alphabetic character or an +# underscore. +# i.e. [a-zA-Z_][a-zA-Z0-9_]* +# My recommendation is not to use underscore as well +# 2. You can prefix any word in following 'list' variables +# with '!' do disable it. +# 3. All lists items will be processed in order they appears in list + +## BUILD: Just some text comment to show in build log +BUILD="Regular build of testing branch" +## DISABLED: is this build disabled ('yes' to disable)? +DISABLED=no +## OE_BRANCH: OE branch to use for this build +OE_BRANCH="testing-next" +## BB_BRANCH: bitbake branch to use for this build +BB_BRANCH="1.10" +## DISTROS: OE distros +DISTROS="angstrom-2008.1 angstrom-2010.x" +## IMAGES: OE images +IMAGES="console-image x11-image opie-image" +## SETS: build all machines in set within same TMPDIR +SETS="armv5 mips32 mips64 ppc sh4 i686" +## MACHINES: OE machines per build set +MACHINES_armv5="qemuarm" +MACHINES_mips32="qemumips qemumipsel" +MACHINES_mips64="qemumips64" +MACHINES_ppc="qemuppc" +MACHINES_sh4="qemush4" +MACHINES_i686="qemux86" +# IMAGE_FSTYPE: What image types to build +IMAGE_FSTYPE="tar.gz" +## TASKS: What to do exactly +# build_clean - do clean builds +# build_incremental - do incremental builds +TASKS="!build_clean build_incremental" -- cgit 1.2.3-korg