From f801d0a00102e77e267c13986b0e5bccad2cd5ad Mon Sep 17 00:00:00 2001 From: Yuri Bushmelev Date: Thu, 2 Dec 2010 14:21:53 +0300 Subject: contrib/testing: add testbuilder script I've added my TestBuilder v0.3 to contrib. Check README file about setup and usage. --- contrib/testing/testbuilder/README | 35 ++ contrib/testing/testbuilder/setup-testbuilder | 100 ++++++ contrib/testing/testbuilder/testbuilder | 383 +++++++++++++++++++++ .../testing/testbuilder/testbuilder.conf.sample | 115 +++++++ 4 files changed, 633 insertions(+) create mode 100644 contrib/testing/testbuilder/README create mode 100755 contrib/testing/testbuilder/setup-testbuilder create mode 100755 contrib/testing/testbuilder/testbuilder create mode 100644 contrib/testing/testbuilder/testbuilder.conf.sample (limited to 'contrib') diff --git a/contrib/testing/testbuilder/README b/contrib/testing/testbuilder/README new file mode 100644 index 0000000000..a8d5aea89c --- /dev/null +++ b/contrib/testing/testbuilder/README @@ -0,0 +1,35 @@ += How to setup and use TestBuilder + +I assume below that you wish install TestBuilder into ~/testbuilder directory. + +1. Copy files from OE contrib/testing/testbuilder directory to any place you + wish to use (e.g. ~/testbuilder) and goto that directory + +2. Run setup-testbuilder script with OE branch and bitbake branch you wish to + use. E.g.: + ./setup-testbuilder "testing-next" "1.10" + + It will clone OE and bitbake trees and prepare build/conf/local.conf file + for you. You can review local.conf after and add some settings there. + +4. Create testbuilder.conf according your needs. You can use + testbuilder.conf.sample as reference. + +5. Run ./testbuilder and wait for build results. Check 'logs' subdir for log + files ('tail logs/B.*' is useful) + +You can run single configured build (e.g. "testing") +./testbuilder -B testing + +Or you can use testbuilder to build specified distro/machine/image +(e.g. angstrom-2008.1/qemuarm/console-image) +./testbuilder -D angstrom-2008.1 -M qemuarm console-image + +If you wish only setup environment and run bitbake by hands you can +use -S option: +./testbuilder -D angstrom-2008.1 -M qemuarm -S + +-- +Yury 'Jay7' Bushmelev +email: jay4mail at gmail com +irc: Jay7 on freenode diff --git a/contrib/testing/testbuilder/setup-testbuilder b/contrib/testing/testbuilder/setup-testbuilder new file mode 100755 index 0000000000..ebf86cfbb6 --- /dev/null +++ b/contrib/testing/testbuilder/setup-testbuilder @@ -0,0 +1,100 @@ +#!/bin/sh + +# TestBuilder setup script for OpenEmbedded +# Copyright (c) 2010 Yuri Bushmelev +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +TB_DIR=`dirname $0` +CFG_FILE="${TB_DIR}/testbuilder.conf" + +if [ "$#" -lt "2" -o "$1" = "-h" -o "$1" = "--help" ]; then + echo "Usage: `basename $0` " + exit 1 +fi + +OE_BRANCH=$1 +BB_BRANCH=$2 + +. "${CFG_FILE}" + +cd "${TB_BASE}" + +echo "OE branch: $OE_BRANCH" +echo "BB branch: $BB_BRANCH" + +## Checkout bitbake +echo "= Fetching bitbake" +git clone -n git://git.openembedded.org/bitbake "${BB_DIR}" +cd "${BB_DIR}" + +if [ -n "${BB_BRANCH}" ]; then + git checkout -b "${BB_BRANCH}" "origin/${BB_BRANCH}" +elif [ -n "${BB_TAG}" ]; then + git checkout -b "${BB_TAG}" "${BB_TAG}" +else + echo "You should specify BB_BRANCH or BB_TAG in ${CFG_FILE}" +fi + +echo "= Fetching OE" +cd "${TB_BASE}" + +git clone -n git://git.openembedded.org/openembedded "${OE_DIR}" +cd "${OE_DIR}" + +if [ -n "${OE_BRANCH}" ]; then + git checkout -b "${OE_BRANCH}" "origin/${OE_BRANCH}" +else + echo "You should specify OE_BRANCH in ${CFG_FILE}" +fi + +cd "${TB_BASE}" +mkdir -p "${BLD_DIR}/conf" + +echo "= Creating bitbake config" +cat >> "${BLD_DIR}/conf/local.conf" << "EOF" +# Testbuilder bitbake local configuration file + +DISTRO ?= "${@bb.fatal('Set DISTRO in your config')}" +MACHINE ?= "${@bb.fatal('Set MACHINE in your config')}" + +DL_DIR ?= "${@bb.fatal('Set DL_DIR in your config')}" +TMPDIR = "${TMP_DIR}" + +BBFILES ?= "${OE_DIR}/recipes/*/*.bb" + +# ENABLE_BINARY_LOCALE_GENERATION = "0" +# GLIBC_GENERATE_LOCALES = "en_US.UTF-8 en_GB.UTF-8 de_DE.UTF-8 fr_FR.UTF-8 ru_RU.UTF-8" +# IMAGE_LINGUAS="en-us en-gb de-de fr-fr ru-ru" + +# jffs2, tar(.gz|bz2), cpio(.gz), cramfs, ext2(.gz), ext3(.gz) +# squashfs, squashfs-lzma +IMAGE_FSTYPES = "jffs2 tar.gz" + +# CCACHE = "${@bb.which(bb.data.getVar('PATH', d, 1), 'ccache') and 'ccache '}" +PARALLEL_MAKE = "-j ${MAKE_NUMBER_THREADS}" +BB_NUMBER_THREADS ?= "${@bb.fatal('Set BB_NUMBER_THREADS in your config')}" + +INHERIT += "rm_work" +#INHERIT += "devshell" +#TERMCMD = ${SCREEN_TERMCMD} + +BBINCLUDELOGS = "yes" + +# OE stats client (make troubleshooting easier) +INHERIT += "oestats-client" +OESTATS_BUILDER = "${TB_BUILDER}" +OESTATS_SERVER = "tinderbox.openembedded.net" + +EOF + +echo "= All is done. Run ${TB_DIR}/testbuilder now" + diff --git a/contrib/testing/testbuilder/testbuilder b/contrib/testing/testbuilder/testbuilder new file mode 100755 index 0000000000..aacaff31d4 --- /dev/null +++ b/contrib/testing/testbuilder/testbuilder @@ -0,0 +1,383 @@ +#!/bin/sh + +# TestBuilder for OpenEmbedded +# Copyright (c) 2010 Yuri Bushmelev +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +VERSION="0.3" +PACKAGE="TestBuilder" + +TB_DIR=`dirname $0` +TB_NAME=`basename $0` +cd "${TB_DIR}" + +tb_usage() { + echo "${PACKAGE} v${VERSION}" + # Run only specified build + echo "Usage: ${TB_NAME} -Vh" + echo " or ${TB_NAME} [-B build ]" + # Behave like DISTRO=distro MACHINE=machine bitbake images + # with -S testbuilder will only setup env and run interactive sub-shell + echo " or ${TB_NAME} [-D distro] [-M machine] [-T /full/path/to/tmpdir] [-S | images]" +} + +CFG_FILE="${TB_DIR}/testbuilder.conf" +. "${CFG_FILE}" + +# Sanity cleanup +ORIG_PATH=${PATH} +ORIG_LD_LIBRARY_PATH=${LD_LIBRARY_PATH} +ORIG_LANG=${LANG} +export ORIG_PATH ORIG_LD_LIBRARY_PATH ORIG_LANG + +# clean LD_LIBRARY_PATH +unset LD_LIBRARY_PATH + +# clean locale +LANG=C + +# include bitbake into PATH +PATH=${BB_DIR}/bin:${PATH} + +BBPATH="${BLD_DIR}:${OE_DIR}" + +TB_LOG="${LOG_DIR}/tb.log" + +#export LD_LIBRARY_PATH LANG PATH +export LD_LIBRARY_PATH LANG + +BB_ENV_EXTRAWHITE="MACHINE DISTRO TB_BUILDER OE_DIR TMP_DIR DL_DIR MAKE_NUMBER_THREADS BB_NUMBER_THREADS" +export BB_ENV_EXTRAWHITE BBPATH TB_BUILDER OE_DIR TMP_DIR DL_DIR MAKE_NUMBER_THREADS BB_NUMBER_THREADS + +mkdir -p ${LOG_DIR} + +### Functions + +# log message +tb_log() { + local _dt=`date "+%F %X (%s)"` + echo "[${_dt}] $@" >&2 + echo "[${_dt}] $@" >> $TB_LOG +} + +# Do vmstat accounting +tb_accounting_vmstat() { + exec vmstat -n "${ACCOUNTING_INTERVAL}" >> "${LOG_DIR}/A.vmstat.log" +} + +# Do accounting by commands (df/ps) +tb_accounting_cmd() { + local _dir=$1; shift + while true; do + [ -n "${DO_ACCOUNTING_DF}" ] && ( df -Pk "${_dir}" | awk 'END {OFS="\t"; print systime(), $4, $6;}' ) >> "${LOG_DIR}/A.df.log" + [ -n "${DO_ACCOUNTING_PS}" ] && ( ps ax | awk 'END {OFS="\t"; print systime(), NR-1;}' ) >> "${LOG_DIR}/A.ps.log" + sleep "${ACCOUNTING_INTERVAL}" + done +} + +# Stop all remaining tasks +tb_kill_all() { + for proc in `jobs | cut -c 2- | cut -d] -f1`; do + kill "%${proc}" + done +} + + +# update git tree +tb_git_update() { + local _dir=$1 + + cd "${_dir}" + tb_log "Updating git tree ${_dir}" + git pull + cd "${TB_DIR}" +} + +# checkout branch from git tree +tb_git_branch() { + local _dir=$1 + local _branch=$2 + + cd "${_dir}" + tb_log "Checking out branch ${_branch} from tree ${_dir}" + git checkout -f ${_branch} + cd "${TB_DIR}" +} + + +# Do build of image-set $bbimages for machine-set $MLIST +# tb_build_machines "$MLIST" $bbimages +tb_build_machines() { + local _MLIST=$1; shift + local _bbimages=$@ + local _machine + local _rc + local _image + local _log + + # Start accounting + trap "tb_kill_all" RETURN + [ -n "${DO_ACCOUNTING_VMSTAT}" ] && tb_accounting_vmstat & + [ -n "${DO_ACCOUNTING_DF}" -o -n "${DO_ACCOUNTING_PS}" ] && tb_accounting_cmd "${TMP_DIR}" & + + for _machine in $_MLIST; do + [ "${_machine}" != "${_machine#!}" ] && continue + + tb_log "${_machine} build started, images: ${_bbimages}" + + export MACHINE=$_machine + + case "${SEPARATE_BUILD}" in + [Yy][Ee][Ss]) + # Build images separately + for _image in $_bbimages; do + tb_log "${_image} build started" + _log="${LOG_DIR}/B.${build}.${DISTRO}.${_machine}.${_image}.log" + [ -f "${_log}" ] && mv "${_log}" "${_log}.0" + bitbake ${BB_OPTS} ${_image} 2>&1 | tee "${_log}" + _rc=$? + tb_log "${_image} build finished. Exit code: ${_rc}." + done + ;; + *) + # Build all images with one bitbake invocation + _log="${LOG_DIR}/B.${build}.${DISTRO}.${_machine}.log" + [ -f "${_log}" ] && mv "${_log}" "${_log}.0" + bitbake ${BB_OPTS} ${_bbimages} 2>&1 | tee "${_log}" + _rc=$? + ;; + esac + + tb_log "${_machine} build finished. Exit code:${_rc}." + done + +} + + +### Main code + +# Parse stages +unset DO_RUN_BUILDS DO_UPDATE_OE DO_UPDATE_BB +for stage in $STAGES; do + [ "${stage}" != "${stage#!}" ] && continue + case ${stage} in + 'run_builds') + DO_RUN_BUILDS=y;; + 'update_oe') + DO_UPDATE_OE=y;; + 'update_bb') + DO_UPDATE_BB=y;; + *) + tb_log "Unknown stage ${stage}.";; + esac +done + +# Parse accounting +unset DO_ACCOUNTING_DF DO_ACCOUNTING_PS DO_ACCOUNTING_VMSTAT +for count in $ACCOUNTING; do +[ "${count}" != "${count#!}" ] && continue + case ${count} in + 'df') + DO_ACCOUNTING_DF=y;; + 'ps') + DO_ACCOUNTING_PS=y;; + 'vmstat') + DO_ACCOUNTING_VMSTAT=y;; + *) + tb_log "Unknown accounting type ${count}.";; + esac +done + +# Parse cmdline +unset ODISTRO OBUILD OMACHINE OIMAGES OTMPDIR DO_RUN_SHELL +while getopts VhSB:D:M:T: opt; do + case $opt in + B) OBUILD="${OPTARG}";; + D) ODISTRO="${OPTARG}";; + M) OMACHINE="${OPTARG}";; + T) OTMPDIR="${OPTARG}";; + S) DO_RUN_SHELL=y;; + *) + tb_usage + exit 2 + ;; + esac +done + +shift $(($OPTIND - 1)) +OIMAGES=$@ + +# Kill all tasks on exit +trap "tb_kill_all" EXIT + +# Check for 'direct' mode +if [ -n "${ODISTRO}" ]; then + DISTRO="${ODISTRO}" + MACHINE="${OMACHINE}" + TMP_DIR=${OTMPDIR:-${CLN_DIR}} + export DISTRO MACHINE TMP_DIR + + if [ -n "${DO_RUN_SHELL}" ]; then + # Run shell if -s option was given + echo "Distro: $DISTRO" + echo "Machine: $MACHINE" + echo "TMPDIR: $TMP_DIR" + echo "Starting shell $TB_SHELL. Use 'exit' or ^D to quit" + [ -n "${TB_PS1}" ] && export PS1=${TB_PS1} + eval $TB_SHELL + else + tb_build_machines "${MACHINE}" ${OIMAGES} + fi + + exit $rc +fi + +# Do stages +[ -n "${DO_UPDATE_BB}" ] && tb_git_update ${BB_DIR} +[ -n "${DO_UPDATE_OE}" ] && tb_git_update ${OE_DIR} +[ -z "${DO_RUN_BUILDS}" ] && exit 0 + +# Use build from cmdline if any +[ -n "$OBUILD" ] && BUILDS=$OBUILD + +# Auto-building mode +tb_log "${PACKAGE} v${VERSION}" + +for build in $BUILDS; do + [ "${build}" != "${build#!}" ] && continue + + eval BUILD="\$BUILD_${build}" + tb_log "= Started build '$build'" + [ -n "$BUILD" ] && tb_log "= $BUILD" + + # change BB branch + eval BB_BRANCH="\$BB_BRANCH_${build}" + if [ -n "${BB_BRANCH}" ]; then + tb_git_branch "$BB_DIR" "$BB_BRANCH" + fi + + # change OE branch + eval OE_BRANCH="\$OE_BRANCH_${build}" + if [ -n "${OE_BRANCH}" ]; then + tb_git_branch "$OE_DIR" "$OE_BRANCH" + fi + + # parse tasks + eval TASKS="\$TASKS_${build}" + unset DO_BUILD_CLEAN DO_BUILD_INCREMENTAL + for task in $TASKS; do + [ "${task}" != "${task#!}" ] && continue + case ${task} in + 'build_clean') + DO_BUILD_CLEAN=y;; + 'build_incremental') + DO_BUILD_INCREMENTAL=y;; + *) + tb_log "Unknown task ${task}.";; + esac + done + + # distros + eval DISTROS="\$DISTROS_${build}" + for distro in $DISTROS; do + [ "${distro}" != "${distro#!}" ] && continue + + export DISTRO=$distro + + # Get build sets + eval BSLIST="\$BUILD_SETS_${build}" + + # Empty $BUILD_SETS is exception - we should use $MACHINES + # to build every machine in separate TMPDIRs + if [ -z "${BSLIST}" ]; then + # Use every machine as separate build set + eval MACHINES="\$MACHINES_${build}" + for machine in $MACHINES; do + [ "${machine}" != "${machine#!}" ] && continue + eval "MACHINE_${machine}=${machine}" + BSLIST="${BSLIST} ${machine}" + done + fi + + tb_log "== Processing distro ${DISTRO}, build set: ${BSLIST}" + + for bset in $BSLIST; do + [ "${bset}" != "${bset#!}" ] && continue + + eval MLIST="\$MACHINES_${build}_${bset}" + [ -z "$MLIST" ] && eval MLIST="\$MACHINES_${build}" + + eval ILIST="\$IMAGES_${build}" + + tb_log "=== Processing set ${bset}, machines: ${MLIST}" + + INC_TMP_DIR="${INC_DIR}/${build}/${distro}/${bset}" + + + # collect images to build in $bbimages + bbimages='' + for image in $ILIST; do + [ "${image}" != "${image#!}" ] && continue + bbimages="${bbimages} ${image}" + done + + # If there is no previous incremental builddir and we are requested + # to do both incremental and clean builds + # then skip clean build and do only incremental because it will be clean build then :) + if [ "${DO_BUILD_INCREMENTAL}" = 'y' \ + -a "${DO_BUILD_CLEAN}" = 'y' \ + -a ! -d "${INC_TMP_DIR}/work" \ + ]; then + tb_log "Will do incremental build instead of clean to populate TMPDIR" + DO_BUILD_CLEAN='skip-once' + fi + + ## Do clean builds + case "${DO_BUILD_CLEAN}" in + 'y') + export TMP_DIR="${CLN_DIR}" + mkdir -p "${TMP_DIR}" + + tb_log "=== Cleaning ${TMP_DIR}" + #rm -rf ${TMP_DIR}/* + find ${TMP_DIR}/* -delete + + tb_log "=== Starting clean builds for machines {$MLIST}" + tb_build_machines "$MLIST" $bbimages + + tb_log "=== Cleaning ${TMP_DIR} again" + find ${TMP_DIR}/* -delete + ;; + 'skip-once') + # Re-enable skipped clean build + DO_BUILD_CLEAN=y + ;; + esac + + ## Do incremental build + case "${DO_BUILD_INCREMENTAL}" in + 'y') + # Switch tmpdir to archive + export TMP_DIR="${INC_TMP_DIR}" + mkdir -p "${TMP_DIR}" + + tb_log "=== Starting incremental builds for machine-set {$MLIST}" + tb_build_machines "$MLIST" $bbimages + ;; + esac + done + done +done + +tb_log "All done." + +exit 0 + diff --git a/contrib/testing/testbuilder/testbuilder.conf.sample b/contrib/testing/testbuilder/testbuilder.conf.sample new file mode 100644 index 0000000000..cf795c2265 --- /dev/null +++ b/contrib/testing/testbuilder/testbuilder.conf.sample @@ -0,0 +1,115 @@ +# Testbuilder config + +### NOTES +# 0. For first time setup look into end of file for "Misc dirs" +# 1. All names in BUILDS and BUILD_SETS variables should +# consists only of alphanumeric characters and underscores, +# and beginning with an alphabetic character or an underscore. +# i.e. [a-zA-Z_][a-zA-Z0-9_]* +# My recommendation is not to use underscore as well +# 2. You can prefix any word in following 'list' variables +# with '!' do disable it. +# 3. All lists items will be processed in order they appears in list + +### BUILDS: Define here your build profiles +# +BUILDS="testing master my" + +## BUILD: Just some text comment to show in build log +BUILD_testing="Regular build of testing branch" +## OE_BRANCH: OE branch to use for this build +OE_BRANCH_testing="testing-next" +## BB_BRANCH: bitbake branch to use for this build +BB_BRANCH_testing="1.10" +## DISTROS: OE distros +DISTROS_testing="angstrom-2008.1 angstrom-2010.x" +## IMAGES: OE images +IMAGES_testing="console-image x11-image opie-image" +## BUILD_SETS: build all machines in set within same TMPDIR +BUILD_SETS_testing="armv5 mips32 mips64 ppc sh4 i686" +## MACHINES: OE machines per build set +MACHINES_testing_armv5="qemuarm" +MACHINES_testing_mips32="qemumips qemumipsel" +MACHINES_testing_mips64="qemumips64" +MACHINES_testing_ppc="qemuppc" +MACHINES_testing_sh4="qemush4" +MACHINES_testing_i686="qemux86" +## TASKS: What to do exactly +# build_clean - do clean builds +# build_incremental - do incremental builds +TASKS_testing="!build_clean build_incremental" + +# Other sample build profile +BUILD_master="Build from master branch" +OE_BRANCH_master="master" +BB_BRANCH_master="master" +# Note that angstrom-2010.x is disabled below ('!' before) +DISTROS_master="!angstrom-2010.x minimal" +IMAGES_master="console-image" +BUILD_SETS_master="zauruses qemux86" +MACHINES_master_zauruses="collie tosa akita" +MACHINES_master_qemu="qemux86" +TASKS_master="build_clean !build_incremental" + +# Sample build from local branch +BUILD_my="Build my local branch" +OE_BRANCH_my="my/work" +BB_BRANCH_my="master" +DISTROS_my="minimal" +IMAGES_my="console-image !my-own-image" +# Note BUILD_SETS_my absence +# Every machine will be built in own TMPDIR +MACHINES_my="tosa akita spitz" +TASKS_my="build_clean !build_incremental" + + +### Common options + +## STAGES: what TB should do +# update_bb - update bitbake git tree +# update_oe - update OE git tree +# run_builds - run builds +STAGES="update_bb update_oe !run_builds" + +## ACCOUNTING: account some useful things +# df - log free disk space +# ps - log processes count +# vmstat - log vm statistics +ACCOUNTING="!df !ps !vmstat" +# Accounting interval (seconds) +ACCOUNTING_INTERVAL="10" + +# Build every image by separate bitbake invocation (lower peak disk usage) +#SEPARATE_BUILD="yes" +# Build all images by single bitbake invocation (slightly faster) +SEPARATE_BUILD="no" + +# Number of parallel bitbake threads +BB_NUMBER_THREADS=2 +# Number of parallel make threads inside one bitbake thread (-j) +MAKE_NUMBER_THREADS=2 + +# Bitbake cmdline options +#BB_OPTS='-k' + +# Direct build invocation shell variables (-S option) +# PS1 for shell mode +TB_PS1="[TB] " +# Use bash with --norc to disable overriding PS1 +TB_SHELL='/bin/bash --norc' +# Use $SHELL variable or fallback to /bin/sh +#TB_SHELL=${SHELL:-'/bin/sh'} + +# Name for oestats-client +TB_BUILDER="You_should_change_this" + +# Misc dirs +TB_BASE=`pwd` +BB_DIR="${TB_BASE}/bitbake" +OE_DIR="${TB_BASE}/openembedded" +DL_DIR="${TB_BASE}/sources" +BLD_DIR="${TB_BASE}/build" +LOG_DIR="${TB_BASE}/logs" +CLN_DIR="/var/tmp/tb-clean-build" # TMPDIR for clean builds +INC_DIR="${BLD_DIR}" # Top dir for incremental builds archive + -- cgit 1.2.3-korg