#!/bin/bash # # This script runs a series of tests (with and without sstate) and reports build time (and tmp/ size) # # Build performance test script # # Copyright 2013 Intel Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # # AUTHORS: # Stefan Stanacar ME=$(basename $0) # # usage and setup # usage () { cat << EOT Usage: $ME [-h] $ME [-c ] [-v] [-m ] [-j ] [-t ] [-i ] [-d ] Options: -h Display this help and exit. -c git checkout before anything else -v Show bitbake output, don't redirect it to a log. -m Value for MACHINE. Default is qemux86. -j Value for PARALLEL_MAKE. Default is 8. -t Value for BB_NUMBER_THREADS. Default is 8. -i Instead of timing against core-image-sato, use -d Use as DL_DIR -p Cherry pick githash onto the commit Note: current working directory must be inside a poky git clone. EOT } if clonedir=$(git rev-parse --show-toplevel); then cd $clonedir else echo "The current working dir doesn't seem to be a poky git clone. Please cd there before running $ME" exit 1 fi IMAGE="core-image-sato" verbose=0 dldir= commit= pmake= cherrypicks= while getopts "hvc:m:j:t:i:d:p:" opt; do case $opt in h) usage exit 0 ;; v) verbose=1 ;; c) commit=$OPTARG ;; m) export MACHINE=$OPTARG ;; j) pmake=$OPTARG ;; t) export BB_NUMBER_THREADS=$OPTARG ;; i) IMAGE=$OPTARG ;; d) dldir=$OPTARG ;; p) cherrypicks="$cherrypicks $OPTARG" ;; *) usage exit 1 ;; esac done #drop cached credentials and test for sudo access without a password sudo -k -n ls > /dev/null 2>&1 reqpass=$? if [ $reqpass -ne 0 ]; then echo "The script requires sudo access to drop caches between builds (echo 3 > /proc/sys/vm/drop_caches)" read -s -p "Please enter your sudo password: " pass echo fi if [ -n "$commit" ]; then echo "git checkout $commit" git pull > /dev/null 2>&1 git checkout $commit || exit 1 git pull > /dev/null 2>&1 fi if [ -n "$cherrypicks" ]; then for c in $cherrypicks; do git cherry-pick $c done fi rev=$(git rev-parse --short HEAD) || exit 1 OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`" BUILDDIR="$OUTDIR/build" resultsfile="$OUTDIR/results.log" bboutput="$OUTDIR/bitbake.log" myoutput="$OUTDIR/output.log" globalres="$clonedir/build-perf-test/globalres.log" mkdir -p $OUTDIR || exit 1 log () { local msg="$1" echo "`date`: $msg" | tee -a $myoutput } # # Config stuff # rev=$(git rev-parse HEAD) || exit 1 log "Git revision is $rev" source ./oe-init-build-env $OUTDIR/build >/dev/null || exit 1 cd $OUTDIR/build [ -n "$MACHINE" ] || export MACHINE="qemux86" [ -n "$BB_NUMBER_THREADS" ] || export BB_NUMBER_THREADS="8" if [ -n "$pmake" ]; then export PARALLEL_MAKE="-j $pmake" else export PARALLEL_MAKE="-j 8" fi if [ -n "$dldir" ]; then echo "DL_DIR = \"$dldir\"" >> conf/local.conf else echo "DL_DIR = \"$clonedir/build-perf-test/downloads\"" >> conf/local.conf fi # Sometimes I've noticed big differences in timings for the same commit, on the same machine # Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy) echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf # # Functions # declare -a TIMES time_count=0 bbtime () { log " Timing: bitbake $1" if [ $verbose -eq 0 ]; then /usr/bin/time -v -o $resultsfile bitbake "$1" >> $bboutput else /usr/bin/time -v -o $resultsfile bitbake "$1" fi ret=$? if [ $ret -eq 0 ]; then t=`grep wall $resultsfile | sed 's/.*m:ss): //'` log " TIME: $t" TIMES[(( time_count++ ))]="$t" else log "ERROR: exit status was non-zero, will report time as 0." TIMES[(( time_count++ ))]="0" fi #time by default overwrites the output file and we want to keep the results #it has an append option but I don't want to clobber the results in the same file i=`ls $OUTDIR/results.log* |wc -l` mv $resultsfile "${resultsfile}.${i}" log "More stats can be found in ${resultsfile}.${i}" } #we don't time bitbake here bbnotime () { log " Running: bitbake $1" if [ $verbose -eq 0 ]; then bitbake "$1" >> $bboutput else bitbake "$1" fi ret=$? if [ $ret -eq 0 ]; then log " Finished bitbake $1" else log "ERROR: exit status was non-zero. Exit.." exit $? fi } do_rmtmp() { log " Removing tmp" rm -rf bitbake.lock pseudodone conf/sanity_info cache tmp } do_rmsstate () { log " Removing sstate-cache" rm -rf sstate-cache } do_sync () { log " Syncing and dropping caches" sync; sync if [ $reqpass -eq 0 ]; then sudo sh -c "echo 3 > /proc/sys/vm/drop_caches" else echo "$pass" | sudo -S sh -c "echo 3 > /proc/sys/vm/drop_caches" echo fi sleep 3 } write_results() { echo -n "`uname -n`,$rev," >> $globalres for i in "${TIMES[@]}"; do echo -n "$i," >> $globalres done echo >> $globalres sed -i '$ s/,$//' $globalres } #### # # Test 1 # Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir (w/o rm_work and w/ rm_work) # Pre: Downloaded sources, no sstate # Steps: # Part1: # - fetchall # - clean build dir # - time bitbake core-image-sato # - collect data # Part2: # - bitbake virtual/kernel -c cleansstate # - time bitbake virtual/kernel # Part3: # - add INHERIT to local.conf # - clean build dir # - build # - report size, remove INHERIT test1_p1 () { log "Running Test 1, part 1/3: Measure wall clock of bitbake $IMAGE and size of tmp/ dir" bbnotime "$IMAGE -c fetchall" do_rmtmp do_rmsstate do_sync bbtime "$IMAGE" log "SIZE of tmp dir is: `du -sh tmp | sed 's/tmp//'`" log "Buildstats are saved in $OUTDIR/buildstats-test1" mv tmp/buildstats $OUTDIR/buildstats-test1 } test1_p2 () { log "Running Test 1, part 2/3: bitbake virtual/kernel -c cleansstate and time bitbake virtual/kernel" bbnotime "virtual/kernel -c cleansstate" do_sync bbtime "virtual/kernel" } test1_p3 () { log "Running Test 1, part 3/3: Build $IMAGE w/o sstate and report size of tmp/dir with rm_work enabled" echo "INHERIT += \"rm_work\"" >> conf/local.conf do_rmtmp do_rmsstate do_sync bbtime "$IMAGE" sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf log "SIZE of tmp dir is: `du -sh tmp | sed 's/tmp//'`" log "Buildstats are saved in $OUTDIR/buildstats-test13" mv tmp/buildstats $OUTDIR/buildstats-test13 } # # Test 2 # Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir # Pre: populated sstate cache test2 () { # Assuming test 1 has run log "Running Test 2: Measure wall clock of bitbake $IMAGE -c rootfs with sstate" do_rmtmp do_sync bbtime "$IMAGE -c rootfs" } # Test 3 # parsing time metrics # # Start with # i) "rm -rf tmp/cache; time bitbake -p" # ii) "rm -rf tmp/cache/default-eglibc/; time bitbake -p" # iii) "time bitbake -p" test3 () { log "Running Test 3: Parsing time metrics (bitbake -p)" log " Removing tmp/cache && cache" rm -rf tmp/cache cache bbtime "-p" log " Removing tmp/cache/default-eglibc/" rm -rf tmp/cache/default-eglibc/ bbtime "-p" bbtime "-p" } # RUN! test1_p1 test1_p2 test1_p3 test2 test3 # if we got til here write to global results write_results log "All done, cleaning up..." do_rmtmp do_rmsstate