util/abuild/abuild

   1 #!/bin/bash
   2 #
   3 #  coreboot autobuild
   4 #
   5 #  This script builds coreboot images for all available targets.
   6 #
   7 #  (C) 2004 by Stefan Reinauer <stepan@openbios.org>
   8 #  (C) 2006-2008 by coresystems GmbH <info@coresystems.de>
   9 #
  10 #  This file is subject to the terms and conditions of the GNU General
  11 #  Public License. See the file COPYING in the main directory of this
  12 #  archive for more details.
  13 #
  14
  15 #set -x # Turn echo on....
  16
  17 ABUILD_DATE="May 27th, 2008"
  18 ABUILD_VERSION="0.7"
  19
  20 # Where shall we place all the build trees?
  21 TARGET=$( pwd )/coreboot-builds
  22 XMLFILE=$( pwd )/abuild.xml
  23
  24 # path to payload. Should be more generic
  25 PAYLOAD=/dev/null
  26
  27 # Lines of error context to be printed in FAILURE case
  28 CONTEXT=5
  29
  30 TESTSUBMISSION="http://qa.coreboot.org/deployment/send.php"
  31
  32 # Number of CPUs to compile on per default
  33 cpus=1
  34
  35 # One might want to adjust these in case of cross compiling
  36 MAKE="make"
  37 PYTHON=python
  38
  39 # this can be changed to xml by -x
  40 mode=text
  41
  42 # silent mode.. no compiler calls, only warnings in the log files.
  43 # this is disabled per default but can be enabled with -s
  44 silent=
  45
  46 # stackprotect mode enabled by -ns option.
  47 stackprotect=false
  48
  49 ARCH=`uname -m | sed -e s/i.86/i386/ -e s/sun4u/sparc64/ \
  50         -e s/arm.*/arm/ -e s/sa110/arm/ -e s/x86_64/amd64/ \
  51         -e "s/Power Macintosh/ppc/"`
  52
  53 trap interrupt INT
  54
  55 function interrupt
  56 {
  57         printf "\n$0: execution interrupted manually.\n"
  58         if [ "$mode" == "xml" ]; then
  59                 printf "$0: deleting incomplete xml output file.\n"
  60         fi
  61         exit 1
  62 }
  63
  64 function debug
  65 {
  66         test "$verbose" == "true" && printf "$*\n"
  67 }
  68
  69 function xml
  70 {
  71         test "$mode" == "xml" && printf "$*\n" >> $XMLFILE
  72 }
  73
  74 function xmlfile
  75 {
  76         test "$mode" == "xml" && {
  77                 printf '<![CDATA[\n'
  78                 cat $1
  79                 printf ']]>\n'
  80         } >> $XMLFILE
  81 }
  82
  83
  84
  85 function vendors
  86 {
  87         # make this a function so we can easily select
  88         # without breaking readability
  89         ls -1 "$LBROOT/src/mainboard" | grep -v CVS
  90 }
  91
  92 function mainboards
  93 {
  94         # make this a function so we can easily select
  95         # without breaking readability
  96
  97         VENDOR=$1
  98
  99         ls -1 $LBROOT/src/mainboard/$VENDOR | grep -v CVS
 100 }
 101
 102 function architecture
 103 {
 104         VENDOR=$1
 105         MAINBOARD=$2
 106         cat $LBROOT/src/mainboard/$VENDOR/$MAINBOARD/Config.lb | \
 107                 grep ^arch | cut -f 2 -d\
 108 }
 109
 110 function create_config
 111 {
 112         VENDOR=$1
 113         MAINBOARD=$2
 114         TARCH=$( architecture $VENDOR $MAINBOARD )
 115         TARGCONFIG=$LBROOT/targets/$VENDOR/$MAINBOARD/Config-abuild.lb
 116
 117         # get a working payload for the board if we have one.
 118         # the --payload option expects a directory containing
 119         # an executable shell script payload.sh
 120         #   Usage: payload.sh [VENDOR] [DEVICE]
 121         # the script returns an absolute path to the payload binary.
 122
 123         if [ -x $payloads/payload.sh ]; then
 124                 PAYLOAD=`$payloads/payload.sh $VENDOR $MAINBOARD`
 125                 printf "Using payload $PAYLOAD\n"
 126         fi
 127
 128         mkdir -p $TARGET
 129
 130         if [ -f $TARGCONFIG ]; then
 131                 cp $TARGCONFIG $TARGET/Config-${VENDOR}_${MAINBOARD}.lb
 132                 printf "Using existing test target $TARGCONFIG"
 133                 xml "  <config>$TARGCONFIG</config>"
 134         else
 135
 136                 printf "  Creating config file..."
 137                 xml "  <config>autogenerated</config>"
 138                 ( cat << EOF
 139 # This will make a target directory of ./VENDOR_MAINBOARD
 140
 141 target VENDOR_MAINBOARD
 142 mainboard VENDOR/MAINBOARD
 143
 144 option CC="CROSSCC"
 145 option CROSS_COMPILE="CROSS_PREFIX"
 146 option HOSTCC="CROSS_HOSTCC"
 147
 148 __COMPRESSION__
 149
 150 EOF
 151                 if [ "$TARCH" == i386 ] ; then
 152                         cat <<EOF
 153 romimage "normal"
 154         option USE_FALLBACK_IMAGE=0
 155         option ROM_IMAGE_SIZE=0x17000
 156         option COREBOOT_EXTRA_VERSION=".0-normal"
 157         payload __PAYLOAD__
 158 end
 159
 160 romimage "fallback"
 161         option USE_FALLBACK_IMAGE=1
 162         option ROM_IMAGE_SIZE=0x17000
 163         option COREBOOT_EXTRA_VERSION=".0-fallback"
 164         payload __PAYLOAD__
 165 end
 166 buildrom ./coreboot.rom ROM_SIZE "normal" "fallback"
 167 EOF
 168                 else
 169                         cat <<EOF
 170 romimage "only"
 171         option COREBOOT_EXTRA_VERSION=".0"
 172         payload __PAYLOAD__
 173 end
 174 buildrom ./coreboot.rom ROM_SIZE "only"
 175 EOF
 176                 fi
 177                 ) > $TARGET/Config-${VENDOR}_${MAINBOARD}.lb
 178         fi
 179
 180         if [ "`which lzma`" != "" -a "$PAYLOAD" != /dev/null ]; then
 181                 COMPRESSION="option CONFIG_COMPRESSED_PAYLOAD_LZMA=1"
 182         else
 183                 COMPRESSION="# no compression"
 184         fi
 185
 186         sed -i.pre -e s:VENDOR:$VENDOR:g \
 187                 -e s:MAINBOARD:$MAINBOARD:g \
 188                 -e s:payload\ __PAYLOAD__:payload\ $PAYLOAD:g \
 189                 -e s:CROSSCC:"$CC":g \
 190                 -e s:CROSS_PREFIX:"$CROSS_COMPILE":g \
 191                 -e s:CROSS_HOSTCC:"$HOSTCC":g \
 192                 -e s:__COMPRESSION__:"$COMPRESSION":g \
 193                 $TARGET/Config-${VENDOR}_${MAINBOARD}.lb
 194         printf " ok\n"
 195 }
 196
 197 function create_builddir
 198 {
 199         VENDOR=$1
 200         MAINBOARD=$2
 201
 202         printf "  Creating builddir..."
 203
 204         target_dir=$TARGET
 205         config_dir=$LBROOT/util/newconfig
 206         yapps2_py=$config_dir/yapps2.py
 207         config_g=$config_dir/config.g
 208         config_lb=Config-${VENDOR}_${MAINBOARD}.lb
 209
 210         cd $target_dir
 211
 212         build_dir=${VENDOR}_${MAINBOARD}
 213         config_py=$build_dir/config.py
 214
 215         if [ ! -d $build_dir ] ; then
 216                 mkdir -p $build_dir
 217         fi
 218         if [ ! -f $config_py ]; then
 219                 $PYTHON $yapps2_py $config_g $config_py &> $build_dir/py.log
 220         fi
 221
 222         # make sure config.py is up-to-date
 223
 224         export PYTHONPATH=$config_dir
 225         $PYTHON $config_py $config_lb $LBROOT &> $build_dir/config.log
 226         if [ $? -eq 0 ]; then
 227                 printf "ok\n"
 228                 xml "  <builddir>ok</builddir>"
 229                 xml ""
 230                 return 0
 231         else
 232                 printf "FAILED! Log excerpt:\n"
 233                 xml "  <builddir>failed</builddir>"
 234                 xml "  <log>"
 235                 xmlfile $build_dir/config.log
 236                 xml "  </log>"
 237                 xml ""
 238                 tail -n $CONTEXT $build_dir/config.log
 239                 return 1
 240         fi
 241 }
 242
 243 function create_buildenv
 244 {
 245         VENDOR=$1
 246         MAINBOARD=$2
 247         create_config $VENDOR $MAINBOARD
 248         create_builddir $VENDOR $MAINBOARD
 249 }
 250
 251 function compile_target
 252 {
 253         VENDOR=$1
 254         MAINBOARD=$2
 255
 256         printf "  Compiling image "
 257         test "$cpus" == "" && printf "in parallel .. "
 258         test "$cpus" == "1" && printf "on 1 cpu .. "
 259         test 0$cpus -gt 1 && printf "on %d cpus in parallel .. " $cpus
 260
 261         CURR=$( pwd )
 262         cd $TARGET/${VENDOR}_${MAINBOARD}
 263         stime=`date +%s`
 264         eval $MAKE $silent -j $cpus &> make.log
 265         ret=$?
 266         etime=`date +%s`
 267         duration=$(( $etime - $stime ))
 268         if [ $ret -eq 0 ]; then
 269                 xml "  <compile>ok</compile>"
 270                 xml "  <compiletime>${duration}s</compiletime>"
 271                 printf "ok\n" > compile.status
 272                 printf "ok. (took ${duration}s)\n"
 273                 cd $CURR
 274                 return 0
 275         else
 276                 xml "  <compile>failed</compile>"
 277                 xml "  <compiletime>${duration}s</compiletime>"
 278                 xml "  <log>"
 279                 xmlfile make.log
 280                 xml "  </log>"
 281
 282                 printf "FAILED after ${duration}s! Log excerpt:\n"
 283                 tail -n $CONTEXT make.log
 284                 cd $CURR
 285                 return 1
 286         fi
 287 }
 288
 289 function built_successfully
 290 {
 291         CURR=`pwd`
 292         status="fail"
 293         if [ -d "$TARGET/${VENDOR}_${MAINBOARD}" ]; then
 294                 cd $TARGET/${VENDOR}_${MAINBOARD}
 295                 if [ -r compile.status ] ; then
 296                         status=`cat compile.status`
 297                 fi
 298                 cd $CURR
 299         fi
 300         [ "$buildall" != "true" -a "$status" == "ok" ]
 301 }
 302
 303 function build_broken
 304 {
 305         CURR=`pwd`
 306         status="yes"
 307         [ -r "$LBROOT/src/mainboard/${VENDOR}/${MAINBOARD}/BROKEN" ] && status="no"
 308         [ "$buildbroken" == "true" -o "$status" == "yes" ]
 309 }
 310
 311 function build_target
 312 {
 313         VENDOR=$1
 314         MAINBOARD=$2
 315         TARCH=$( architecture $VENDOR $MAINBOARD )
 316
 317         # default setting
 318
 319         if [ "`uname -s`" == Darwin ]; then
 320                 # Darwin requires i386-elf-[binuitils|gcc] from MacPorts
 321                 # and a link from i386-elf-gcc-<version> to i386-elf-gcc
 322                 CC="$TARCH-elf-gcc -Wa,--divide"
 323                 CROSS_COMPILE="$TARCH-elf-"
 324         else
 325                 CC='$(CROSS_COMPILE)gcc'
 326                 CROSS_COMPILE=''
 327         fi
 328
 329         if  [ "$stackprotect" = "true" ]; then
 330                 CC="$CC -fno-stack-protector"
 331         fi
 332
 333         HOSTCC='gcc'
 334
 335         printf "Processing mainboard/$VENDOR/$MAINBOARD"
 336
 337         xml "<mainboard>"
 338         xml ""
 339         xml "  <vendor>$VENDOR</vendor>"
 340         xml "  <device>$MAINBOARD</device>"
 341         xml ""
 342         xml "  <architecture>$TARCH</architecture>"
 343         xml ""
 344
 345         [ -r "$LBROOT/src/mainboard/${VENDOR}/${MAINBOARD}/abuild.info" ] && \
 346                 source $LBROOT/src/mainboard/${VENDOR}/${MAINBOARD}/abuild.info
 347
 348         if [ "$ARCH" == "$TARCH" ]; then
 349                 printf " ($TARCH: ok)\n"
 350         else
 351                 found_crosscompiler=false
 352                 if [ "$ARCH" == amd64 -a "$TARCH" == i386 ]; then
 353                         CC="gcc -m32"
 354                         found_crosscompiler=true
 355                 fi
 356                 if [ "$ARCH" == ppc64 -a "$TARCH" == ppc ]; then
 357                         CC="gcc -m32"
 358                         found_crosscompiler=true
 359                 fi
 360                 if [ "$found_crosscompiler" == "false" -a "$TARCH" == ppc ];then
 361                         for prefix in powerpc-eabi- powerpc-linux- ppc_74xx- \
 362                             powerpc-7450-linux-gnu-; do
 363                                 if ${prefix}gcc --version > /dev/null 2> /dev/null ; then
 364                                         found_crosscompiler=true
 365                                         CROSS_COMPILE=$prefix
 366                                 fi
 367                         done
 368                 fi
 369
 370
 371                 # TBD: look for suitable cross compiler suite
 372                 # cross-$TARCH-gcc and cross-$TARCH-ld
 373
 374                 # Check result:
 375                 if [ $found_crosscompiler == "false" ]; then
 376                         printf " ($TARCH: skipped, we're $ARCH)\n\n"
 377                         xml "  <status>notbuilt</status>"
 378                         xml ""
 379                         xml "</mainboard>"
 380
 381                         return 0
 382                 else
 383                         printf " ($TARCH: ok, we're $ARCH)\n"
 384                         xml "  <compiler>"
 385                         xml "    <path>`which ${CROSS_COMPILE}gcc`</path>"
 386                         xml "    <version>`${CROSS_COMPILE}gcc --version | head -1`</version>"
 387                         xml "  </compiler>"
 388                         xml ""
 389                 fi
 390         fi
 391
 392         built_successfully $VENDOR $MAINBOARD && \
 393         {
 394                 printf " ( mainboard/$VENDOR/$MAINBOARD previously ok )\n\n"
 395                 xml "  <status>previouslyok</status>"
 396                 xml ""
 397                 xml "</mainboard>"
 398                 return 0
 399         }
 400
 401         build_broken $VENDOR $MAINBOARD || \
 402         {
 403                 printf " ( broken mainboard/$VENDOR/$MAINBOARD skipped )\n\n"
 404                 xml "  <status>knownbroken</status>"
 405                 xml ""
 406                 xml "</mainboard>"
 407                 return 0
 408         }
 409
 410         create_buildenv $VENDOR $MAINBOARD
 411         if [ $? -eq 0 ]; then
 412                 compile_target $VENDOR $MAINBOARD &&
 413                         xml "  <status>ok</status>" ||
 414                         xml "<status>broken</status>"
 415         fi
 416
 417         xml ""
 418         xml "</mainboard>"
 419
 420         printf "\n"
 421 }
 422
 423 function test_target
 424 {
 425         VENDOR=$1
 426         MAINBOARD=$2
 427
 428         if [ "$hwtest" != "true" ]; then
 429                 return 0
 430         fi
 431
 432         # image does not exist. we silently skip the patch.
 433         if [ ! -r "$TARGET/${VENDOR}_${MAINBOARD}/coreboot.rom" ]; then
 434                 return 0
 435         fi
 436
 437         which curl &> /dev/null
 438         if [ $? != 0 ]; then
 439                 printf "curl is not installed but required for test submission.  skipping test.\n\n"
 440                 return 0
 441         fi
 442
 443         CURR=`pwd`
 444         if [ -r "$TARGET/${VENDOR}_${MAINBOARD}/tested" ]; then
 445                 printf "Testing image for board $VENDOR $MAINBOARD skipped (previously submitted).\n\n"
 446                 return 0
 447         fi
 448         # touch $TARGET/${VENDOR}_${MAINBOARD}/tested
 449
 450         printf "Submitting image for board $VENDOR $MAINBOARD to test system...\n"
 451
 452         curl -f -F "romfile=@$TARGET/${VENDOR}_${MAINBOARD}/coreboot.rom" \
 453                 -F "mode=abuild" -F "mainboard=${VENDOR}_${MAINBOARD}" -F "submit=Upload" \
 454                 "http://qa.coreboot.org/deployment/send.php"
 455
 456         printf "\n"
 457         return 0
 458 }
 459
 460 function myhelp
 461 {
 462         printf "Usage: $0 [-v] [-a] [-b] [-t <vendor/board>] [-p <dir>] [lbroot]\n"
 463         printf "       $0 [-V|--version]\n"
 464         printf "       $0 [-h|--help]\n\n"
 465
 466         printf "Options:\n"
 467         printf "    [-v|--verbose]                print more messages\n"
 468         printf "    [-a|--all]                    build previously succeeded ports as well\n"
 469         printf "    [-b|--broken]                 attempt to build ports that are known broken\n"
 470         printf "    [-t|--target <vendor/board>]  attempt to build target vendor/board only\n"
 471         printf "    [-p|--payloads <dir>]         use payloads in <dir> to build images\n"
 472         printf "    [-V|--version]                print version number and exit\n"
 473         printf "    [-h|--help]                   print this help and exit\n"
 474         printf "    [-x|--xml]                    write xml log file \n"
 475         printf "                                  (defaults to $XMLFILE)\n"
 476         printf "    [-T|--test]                   submit image(s) to automated test system\n"
 477         printf "    [-c|--cpus <numcpus>]         build on <numcpus> at the same time\n"
 478         printf "    [-s|--silent]                 omit compiler calls in logs\n"
 479         printf "    [-ns|--nostackprotect]        use gcc -fno-stack-protector option\n"
 480         printf "    [lbroot]                      absolute path to coreboot sources\n"
 481         printf "                                  (defaults to $LBROOT)\n\n"
 482 }
 483
 484 function myversion
 485 {
 486         cat << EOF
 487
 488 coreboot autobuild v$ABUILD_VERSION ($ABUILD_DATE)
 489
 490 Copyright (C) 2004 by Stefan Reinauer <stepan@openbios.org>
 491 Copyright (C) 2006-2008 by coresystems GmbH <info@coresystems.de>
 492
 493 This program is free software; you may redistribute it under the terms
 494 of the GNU General Public License. This program has absolutely no
 495 warranty.
 496
 497 EOF
 498 }
 499
 500 # default options
 501 target=""
 502 buildall=false
 503 LBROOT=$( cd ../..; pwd )
 504 verbose=false
 505
 506 # parse parameters.. try to find out whether we're running GNU getopt
 507 getoptbrand="`getopt -V`"
 508 if [ "${getoptbrand:0:6}" == "getopt" ]; then
 509         # Detected GNU getopt that supports long options.
 510         args=`getopt -l version,verbose,help,all,target:,broken,payloads:,test,cpus:,silent,xml Vvhat:bp:Tc:sx -- "$@"`
 511         eval set "$args"
 512 else
 513         # Detected non-GNU getopt
 514         args=`getopt Vvhat:bp:Tc:sx $*`
 515         set -- $args
 516 fi
 517
 518 if [ $? != 0 ]; then
 519         myhelp
 520         exit 1
 521 fi
 522
 523 while true ; do
 524         case "$1" in
 525                 -x|--xml)       shift; mode=xml; rm -f $XMLFILE ;;
 526                 -t|--target)    shift; target="$1"; shift;;
 527                 -a|--all)       shift; buildall=true;;
 528                 -b|--broken)    shift; buildbroken=true;;
 529                 -v|--verbose)   shift; verbose=true;;
 530                 -V|--version)   shift; myversion; exit 0;;
 531                 -h|--help)      shift; myversion; myhelp; exit 0;;
 532                 -p|--payloads)  shift; payloads="$1"; shift;;
 533                 -T|--test)      shift; hwtest=true;;
 534                 -c|--cpus)      shift; cpus="$1"; test "$cpus" == "max" && cpus=""; shift;;
 535                 -s|--silent)    shift; silent="-s";;
 536                 -ns|--nostackprotect) shift; stackprotect=true;;
 537                 --)             shift; break;;
 538                 -*)             printf "Invalid option\n\n"; myhelp; exit 1;;
 539                 *)              break;;
 540         esac
 541 done
 542
 543 # /path/to/freebios2/
 544 test -z "$1" || LBROOT=$1
 545
 546 debug "LBROOT=$LBROOT"
 547
 548 xml '<?xml version="1.0" encoding="utf-8"?>'
 549 xml '<abuild>'
 550
 551 if [ "$target" != "" ]; then
 552         # build a single board
 553         VENDOR=`printf $target|cut -f1 -d/`
 554         MAINBOARD=`printf $target|cut -f2 -d/`
 555         build_target $VENDOR $MAINBOARD
 556         test_target $VENDOR $MAINBOARD
 557 else
 558         # build all boards per default
 559         for VENDOR in $( vendors ); do
 560                 for MAINBOARD in $( mainboards $VENDOR ); do
 561                         build_target $VENDOR $MAINBOARD
 562                         test_target $VENDOR $MAINBOARD
 563                 done
 564         done
 565 fi
 566 xml '</abuild>'
 567