Instead of just
[coreboot.git] / util / abuild / abuild
1 #!/bin/bash
2 #
3 #  coreboot autobuild
4 #
5 #  This script builds coreboot images for all available targets.
6 #
7 #  (C) 2004 by Stefan Reinauer <stepan@openbios.org>
8 #  (C) 2006-2009 by coresystems GmbH <info@coresystems.de>
9 #
10 #  This file is subject to the terms and conditions of the GNU General
11 #  Public License. See the file COPYING in the main directory of this
12 #  archive for more details.
13 #     
14
15 #set -x # Turn echo on....
16
17 ABUILD_DATE="April 10th, 2009"
18 ABUILD_VERSION="0.8.1"
19
20 # Where shall we place all the build trees?
21 TARGET=$( pwd )/coreboot-builds
22 XMLFILE=$( pwd )/abuild.xml
23
24 # path to payload. Should be more generic
25 PAYLOAD=/dev/null
26
27 # Lines of error context to be printed in FAILURE case
28 CONTEXT=5
29
30 TESTSUBMISSION="http://qa.coreboot.org/deployment/send.php"
31
32 # Number of CPUs to compile on per default
33 cpus=1
34
35 # Configure-only mode
36 configureonly=0
37
38 # One might want to adjust these in case of cross compiling
39 MAKE="make"
40 PYTHON=python
41
42 # this can be changed to xml by -x
43 mode=text
44
45 # silent mode.. no compiler calls, only warnings in the log files.
46 # this is disabled per default but can be enabled with -s
47 silent=
48
49 # clang mode enabled by -sb option.
50 scanbuild=false
51
52 # stackprotect mode enabled by -ns option.
53 stackprotect=false
54
55 # loglevel changed with -l / --loglevel option
56 loglevel=default
57
58 ARCH=`uname -m | sed -e s/i.86/i386/ -e s/sun4u/sparc64/ \
59         -e s/i86pc/i386/ \
60         -e s/arm.*/arm/ -e s/sa110/arm/ -e s/x86_64/amd64/ \
61         -e "s/Power Macintosh/ppc/"`
62
63 trap interrupt INT
64
65 function interrupt
66 {
67         printf "\n$0: execution interrupted manually.\n"
68         if [ "$mode" == "xml" ]; then
69                 printf "$0: deleting incomplete xml output file.\n"
70         fi
71         exit 1
72 }
73
74 function debug
75 {
76         test "$verbose" == "true" && printf "$*\n"
77 }
78
79 function xml
80 {
81         test "$mode" == "xml" && printf "$*\n" >> $XMLFILE
82 }
83
84 function xmlfile
85 {
86         test "$mode" == "xml" && { 
87                 printf '<![CDATA[\n'
88                 cat $1
89                 printf ']]>\n' 
90         } >> $XMLFILE
91 }
92
93
94
95 function vendors
96 {
97         # make this a function so we can easily select
98         # without breaking readability
99         ls -1 "$LBROOT/src/mainboard" | grep -v CVS
100 }
101
102 function mainboards
103 {
104         # make this a function so we can easily select
105         # without breaking readability
106         
107         VENDOR=$1
108         
109         ls -1 $LBROOT/src/mainboard/$VENDOR | grep -v CVS 
110 }
111
112 function architecture
113 {
114         VENDOR=$1
115         MAINBOARD=$2
116         ARCH=`cat $LBROOT/src/mainboard/$VENDOR/$MAINBOARD/Config.lb | \
117                 grep ^arch | cut -f 2 -d\ `
118         echo $ARCH | sed s/ppc/powerpc/
119 }
120
121 function create_config
122 {
123         VENDOR=$1
124         MAINBOARD=$2
125         CONFIG=$3
126         TARCH=$( architecture $VENDOR $MAINBOARD )
127         TARGCONFIG=$LBROOT/targets/$VENDOR/$MAINBOARD/Config-abuild.lb
128
129         if [ "$CONFIG" != "" ]; then
130                 TARGCONFIG=$LBROOT/targets/$VENDOR/$MAINBOARD/$CONFIG
131         fi
132
133         # get a working payload for the board if we have one.
134         # the --payload option expects a directory containing 
135         # a shell script payload.sh
136         #   Usage: payload.sh [VENDOR] [DEVICE]
137         # the script returns an absolute path to the payload binary.
138
139         if [ -f $payloads/payload.sh ]; then
140                 PAYLOAD=`sh $payloads/payload.sh $VENDOR $MAINBOARD`
141                 printf "Using payload $PAYLOAD\n"
142         fi
143         
144         mkdir -p $TARGET
145
146         if [ -f $TARGCONFIG ]; then
147                 cp $TARGCONFIG $TARGET/Config-${VENDOR}_${MAINBOARD}.lb
148                 printf "Using existing test target $TARGCONFIG"
149                 xml "  <config>$TARGCONFIG</config>"
150         else
151
152                 printf "  Creating config file..."
153                 xml "  <config>autogenerated</config>"
154                 ( cat << EOF
155 # This will make a target directory of ./VENDOR_MAINBOARD
156
157 target VENDOR_MAINBOARD
158 mainboard VENDOR/MAINBOARD
159
160 option CC="CROSSCC"
161 option CROSS_COMPILE="CROSS_PREFIX"
162 option HOSTCC="CROSS_HOSTCC"
163
164 __COMPRESSION__
165 __LOGLEVEL__
166
167 EOF
168                 if [ "$TARCH" == i386 ] ; then
169                         cat <<EOF
170 romimage "normal"
171         option USE_FALLBACK_IMAGE=0
172         option ROM_IMAGE_SIZE=0x17000
173         option COREBOOT_EXTRA_VERSION=".0-normal"
174         payload __PAYLOAD__
175 end
176
177 romimage "fallback" 
178         option USE_FALLBACK_IMAGE=1
179         option ROM_IMAGE_SIZE=0x17000
180         option COREBOOT_EXTRA_VERSION=".0-fallback"
181         payload __PAYLOAD__
182 end
183 buildrom ./coreboot.rom ROM_SIZE "normal" "fallback"
184 EOF
185                 else
186                         cat <<EOF
187 romimage "only"
188         option COREBOOT_EXTRA_VERSION=".0"
189         payload __PAYLOAD__
190 end
191 buildrom ./coreboot.rom ROM_SIZE "only"
192 EOF
193                 fi 
194                 ) > $TARGET/Config-${VENDOR}_${MAINBOARD}.lb
195         fi
196
197         if [ "$loglevel" != "default" ]; then
198                 LOGLEVEL1="option MAXIMUM_CONSOLE_LOGLEVEL=$loglevel"
199                 LOGLEVEL2="option DEFAULT_CONSOLE_LOGLEVEL=$loglevel"
200         else
201                 LOGLEVEL1="# no loglevel override"
202                 LOGLEVEL2=""
203         fi
204
205         if [ "`which lzma`" != "" -a "$PAYLOAD" != /dev/null ]; then
206                 COMPRESSION="option CONFIG_COMPRESSED_PAYLOAD_LZMA=1"
207         else
208                 COMPRESSION="# no compression"
209         fi
210
211         cp $TARGET/Config-${VENDOR}_${MAINBOARD}.lb $TARGET/Config-${VENDOR}_${MAINBOARD}.lb.pre
212         sed -e s:VENDOR:$VENDOR:g \
213                 -e s:MAINBOARD:$MAINBOARD:g \
214                 -e s:payload\ __PAYLOAD__:payload\ $PAYLOAD:g \
215                 -e s:CROSSCC:"$CC":g \
216                 -e s:CROSS_PREFIX:"$CROSS_COMPILE":g \
217                 -e s:CROSS_HOSTCC:"$HOSTCC":g \
218                 -e s:__COMPRESSION__:"$COMPRESSION":g \
219                 -e s:__LOGLEVEL__:"$LOGLEVEL1"\
220 "$LOGLEVEL2":g \
221                 $TARGET/Config-${VENDOR}_${MAINBOARD}.lb.pre > $TARGET/Config-${VENDOR}_${MAINBOARD}.lb
222         printf " ok\n"
223 }
224
225 function create_builddir
226 {       
227         VENDOR=$1
228         MAINBOARD=$2
229         
230         printf "  Creating builddir..."
231
232         target_dir=$TARGET
233         config_dir=$LBROOT/util/newconfig
234         yapps2_py=$config_dir/yapps2.py
235         config_g=$config_dir/config.g
236         config_lb=Config-${VENDOR}_${MAINBOARD}.lb
237
238         cd $target_dir
239
240         build_dir=${VENDOR}_${MAINBOARD}
241         config_py=$build_dir/config.py
242
243         if [ ! -d $build_dir ] ; then
244                 mkdir -p $build_dir
245         fi
246         if [ ! -f $config_py ]; then
247                 $PYTHON $yapps2_py $config_g $config_py &> $build_dir/py.log
248         fi
249
250         # make sure config.py is up-to-date
251
252         export PYTHONPATH=$config_dir
253         $PYTHON $config_py $config_lb $LBROOT &> $build_dir/config.log
254         if [ $? -eq 0 ]; then
255                 printf "ok\n"
256                 xml "  <builddir>ok</builddir>"
257                 xml "  <log>"
258                 xmlfile $build_dir/config.log
259                 xml "  </log>"
260                 xml ""
261                 return 0
262         else
263                 printf "FAILED! Log excerpt:\n"
264                 xml "  <builddir>failed</builddir>"
265                 xml "  <log>"
266                 xmlfile $build_dir/config.log
267                 xml "  </log>"
268                 xml ""
269                 tail -n $CONTEXT $build_dir/config.log
270                 return 1
271         fi
272 }
273
274 function create_buildenv
275 {
276         VENDOR=$1
277         MAINBOARD=$2
278         CONFIG=$3
279         create_config $VENDOR $MAINBOARD $CONFIG
280         create_builddir $VENDOR $MAINBOARD
281 }
282
283 function compile_target
284 {       
285         VENDOR=$1
286         MAINBOARD=$2
287
288         printf "  Compiling image "
289         test "$cpus" == "" && printf "in parallel .. "
290         test "$cpus" == "1" && printf "on 1 cpu .. "
291         test 0$cpus -gt 1 && printf "on %d cpus in parallel .. " $cpus
292
293         CURR=$( pwd )
294         cd $TARGET/${VENDOR}_${MAINBOARD}
295         stime=`perl -e 'print time();'`
296         eval $MAKE $silent -j $cpus &> make.log
297         ret=$?
298         etime=`perl -e 'print time();'`
299         duration=$(( $etime - $stime ))
300         if [ $ret -eq 0 ]; then
301                 xml "  <compile>ok</compile>"
302                 xml "  <compiletime>${duration}s</compiletime>"
303                 xml "  <log>"
304                 xmlfile make.log
305                 xml "  </log>"
306                 printf "ok\n" > compile.status
307                 printf "ok. (took ${duration}s)\n"
308                 cd $CURR
309                 return 0
310         else
311                 xml "  <compile>failed</compile>"
312                 xml "  <compiletime>${duration}s</compiletime>"
313                 xml "  <log>"
314                 xmlfile make.log
315                 xml "  </log>"
316
317                 printf "FAILED after ${duration}s! Log excerpt:\n"
318                 tail -n $CONTEXT make.log
319                 cd $CURR
320                 return 1
321         fi
322 }
323
324 function built_successfully
325 {
326         CURR=`pwd`
327         status="fail"
328         if [ -d "$TARGET/${VENDOR}_${MAINBOARD}" ]; then
329                 cd $TARGET/${VENDOR}_${MAINBOARD}
330                 if [ -r compile.status ] ; then
331                         status=`cat compile.status`
332                 fi
333                 cd $CURR
334         fi
335         [ "$buildall" != "true" -a "$status" == "ok" ]
336 }
337
338 function build_broken
339 {
340         CURR=`pwd`
341         status="yes"
342         [ -r "$LBROOT/src/mainboard/${VENDOR}/${MAINBOARD}/BROKEN" ] && status="no"
343         [ "$buildbroken" == "true" -o "$status" == "yes" ]
344 }
345
346 function build_target
347 {
348         VENDOR=$1
349         MAINBOARD=$2
350         CONFIG=$3
351         TARCH=$( architecture $VENDOR $MAINBOARD )
352
353         # Allow architecture override in an abuild.info file.
354         # This is used for the Motorola Sandpoint, which is not a real target
355         # but a skeleton target for the Sandpoint X3.
356         [ -r "$LBROOT/src/mainboard/${VENDOR}/${MAINBOARD}/abuild.info" ] && \
357                 source $LBROOT/src/mainboard/${VENDOR}/${MAINBOARD}/abuild.info
358
359         # default setting
360
361         CC="${CROSS_COMPILE}gcc"
362         CROSS_COMPILE=""
363         found_crosscompiler=false
364         if which $TARCH-elf-gcc 2>/dev/null >/dev/null; then
365                 # i386-elf target needs --divide, for i386-linux, that's the default
366                 if [ "$TARCH" = "i386" ]; then
367                         CC="$CC -Wa,--divide"
368                 fi
369                 CROSS_COMPILE="$TARCH-elf-"
370                 echo using $CROSS_COMPILE$CC
371                 found_crosscompiler=true
372         fi
373
374         HOSTCC='gcc'
375
376         printf "Processing mainboard/$VENDOR/$MAINBOARD"
377
378         xml "<mainboard>"
379         xml ""
380         xml "  <vendor>$VENDOR</vendor>"
381         xml "  <device>$MAINBOARD</device>"
382         xml ""
383         xml "  <architecture>$TARCH</architecture>"
384         xml ""
385
386         if [ "$ARCH" = "$TARCH" -o $found_crosscompiler = true ]; then
387                 printf " ($TARCH: ok)\n"
388         else
389                 found_crosscompiler=false
390                 if [ "$ARCH" == amd64 -a "$TARCH" == i386 ]; then
391                         CC="gcc -m32"
392                         found_crosscompiler=true
393                 fi
394                 if [ "$ARCH" == ppc64 -a "$TARCH" == ppc ]; then
395                         CC="gcc -m32"
396                         found_crosscompiler=true
397                 fi
398                 if [ "$found_crosscompiler" == "false" -a "$TARCH" == ppc ];then
399                         for prefix in powerpc-eabi- powerpc-linux- ppc_74xx- \
400                             powerpc-7450-linux-gnu- powerpc-elf-; do
401                                 if ${prefix}gcc --version > /dev/null 2> /dev/null ; then
402                                         found_crosscompiler=true
403                                         CROSS_COMPILE=$prefix
404                                 fi
405                         done
406                 fi
407
408         
409                 # TBD: look for suitable cross compiler suite
410                 # cross-$TARCH-gcc and cross-$TARCH-ld
411                 
412                 # Check result:
413                 if [ $found_crosscompiler == "false" ]; then
414                         printf " ($TARCH: skipped, we're $ARCH)\n\n"
415                         xml "  <status>notbuilt</status>"
416                         xml ""
417                         xml "</mainboard>"
418                 
419                         return 0
420                 else
421                         printf " ($TARCH: ok, we're $ARCH with a ${CROSS_COMPILE} cross compiler)\n"
422                         xml "  <compiler>"
423                         xml "    <path>`which ${CROSS_COMPILE}gcc`</path>"
424                         xml "    <version>`${CROSS_COMPILE}gcc --version | head -1`</version>"
425                         xml "  </compiler>"
426                         xml ""
427                 fi
428         fi
429
430         CC=${CROSS_COMPILE}$CC
431
432         if  [ "$stackprotect" = "true" ]; then
433                 CC="$CC -fno-stack-protector"
434         fi
435
436         if  [ "$scanbuild" = "true" ]; then
437                 ccwrap=`mktemp`
438                 mkdir -p $TARGET/${VENDOR}_${MAINBOARD}
439                 mkdir -p $TARGET/scan-build-results-tmp
440                 mv $ccwrap $TARGET/${VENDOR}_${MAINBOARD}
441                 ccwrap=$TARGET/${VENDOR}_${MAINBOARD}/`basename $ccwrap`
442                 echo '#!/bin/sh' > $ccwrap
443                 echo $CC' "$@"' >> $ccwrap
444                 chmod +x $ccwrap
445                 origMAKE=$MAKE
446                 MAKE="scan-build --use-cc=$ccwrap -o $TARGET/scan-build-results-tmp -analyze-headers $MAKE GCC=$ccwrap"
447                 CC="\$(CC)"
448                 HOSTCC="CCC_CC=$HOSTCC \$(CC)"
449         fi
450
451         built_successfully $VENDOR $MAINBOARD && \
452         {
453                 printf " ( mainboard/$VENDOR/$MAINBOARD previously ok )\n\n"
454                 xml "  <status>previouslyok</status>"
455                 xml ""
456                 xml "</mainboard>"
457                 return 0
458         }
459
460         build_broken $VENDOR $MAINBOARD || \
461         {
462                 printf " ( broken mainboard/$VENDOR/$MAINBOARD skipped )\n\n"
463                 xml "  <status>knownbroken</status>"
464                 xml ""
465                 xml "</mainboard>"
466                 return 0
467         }
468         
469         create_buildenv $VENDOR $MAINBOARD $CONFIG
470         if [ $? -eq 0  -a  $configureonly -eq 0 ]; then
471                 compile_target $VENDOR $MAINBOARD && 
472                         xml "  <status>ok</status>" ||
473                         xml "<status>broken</status>"
474                 if [ "$scanbuild" = "true" ]; then
475                         mv `dirname $TARGET/scan-build-results-tmp/*/index.html` $TARGET/${VENDOR}_${MAINBOARD}-scanbuild
476                         MAKE=$origMAKE
477                 fi
478         fi
479
480         xml ""
481         xml "</mainboard>"
482
483         printf "\n"
484 }
485
486 function test_target
487 {
488         VENDOR=$1
489         MAINBOARD=$2
490
491         if [ "$hwtest" != "true" ]; then
492                 return 0
493         fi
494
495         # image does not exist. we silently skip the patch.
496         if [ ! -r "$TARGET/${VENDOR}_${MAINBOARD}/coreboot.rom" ]; then
497                 return 0
498         fi
499
500         which curl &> /dev/null
501         if [ $? != 0 ]; then
502                 printf "curl is not installed but required for test submission.  skipping test.\n\n"
503                 return 0
504         fi
505
506         CURR=`pwd`
507         if [ -r "$TARGET/${VENDOR}_${MAINBOARD}/tested" ]; then
508                 printf "Testing image for board $VENDOR $MAINBOARD skipped (previously submitted).\n\n"
509                 return 0
510         fi
511         # touch $TARGET/${VENDOR}_${MAINBOARD}/tested
512
513         printf "Submitting image for board $VENDOR $MAINBOARD to test system...\n"
514
515         curl -f -F "romfile=@$TARGET/${VENDOR}_${MAINBOARD}/coreboot.rom" \
516                 -F "mode=abuild" -F "mainboard=${VENDOR}_${MAINBOARD}" -F "submit=Upload" \
517                 "http://qa.coreboot.org/deployment/send.php"
518
519         printf "\n"
520         return 0
521 }
522
523 function remove_target
524 {
525         if [ "$remove" != "true" ]; then
526                 return 0
527         fi
528
529         VENDOR=$1
530         MAINBOARD=$2
531
532         # Save the generated coreboot.rom file of each board.
533         if [ -r "$TARGET/${VENDOR}_${MAINBOARD}/coreboot.rom" ]; then
534                 cp $TARGET/${VENDOR}_${MAINBOARD}/coreboot.rom \
535                    ${VENDOR}_${MAINBOARD}_coreboot.rom
536         fi
537
538         printf "Removing build dir for board $VENDOR $MAINBOARD...\n"
539         rm -rf $TARGET/${VENDOR}_${MAINBOARD}
540
541         return 0
542 }
543
544 function myhelp
545 {
546         printf "Usage: $0 [-v] [-a] [-b] [-r] [-t <vendor/board>] [-p <dir>] [lbroot]\n"
547         printf "       $0 [-V|--version]\n"
548         printf "       $0 [-h|--help]\n\n"
549
550         printf "Options:\n"
551         printf "    [-v|--verbose]                print more messages\n"
552         printf "    [-a|--all]                    build previously succeeded ports as well\n"
553         printf "    [-b|--broken]                 attempt to build ports that are known broken\n"
554         printf "    [-r|--remove]                 remove output dir after build\n"
555         printf "    [-t|--target <vendor/board>]  attempt to build target vendor/board only\n"
556         printf "    [-p|--payloads <dir>]         use payloads in <dir> to build images\n"
557         printf "    [-V|--version]                print version number and exit\n"
558         printf "    [-h|--help]                   print this help and exit\n"
559         printf "    [-x|--xml]                    write xml log file \n"
560         printf "                                  (defaults to $XMLFILE)\n"
561         printf "    [-T|--test]                   submit image(s) to automated test system\n"
562         printf "    [-c|--cpus <numcpus>]         build on <numcpus> at the same time\n"
563         printf "    [-s|--silent]                 omit compiler calls in logs\n"
564         printf "    [-ns|--nostackprotect]        use gcc -fno-stack-protector option\n"
565         printf "    [-sb|--scan-build]            use clang's static analyzer\n"
566         printf "    [-C|--config]                 configure-only mode\n"
567         printf "    [-l|--loglevel <num>]         set loglevel\n"
568         printf "    [lbroot]                      absolute path to coreboot sources\n"
569         printf "                                  (defaults to $LBROOT)\n\n"
570 }
571
572 function myversion 
573 {
574         cat << EOF
575
576 coreboot autobuild v$ABUILD_VERSION ($ABUILD_DATE)
577
578 Copyright (C) 2004 by Stefan Reinauer <stepan@openbios.org>
579 Copyright (C) 2006-2008 by coresystems GmbH <info@coresystems.de>
580
581 This program is free software; you may redistribute it under the terms
582 of the GNU General Public License. This program has absolutely no
583 warranty.
584
585 EOF
586 }
587
588 # default options
589 target=""
590 buildall=false
591 LBROOT=$( cd ../..; pwd )
592 verbose=false
593
594 # parse parameters.. try to find out whether we're running GNU getopt
595 getoptbrand="`getopt -V`"
596 if [ "${getoptbrand:0:6}" == "getopt" ]; then
597         # Detected GNU getopt that supports long options.
598         args=`getopt -l version,verbose,help,all,target:,broken,payloads:,test,cpus:,silent,xml,config,loglevel Vvhat:bp:Tc:sxCl: -- "$@"`
599         eval set "$args"
600 else
601         # Detected non-GNU getopt
602         args=`getopt Vvhat:bp:Tc:sxCl: $*`
603         set -- $args
604 fi
605
606 if [ $? != 0 ]; then
607         myhelp
608         exit 1
609 fi
610
611 while true ; do
612         case "$1" in
613                 -x|--xml)       shift; mode=xml; rm -f $XMLFILE ;;
614                 -t|--target)    shift; target="$1"; shift;;
615                 -a|--all)       shift; buildall=true;;
616                 -b|--broken)    shift; buildbroken=true;;
617                 -r|--remove)    shift; remove=true; shift;;
618                 -v|--verbose)   shift; verbose=true;;
619                 -V|--version)   shift; myversion; exit 0;;
620                 -h|--help)      shift; myversion; myhelp; exit 0;;
621                 -p|--payloads)  shift; payloads="$1"; shift;;
622                 -T|--test)      shift; hwtest=true;;
623                 -c|--cpus)      shift; cpus="$1"; test "$cpus" == "max" && cpus=""; shift;;
624                 -s|--silent)    shift; silent="-s";;
625                 -ns|--nostackprotect) shift; stackprotect=true;;
626                 -sb|--scan-build) shift; scanbuild=true;;
627                 -C|--config)    shift; configureonly=1;;
628                 -l|--loglevel)  shift; loglevel="$1"; shift;;
629                 --)             shift; break;;
630                 -*)             printf "Invalid option\n\n"; myhelp; exit 1;;
631                 *)              break;;
632         esac
633 done
634
635 # /path/to/freebios2/
636 test -z "$1" || LBROOT=$1
637
638 debug "LBROOT=$LBROOT"
639
640 xml '<?xml version="1.0" encoding="utf-8"?>'
641 xml '<abuild>'
642
643 if [ "$target" != "" ]; then
644         # build a single board
645         VENDOR=`printf $target|cut -f1 -d/`
646         MAINBOARD=`printf $target|cut -f2 -d/`
647         CONFIG=`printf $target|cut -f3 -d/`
648         build_target $VENDOR $MAINBOARD $CONFIG
649         test_target $VENDOR $MAINBOARD
650 else
651         # build all boards per default
652         for VENDOR in $( vendors ); do
653                 for MAINBOARD in $( mainboards $VENDOR ); do
654                         build_target $VENDOR $MAINBOARD
655                         test_target $VENDOR $MAINBOARD
656                         remove_target $VENDOR $MAINBOARD
657                 done
658         done
659 fi
660 xml '</abuild>'
661