openwrt/flashrom
1
0
Fork 0
mirror of https://review.coreboot.org/flashrom.git synced 2025-04-27 15:12:36 +02:00
Code Issues Releases Wiki Activity

Add Manibuilder

Browse Source 
Add a set of Dockerfiles for build testing. If you have an x86 machine
and ~20GiB free disk space, run `make register` and `make -jxx` in
util/manibuilder and go eat some pizza. The former runs a privileged
docker container to set binfmt_misc up for qemu (read the code, don't
trust it).

Regarding the build targets, this is the original state of Manibuilder
as it was used to build-test `flashrom-1.0`. Some fixes to the frame-
work were applied, but fixups for the targets will be done in separate
patches to maintain their original state for reference.

Change-Id: I60863a5c7d70dde71486fccb66cb59b30ba4d982
Signed-off-by: Nico Huber <nico.h@gmx.de>
Reviewed-on: https://review.coreboot.org/c/flashrom/+/23005
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Angel Pons <th3fanbus@gmail.com>
This commit is contained in:
Nico Huber 2017-12-22 02:08:57 +01:00 committed by Angel Pons
parent dc7c99b99e
commit 3732ff6b7b
13 changed files with 470 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
util/manibuilder/Dockerfile.alpine Normal file
View File

@ -0,0 +1,17 @@
FROM manibase
RUN \
adduser -D mani mani && \
apk update && \
apk add build-base linux-headers git ccache \
pciutils-dev libusb-compat-dev libusb-dev
USER mani
RUN \
cd && \
mkdir .ccache && chown mani:mani .ccache && \
git clone https://review.coreboot.org/flashrom.git
ENV DEVSHELL /bin/sh
COPY mani-wrapper.sh /home/mani/
ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

55
util/manibuilder/Dockerfile.anita Normal file
View File

@ -0,0 +1,55 @@
FROM debian:stable
ARG PKG_PATH=http://cdn.netbsd.org/pub/pkgsrc/packages/NetBSD/amd64/7.1/All
ARG INST_IMG=http://ftp.de.netbsd.org/pub/NetBSD/NetBSD-7.1/amd64/
ARG EXTRA_PKG=""
RUN \
useradd -p locked -m mani && \
apt-get -qq update && \
apt-get -qq upgrade && \
apt-get -qq dist-upgrade && \
apt-get -qqy install git python python-pexpect \
genisoimage qemu-system && \
apt-get clean && \
git clone https://github.com/gson1703/anita.git && \
cd anita && python setup.py install
USER mani
RUN cd && mkdir .ccache && chown mani:mani .ccache && \
anita --sets kern-GENERIC,modules,base,etc,comp \
--disk-size 1024M --memory-size=128M install ${INST_IMG} && \
rm -rf work-*/download
RUN cd && anita --persist --run \
"echo 'dhcpcd' >init && \
echo 'export PKG_PATH=${PKG_PATH}' >>init && \
. ./init && \
pkg_add gmake git-base ccache pciutils libusb1 libusb-compat libftdi \
${EXTRA_PKG} && \
git config --global --add http.sslVerify false && \
git clone https://review.coreboot.org/flashrom.git" \
boot ${INST_IMG}
RUN cd && dd if=/dev/zero bs=1M count=64 of=cache.img && \
anita --vmm-args '-hdb cache.img' --persist --run \
"echo fdisk -0 -s 169/63/131009 -f -u wd1 && \
newfs wd1a && \
mkdir .ccache && \
mount /dev/wd1a .ccache && \
ccache -M 60M && \
umount .ccache && \
echo 'manitest() {' >>init && \
echo ' fsck -y /dev/wd1a' >>init && \
echo ' mount /dev/wd1a ~/.ccache' >>init && \
echo ' (cd ~/flashrom && eval \" \$*\")' >>init && \
echo ' ret=\$?' >>init && \
echo ' umount ~/.ccache' >>init && \
echo ' return \$ret' >>init && \
echo '}' >>init" \
boot ${INST_IMG} && \
gzip cache.img
COPY anita-wrapper.sh /home/mani/mani-wrapper.sh
ENV INST_IMG ${INST_IMG}
ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

16
util/manibuilder/Dockerfile.centos Normal file
View File

@ -0,0 +1,16 @@
FROM manibase
RUN \
useradd -p locked -m mani && \
yum install -q -y git gcc pciutils-devel libusb-devel libusbx-devel && \
yum clean -q -y all
USER mani
RUN \
cd && \
mkdir .ccache && chown mani:mani .ccache && \
git clone https://review.coreboot.org/flashrom.git
ENV DEVSHELL /bin/bash
COPY mani-wrapper.sh /home/mani/
ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

20
util/manibuilder/Dockerfile.debian-debootstrap Normal file
View File

@ -0,0 +1,20 @@
FROM manibase
RUN \
useradd -p locked -m mani && \
apt-get -qq update && \
apt-get -qq upgrade && \
apt-get -qq dist-upgrade && \
apt-get -qqy install gcc make git doxygen ccache \
libpci-dev libusb-dev libftdi-dev libusb-1.0-0-dev && \
apt-get clean
USER mani
RUN \
cd && \
mkdir .ccache && chown mani:mani .ccache && \
git clone https://review.coreboot.org/flashrom.git
ENV DEVSHELL /bin/bash
COPY mani-wrapper.sh /home/mani/
ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

29
util/manibuilder/Dockerfile.djgpp Normal file
View File

@ -0,0 +1,29 @@
FROM anibali/djgpp:6.1.0
USER root
RUN \
userdel appuser && \
useradd -p locked -m mani && \
zypper -q install -y tar make git ccache
USER mani
RUN cd && \
mkdir .ccache && chown mani:users .ccache && \
git clone https://review.coreboot.org/flashrom.git && \
git clone https://git.kernel.org/pub/scm/utils/pciutils/pciutils.git && \
cd pciutils && \
git checkout v3.5.6 && \
curl https://flashrom.org/images/6/6a/Pciutils-3.5.6.patch.gz | zcat | git apply && \
make ZLIB=no DNS=no HOST=i386-djgpp-djgpp \
CROSS_COMPILE=i586-pc-msdosdjgpp- \
PREFIX=/ DESTDIR=$PWD/../ \
STRIP="--strip-program=i586-pc-msdosdjgpp-strip -s" \
install install-lib && \
cd ../ && \
curl https://flashrom.org/images/3/3d/Libgetopt.tar.gz | zcat | tar x && \
cd libgetopt && \
make && cp libgetopt.a ../lib/ && cp getopt.h ../include/
ENV DEVSHELL /bin/bash
COPY mani-wrapper.sh /home/mani/
ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

17
util/manibuilder/Dockerfile.fedora Normal file
View File

@ -0,0 +1,17 @@
FROM manibase
RUN \
useradd -p locked -m mani && \
dnf install -q -y git gcc ccache make \
pciutils-devel libusb-devel libusbx-devel libftdi-devel && \
dnf clean -q -y all
USER mani
RUN \
cd && \
mkdir .ccache && chown mani:mani .ccache && \
git clone https://review.coreboot.org/flashrom.git
ENV DEVSHELL /bin/bash
COPY mani-wrapper.sh /home/mani/
ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

3
util/manibuilder/Dockerfile.qemu-user-static Normal file
View File

@ -0,0 +1,3 @@
FROM multiarch/qemu-user-static:register
RUN sed -i -e's/ mipsn32 mipsn32el / /' /qemu-binfmt-conf.sh

33
util/manibuilder/Dockerfile.ubuntu-debootstrap Normal file
View File

@ -0,0 +1,33 @@
FROM manibase
RUN \
useradd -p locked -m mani && \
if grep -q main /etc/apt/sources.list; then \
if ! grep -q universe /etc/apt/sources.list; then \
sed -i -e 's/ main$/ main universe/' \
/etc/apt/sources.list || exit 1; \
fi; \
else \
url="http://ports.ubuntu.com/" && \
cn="$(sed -ne's/DISTRIB_CODENAME=//p' /etc/lsb-release)" && \
for t in "" "-updates" "-security"; do \
echo "deb ${url} ${cn}${t} main universe" \
>>/etc/apt/sources.list || exit 1; \
done; \
fi && \
apt-get -qq update && \
apt-get -qq upgrade && \
apt-get -qq dist-upgrade && \
apt-get -qqy install gcc make git doxygen ccache \
libpci-dev libusb-dev libftdi-dev libusb-1.0-0-dev && \
apt-get clean
USER mani
RUN \
cd && \
mkdir .ccache && chown mani:mani .ccache && \
git clone https://review.coreboot.org/flashrom.git
ENV DEVSHELL /bin/bash
COPY mani-wrapper.sh /home/mani/
ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

97
util/manibuilder/Makefile Normal file
View File

@ -0,0 +1,97 @@
QUIET_TEST := @
include Makefile.targets
CC := ccache cc
MAKECMD := make
spc :=
spc := $(spc) $(spc)
stem = $(word 1,$(subst :,$(spc),$(subst \:,$(spc),$(1))))
ident = $(subst :,_,$(subst \:,_,$(1)))
PKGSRC_MIRROR = http://cdn.netbsd.org/
NETBSD_MIRROR = http://ftp.de.netbsd.org/
anita\:amd64-build: PKGSRC_PATH=pub/pkgsrc/packages/NetBSD/amd64/7.1/All
anita\:amd64-build: NETBSD_IMAGE=pub/NetBSD/NetBSD-7.1/amd64/
anita\:i386-build: PKGSRC_PATH=pub/pkgsrc/packages/NetBSD/i386/7.1/All
anita\:i386-build: NETBSD_IMAGE=pub/NetBSD/NetBSD-7.1/i386/
$(addsuffix -build,$(ANITA_TAGS)): %-build: Dockerfile.anita anita-wrapper.sh
$(QUIET_SETUP)docker build . -f $< -t mani/$* \
--build-arg PKG_PATH=$(PKGSRC_MIRROR)$(PKGSRC_PATH) \
--build-arg INST_IMG=$(NETBSD_MIRROR)$(NETBSD_IMAGE)
define build_template
Dockerfile.$(call ident,$(1)): Dockerfile.$(call stem,$(1)) mani-wrapper.sh
$(QUIET_SETUP)sed -e 's|^FROM manibase|FROM $(2)/$(1)|' $$< >$$@
.INTERMEDIATE: Dockerfile.$(call ident,$(1))
$(1)-build: Dockerfile.$(call ident,$(1))
$(QUIET_SETUP)docker build . -f $$< -t mani/$(1)
endef
$(foreach tag,$(MULTIARCH_TAGS), \
$(eval $(call build_template,$(tag),multiarch)))
djgpp\:6.1.0-build: %-build: Dockerfile.djgpp mani-wrapper.sh
$(QUIET_SETUP)docker build . -f $< -t mani/$*
$(addsuffix -check-build,$(ALL_TAGS)): %-check-build:
$(QUIET_SETUP)\
[ $$(docker image ls -q mani/$*) ] \
|| $(MAKE) $*-build $(if $(QUIET_SETUP),>/dev/null 2>/dev/null)
$(filter centos%,$(MULTIARCH_TAGS)) anita\:sparc: CC=cc
djgpp\:6.1.0: CC=ccache i586-pc-msdosdjgpp-gcc
djgpp\:6.1.0: STRIP=i586-pc-msdosdjgpp-strip
djgpp\:6.1.0: LIBS_BASE=../
djgpp\:6.1.0: TARGET=strip
$(ANITA_TAGS): MAKECMD=gmake
$(ALL_TAGS): export QUIET_SETUP=$(QUIET_TEST)
$(ALL_TAGS): %: %-check-build
$(QUIET_TEST)docker rm -f mani_$(call ident,$*) >/dev/null 2>&1 || true
$(QUIET_TEST)\
docker run \
--env IDENT=$(call ident,$*) \
--volume manicache:/home/mani/.ccache \
--name mani_$(call ident,$*) mani/$* \
"git fetch origin $${TEST_REVISION:-master} && \
git checkout FETCH_HEAD && \
$(MAKECMD) clean && $(MAKECMD) -j$${CPUS:-1} CC='$(CC)' \
$(if $(STRIP),STRIP='$(STRIP)') \
$(if $(LIBS_BASE),LIBS_BASE='$(LIBS_BASE)') \
$(TARGET)" \
$(if $(QUIET_TEST),>/dev/null 2>&1) || echo $*: $$?
$(addsuffix -shell,$(ALL_TAGS)): %-shell: %-check-build
$(QUIET_SETUP)\
if [ $$(docker ps -a -q -f name=mani_$(call ident,$*)) ]; then \
docker commit mani_$(call ident,$*) mani_run/$* && \
docker run --rm -it \
--env IDENT=$(call ident,$*) \
--volume manicache:/home/mani/.ccache \
--entrypoint /bin/sh mani_run/$* \
/home/mani/mani-wrapper.sh \
$(patsubst %,"%",$(SHELL_ARG)); \
docker image rm mani_run/$*; \
else \
docker run --rm -it \
--env IDENT=$(call ident,$*) \
--volume manicache:/home/mani/.ccache \
mani/$* $(patsubst %,"%",$(SHELL_ARG)); \
fi
.PHONY: $(foreach s,-build -check-build -shell, $(addsuffix $(s),$(ALL_TAGS)))
register:
docker build . \
-f Dockerfile.qemu-user-static \
-t mani/qemu-user-static:register
docker run --rm --privileged mani/qemu-user-static:register --reset
.PHONY: register

85
util/manibuilder/Makefile.targets Normal file
View File

@ -0,0 +1,85 @@
ANITA_TAGS := anita\:amd64 anita\:i386
MULTIARCH_TAGS := \
centos\:7.3-aarch64-clean centos\:7.3-amd64-clean \
centos\:7.2-amd64-clean \
$(foreach a,x86_64 ppc64le aarch64, \
$(foreach v,25 24, \
fedora\:$(v)-$(a))) \
$(foreach a,ppc64el arm64 armhf mips mipsel amd64 i386, \
$(foreach v,sid buster stretch, \
debian-debootstrap\:$(a)-$(v))) \
$(foreach a,ppc64el arm64 armhf amd64 i386, \
$(foreach v,bionic xenial, \
ubuntu-debootstrap\:$(a)-$(v))) \
ubuntu-debootstrap\:powerpc-xenial \
$(foreach a,aarch64 armhf amd64 i386, \
$(foreach v,v3.8 v3.7 v3.6, \
alpine\:$(a)-$(v))) \
OTHER_TAGS := djgpp\:6.1.0
ALL_TAGS := $(ANITA_TAGS) $(MULTIARCH_TAGS) $(OTHER_TAGS)
arch_filter = $(sort \
$(foreach arch,$(1), \
$(filter-out $(subst $(arch),,$(MULTIARCH_TAGS)),$(MULTIARCH_TAGS))))
machine_map = \
$(if $(filter i386 i686 x86,$(1)),i386 x86, \
$(if $(filter x86_64,$(1)),amd64 i386 x86, \
$(if $(filter armv7l armv6l,$(1)),armhf, \
$(if $(filter aarch64,$(1)),aarch64 arm64, \
$(if $(filter ppc64le,$(1)),ppc64le ppc64el, \
$(if $(filter ppc,$(1)),powerpc, \
$(if $(filter mips,$(1)),mips mipsel, \
$(1))))))))
NATIVE_TAGS := $(call arch_filter,$(call machine_map,$(shell uname -m)))
# rather arbitrary selection of images that seem to work (focus on amd64)
DEFAULT_TAGS := \
anita\:amd64 \
djgpp\:6.1.0 \
fedora\:25-x86_64 \
fedora\:25-ppc64le \
fedora\:25-aarch64 \
fedora\:24-x86_64 \
centos\:7.3-aarch64-clean \
centos\:7.3-amd64-clean \
centos\:7.2-amd64-clean \
debian-debootstrap\:ppc64el-stretch \
debian-debootstrap\:armhf-stretch \
debian-debootstrap\:mips-stretch \
debian-debootstrap\:mipsel-stretch \
debian-debootstrap\:amd64-stretch \
debian-debootstrap\:i386-stretch \
debian-debootstrap\:amd64-sid \
ubuntu-debootstrap\:arm64-xenial \
ubuntu-debootstrap\:amd64-xenial \
ubuntu-debootstrap\:powerpc-xenial \
ubuntu-debootstrap\:amd64-bionic \
alpine\:amd64-v3.7 \
alpine\:amd64-v3.8 \
# also run all native tests by default
DEFAULT_TAGS += $(filter-out $(DEFAULT_TAGS),$(NATIVE_TAGS))
default: $(DEFAULT_TAGS)
native: $(NATIVE_TAGS)
all: $(ALL_TAGS)
show-default:
@printf "%s\n" $(DEFAULT_TAGS)
show-native:
@printf "%s\n" $(NATIVE_TAGS)
show-all:
@printf "%s\n" $(ALL_TAGS)
.PHONY: default native all
.PHONY: show-default show-native show-all
.PHONY: $(ALL_TAGS)

72
util/manibuilder/README.md Normal file
View File

@ -0,0 +1,72 @@
Manibuilder
===========
Manibuilder is a set of Dockerfiles for manic build testing, hold
together by some make-foo. Most of the Dockerfiles make use of
*multiarch* images. This way we can test building on many platforms
supported by *Qemu*. The idea is to test in environments as close
as possible to those of potential users, i.e. no cross-compiling
(with some exceptions).
Make targets
------------
For each supported target OS/version/architecture exists a *tag*
target, for instance `alpine:amd64-v3.7`. These targets will
automatically check for existence of their respective *Docker*
images (sub target <tag>-check-build), and build them if necessary
(<tag>-build). Finally, flashrom revision `$(TEST_REVISION)` is
fetched and build tested.
The results will be kept by *Docker* as stopped containers and
can be accessed with the <tag>-shell target.
There are some additional targets that form sets of the *tag*
targets:
* default: runs a preselected subset of all supported tags.
* native: runs all tags native to the host architecture.
* all: runs all supported tags.
For each of these show-<set> lists the included *tags*.
For preparation of *Qemu* for the *multiarch* images, there is the
`register` target. It has to be run once per boot, though as it
uses a privileged *Docker* container, that is kept as a manual step.
Usage example
-------------
The most common use case may be testing the current upstream
*master* branch which is the default for `$(TEST_REVISION)`.
You'll need roughly 20GiB for the *Docker* images. Might look
like this:
$ # have to register Qemu first:
$ make register
[...]
$ # run the default target:
$ make -j4
debian-debootstrap:mips-stretch: 2
debian-debootstrap:mips-sid: 2
debian-debootstrap:mips-buster: 2
ubuntu-debootstrap:powerpc-xenial: 2
djgpp:6.1.0: 2
For each *tag* that returns with a non-zero exit code, the *tag*
and actual exit code is printed. An exit code of `2` is most likely
as that is what *make* returns on failure. Other exit codes might
hint towards a problem in the setup. Failing *tags* can then be
investigated individually with the <tag>-shell target, e.g.:
$ make debian-debootstrap:mips-sid-shell
[...]
mani@63536fc102a5:~/flashrom$ make
[...]
cc -MMD -Os -Wall -Wshadow -Werror -I/usr/include/libusb-1.0 -D'CONFIG_DEFAULT_PROGRAMMER=PROGRAMMER_INVALID' -D'CONFIG_DEFAULT_PROGRAMMER_ARGS="''"' -D'CONFIG_SERPROG=1' -D'CONFIG_PONY_SPI=1' -D'CONFIG_BITBANG_SPI=1' -D'CONFIG_GFXNVIDIA=1' -D'CONFIG_SATASII=1' -D'CONFIG_ATAVIA=1' -D'CONFIG_IT8212=1' -D'CONFIG_FT2232_SPI=1' -D'CONFIG_USBBLASTER_SPI=1' -D'CONFIG_PICKIT2_SPI=1' -D'HAVE_FT232H=1' -D'CONFIG_DUMMY=1' -D'CONFIG_DRKAISER=1' -D'CONFIG_NICINTEL=1' -D'CONFIG_NICINTEL_SPI=1' -D'CONFIG_NICINTEL_EEPROM=1' -D'CONFIG_OGP_SPI=1' -D'CONFIG_BUSPIRATE_SPI=1' -D'CONFIG_DEDIPROG=1' -D'CONFIG_DEVELOPERBOX_SPI=1' -D'CONFIG_LINUX_MTD=1' -D'CONFIG_LINUX_SPI=1' -D'CONFIG_CH341A_SPI=1' -D'CONFIG_DIGILENT_SPI=1' -D'NEED_PCI=1' -D'NEED_RAW_ACCESS=1' -D'NEED_LIBUSB0=1' -D'NEED_LIBUSB1=1' -D'HAVE_UTSNAME=1' -D'HAVE_CLOCK_GETTIME=1' -D'FLASHROM_VERSION="p1.0-141-g9cecc7e"' -o libflashrom.o -c libflashrom.c
libflashrom.c:386:12: error: 'flashrom_layout_parse_fmap' defined but not used [-Werror=unused-function]
static int flashrom_layout_parse_fmap(struct flashrom_layout **layout,
^~~~~~~~~~~~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors
make: *** [Makefile:1075: libflashrom.o] Error 1
$ # uh-huh, might be a problem with big-endian #if foo

17
util/manibuilder/anita-wrapper.sh Normal file
View File

@ -0,0 +1,17 @@
#!/bin/sh
cd
[ "${IDENT}" ] || IDENT=$(mktemp -u XXXXXXXX)
CCACHE=.ccache/anita-${IDENT}.img
[ -f ${CCACHE} ] || zcat cache.img.gz >${CCACHE}
if [ $# -eq 0 ]; then
exec anita --vmm-args "-hdb ${CCACHE}" interact ${INST_IMG}
else
exec anita --vmm-args "-hdb ${CCACHE}" --persist \
--run ". ./init && manitest \"$*\"" \
boot ${INST_IMG}
fi

9
util/manibuilder/mani-wrapper.sh Normal file
View File

@ -0,0 +1,9 @@
#!/bin/sh
cd /home/mani/flashrom/
if [ $# -eq 0 ]; then
exec "${DEVSHELL}"
else
exec "${DEVSHELL}" -c "$*"
fi