Add Manibuilder

Add a set of Dockerfiles for build testing. If you have an x86 machine
and ~20GiB free disk space, run `make register` and `make -jxx` in
util/manibuilder and go eat some pizza. The former runs a privileged
docker container to set binfmt_misc up for qemu (read the code, don't
trust it).

Regarding the build targets, this is the original state of Manibuilder
as it was used to build-test `flashrom-1.0`. Some fixes to the frame-
work were applied, but fixups for the targets will be done in separate
patches to maintain their original state for reference.

Change-Id: I60863a5c7d70dde71486fccb66cb59b30ba4d982
Signed-off-by: Nico Huber <nico.h@gmx.de>
Reviewed-on: https://review.coreboot.org/c/flashrom/+/23005
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Angel Pons <th3fanbus@gmail.com>

util/manibuilder/Dockerfile.alpine

@@ -0,0 +1,17 @@
+FROM manibase
+
+RUN \
+	adduser -D mani mani && \
+	apk update && \
+	apk add build-base linux-headers git ccache \
+		pciutils-dev libusb-compat-dev libusb-dev
+
+USER mani
+RUN \
+	cd && \
+	mkdir .ccache && chown mani:mani .ccache && \
+	git clone https://review.coreboot.org/flashrom.git
+
+ENV DEVSHELL /bin/sh
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

util/manibuilder/Dockerfile.anita

@@ -0,0 +1,55 @@
+FROM debian:stable
+
+ARG PKG_PATH=http://cdn.netbsd.org/pub/pkgsrc/packages/NetBSD/amd64/7.1/All
+ARG INST_IMG=http://ftp.de.netbsd.org/pub/NetBSD/NetBSD-7.1/amd64/
+ARG EXTRA_PKG=""
+
+RUN \
+	useradd -p locked -m mani && \
+	apt-get -qq update && \
+	apt-get -qq upgrade && \
+	apt-get -qq dist-upgrade && \
+	apt-get -qqy install git python python-pexpect \
+			     genisoimage qemu-system && \
+	apt-get clean && \
+	git clone https://github.com/gson1703/anita.git && \
+	cd anita && python setup.py install
+
+USER mani
+RUN cd && mkdir .ccache && chown mani:mani .ccache && \
+	  anita --sets kern-GENERIC,modules,base,etc,comp \
+		--disk-size 1024M --memory-size=128M install ${INST_IMG} && \
+	  rm -rf work-*/download
+
+RUN cd && anita --persist --run \
+"echo 'dhcpcd'                                   >init && \
+ echo 'export PKG_PATH=${PKG_PATH}'             >>init && \
+ . ./init && \
+ pkg_add gmake git-base ccache pciutils libusb1 libusb-compat libftdi \
+         ${EXTRA_PKG} && \
+ git config --global --add http.sslVerify false && \
+ git clone https://review.coreboot.org/flashrom.git" \
+		boot ${INST_IMG}
+
+RUN cd && dd if=/dev/zero bs=1M count=64 of=cache.img && \
+	  anita --vmm-args '-hdb cache.img' --persist --run \
+"echo fdisk -0 -s 169/63/131009 -f -u wd1 && \
+ newfs wd1a && \
+ mkdir .ccache && \
+ mount /dev/wd1a .ccache && \
+ ccache -M 60M && \
+ umount .ccache && \
+ echo 'manitest() {'                            >>init && \
+ echo '    fsck -y /dev/wd1a'                   >>init && \
+ echo '    mount /dev/wd1a ~/.ccache'           >>init && \
+ echo '    (cd ~/flashrom && eval \" \$*\")'    >>init && \
+ echo '    ret=\$?'                             >>init && \
+ echo '    umount ~/.ccache'                    >>init && \
+ echo '    return \$ret'                        >>init && \
+ echo '}'                                       >>init" \
+		boot ${INST_IMG} && \
+	  gzip cache.img
+
+COPY anita-wrapper.sh /home/mani/mani-wrapper.sh
+ENV INST_IMG ${INST_IMG}
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

util/manibuilder/Dockerfile.centos

@@ -0,0 +1,16 @@
+FROM manibase
+
+RUN \
+	useradd -p locked -m mani && \
+	yum install -q -y git gcc pciutils-devel libusb-devel libusbx-devel && \
+	yum clean -q -y all
+
+USER mani
+RUN \
+	cd && \
+	mkdir .ccache && chown mani:mani .ccache && \
+	git clone https://review.coreboot.org/flashrom.git
+
+ENV DEVSHELL /bin/bash
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

util/manibuilder/Dockerfile.debian-debootstrap

@@ -0,0 +1,20 @@
+FROM manibase
+
+RUN \
+	useradd -p locked -m mani && \
+	apt-get -qq update && \
+	apt-get -qq upgrade && \
+	apt-get -qq dist-upgrade && \
+	apt-get -qqy install gcc make git doxygen ccache \
+		libpci-dev libusb-dev libftdi-dev libusb-1.0-0-dev && \
+	apt-get clean
+
+USER mani
+RUN \
+	cd && \
+	mkdir .ccache && chown mani:mani .ccache && \
+	git clone https://review.coreboot.org/flashrom.git
+
+ENV DEVSHELL /bin/bash
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

util/manibuilder/Dockerfile.djgpp

@@ -0,0 +1,29 @@
+FROM anibali/djgpp:6.1.0
+
+USER root
+RUN \
+	userdel appuser && \
+	useradd -p locked -m mani && \
+	zypper -q install -y tar make git ccache
+
+USER mani
+RUN cd && \
+	mkdir .ccache && chown mani:users .ccache && \
+	git clone https://review.coreboot.org/flashrom.git && \
+	git clone https://git.kernel.org/pub/scm/utils/pciutils/pciutils.git && \
+	cd pciutils && \
+		git checkout v3.5.6 && \
+		curl https://flashrom.org/images/6/6a/Pciutils-3.5.6.patch.gz | zcat | git apply && \
+		make ZLIB=no DNS=no HOST=i386-djgpp-djgpp \
+			CROSS_COMPILE=i586-pc-msdosdjgpp- \
+			PREFIX=/ DESTDIR=$PWD/../ \
+			STRIP="--strip-program=i586-pc-msdosdjgpp-strip -s" \
+			install install-lib && \
+		cd ../ && \
+	curl https://flashrom.org/images/3/3d/Libgetopt.tar.gz | zcat | tar x && \
+	cd libgetopt && \
+		make && cp libgetopt.a ../lib/ && cp getopt.h ../include/
+
+ENV DEVSHELL /bin/bash
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

util/manibuilder/Dockerfile.fedora

@@ -0,0 +1,17 @@
+FROM manibase
+
+RUN \
+	useradd -p locked -m mani && \
+	dnf install -q -y git gcc ccache make \
+		pciutils-devel libusb-devel libusbx-devel libftdi-devel && \
+	dnf clean -q -y all
+
+USER mani
+RUN \
+	cd && \
+	mkdir .ccache && chown mani:mani .ccache && \
+	git clone https://review.coreboot.org/flashrom.git
+
+ENV DEVSHELL /bin/bash
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

util/manibuilder/Dockerfile.qemu-user-static

@@ -0,0 +1,3 @@
+FROM multiarch/qemu-user-static:register
+
+RUN sed -i -e's/ mipsn32 mipsn32el / /' /qemu-binfmt-conf.sh

util/manibuilder/Dockerfile.ubuntu-debootstrap

@@ -0,0 +1,33 @@
+FROM manibase
+
+RUN \
+	useradd -p locked -m mani && \
+	if grep -q main /etc/apt/sources.list; then \
+		if ! grep -q universe /etc/apt/sources.list; then \
+			sed -i -e 's/ main$/ main universe/' \
+				/etc/apt/sources.list || exit 1; \
+		fi; \
+	else \
+		url="http://ports.ubuntu.com/" && \
+		cn="$(sed -ne's/DISTRIB_CODENAME=//p' /etc/lsb-release)" && \
+		for t in "" "-updates" "-security"; do \
+			echo "deb ${url} ${cn}${t} main universe" \
+				>>/etc/apt/sources.list || exit 1; \
+		done; \
+	fi && \
+	apt-get -qq update && \
+	apt-get -qq upgrade && \
+	apt-get -qq dist-upgrade && \
+	apt-get -qqy install gcc make git doxygen ccache \
+		libpci-dev libusb-dev libftdi-dev libusb-1.0-0-dev && \
+	apt-get clean
+
+USER mani
+RUN \
+	cd && \
+	mkdir .ccache && chown mani:mani .ccache && \
+	git clone https://review.coreboot.org/flashrom.git
+
+ENV DEVSHELL /bin/bash
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]

util/manibuilder/Makefile

@@ -0,0 +1,97 @@
+QUIET_TEST := @
+
+include Makefile.targets
+
+CC := ccache cc
+MAKECMD := make
+
+spc :=
+spc := $(spc) $(spc)
+
+stem = $(word 1,$(subst :,$(spc),$(subst \:,$(spc),$(1))))
+ident = $(subst :,_,$(subst \:,_,$(1)))
+
+PKGSRC_MIRROR = http://cdn.netbsd.org/
+NETBSD_MIRROR = http://ftp.de.netbsd.org/
+
+anita\:amd64-build: PKGSRC_PATH=pub/pkgsrc/packages/NetBSD/amd64/7.1/All
+anita\:amd64-build: NETBSD_IMAGE=pub/NetBSD/NetBSD-7.1/amd64/
+
+anita\:i386-build: PKGSRC_PATH=pub/pkgsrc/packages/NetBSD/i386/7.1/All
+anita\:i386-build: NETBSD_IMAGE=pub/NetBSD/NetBSD-7.1/i386/
+
+$(addsuffix -build,$(ANITA_TAGS)): %-build: Dockerfile.anita anita-wrapper.sh
+	$(QUIET_SETUP)docker build . -f $< -t mani/$* \
+		--build-arg PKG_PATH=$(PKGSRC_MIRROR)$(PKGSRC_PATH) \
+		--build-arg INST_IMG=$(NETBSD_MIRROR)$(NETBSD_IMAGE)
+
+define build_template
+Dockerfile.$(call ident,$(1)): Dockerfile.$(call stem,$(1)) mani-wrapper.sh
+	$(QUIET_SETUP)sed -e 's|^FROM manibase|FROM $(2)/$(1)|' $$< >$$@
+
+.INTERMEDIATE: Dockerfile.$(call ident,$(1))
+
+$(1)-build: Dockerfile.$(call ident,$(1))
+	$(QUIET_SETUP)docker build . -f $$< -t mani/$(1)
+endef
+
+$(foreach tag,$(MULTIARCH_TAGS), \
+	$(eval $(call build_template,$(tag),multiarch)))
+
+djgpp\:6.1.0-build: %-build: Dockerfile.djgpp mani-wrapper.sh
+	$(QUIET_SETUP)docker build . -f $< -t mani/$*
+
+$(addsuffix -check-build,$(ALL_TAGS)): %-check-build:
+	$(QUIET_SETUP)\
+	[ $$(docker image ls -q mani/$*) ] \
+		|| $(MAKE) $*-build $(if $(QUIET_SETUP),>/dev/null 2>/dev/null)
+
+$(filter centos%,$(MULTIARCH_TAGS)) anita\:sparc: CC=cc
+djgpp\:6.1.0: CC=ccache i586-pc-msdosdjgpp-gcc
+djgpp\:6.1.0: STRIP=i586-pc-msdosdjgpp-strip
+djgpp\:6.1.0: LIBS_BASE=../
+djgpp\:6.1.0: TARGET=strip
+$(ANITA_TAGS): MAKECMD=gmake
+$(ALL_TAGS): export QUIET_SETUP=$(QUIET_TEST)
+$(ALL_TAGS): %: %-check-build
+	$(QUIET_TEST)docker rm -f mani_$(call ident,$*) >/dev/null 2>&1 || true
+	$(QUIET_TEST)\
+	docker run \
+		--env IDENT=$(call ident,$*) \
+		--volume manicache:/home/mani/.ccache \
+		--name mani_$(call ident,$*) mani/$* \
+		"git fetch origin $${TEST_REVISION:-master} && \
+		 git checkout FETCH_HEAD && \
+		 $(MAKECMD) clean && $(MAKECMD) -j$${CPUS:-1} CC='$(CC)' \
+		 $(if $(STRIP),STRIP='$(STRIP)') \
+		 $(if $(LIBS_BASE),LIBS_BASE='$(LIBS_BASE)') \
+		 $(TARGET)" \
+		$(if $(QUIET_TEST),>/dev/null 2>&1) || echo $*: $$?
+
+$(addsuffix -shell,$(ALL_TAGS)): %-shell: %-check-build
+	$(QUIET_SETUP)\
+	if [ $$(docker ps -a -q -f name=mani_$(call ident,$*)) ]; then \
+		docker commit mani_$(call ident,$*) mani_run/$* && \
+		docker run --rm -it \
+			--env IDENT=$(call ident,$*) \
+			--volume manicache:/home/mani/.ccache \
+			--entrypoint /bin/sh mani_run/$* \
+			/home/mani/mani-wrapper.sh \
+			$(patsubst %,"%",$(SHELL_ARG)); \
+		docker image rm mani_run/$*; \
+	else \
+		docker run --rm -it \
+			--env IDENT=$(call ident,$*) \
+			--volume manicache:/home/mani/.ccache \
+			mani/$* $(patsubst %,"%",$(SHELL_ARG)); \
+	fi
+
+.PHONY: $(foreach s,-build -check-build -shell, $(addsuffix $(s),$(ALL_TAGS)))
+
+register:
+	docker build . \
+		-f Dockerfile.qemu-user-static \
+		-t mani/qemu-user-static:register
+	docker run --rm --privileged mani/qemu-user-static:register --reset
+
+.PHONY: register

util/manibuilder/Makefile.targets

@@ -0,0 +1,85 @@
+ANITA_TAGS := anita\:amd64 anita\:i386
+
+MULTIARCH_TAGS := \
+	centos\:7.3-aarch64-clean centos\:7.3-amd64-clean \
+	centos\:7.2-amd64-clean \
+	$(foreach a,x86_64 ppc64le aarch64, \
+	  $(foreach v,25 24, \
+	    fedora\:$(v)-$(a))) \
+	$(foreach a,ppc64el arm64 armhf mips mipsel amd64 i386, \
+	  $(foreach v,sid buster stretch, \
+	    debian-debootstrap\:$(a)-$(v))) \
+	$(foreach a,ppc64el arm64 armhf amd64 i386, \
+	  $(foreach v,bionic xenial, \
+	    ubuntu-debootstrap\:$(a)-$(v))) \
+	ubuntu-debootstrap\:powerpc-xenial \
+	$(foreach a,aarch64 armhf amd64 i386, \
+	  $(foreach v,v3.8 v3.7 v3.6, \
+	    alpine\:$(a)-$(v))) \
+
+OTHER_TAGS := djgpp\:6.1.0
+
+ALL_TAGS := $(ANITA_TAGS) $(MULTIARCH_TAGS) $(OTHER_TAGS)
+
+arch_filter = $(sort \
+	$(foreach arch,$(1), \
+	  $(filter-out $(subst $(arch),,$(MULTIARCH_TAGS)),$(MULTIARCH_TAGS))))
+
+machine_map = \
+	$(if $(filter i386 i686 x86,$(1)),i386 x86, \
+	$(if $(filter x86_64,$(1)),amd64 i386 x86, \
+	$(if $(filter armv7l armv6l,$(1)),armhf, \
+	$(if $(filter aarch64,$(1)),aarch64 arm64, \
+	$(if $(filter ppc64le,$(1)),ppc64le ppc64el, \
+	$(if $(filter ppc,$(1)),powerpc, \
+	$(if $(filter mips,$(1)),mips mipsel, \
+	$(1))))))))
+
+NATIVE_TAGS := $(call arch_filter,$(call machine_map,$(shell uname -m)))
+
+# rather arbitrary selection of images that seem to work (focus on amd64)
+DEFAULT_TAGS := \
+	anita\:amd64 \
+	djgpp\:6.1.0 \
+	fedora\:25-x86_64 \
+	fedora\:25-ppc64le \
+	fedora\:25-aarch64 \
+	fedora\:24-x86_64 \
+	centos\:7.3-aarch64-clean \
+	centos\:7.3-amd64-clean \
+	centos\:7.2-amd64-clean \
+	debian-debootstrap\:ppc64el-stretch \
+	debian-debootstrap\:armhf-stretch \
+	debian-debootstrap\:mips-stretch \
+	debian-debootstrap\:mipsel-stretch \
+	debian-debootstrap\:amd64-stretch \
+	debian-debootstrap\:i386-stretch \
+	debian-debootstrap\:amd64-sid \
+	ubuntu-debootstrap\:arm64-xenial \
+	ubuntu-debootstrap\:amd64-xenial \
+	ubuntu-debootstrap\:powerpc-xenial \
+	ubuntu-debootstrap\:amd64-bionic \
+	alpine\:amd64-v3.7 \
+	alpine\:amd64-v3.8 \
+
+# also run all native tests by default
+DEFAULT_TAGS += $(filter-out $(DEFAULT_TAGS),$(NATIVE_TAGS))
+
+default: $(DEFAULT_TAGS)
+
+native: $(NATIVE_TAGS)
+
+all: $(ALL_TAGS)
+
+show-default:
+	@printf "%s\n" $(DEFAULT_TAGS)
+
+show-native:
+	@printf "%s\n" $(NATIVE_TAGS)
+
+show-all:
+	@printf "%s\n" $(ALL_TAGS)
+
+.PHONY: default native all
+.PHONY: show-default show-native show-all
+.PHONY: $(ALL_TAGS)

util/manibuilder/README.md

@@ -0,0 +1,72 @@
+Manibuilder
+===========
+
+Manibuilder is a set of Dockerfiles for manic build testing, hold
+together by some make-foo. Most of the Dockerfiles make use of
+*multiarch* images. This way we can test building on many platforms
+supported by *Qemu*. The idea is to test in environments as close
+as possible to those of potential users, i.e. no cross-compiling
+(with some exceptions).
+
+Make targets
+------------
+
+For each supported target OS/version/architecture exists a *tag*
+target, for instance `alpine:amd64-v3.7`. These targets will
+automatically check for existence of their respective *Docker*
+images (sub target <tag>-check-build), and build them if necessary
+(<tag>-build). Finally, flashrom revision `$(TEST_REVISION)` is
+fetched and build tested.
+
+The results will be kept by *Docker* as stopped containers and
+can be accessed with the <tag>-shell target.
+
+There are some additional targets that form sets of the *tag*
+targets:
+
+* default:  runs a preselected subset of all supported tags.
+* native:   runs all tags native to the host architecture.
+* all:      runs all supported tags.
+
+For each of these show-<set> lists the included *tags*.
+
+For preparation of *Qemu* for the *multiarch* images, there is the
+`register` target. It has to be run once per boot, though as it
+uses a privileged *Docker* container, that is kept as a manual step.
+
+Usage example
+-------------
+
+The most common use case may be testing the current upstream
+*master* branch which is the default for `$(TEST_REVISION)`.
+You'll need roughly 20GiB for the *Docker* images. Might look
+like this:
+
+    $ # have to register Qemu first:
+    $ make register
+    [...]
+    $ # run the default target:
+    $ make -j4
+    debian-debootstrap:mips-stretch: 2
+    debian-debootstrap:mips-sid: 2
+    debian-debootstrap:mips-buster: 2
+    ubuntu-debootstrap:powerpc-xenial: 2
+    djgpp:6.1.0: 2
+
+For each *tag* that returns with a non-zero exit code, the *tag*
+and actual exit code is printed. An exit code of `2` is most likely
+as that is what *make* returns on failure. Other exit codes might
+hint towards a problem in the setup. Failing *tags* can then be
+investigated individually with the <tag>-shell target, e.g.:
+
+    $ make debian-debootstrap:mips-sid-shell
+    [...]
+    mani@63536fc102a5:~/flashrom$ make
+    [...]
+    cc -MMD -Os -Wall -Wshadow -Werror -I/usr/include/libusb-1.0  -D'CONFIG_DEFAULT_PROGRAMMER=PROGRAMMER_INVALID' -D'CONFIG_DEFAULT_PROGRAMMER_ARGS="''"' -D'CONFIG_SERPROG=1' -D'CONFIG_PONY_SPI=1' -D'CONFIG_BITBANG_SPI=1' -D'CONFIG_GFXNVIDIA=1' -D'CONFIG_SATASII=1' -D'CONFIG_ATAVIA=1' -D'CONFIG_IT8212=1' -D'CONFIG_FT2232_SPI=1' -D'CONFIG_USBBLASTER_SPI=1' -D'CONFIG_PICKIT2_SPI=1' -D'HAVE_FT232H=1'  -D'CONFIG_DUMMY=1' -D'CONFIG_DRKAISER=1' -D'CONFIG_NICINTEL=1' -D'CONFIG_NICINTEL_SPI=1' -D'CONFIG_NICINTEL_EEPROM=1' -D'CONFIG_OGP_SPI=1' -D'CONFIG_BUSPIRATE_SPI=1' -D'CONFIG_DEDIPROG=1' -D'CONFIG_DEVELOPERBOX_SPI=1' -D'CONFIG_LINUX_MTD=1' -D'CONFIG_LINUX_SPI=1' -D'CONFIG_CH341A_SPI=1' -D'CONFIG_DIGILENT_SPI=1' -D'NEED_PCI=1'  -D'NEED_RAW_ACCESS=1' -D'NEED_LIBUSB0=1' -D'NEED_LIBUSB1=1' -D'HAVE_UTSNAME=1' -D'HAVE_CLOCK_GETTIME=1' -D'FLASHROM_VERSION="p1.0-141-g9cecc7e"' -o libflashrom.o -c libflashrom.c
+    libflashrom.c:386:12: error: 'flashrom_layout_parse_fmap' defined but not used [-Werror=unused-function]
+     static int flashrom_layout_parse_fmap(struct flashrom_layout **layout,
+            ^~~~~~~~~~~~~~~~~~~~~~~~~~
+    cc1: all warnings being treated as errors
+    make: *** [Makefile:1075: libflashrom.o] Error 1
+    $ # uh-huh, might be a problem with big-endian #if foo

util/manibuilder/anita-wrapper.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+cd
+
+[ "${IDENT}" ] || IDENT=$(mktemp -u XXXXXXXX)
+
+CCACHE=.ccache/anita-${IDENT}.img
+
+[ -f ${CCACHE} ] || zcat cache.img.gz >${CCACHE}
+
+if [ $# -eq 0 ]; then
+	exec anita --vmm-args "-hdb ${CCACHE}" interact ${INST_IMG}
+else
+	exec anita --vmm-args "-hdb ${CCACHE}" --persist \
+		--run ". ./init && manitest \"$*\"" \
+		boot ${INST_IMG}
+fi

util/manibuilder/mani-wrapper.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+cd /home/mani/flashrom/
+
+if [ $# -eq 0 ]; then
+	exec "${DEVSHELL}"
+else
+	exec "${DEVSHELL}" -c "$*"
+fi