Add docker configurations used by the buildbots.

These are the scripts I use to create the docker images for
the build bots and run them.

llvm-svn: 347244
This commit is contained in:
Eric Fiselier 2018-11-19 18:43:31 +00:00
parent c01fc1c696
commit 9c97ca1b99
7 changed files with 676 additions and 0 deletions

View File

@ -0,0 +1,109 @@
#!/bin/bash
#===- libcxx/utils/docker/build_docker_image.sh ----------------------------===//
#
# The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
#===----------------------------------------------------------------------===//
set -e
IMAGE_SOURCE=""
DOCKER_REPOSITORY=""
DOCKER_TAG=""
function show_usage() {
cat << EOF
Usage: build_docker_image.sh [options] [-- [cmake_args]...]
Available options:
General:
-h|--help show this help message
Docker-specific:
-s|--source image source dir (i.e. debian8, nvidia-cuda, etc)
-d|--docker-repository docker repository for the image
-t|--docker-tag docker tag for the image
Required options: --source and --docker-repository.
For example, running:
$ build_docker_image.sh -s debian9 -d mydocker/debian9-clang -t latest
will produce two docker images:
mydocker/debian9-clang-build:latest - an intermediate image used to compile
clang.
mydocker/clang-debian9:latest - a small image with preinstalled clang.
Please note that this example produces a not very useful installation, since it
doesn't override CMake defaults, which produces a Debug and non-boostrapped
version of clang.
EOF
}
while [[ $# -gt 0 ]]; do
case "$1" in
-h|--help)
show_usage
exit 0
;;
-s|--source)
shift
IMAGE_SOURCE="$1"
shift
;;
-d|--docker-repository)
shift
DOCKER_REPOSITORY="$1"
shift
;;
-t|--docker-tag)
shift
DOCKER_TAG="$1"
shift
;;
*)
echo "Unknown argument $1"
exit 1
;;
esac
done
command -v docker >/dev/null ||
{
echo "Docker binary cannot be found. Please install Docker to use this script."
exit 1
}
if [ "$IMAGE_SOURCE" == "" ]; then
echo "Required argument missing: --source"
exit 1
fi
if [ "$DOCKER_REPOSITORY" == "" ]; then
echo "Required argument missing: --docker-repository"
exit 1
fi
SOURCE_DIR=$(dirname $0)
if [ ! -d "$SOURCE_DIR/$IMAGE_SOURCE" ]; then
echo "No sources for '$IMAGE_SOURCE' were found in $SOURCE_DIR"
exit 1
fi
BUILD_DIR=$(mktemp -d)
trap "rm -rf $BUILD_DIR" EXIT
echo "Using a temporary directory for the build: $BUILD_DIR"
cp -r "$SOURCE_DIR/$IMAGE_SOURCE" "$BUILD_DIR/$IMAGE_SOURCE"
cp -r "$SOURCE_DIR/scripts" "$BUILD_DIR/scripts"
if [ "$DOCKER_TAG" != "" ]; then
DOCKER_TAG=":$DOCKER_TAG"
fi
echo "Building ${DOCKER_REPOSITORY}${DOCKER_TAG} from $IMAGE_SOURCE"
docker build -t "${DOCKER_REPOSITORY}${DOCKER_TAG}" \
-f "$BUILD_DIR/$IMAGE_SOURCE/Dockerfile" \
"$BUILD_DIR"
echo "Done"

View File

@ -0,0 +1,113 @@
#===- libcxx/utils/docker/debian9/Dockerfile -------------------------===//
#
# The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
#===----------------------------------------------------------------------===//
# Setup the base builder image with the packages we'll need to build GCC and Clang from source.
FROM launcher.gcr.io/google/debian9:latest as builder-base
LABEL maintainer "libc++ Developers"
RUN apt-get update && \
apt-get install -y --no-install-recommends \
ca-certificates \
gnupg \
build-essential \
wget \
subversion \
unzip \
automake \
python \
cmake \
ninja-build \
curl \
git \
gcc-multilib \
g++-multilib \
libc6-dev \
bison \
flex \
libtool \
autoconf \
binutils-dev \
binutils-gold \
software-properties-common && \
update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.gold" 20 && \
update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.bfd" 10
# Build GCC 4.9 for testing our C++11 against
FROM builder-base as gcc-49-builder
LABEL maintainer "libc++ Developers"
ADD scripts/build_gcc.sh /tmp/build_gcc.sh
RUN git clone --depth=1 --branch gcc-4_9_4-release git://gcc.gnu.org/git/gcc.git /tmp/gcc-4.9.4
RUN cd /tmp/gcc-4.9.4/ && ./contrib/download_prerequisites
RUN /tmp/build_gcc.sh --source /tmp/gcc-4.9.4 --to /opt/gcc-4.9.4
# Build GCC ToT for testing in all dialects.
FROM builder-base as gcc-tot-builder
LABEL maintainer "libc++ Developers"
ADD scripts/build_gcc.sh /tmp/build_gcc.sh
RUN git clone --depth=1 git://gcc.gnu.org/git/gcc.git /tmp/gcc-tot
RUN cd /tmp/gcc-tot && ./contrib/download_prerequisites
RUN /tmp/build_gcc.sh --source /tmp/gcc-tot --to /opt/gcc-tot
# Build LLVM 4.0 which is used to test against a "legacy" compiler.
FROM builder-base as llvm-4-builder
LABEL maintainer "libc++ Developers"
ADD scripts/checkout_git.sh /tmp/checkout_git.sh
ADD scripts/build_install_llvm.sh /tmp/build_install_llvm.sh
RUN /tmp/checkout_git.sh --to /tmp/llvm-4.0 -p clang -p compiler-rt --branch release_40
RUN /tmp/build_install_llvm.sh \
--install /opt/llvm-4.0 \
--source /tmp/llvm-4.0 \
--build /tmp/build-llvm-4.0 \
-i install-clang -i install-clang-headers \
-i install-compiler-rt \
-- \
-DCMAKE_BUILD_TYPE=RELEASE \
-DLLVM_ENABLE_ASSERTIONS=ON
# Stage 2. Produce a minimal release image with build results.
FROM launcher.gcr.io/google/debian9:latest
LABEL maintainer "libc++ Developers"
# Copy over the GCC and Clang installations
COPY --from=gcc-49-builder /opt/gcc-4.9.4 /opt/gcc-4.9.4
COPY --from=gcc-tot-builder /opt/gcc-tot /opt/gcc-tot
COPY --from=llvm-4-builder /opt/llvm-4.0 /opt/llvm-4.0
RUN ln -s /opt/gcc-4.9.4/bin/gcc /usr/local/bin/gcc-4.9 && \
ln -s /opt/gcc-4.9.4/bin/g++ /usr/local/bin/g++-4.9
RUN apt-get update && \
apt-get install -y \
ca-certificates \
gnupg \
build-essential \
apt-transport-https \
curl \
software-properties-common
RUN apt-get install -y --no-install-recommends \
systemd \
sysvinit-utils \
cmake \
subversion \
git \
ninja-build \
gcc-multilib \
g++-multilib \
python \
buildbot-slave
ADD scripts /libcxx-scripts/
RUN /libcxx-scripts/install_clang_packages.sh

View File

@ -0,0 +1,91 @@
#!/usr/bin/env bash
#===- libcxx/utils/docker/scripts/build-gcc.sh ----------------------------===//
#
# The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
#===-----------------------------------------------------------------------===//
set -e
function show_usage() {
cat << EOF
Usage: build-gcc.sh [options]
Run autoconf with the specified arguments. Used inside docker container.
Available options:
-h|--help show this help message
--source the source path from which to run the configuration.
--to destination directory where to install the targets.
Required options: --to, at least one --install-target.
All options after '--' are passed to CMake invocation.
EOF
}
GCC_INSTALL_DIR=""
GCC_SOURCE_DIR=""
while [[ $# -gt 0 ]]; do
case "$1" in
--to)
shift
GCC_INSTALL_DIR="$1"
shift
;;
--source)
shift
GCC_SOURCE_DIR="$1"
shift
;;
-h|--help)
show_usage
exit 0
;;
*)
echo "Unknown option: $1"
exit 1
esac
done
if [ "$GCC_INSTALL_DIR" == "" ]; then
echo "No install directory. Please specify the --to argument."
exit 1
fi
if [ "$GCC_SOURCE_DIR" == "" ]; then
echo "No source directory. Please specify the --source argument."
exit 1
fi
GCC_NAME=`basename $GCC_SOURCE_DIR`
GCC_BUILD_DIR="/tmp/gcc-build-root/build-$GCC_NAME"
mkdir -p "$GCC_INSTALL_DIR"
mkdir -p "$GCC_BUILD_DIR"
pushd "$GCC_BUILD_DIR"
# Run the build as specified in the build arguments.
echo "Running configuration"
$GCC_SOURCE_DIR/configure --prefix=$GCC_INSTALL_DIR \
--disable-bootstrap --disable-libgomp --disable-libitm \
--disable-libvtv --disable-libcilkrts --disable-libmpx \
--disable-liboffloadmic --disable-libcc1 --enable-languages=c,c++
NPROC=`nproc`
echo "Running build with $NPROC threads"
make -j$NPROC
echo "Installing to $GCC_INSTALL_DIR"
make install -j$NPROC
popd
# Cleanup.
rm -rf "$GCC_BUILD_DIR"
echo "Done"

View File

@ -0,0 +1,114 @@
#!/usr/bin/env bash
#===- llvm/utils/docker/scripts/build_install_llvm.sh ---------------------===//
#
# The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
#===-----------------------------------------------------------------------===//
set -e
function show_usage() {
cat << EOF
Usage: build_install_llvm.sh [options] -- [cmake-args]
Run cmake with the specified arguments. Used inside docker container.
Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into
the directory specified by --to option.
Available options:
-h|--help show this help message
-i|--install-target name of a cmake install target to build and include in
the resulting archive. Can be specified multiple times.
--install destination directory where to install the targets.
--source location of the source tree.
--build location to use as the build directory.
Required options: --to, --source, --build, and at least one --install-target.
All options after '--' are passed to CMake invocation.
EOF
}
CMAKE_ARGS=""
CMAKE_INSTALL_TARGETS=""
CLANG_INSTALL_DIR=""
CLANG_SOURCE_DIR=""
CLANG_BUILD_DIR=""
while [[ $# -gt 0 ]]; do
case "$1" in
-i|--install-target)
shift
CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1"
shift
;;
--source)
shift
CLANG_SOURCE_DIR="$1"
shift
;;
--build)
shift
CLANG_BUILD_DIR="$1"
shift
;;
--install)
shift
CLANG_INSTALL_DIR="$1"
shift
;;
--)
shift
CMAKE_ARGS="$*"
shift $#
;;
-h|--help)
show_usage
exit 0
;;
*)
echo "Unknown option: $1"
exit 1
esac
done
if [ "$CLANG_SOURCE_DIR" == "" ]; then
echo "No source directory. Please pass --source."
exit 1
fi
if [ "$CLANG_BUILD_DIR" == "" ]; then
echo "No build directory. Please pass --build"
exit 1
fi
if [ "$CMAKE_INSTALL_TARGETS" == "" ]; then
echo "No install targets. Please pass one or more --install-target."
exit 1
fi
if [ "$CLANG_INSTALL_DIR" == "" ]; then
echo "No install directory. Please specify the --to argument."
exit 1
fi
echo "Building in $CLANG_BUILD_DIR"
mkdir -p "$CLANG_BUILD_DIR"
pushd "$CLANG_BUILD_DIR"
# Run the build as specified in the build arguments.
echo "Running build"
cmake -GNinja \
-DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \
$CMAKE_ARGS \
"$CLANG_SOURCE_DIR"
ninja $CMAKE_INSTALL_TARGETS
popd
# Cleanup.
rm -rf "$CLANG_BUILD_DIR"
echo "Done"

View File

@ -0,0 +1,130 @@
#!/usr/bin/env bash
#===- llvm/utils/docker/scripts/checkout.sh ---------------------===//
#
# The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
#===-----------------------------------------------------------------------===//
set -e
function show_usage() {
cat << EOF
Usage: checkout.sh [options]
Checkout svn sources into /tmp/clang-build/src. Used inside a docker container.
Available options:
-h|--help show this help message
-b|--branch svn branch to checkout, i.e. 'trunk',
'branches/release_40'
(default: 'trunk')
-p|--llvm-project name of an svn project to checkout.
For clang, please use 'clang', not 'cfe'.
Project 'llvm' is always included and ignored, if
specified.
Can be specified multiple times.
EOF
}
LLVM_BRANCH=""
# We always checkout llvm
LLVM_PROJECTS="llvm"
SOURCE_DIR=""
function contains_project() {
local TARGET_PROJ="$1"
local PROJ
for PROJ in $LLVM_PROJECTS; do
if [ "$PROJ" == "$TARGET_PROJ" ]; then
return 0
fi
done
return 1
}
while [[ $# -gt 0 ]]; do
case "$1" in
--to)
shift
SOURCE_DIR="$1"
shift
;;
-b|--branch)
shift
LLVM_BRANCH="$1"
shift
;;
-p|--llvm-project)
shift
PROJ="$1"
shift
if [ "$PROJ" == "cfe" ]; then
PROJ="clang"
fi
if ! contains_project "$PROJ" ; then
if [ "$PROJ" == "clang-tools-extra" ] && [ ! contains_project "clang" ]; then
echo "Project 'clang-tools-extra' specified before 'clang'. Adding 'clang' to a list of projects first."
LLVM_PROJECTS="$LLVM_PROJECTS clang"
fi
LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
else
echo "Project '$PROJ' is already enabled, ignoring extra occurrences."
fi
;;
-h|--help)
show_usage
exit 0
;;
*)
echo "Unknown option: $1"
exit 1
esac
done
if [ "$SOURCE_DIR" == "" ]; then
echo "Must specify checkout directory using --to"
exit 1
fi
if [ "$LLVM_BRANCH" == "" ]; then
GIT_BRANCH_ARG=""
else
GIT_BRANCH_ARG="--branch $LLVM_BRANCH"
fi
if [ "$LLVM_SVN_REV" != "" ]; then
SVN_REV_ARG="-r$LLVM_SVN_REV"
echo "Checking out svn revision r$LLVM_SVN_REV."
else
SVN_REV_ARG=""
echo "Checking out latest svn revision."
fi
# Get the sources from svn.
echo "Checking out sources from git"
for LLVM_PROJECT in $LLVM_PROJECTS; do
if [ "$LLVM_PROJECT" == "llvm" ]; then
CHECKOUT_DIR="$SOURCE_DIR"
elif [ "$LLVM_PROJECT" == "libcxx" ] || [ "$LLVM_PROJECT" == "libcxxabi" ] || [ "$LLVM_PROJECT" == "compiler-rt" ]; then
CHECKOUT_DIR="$SOURCE_DIR/projects/$LLVM_PROJECT"
elif [ "$LLVM_PROJECT" == "clang" ]; then
CHECKOUT_DIR="$SOURCE_DIR/tools/clang"
elif [ "$LLVM_PROJECT" == "clang-tools-extra" ]; then
CHECKOUT_DIR="$SOURCE_DIR/tools/clang/tools/extra"
else
CHECKOUT_DIR="$SOURCE_DIR/$LLVM_PROJECT"
fi
echo "Checking out https://git.llvm.org/git/$LLVM_PROJECT to $CHECKOUT_DIR"
git clone --depth=1 $GIT_BRANCH_ARG \
"https://git.llvm.org/git/$LLVM_PROJECT.git" \
"$CHECKOUT_DIR"
done
echo "Done"

View File

@ -0,0 +1,64 @@
#!/usr/bin/env bash
#===- libcxx/utils/docker/scripts/install_clang_package.sh -----------------===//
#
# The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
#===-----------------------------------------------------------------------===//
set -e
function show_usage() {
cat << EOF
Usage: install_clang_package.sh [options]
Install
Available options:
-h|--help show this help message
--version the numeric version of the package to use.
EOF
}
VERSION=""
while [[ $# -gt 0 ]]; do
case "$1" in
--version)
shift
VERSION="$1"
shift
;;
-h|--help)
show_usage
exit 0
;;
*)
echo "Unknown option: $1"
exit 1
esac
done
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
add-apt-repository -s "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main"
apt-get update
apt-get install -y --no-install-recommends clang
echo "Testing clang version..."
clang --version
echo "Testing clang++ version..."
clang++ --version
# Figure out the libc++ and libc++abi package versions that we want.
if [ "$VERSION" == "" ]; then
VERSION="$(apt-cache search 'libc\+\+-[0-9]-dev' | awk '{print $1}' | awk -F- '{print $2}')"
echo "Installing version '$VERSION'"
fi
apt-get install -y --no-install-recommends "libc++-$VERSION-dev" "libc++abi-$VERSION-dev"
echo "Done"

View File

@ -0,0 +1,55 @@
#!/usr/bin/env bash
set -x
BOT_DIR=/b
BOT_NAME=$1
BOT_PASS=$2
mkdir -p $BOT_DIR
#curl "https://repo.stackdriver.com/stack-install.sh" | bash -s -- --write-gcm
apt-get update -y
apt-get upgrade -y
systemctl set-property buildslave.service TasksMax=100000
buildslave stop $BOT_DIR
chown buildbot:buildbot $BOT_DIR
echo "Connecting as $BOT_NAME"
buildslave create-slave --allow-shutdown=signal $BOT_DIR lab.llvm.org:9990 $BOT_NAME $BOT_PASS
echo "Eric Fiselier <ericwf@google.com>" > $BOT_DIR/info/admin
{
uname -a | head -n1
cmake --version | head -n1
g++ --version | head -n1
ld --version | head -n1
date
lscpu
} > $BOT_DIR/info/host
echo "SLAVE_RUNNER=/usr/bin/buildslave
SLAVE_ENABLED[1]=\"1\"
SLAVE_NAME[1]=\"buildslave1\"
SLAVE_USER[1]=\"buildbot\"
SLAVE_BASEDIR[1]=\"$BOT_DIR\"
SLAVE_OPTIONS[1]=\"\"
SLAVE_PREFIXCMD[1]=\"\"" > /etc/default/buildslave
chown -R buildbot:buildbot $BOT_DIR
systemctl daemon-reload
service buildslave restart
sleep 30
cat $BOT_DIR/twistd.log
grep "slave is ready" $BOT_DIR/twistd.log || shutdown now
# GCE can restart instance after 24h in the middle of the build.
# Gracefully restart before that happen.
sleep 72000
while pkill -SIGHUP buildslave; do sleep 5; done;
shutdown now