#!/bin/bash # # Verify that an assortment of known good reference pools can be imported # using different versions of the ZoL code. # # By default references pools for the major ZFS implementation will be # checked against the most recent ZoL tags and the master development branch. # Alternate tags or branches may be verified with the '-s option. # Passing the keyword "installed" will instruct the script to test whatever # version is installed. # # Preferentially a reference pool is used for all tests. However, if one # does not exist and the pool-tag matches one of the src-tags then a new # reference pool will be created using binaries from that source build. # This is particularly useful when you need to test your changes before # opening a pull request. The keyword 'all' can be used as short hand # refer to all available reference pools. # # New reference pools may be added by placing a bzip2 compressed tarball # of the pool in the scripts/zfs-images directory and then passing # the -p option. To increase the test coverage reference pools # should be collected for all the major ZFS implementations. Having these # pools easily available is also helpful to the developers. # # Care should be taken to run these tests with a kernel supported by all # the listed tags. Otherwise build failure will cause false positives. # # # EXAMPLES: # # The following example will verify the zfs-0.6.2 tag, the master branch, # and the installed zfs version can correctly import the listed pools. # Note there is no reference pool available for master and installed but # because binaries are available one is automatically constructed. The # working directory is also preserved between runs (-k) preventing the # need to rebuild from source for multiple runs. # # zimport.sh -k -f /var/tmp/zimport \ # -s "zfs-0.6.2 master installed" \ # -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed" # # --------------------- ZFS on Linux Source Versions -------------- # zfs-0.6.2 master 0.6.2-175_g36eb554 # ----------------------------------------------------------------- # Clone SPL Local Local Skip # Clone ZFS Local Local Skip # Build SPL Pass Pass Skip # Build ZFS Pass Pass Skip # ----------------------------------------------------------------- # zevo-1.1.1 Pass Pass Pass # zol-0.6.2 Pass Pass Pass # zol-0.6.2-173 Fail Pass Pass # master Pass Pass Pass # installed Pass Pass Pass # BASE_DIR=$(dirname "$0") SCRIPT_COMMON=common.sh if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then . "${BASE_DIR}/${SCRIPT_COMMON}" else echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 fi PROG=zimport.sh SRC_TAGS="zfs-0.6.5.11 master" POOL_TAGS="all master" POOL_CREATE_OPTIONS= TEST_DIR=$(mktemp -u -d -p /var/tmp zimport.XXXXXXXX) KEEP="no" VERBOSE="no" COLOR="yes" REPO="https://github.com/zfsonlinux" IMAGES_DIR="$SCRIPTDIR/zfs-images/" IMAGES_TAR="https://github.com/zfsonlinux/zfs-images/tarball/master" ERROR=0 CONFIG_LOG="configure.log" CONFIG_OPTIONS=${CONFIG_OPTIONS:-""} MAKE_LOG="make.log" MAKE_OPTIONS=${MAKE_OPTIONS:-"-s -j$(nproc)"} COLOR_GREEN="\033[0;32m" COLOR_RED="\033[0;31m" COLOR_BROWN="\033[0;33m" COLOR_RESET="\033[0m" usage() { cat << EOF USAGE: zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag] [-f path] [-o options] DESCRIPTION: ZPOOL import verification tests OPTIONS: -h Show this message -v Verbose -c No color -k Keep temporary directory -r Source repository ($REPO) -s ... Verify ZoL versions with the listed tags -i Pool image directory -p ... Verify pools created with the listed tags -f Temporary directory to use -o Additional options to pass to 'zpool create' EOF } while getopts 'hvckr:s:i:p:f:o:?' OPTION; do case $OPTION in h) usage exit 1 ;; v) VERBOSE="yes" ;; c) COLOR="no" ;; k) KEEP="yes" ;; r) REPO="$OPTARG" ;; s) SRC_TAGS="$OPTARG" ;; i) IMAGES_DIR="$OPTARG" ;; p) POOL_TAGS="$OPTARG" ;; f) TEST_DIR="$OPTARG" ;; o) POOL_CREATE_OPTIONS="$OPTARG" ;; ?) usage exit 1 ;; esac done # # Verify the module start is not loaded # if lsmod | grep zfs >/dev/null; then echo "ZFS modules must be unloaded" exit 1 fi # # Create a random directory tree of files and sub-directories to # to act as a copy source for the various regression tests. # populate() { local ROOT=$1 local MAX_DIR_SIZE=$2 local MAX_FILE_SIZE=$3 # shellcheck disable=SC2086 mkdir -p $ROOT/{a,b,c,d,e,f,g}/{h,i} DIRS=$(find "$ROOT") for DIR in $DIRS; do COUNT=$((RANDOM % MAX_DIR_SIZE)) # shellcheck disable=SC2034 for i in $(seq $COUNT); do FILE=$(mktemp -p "$DIR") SIZE=$((RANDOM % MAX_FILE_SIZE)) dd if=/dev/urandom of="$FILE" bs=1k \ count="$SIZE" &>/dev/null done done return 0 } SRC_DIR=$(mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX) trap 'rm -Rf "$SRC_DIR"' INT TERM EXIT populate "$SRC_DIR" 10 100 SRC_DIR="$TEST_DIR/src" SRC_DIR_SPL="$SRC_DIR/spl" SRC_DIR_ZFS="$SRC_DIR/zfs" if [ "$COLOR" = "no" ]; then COLOR_GREEN="" COLOR_BROWN="" COLOR_RED="" COLOR_RESET="" fi pass_nonewline() { echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t" } skip_nonewline() { echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t" } fail_nonewline() { echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t" } # # Log a failure message, cleanup, and return an error. # fail() { echo -e "$PROG: $1" >&2 $ZFS_SH -u >/dev/null 2>&1 exit 1 } # # Set several helper variables which are derived from a source tag. # # SPL_TAG - The tag zfs-x.y.z is translated to spl-x.y.z. # SPL_DIR - The spl directory name. # SPL_URL - The spl github URL to fetch the tarball. # ZFS_TAG - The passed zfs-x.y.z tag # ZFS_DIR - The zfs directory name # ZFS_URL - The zfs github URL to fetch the tarball # src_set_vars() { local TAG=$1 SPL_TAG="${TAG//zfs/spl}" SPL_DIR="$SRC_DIR_SPL/$SPL_TAG" SPL_URL="$REPO/spl/tarball/$SPL_TAG" ZFS_TAG="$TAG" ZFS_DIR="$SRC_DIR_ZFS/$ZFS_TAG" ZFS_URL="$REPO/zfs/tarball/$ZFS_TAG" if [ "$TAG" = "installed" ]; then ZPOOL_CMD=$(which zpool) ZFS_CMD=$(which zfs) ZFS_SH="/usr/share/zfs/zfs.sh" else ZPOOL_CMD="./cmd/zpool/zpool" ZFS_CMD="./cmd/zfs/zfs" ZFS_SH="./scripts/zfs.sh" fi } # # Set several helper variables which are derived from a pool name such # as zol-0.6.x, zevo-1.1.1, etc. These refer to example pools from various # ZFS implementations which are used to verify compatibility. # # POOL_TAG - The example pools name in scripts/zfs-images/. # POOL_BZIP - The full path to the example bzip2 compressed pool. # POOL_DIR - The top level test path for this pool. # POOL_DIR_PRISTINE - The directory containing a pristine version of the pool. # POOL_DIR_COPY - The directory containing a working copy of the pool. # POOL_DIR_SRC - Location of a source build if it exists for this pool. # pool_set_vars() { local TAG=$1 POOL_TAG=$TAG POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2 POOL_DIR=$TEST_DIR/pools/$POOL_TAG POOL_DIR_PRISTINE=$POOL_DIR/pristine POOL_DIR_COPY=$POOL_DIR/copy POOL_DIR_SRC="$SRC_DIR_ZFS/${POOL_TAG//zol/zfs}" } # # Construct a non-trivial pool given a specific version of the source. More # interesting pools provide better test coverage so this function should # extended as needed to create more realistic pools. # pool_create() { pool_set_vars "$1" src_set_vars "$1" if [ "$POOL_TAG" != "installed" ]; then cd "$POOL_DIR_SRC" || fail "Failed 'cd $POOL_DIR_SRC'" fi $ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" || \ fail "Failed to load kmods" # Create a file vdev RAIDZ pool. truncate -s 1G \ "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \ "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \ fail "Failed 'truncate -s 1G ...'" # shellcheck disable=SC2086 $ZPOOL_CMD create $POOL_CREATE_OPTIONS "$POOL_TAG" raidz \ "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \ "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \ fail "Failed '$ZPOOL_CMD create $POOL_CREATE_OPTIONS $POOL_TAG ...'" # Create a pool/fs filesystem with some random contents. $ZFS_CMD create "$POOL_TAG/fs" || \ fail "Failed '$ZFS_CMD create $POOL_TAG/fs'" populate "/$POOL_TAG/fs/" 10 100 # Snapshot that filesystem, clone it, remove the files/dirs, # replace them with new files/dirs. $ZFS_CMD snap "$POOL_TAG/fs@snap" || \ fail "Failed '$ZFS_CMD snap $POOL_TAG/fs@snap'" $ZFS_CMD clone "$POOL_TAG/fs@snap" "$POOL_TAG/clone" || \ fail "Failed '$ZFS_CMD clone $POOL_TAG/fs@snap $POOL_TAG/clone'" # shellcheck disable=SC2086 rm -Rf /$POOL_TAG/clone/* populate "/$POOL_TAG/clone/" 10 100 # Scrub the pool, delay slightly, then export it. It is now # somewhat interesting for testing purposes. $ZPOOL_CMD scrub "$POOL_TAG" || \ fail "Failed '$ZPOOL_CMD scrub $POOL_TAG'" sleep 10 $ZPOOL_CMD export "$POOL_TAG" || \ fail "Failed '$ZPOOL_CMD export $POOL_TAG'" $ZFS_SH -u || fail "Failed to unload kmods" } # If the zfs-images directory doesn't exist fetch a copy from Github then # cache it in the $TEST_DIR and update $IMAGES_DIR. if [ ! -d "$IMAGES_DIR" ]; then IMAGES_DIR="$TEST_DIR/zfs-images" mkdir -p "$IMAGES_DIR" curl -sL "$IMAGES_TAR" | \ tar -xz -C "$IMAGES_DIR" --strip-components=1 || \ fail "Failed to download pool images" fi # Given the available images in the zfs-images directory substitute the # list of available images for the reserved keyword 'all'. for TAG in $POOL_TAGS; do if [ "$TAG" = "all" ]; then # shellcheck disable=SC2010 ALL_TAGS=$(ls "$IMAGES_DIR" | grep "tar.bz2" | \ sed 's/.tar.bz2//' | tr '\n' ' ') NEW_TAGS="$NEW_TAGS $ALL_TAGS" else NEW_TAGS="$NEW_TAGS $TAG" fi done POOL_TAGS="$NEW_TAGS" if [ "$VERBOSE" = "yes" ]; then echo "---------------------------- Options ----------------------------" echo "VERBOSE=$VERBOSE" echo "KEEP=$KEEP" echo "REPO=$REPO" echo "SRC_TAGS=$SRC_TAGS" echo "POOL_TAGS=$POOL_TAGS" echo "PATH=$TEST_DIR" echo "POOL_CREATE_OPTIONS=$POOL_CREATE_OPTIONS" echo fi if [ ! -d "$TEST_DIR" ]; then mkdir -p "$TEST_DIR" fi if [ ! -d "$SRC_DIR" ]; then mkdir -p "$SRC_DIR" fi # Print a header for all tags which are being tested. echo "--------------------- ZFS on Linux Source Versions --------------" printf "%-16s" " " for TAG in $SRC_TAGS; do src_set_vars "$TAG" if [ "$TAG" = "installed" ]; then ZFS_VERSION=$(modinfo zfs | awk '/version:/ { print $2; exit }') if [ -n "$ZFS_VERSION" ]; then printf "%-16s" "$ZFS_VERSION" else fail "ZFS is not installed" fi else printf "%-16s" "$TAG" fi done echo -e "\n-----------------------------------------------------------------" # # Attempt to generate the tarball from your local git repository, if that # fails then attempt to download the tarball from Github. # printf "%-16s" "Clone SPL" for TAG in $SRC_TAGS; do src_set_vars "$TAG" if [ -d "$SPL_DIR" ]; then skip_nonewline elif [ "$SPL_TAG" = "installed" ]; then skip_nonewline else cd "$SRC_DIR" || fail "Failed 'cd $SRC_DIR'" if [ ! -d "$SRC_DIR_SPL" ]; then mkdir -p "$SRC_DIR_SPL" fi git archive --format=tar --prefix="$SPL_TAG/ $SPL_TAG" \ -o "$SRC_DIR_SPL/$SPL_TAG.tar" &>/dev/null || \ rm "$SRC_DIR_SPL/$SPL_TAG.tar" if [ -s "$SRC_DIR_SPL/$SPL_TAG.tar" ]; then tar -xf "$SRC_DIR_SPL/$SPL_TAG.tar" -C "$SRC_DIR_SPL" rm "$SRC_DIR_SPL/$SPL_TAG.tar" echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t" else mkdir -p "$SPL_DIR" || fail "Failed to create $SPL_DIR" curl -sL "$SPL_URL" | tar -xz -C "$SPL_DIR" \ --strip-components=1 || \ fail "Failed to download $SPL_URL" echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t" fi fi done printf "\n" # # Attempt to generate the tarball from your local git repository, if that # fails then attempt to download the tarball from Github. # printf "%-16s" "Clone ZFS" for TAG in $SRC_TAGS; do src_set_vars "$TAG" if [ -d "$ZFS_DIR" ]; then skip_nonewline elif [ "$ZFS_TAG" = "installed" ]; then skip_nonewline else cd "$SRC_DIR" || fail "Failed 'cd $SRC_DIR'" if [ ! -d "$SRC_DIR_ZFS" ]; then mkdir -p "$SRC_DIR_ZFS" fi git archive --format=tar --prefix="$ZFS_TAG/ $ZFS_TAG" \ -o "$SRC_DIR_ZFS/$ZFS_TAG.tar" &>/dev/null || \ rm "$SRC_DIR_ZFS/$ZFS_TAG.tar" if [ -s "$SRC_DIR_ZFS/$ZFS_TAG.tar" ]; then tar -xf "$SRC_DIR_ZFS/$ZFS_TAG.tar" -C "$SRC_DIR_ZFS" rm "$SRC_DIR_ZFS/$ZFS_TAG.tar" echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t" else mkdir -p "$ZFS_DIR" || fail "Failed to create $ZFS_DIR" curl -sL "$ZFS_URL" | tar -xz -C "$ZFS_DIR" \ --strip-components=1 || \ fail "Failed to download $ZFS_URL" echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t" fi fi done printf "\n" # Build the listed tags printf "%-16s" "Build SPL" for TAG in $SRC_TAGS; do src_set_vars "$TAG" if [ -f "$SPL_DIR/module/spl/spl.ko" ]; then skip_nonewline elif [ "$SPL_TAG" = "installed" ]; then skip_nonewline else cd "$SPL_DIR" || fail "Failed 'cd $SPL_DIR'" make distclean &>/dev/null ./autogen.sh >>"$CONFIG_LOG" 2>&1 || \ fail "Failed SPL 'autogen.sh'" # shellcheck disable=SC2086 ./configure $CONFIG_OPTIONS >>"$CONFIG_LOG" 2>&1 || \ fail "Failed SPL 'configure $CONFIG_OPTIONS'" # shellcheck disable=SC2086 make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || \ fail "Failed SPL 'make $MAKE_OPTIONS'" pass_nonewline fi done printf "\n" # Build the listed tags printf "%-16s" "Build ZFS" for TAG in $SRC_TAGS; do src_set_vars "$TAG" if [ -f "$ZFS_DIR/module/zfs/zfs.ko" ]; then skip_nonewline elif [ "$ZFS_TAG" = "installed" ]; then skip_nonewline else cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'" make distclean &>/dev/null ./autogen.sh >>"$CONFIG_LOG" 2>&1 || \ fail "Failed ZFS 'autogen.sh'" # shellcheck disable=SC2086 ./configure --with-spl="$SPL_DIR" $CONFIG_OPTIONS \ >>"$CONFIG_LOG" 2>&1 || \ fail "Failed ZFS 'configure $CONFIG_OPTIONS'" # shellcheck disable=SC2086 make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || \ fail "Failed ZFS 'make $MAKE_OPTIONS'" pass_nonewline fi done printf "\n" echo "-----------------------------------------------------------------" # Either create a new pool using 'zpool create', or alternately restore an # existing pool from another ZFS implementation for compatibility testing. for TAG in $POOL_TAGS; do pool_set_vars "$TAG" SKIP=0 printf "%-16s" "$POOL_TAG" rm -Rf "$POOL_DIR" mkdir -p "$POOL_DIR_PRISTINE" # Use the existing compressed image if available. if [ -f "$POOL_BZIP" ]; then tar -xjf "$POOL_BZIP" -C "$POOL_DIR_PRISTINE" \ --strip-components=1 || \ fail "Failed 'tar -xjf $POOL_BZIP" # Use the installed version to create the pool. elif [ "$TAG" = "installed" ]; then pool_create "$TAG" # A source build is available to create the pool. elif [ -d "$POOL_DIR_SRC" ]; then pool_create "$TAG" else SKIP=1 fi # Verify 'zpool import' works for all listed source versions. for SRC_TAG in $SRC_TAGS; do if [ $SKIP -eq 1 ]; then skip_nonewline continue fi src_set_vars "$SRC_TAG" if [ "$SRC_TAG" != "installed" ]; then cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'" fi $ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY" cp -a --sparse=always "$POOL_DIR_PRISTINE" \ "$POOL_DIR_COPY" || \ fail "Failed to copy $POOL_DIR_PRISTINE to $POOL_DIR_COPY" POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" | \ awk '/pool:/ { print $2; exit 0 }') $ZPOOL_CMD import -N -d "$POOL_DIR_COPY" \ "$POOL_NAME" &>/dev/null if [ $? -ne 0 ]; then fail_nonewline ERROR=1 else $ZPOOL_CMD export "$POOL_NAME" || \ fail "Failed to export pool" pass_nonewline fi rm -Rf "$POOL_DIR_COPY" $ZFS_SH -u || fail "Failed to unload kmods" done printf "\n" done if [ "$KEEP" = "no" ]; then rm -Rf "$TEST_DIR" fi exit $ERROR