Refactor checksum operations in tests

md5sum in particular but also sha256sum to a lesser extent is used
in several areas of the test suite for computing checksums. The vast
majority of invocations are followed by `| awk '{ print $1 }'`.

Introduce functions to wrap up `md5sum $file | awk '{ print $1 }'` and
likewise for sha256sum. These also serve as a convenient interface for
alternative implementations on other platforms.

Reviewed-by: Igor Kozhukhov <igor@dilos.org>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Ryan Moeller <ryan@ixsystems.com>
Closes #9280

tests/zfs-tests/include/libtest.shlib

@@ -3575,3 +3575,25 @@ function mdb_ctf_set_int
 
 	return 0
 }
+
+#
+# Compute MD5 digest for given file or stdin if no file given.
+# Note: file path must not contain spaces
+#
+function md5digest
+{
+	typeset file=$1
+
+	md5sum -b $file | awk '{ print $1 }'
+}
+
+#
+# Compute SHA256 digest for given file or stdin if no file given.
+# Note: file path must not contain spaces
+#
+function sha256digest
+{
+	typeset file=$1
+
+	sha256sum -b $file | awk '{ print $1 }'
+}

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_from_encrypted.ksh

@@ -59,7 +59,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
 	"-o keyformat=passphrase $TESTPOOL/$TESTFS2"
 
 log_must mkfile 1M /$TESTPOOL/$TESTFS2/$TESTFILE0
-typeset checksum=$(md5sum /$TESTPOOL/$TESTFS2/$TESTFILE0 | awk '{ print $1 }')
+typeset checksum=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
 
 log_must zfs snapshot $snap
 
@@ -69,14 +69,14 @@ log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS1/c1"
 crypt=$(get_prop encryption $TESTPOOL/$TESTFS1/c1)
 [[ "$crypt" == "off" ]] || log_fail "Received unencrypted stream as encrypted"
 
-typeset cksum1=$(md5sum /$TESTPOOL/$TESTFS1/c1/$TESTFILE0 | awk '{ print $1 }')
+typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
 [[ "$cksum1" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum1 != $checksum)"
 
 log_note "Verify ZFS can receive into an encrypted child"
 log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS2/c1"
 
-typeset cksum2=$(md5sum /$TESTPOOL/$TESTFS2/c1/$TESTFILE0 | awk '{ print $1 }')
+typeset cksum2=$(md5digest /$TESTPOOL/$TESTFS2/c1/$TESTFILE0)
 [[ "$cksum2" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum2 != $checksum)"
 

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_raw.ksh

@@ -60,8 +60,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
 	"-o keyformat=passphrase $TESTPOOL/$TESTFS1"
 
 log_must mkfile 1M /$TESTPOOL/$TESTFS1/$TESTFILE0
-typeset checksum=$(md5sum /$TESTPOOL/$TESTFS1/$TESTFILE0 | \
-	awk '{ print $1 }')
+typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
 
 log_must zfs snapshot $snap
 
@@ -74,7 +73,7 @@ keystatus=$(get_prop keystatus $TESTPOOL/$TESTFS2)
 
 log_must eval "echo $passphrase | zfs mount -l $TESTPOOL/$TESTFS2"
 
-typeset cksum1=$(md5sum /$TESTPOOL/$TESTFS2/$TESTFILE0 | awk '{ print $1 }')
+typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
 [[ "$cksum1" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum1 != $checksum)"
 
@@ -85,8 +84,7 @@ keystatus=$(get_prop keystatus $TESTPOOL/$TESTFS1/c1)
 	log_fail "Expected keystatus unavailable, got $keystatus"
 
 log_must eval "echo $passphrase | zfs mount -l $TESTPOOL/$TESTFS1/c1"
-typeset cksum2=$(md5sum /$TESTPOOL/$TESTFS1/c1/$TESTFILE0 | \
-	awk '{ print $1 }')
+typeset cksum2=$(md5digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
 [[ "$cksum2" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum2 != $checksum)"
 

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_raw_incremental.ksh

@@ -69,7 +69,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
 log_must zfs snapshot $snap1
 
 log_must mkfile 1M /$TESTPOOL/$TESTFS1/$TESTFILE0
-typeset checksum=$(md5sum /$TESTPOOL/$TESTFS1/$TESTFILE0 | awk '{ print $1 }')
+typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
 
 log_must zfs snapshot $snap2
 
@@ -89,7 +89,7 @@ log_must zfs unload-key $TESTPOOL/$TESTFS2
 log_must eval "zfs receive $TESTPOOL/$TESTFS2 < $ibackup"
 log_must eval "echo $passphrase2 | zfs mount -l $TESTPOOL/$TESTFS2"
 
-typeset cksum1=$(md5sum /$TESTPOOL/$TESTFS2/$TESTFILE0 | awk '{ print $1 }')
+typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
 [[ "$cksum1" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum1 != $checksum)"
 

tests/zfs-tests/tests/functional/cli_root/zpool_import/import_cachefile_shared_device.ksh

@@ -50,7 +50,7 @@ function dev_checksum
 
 	log_note "Compute checksum of '$dev'"
 
-	checksum=$(md5sum $dev)
+	checksum=$(md5digest $dev)
 	if [[ $? -ne 0 ]]; then
 		log_fail "Failed to compute checksum of '$dev'"
 		return 1

tests/zfs-tests/tests/functional/cli_root/zpool_import/zpool_import.kshlib

@@ -79,10 +79,10 @@ function write_some_data
 
 #
 # Create/overwrite a few datasets with files.
-# Apply md5sum on all the files and store checksums in a file.
+# Checksum all the files and store digests in a file.
 #
 # newdata: overwrite existing files if false.
-# md5file: file where to store md5sums
+# md5file: file where to store md5 digests
 # datasetname: base name for datasets
 #
 function _generate_data_common
@@ -102,7 +102,10 @@ function _generate_data_common
 		for j in {1..$files}; do
 			typeset file="/$pool/$datasetname$i/file$j"
 			dd if=/dev/urandom of=$file bs=128k count=$blocks > /dev/null
-			[[ -n $md5file ]] && md5sum $file >> $md5file
+			if [[ -n $md5file ]]; then
+				typeset cksum=$(md5digest $file)
+				echo $cksum $file >> $md5file
+			fi
 		done
 		( $newdata ) && sync_pool "$pool"
 	done
@@ -140,8 +143,15 @@ function verify_data_md5sums
 		return 1
 	fi
 
-	md5sum -c --quiet $md5file
-	return $?
+	cat $md5file | \
+	while read digest file; do
+		typeset digest1=$(md5digest $file)
+		if [[ "$digest1" != "$digest" ]]; then
+			return 1
+		fi
+	done
+
+	return 0
 }
 
 #

tests/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_003_pos.ksh

@@ -42,7 +42,6 @@ verify_runnable "global"
 function cleanup
 {
 	log_must zinject -c all
-	rm -f $TESTFILE_MD5 2>/dev/null
 	# bring back removed disk online for further tests
 	insert_disk $REMOVED_DISK $scsi_host
 	poolexists $TESTPOOL && destroy_pool $TESTPOOL
@@ -64,9 +63,8 @@ log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "unavail"
 
 # 3. Write a test file to the pool and calculate its checksum.
 TESTFILE=/$TESTPOOL/data
-TESTFILE_MD5=$(mktemp --tmpdir=/var/tmp)
 log_must generate_random_file /$TESTPOOL/data $LARGE_FILE_SIZE
-log_must md5sum $TESTFILE > $TESTFILE_MD5
+TESTFILE_MD5=$(md5digest $TESTFILE)
 
 # 4. Execute scrub.
 # add delay to I/O requests for remaining disk in pool
@@ -90,12 +88,13 @@ log_must is_scan_restarted $TESTPOOL
 
 # 8. Put another device offline and check if the test file checksum is correct.
 log_must zpool offline $TESTPOOL $DISK2
-log_must md5sum -c $TESTFILE_MD5
+CHECK_MD5=$(md5digest $TESTFILE)
+[[ $CHECK_MD5 == $TESTFILE_MD5 ]] || \
+    log_fail "Checksums differ ($CHECK_MD5 != $TESTFILE_MD5)"
 log_must zpool online $TESTPOOL $DISK2
 sleep 1
 
 # clean up
-rm -f $TESTFILE_MD5 2>/dev/null
 log_must zpool destroy $TESTPOOL
 
 log_pass "Zpool reopen test successful"

tests/zfs-tests/tests/functional/history/history_003_pos.ksh

@@ -65,9 +65,7 @@ log_must zpool create $spool $VDEV0
 log_must zfs create $spool/$sfs
 
 typeset -i orig_count=$(zpool history $spool | wc -l)
-typeset orig_md5=$(zpool history $spool | head -2 | md5sum | \
-    awk '{print $1}')
-
+typeset orig_md5=$(zpool history $spool | head -2 | md5digest)
 typeset -i i=0
 while ((i < 300)); do
 	zfs set compression=off $spool/$sfs
@@ -82,7 +80,7 @@ done
 TMPFILE=$TEST_BASE_DIR/spool.$$
 zpool history $spool >$TMPFILE
 typeset -i entry_count=$(wc -l $TMPFILE | awk '{print $1}')
-typeset final_md5=$(head -2 $TMPFILE | md5sum | awk '{print $1}')
+typeset final_md5=$(head -2 $TMPFILE | md5digest)
 
 grep 'zpool create' $TMPFILE >/dev/null 2>&1 ||
     log_fail "'zpool create' was not found in pool history"

tests/zfs-tests/tests/functional/rsend/rsend.kshlib

@@ -158,14 +158,9 @@ function cmp_md5s {
 	typeset file1=$1
 	typeset file2=$2
 
-	eval md5sum $file1 | awk '{ print $1 }'	> $BACKDIR/md5_file1
-	eval md5sum $file2 | awk '{ print $1 }'	> $BACKDIR/md5_file2
-	diff $BACKDIR/md5_file1 $BACKDIR/md5_file2
-	typeset -i ret=$?
-
-	rm -f $BACKDIR/md5_file1 $BACKDIR/md5_file2
-
-	return $ret
+	typeset sum1=$(md5digest $file1)
+	typeset sum2=$(md5digest $file2)
+	test "$sum1" = "$sum2"
 }
 
 #

tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh

@@ -49,8 +49,8 @@ typeset megs=8
 log_must zfs create -V 256m -o compress=lz4 $vol
 
 write_compressible $BACKDIR ${megs}m 2
-md5_1=$(md5sum $data1 | awk '{print $1}')
-md5_2=$(md5sum $data2 | awk '{print $1}')
+md5_1=$(md5digest $data1)
+md5_2=$(md5digest $data2)
 
 log_must dd if=$data1 of=$voldev bs=1024k
 log_must zfs snapshot $vol@snap
@@ -60,8 +60,7 @@ log_must eval "zfs recv -d $POOL2 <$BACKDIR/full"
 
 verify_stream_size $BACKDIR/full $vol
 verify_stream_size $BACKDIR/full $vol2
-md5=$(dd if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5sum | \
-    awk '{print $1}')
+md5=$(dd if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5digest)
 [[ $md5 = $md5_1 ]] || log_fail "md5 mismatch: $md5 != $md5_1"
 
 # Repeat, for an incremental send
@@ -73,8 +72,7 @@ log_must eval "zfs recv -d $POOL2 <$BACKDIR/inc"
 
 verify_stream_size $BACKDIR/inc $vol 90 $vol@snap
 verify_stream_size $BACKDIR/inc $vol2 90 $vol2@snap
-md5=$(dd skip=$megs if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5sum | \
-    awk '{print $1}')
+md5=$(dd skip=$megs if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5digest)
 [[ $md5 = $md5_2 ]] || log_fail "md5 mismatch: $md5 != $md5_2"
 
 log_pass "Verify compressed send works with volumes"

tests/zfs-tests/tests/functional/rsend/send-wDR_encrypted_zvol.ksh

@@ -86,8 +86,8 @@ block_device_wait
 
 log_must mount $recvdev $recvmnt
 
-md5_1=$(cat $mntpnt/* | md5sum | awk '{print $1}')
-md5_2=$(cat $recvmnt/* | md5sum | awk '{print $1}')
+md5_1=$(cat $mntpnt/* | md5digest)
+md5_2=$(cat $recvmnt/* | md5digest)
 [[ "$md5_1" == "$md5_2" ]] || log_fail "md5 mismatch: $md5_1 != $md5_2"
 
 log_pass "zfs can receive raw, recursive, and deduplicated send streams"

tests/zfs-tests/tests/functional/rsend/send_encrypted_props.ksh

@@ -75,7 +75,7 @@ log_must zfs create -o keyformat=passphrase -o keylocation=file://$keyfile \
 
 log_must mkfile 1M /$TESTPOOL/ds/$TESTFILE0
 log_must cp /$TESTPOOL/ds/$TESTFILE0 /$TESTPOOL/crypt/$TESTFILE0
-typeset cksum=$(md5sum /$TESTPOOL/ds/$TESTFILE0 | awk '{  print $1 }')
+typeset cksum=$(md5digest /$TESTPOOL/ds/$TESTFILE0)
 
 log_must zfs snap -r $snap
 log_must zfs snap -r $esnap
@@ -127,7 +127,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds"
 log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
 log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile"
 log_must test "$(get_prop 'mounted' $ds)" == "yes"
-recv_cksum=$(md5sum /$ds/$TESTFILE0 | awk '{ print $1 }')
+recv_cksum=$(md5digest /$ds/$TESTFILE0)
 log_must test "$recv_cksum" == "$cksum"
 log_must zfs destroy -r $ds
 
@@ -143,7 +143,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds"
 log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
 log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile"
 log_must test "$(get_prop 'mounted' $ds)" == "yes"
-recv_cksum=$(md5sum /$ds/$TESTFILE0 | awk '{ print $1 }')
+recv_cksum=$(md5digest /$ds/$TESTFILE0)
 log_must test "$recv_cksum" == "$cksum"
 log_must zfs destroy -r $ds
 
@@ -161,7 +161,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds"
 log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
 log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile"
 log_must test "$(get_prop 'mounted' $ds)" == "yes"
-recv_cksum=$(md5sum /$ds/$TESTFILE0 | awk '{ print $1 }')
+recv_cksum=$(md5digest /$ds/$TESTFILE0)
 log_must test "$recv_cksum" == "$cksum"
 log_must zfs destroy -r $ds
 
@@ -175,7 +175,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$TESTPOOL/crypt"
 log_must test "$(get_prop 'encryption' $ds)" == "aes-256-ccm"
 log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
 log_must test "$(get_prop 'mounted' $ds)" == "yes"
-recv_cksum=$(md5sum /$ds/$TESTFILE0 | awk '{ print $1 }')
+recv_cksum=$(md5digest /$ds/$TESTFILE0)
 log_must test "$recv_cksum" == "$cksum"
 log_must zfs destroy -r $ds
 
@@ -189,7 +189,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$TESTPOOL/crypt"
 log_must test "$(get_prop 'encryption' $ds)" == "aes-256-ccm"
 log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
 log_must test "$(get_prop 'mounted' $ds)" == "yes"
-recv_cksum=$(md5sum /$ds/$TESTFILE0 | awk '{ print $1 }')
+recv_cksum=$(md5digest /$ds/$TESTFILE0)
 log_must test "$recv_cksum" == "$cksum"
 log_must zfs destroy -r $ds
 

tests/zfs-tests/tests/functional/slog/slog_replay_fs_001.ksh

@@ -58,14 +58,8 @@
 
 verify_runnable "global"
 
-function cleanup_fs
-{
-	rm -f $TESTDIR/checksum
-	cleanup
-}
-
 log_assert "Replay of intent log succeeds."
-log_onexit cleanup_fs
+log_onexit cleanup
 log_must setup
 
 #
@@ -115,7 +109,7 @@ log_must rmdir /$TESTPOOL/$TESTFS/dir_to_delete
 # Create a simple validation payload
 log_must mkdir -p $TESTDIR
 log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS/payload bs=1k count=8
-log_must eval "sha256sum -b /$TESTPOOL/$TESTFS/payload >$TESTDIR/checksum"
+typeset checksum=$(sha256digest /$TESTPOOL/$TESTFS/payload)
 
 # TX_WRITE (small file with ordering)
 log_must mkfile 1k /$TESTPOOL/$TESTFS/small_file
@@ -210,6 +204,8 @@ log_note "Verify working set diff:"
 log_must diff -r /$TESTPOOL/$TESTFS $TESTDIR/copy
 
 log_note "Verify file checksum:"
-log_must sha256sum -c $TESTDIR/checksum
+typeset checksum1=$(sha256digest /$TESTPOOL/$TESTFS/payload)
+[[ "$checksum1" == "$checksum" ]] || \
+    log_fail "checksum mismatch ($checksum1 != $checksum)"
 
 log_pass "Replay of intent log succeeds."

tests/zfs-tests/tests/functional/slog/slog_replay_volume.ksh

@@ -128,7 +128,7 @@ fi
 #
 # 4. Generate checksums for all ext4 files.
 #
-log_must sha256sum -b $MNTPNT/* >$TESTDIR/checksum
+typeset checksum=$(cat $MNTPNT/* | sha256digest)
 
 #
 # 5. Unmount filesystem and export the pool
@@ -160,6 +160,8 @@ log_note "Verify current block usage:"
 log_must zdb -bcv $TESTPOOL
 
 log_note "Verify checksums"
-log_must sha256sum -c $TESTDIR/checksum
+typeset checksum1=$(cat $MNTPNT/* | sha256digest)
+[[ "$checksum1" == "$checksum" ]] || \
+    log_fail "checksum mismatch ($checksum1 != $checksum)"
 
 log_pass "Replay of intent log succeeds."