folkslinux
/
zfs
mirror of https://github.com/openzfs/zfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

OpenZFS 7247 - zfs receive of deduplicated stream fails 

Authored by: Chris Williamson <chris.williamson@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Dan Kimmel <dan.kimmel@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Reviewed-by: loli10K <ezomori.nozomu@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Ported-by: George Melikov <mail@gmelikov.ru>

OpenZFS-issue: https://www.illumos.org/issues/7247
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/2ad25b4
Closes #5689 

Porting notes:
- tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh
  renamed as zfs_receive_015_pos.ksh, zfs_receive_013_pos.ksh is now
  used for OpenZFS test.
- libzfs_sendrecv.c: SMALLEST_POSSIBLE_MAX_DDT_MB is always used
  for all 32-bit builds.
This commit is contained in:
George Melikov 2017-02-04 20:10:24 +03:00 committed by Brian Behlendorf
parent 57c6a9174d
commit 2e0e443ac4
6 changed files with 147 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/libzfs/libzfs_sendrecv.c
View File

@ -245,12 +245,16 @@ cksummer(void *arg)
int outfd;
dedup_table_t ddt;
zio_cksum_t stream_cksum;
uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
uint64_t numbuckets;
#ifdef _ILP32
ddt.max_ddt_size = SMALLEST_POSSIBLE_MAX_DDT_MB << 20;
#else
uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
ddt.max_ddt_size =
MAX((physmem * MAX_DDT_PHYSMEM_PERCENT) / 100,
SMALLEST_POSSIBLE_MAX_DDT_MB << 20);
#endif
numbuckets = ddt.max_ddt_size / (sizeof (dedup_entry_t));

44
module/zfs/dmu_send.c
View File

@ -3230,6 +3230,9 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
dsl_dataset_phys(origin_head)->ds_flags &=
~DS_FLAG_INCONSISTENT;
drc->drc_newsnapobj =
dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
dsl_dataset_rele(origin_head, FTAG);
dsl_destroy_head_sync_impl(drc->drc_ds, tx);
@ -3265,8 +3268,9 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
DS_FIELD_RESUME_TONAME, tx);
}
drc->drc_newsnapobj =
dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
}
drc->drc_newsnapobj = dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
zvol_create_minors(dp->dp_spa, drc->drc_tofs, B_TRUE);
/*
* Release the hold from dmu_recv_begin. This must be done before
@ -3310,8 +3314,6 @@ static int dmu_recv_end_modified_blocks = 3;
static int
dmu_recv_existing_end(dmu_recv_cookie_t *drc)
{
int error;
#ifdef _KERNEL
/*
* We will be destroying the ds; make sure its origin is unmounted if
@ -3322,23 +3324,30 @@ dmu_recv_existing_end(dmu_recv_cookie_t *drc)
zfs_destroy_unmount_origin(name);
#endif
error = dsl_sync_task(drc->drc_tofs,
return (dsl_sync_task(drc->drc_tofs,
dmu_recv_end_check, dmu_recv_end_sync, drc,
dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL);
if (error != 0)
dmu_recv_cleanup_ds(drc);
return (error);
dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL));
}
static int
dmu_recv_new_end(dmu_recv_cookie_t *drc)
{
return (dsl_sync_task(drc->drc_tofs,
dmu_recv_end_check, dmu_recv_end_sync, drc,
dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL));
}
int
dmu_recv_end(dmu_recv_cookie_t *drc, void *owner)
{
int error;
error = dsl_sync_task(drc->drc_tofs,
dmu_recv_end_check, dmu_recv_end_sync, drc,
dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL);
drc->drc_owner = owner;
if (drc->drc_newfs)
error = dmu_recv_new_end(drc);
else
error = dmu_recv_existing_end(drc);
if (error != 0) {
dmu_recv_cleanup_ds(drc);
@ -3350,17 +3359,6 @@ dmu_recv_new_end(dmu_recv_cookie_t *drc)
return (error);
}
int
dmu_recv_end(dmu_recv_cookie_t *drc, void *owner)
{
drc->drc_owner = owner;
if (drc->drc_newfs)
return (dmu_recv_new_end(drc));
else
return (dmu_recv_existing_end(drc));
}
/*
* Return TRUE if this objset is currently being received into.
*/

2
tests/runfiles/linux.run
View File

@ -150,7 +150,7 @@ tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos',
'zfs_receive_005_neg', 'zfs_receive_006_pos',
'zfs_receive_007_neg', 'zfs_receive_008_pos', 'zfs_receive_009_neg',
'zfs_receive_010_pos', 'zfs_receive_011_pos', 'zfs_receive_012_pos',
'zfs_receive_013_pos', 'zfs_receive_014_pos']
'zfs_receive_013_pos', 'zfs_receive_014_pos', 'zfs_receive_015_pos']
# DISABLED:
# zfs_rename_006_pos - https://github.com/zfsonlinux/zfs/issues/5647

3
tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am
View File

@ -15,4 +15,5 @@ dist_pkgdata_SCRIPTS = \
zfs_receive_011_pos.ksh \
zfs_receive_012_pos.ksh \
zfs_receive_013_pos.ksh \
zfs_receive_014_pos.ksh
zfs_receive_014_pos.ksh \
zfs_receive_015_pos.ksh

80
tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh
View File

@ -15,69 +15,59 @@
#
#
# Copyright 2016, loli10K. All rights reserved.
# Copyright (c) 2015 by Delphix. All rights reserved.
#
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
. $STF_SUITE/tests/functional/cli_root/cli_common.kshlib
#
# DESCRIPTION:
# Verify ZFS can receive custom properties on both filesystems and
# snapshots from full and incremental streams.
# Verifying 'zfs receive' works correctly on deduplicated streams
#
# STRATEGY:
# 1. Create a filesystem.
# 2. Snapshot the filesystem.
# 3. Set custom properties on both the fs and snapshots.
# 4. Create different send streams with the properties.
# 5. Receive the send streams and verify the properties.
# 1. Create some snapshots with duplicated data
# 2. Send a deduplicated stream of the last snapshot
# 3. Attempt to receive the deduplicated stream
#
verify_runnable "both"
typeset streamfile_full=/var/tmp/streamfile_full.$$
typeset streamfile_incr=/var/tmp/streamfile_incr.$$
orig=$TESTPOOL/$TESTFS1
dest=$TESTPOOL/$TESTFS2
typeset user_prop=$(valid_user_property 8)
typeset value=$(user_property_value 8)
src_fs=$TESTPOOL/drecvsrc
temppool=recvtank
dst_fs=$temppool/drecvdest
streamfile=/var/tmp/drecvstream.$$
tpoolfile=/temptank.$$
function cleanup
{
log_must $RM $streamfile_full
log_must $RM $streamfile_incr
log_must $ZFS destroy -rf $TESTPOOL/$TESTFS1
log_must $ZFS destroy -rf $TESTPOOL/$TESTFS2
for fs in $src_fs $dst_fs; do
datasetexists $fs && log_must $ZFS destroy -rf $fs
done
$ZPOOL destroy $temppool
[[ -f $streamfile ]] && log_must $RM -f $streamfile
[[ -f $tpoolfile ]] && log_must $RM -f $tpoolfile
}
log_assert "ZFS can receive custom properties."
log_assert "Verifying 'zfs receive' works correctly on deduplicated streams"
log_onexit cleanup
# 1. Create a filesystem.
log_must $ZFS create $orig
truncate -s 100M $tpoolfile
log_must $ZPOOL create $temppool $tpoolfile
log_must $ZFS create $src_fs
src_mnt=$(get_prop mountpoint $src_fs) || log_fail "get_prop mountpoint $src_fs"
# 2. Snapshot the filesystem.
log_must $ZFS snapshot $orig@snap1
log_must $ZFS snapshot $orig@snap2
log_must $ZFS snapshot $orig@snap3
echo blah > $src_mnt/blah
$ZFS snapshot $src_fs@base
# 3. Set custom properties on both the fs and snapshots.
log_must eval "$ZFS set '$user_prop'='$value' $orig"
log_must eval "$ZFS set '$user_prop:snap1'='$value:snap1' $orig@snap1"
log_must eval "$ZFS set '$user_prop:snap2'='$value:snap2' $orig@snap2"
log_must eval "$ZFS set '$user_prop:snap3'='$value:snap3' $orig@snap3"
echo grumble > $src_mnt/grumble
echo blah > $src_mnt/blah2
$ZFS snapshot $src_fs@snap2
# 4. Create different send streams with the properties.
log_must eval "$ZFS send -p $orig@snap1 > $streamfile_full"
log_must eval "$ZFS send -p -I $orig@snap1 $orig@snap3 > $streamfile_incr"
echo grumble > $src_mnt/mumble
echo blah > $src_mnt/blah3
$ZFS snapshot $src_fs@snap3
# 5. Receive the send streams and verify the properties.
log_must eval "$ZFS recv $dest < $streamfile_full"
log_must eval "check_user_prop $dest $user_prop '$value'"
log_must eval "check_user_prop $dest@snap1 '$user_prop:snap1' '$value:snap1'"
log_must eval "$ZFS recv $dest < $streamfile_incr"
log_must eval "check_user_prop $dest@snap2 '$user_prop:snap2' '$value:snap2'"
log_must eval "check_user_prop $dest@snap3 '$user_prop:snap3' '$value:snap3'"
log_must eval "$ZFS send -D -R $src_fs@snap3 > $streamfile"
log_must eval "$ZFS receive -v $dst_fs < $streamfile"
log_pass "ZFS can receive custom properties passed."
cleanup
log_pass "Verifying 'zfs receive' works correctly on deduplicated streams"

83
tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_015_pos.ksh Executable file
View File

@ -0,0 +1,83 @@
#!/bin/ksh -p
#
# CDDL HEADER START
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
# CDDL HEADER END
#
#
# Copyright 2016, loli10K. All rights reserved.
#
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
#
# DESCRIPTION:
# Verify ZFS can receive custom properties on both filesystems and
# snapshots from full and incremental streams.
#
# STRATEGY:
# 1. Create a filesystem.
# 2. Snapshot the filesystem.
# 3. Set custom properties on both the fs and snapshots.
# 4. Create different send streams with the properties.
# 5. Receive the send streams and verify the properties.
#
verify_runnable "both"
typeset streamfile_full=/var/tmp/streamfile_full.$$
typeset streamfile_incr=/var/tmp/streamfile_incr.$$
orig=$TESTPOOL/$TESTFS1
dest=$TESTPOOL/$TESTFS2
typeset user_prop=$(valid_user_property 8)
typeset value=$(user_property_value 8)
function cleanup
{
log_must $RM $streamfile_full
log_must $RM $streamfile_incr
log_must $ZFS destroy -rf $TESTPOOL/$TESTFS1
log_must $ZFS destroy -rf $TESTPOOL/$TESTFS2
}
log_assert "ZFS can receive custom properties."
log_onexit cleanup
# 1. Create a filesystem.
log_must $ZFS create $orig
# 2. Snapshot the filesystem.
log_must $ZFS snapshot $orig@snap1
log_must $ZFS snapshot $orig@snap2
log_must $ZFS snapshot $orig@snap3
# 3. Set custom properties on both the fs and snapshots.
log_must eval "$ZFS set '$user_prop'='$value' $orig"
log_must eval "$ZFS set '$user_prop:snap1'='$value:snap1' $orig@snap1"
log_must eval "$ZFS set '$user_prop:snap2'='$value:snap2' $orig@snap2"
log_must eval "$ZFS set '$user_prop:snap3'='$value:snap3' $orig@snap3"
# 4. Create different send streams with the properties.
log_must eval "$ZFS send -p $orig@snap1 > $streamfile_full"
log_must eval "$ZFS send -p -I $orig@snap1 $orig@snap3 > $streamfile_incr"
# 5. Receive the send streams and verify the properties.
log_must eval "$ZFS recv $dest < $streamfile_full"
log_must eval "check_user_prop $dest $user_prop '$value'"
log_must eval "check_user_prop $dest@snap1 '$user_prop:snap1' '$value:snap1'"
log_must eval "$ZFS recv $dest < $streamfile_incr"
log_must eval "check_user_prop $dest@snap2 '$user_prop:snap2' '$value:snap2'"
log_must eval "check_user_prop $dest@snap3 '$user_prop:snap3' '$value:snap3'"
log_pass "ZFS can receive custom properties passed."