OpenZFS 5704 - libzfs can only handle 255 file descriptors

Authored by: Simon Klinkert <simon.klinkert@gmail.com>
Reviewed by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
Reviewed by: John Kennedy <john.kennedy@delphix.com>
Approved by: Richard Lowe <richlowe@richlowe.net>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed by: Ned Bass <bass6@llnl.gov>
Reviewed-by: Chunwei Chen <david.chen@osnexus.com>
Ported-by: George Melikov <mail@gmelikov.ru>

OpenZFS-issue: https://www.illumos.org/issues/5704
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/bde3d61
Closes #5767

configure.ac

@@ -246,6 +246,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/large_files/Makefile
 	tests/zfs-tests/tests/functional/largest_pool/Makefile
 	tests/zfs-tests/tests/functional/link_count/Makefile
+	tests/zfs-tests/tests/functional/libzfs/Makefile
 	tests/zfs-tests/tests/functional/migration/Makefile
 	tests/zfs-tests/tests/functional/mmap/Makefile
 	tests/zfs-tests/tests/functional/mount/Makefile

tests/runfiles/linux.run

@@ -666,3 +666,6 @@ tests = ['zvol_cli_001_pos', 'zvol_cli_002_pos', 'zvol_cli_003_neg']
 #[tests/functional/zvol/zvol_swap]
 #tests = ['zvol_swap_001_pos', 'zvol_swap_002_pos', 'zvol_swap_003_pos',
 #    'zvol_swap_004_pos', 'zvol_swap_005_pos', 'zvol_swap_006_pos']
+
+[tests/functional/libzfs]
+tests = ['many_fds']

tests/zfs-tests/tests/functional/Makefile.am

@@ -23,6 +23,7 @@ SUBDIRS = \
 	inuse \
 	large_files \
 	largest_pool \
+	libzfs \
 	link_count \
 	migration \
 	mmap \

tests/zfs-tests/tests/functional/libzfs/.gitignore

@@ -0,0 +1 @@
+/many_fds

tests/zfs-tests/tests/functional/libzfs/Makefile.am

@@ -0,0 +1,24 @@
+include $(top_srcdir)/config/Rules.am
+
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/libzfs
+
+pkgexecdir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/libzfs
+
+dist_pkgdata_SCRIPTS = \
+	cleanup.ksh \
+	setup.ksh
+
+DEFAULT_INCLUDES += \
+	-I$(top_srcdir)/include \
+	-I$(top_srcdir)/lib/libspl/include
+
+many_fds_LDADD = \
+	$(top_builddir)/lib/libnvpair/libnvpair.la \
+	$(top_builddir)/lib/libuutil/libuutil.la \
+	$(top_builddir)/lib/libzpool/libzpool.la \
+	$(top_builddir)/lib/libzfs/libzfs.la \
+	$(top_builddir)/lib/libzfs_core/libzfs_core.la
+
+pkgexec_PROGRAMS = many_fds
+many_fds_SOURCES = many_fds.c
+

tests/zfs-tests/tests/functional/libzfs/cleanup.ksh

@@ -0,0 +1,34 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+default_cleanup

tests/zfs-tests/tests/functional/libzfs/many_fds.c

@@ -0,0 +1,73 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (C) 2015 STRATO AG.
+ */
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <libzfs.h>
+#include <sys/resource.h>
+#include <errno.h>
+
+/*
+ * Check if libzfs works with more than 255 held file handles.
+ */
+/* ARGSUSED */
+int
+main(int argc, char **argv)
+{
+	int i;
+	struct rlimit limit;
+	libzfs_handle_t *h;
+
+	limit.rlim_cur = 65535;
+	limit.rlim_max = 65535;
+
+	if (setrlimit(RLIMIT_NOFILE, &limit) != 0) {
+		(void) printf("many_fds: setrlimit() failed with errno=%d\n",
+		    errno);
+		exit(1);
+	}
+
+	for (i = 0; i < 255; ++i) {
+		int fd = open("/dev/null", O_RDONLY);
+		if (fd == -1) {
+			(void) printf("open failed with errno=%d\n", errno);
+			return (1);
+		}
+	}
+
+	h = libzfs_init();
+
+	if (h != NULL) {
+		libzfs_fini(h);
+		return (0);
+	} else {
+		(void) printf("many_fds: libzfs_init() failed with errno=%d\n",
+		    errno);
+		return (1);
+	}
+}

tests/zfs-tests/tests/functional/libzfs/setup.ksh

@@ -0,0 +1,35 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+DISK=${DISKS%% *}
+default_setup $DISK