libzfs_core: add ZFS_IOC_TRACE envvar to enable ioctl tracing

lib/libzfs/libzfs_dataset.c

@@ -4911,7 +4911,7 @@ zfs_smb_acl_mgmt(libzfs_handle_t *hdl, char *dataset, char *path,
 	default:
 		return (-1);
 	}
-	error = ioctl(hdl->libzfs_fd, ZFS_IOC_SMB_ACL, &zc);
+	error = lzc_ioctl_fd(hdl->libzfs_fd, ZFS_IOC_SMB_ACL, &zc);
 	nvlist_free(nvlist);
 	return (error);
 }

lib/libzfs/libzfs_util.c

@@ -785,6 +785,12 @@ zpool_standard_error_fmt(libzfs_handle_t *hdl, int error, const char *fmt, ...)
 	return (-1);
 }
 
+int
+zfs_ioctl(libzfs_handle_t *hdl, int request, zfs_cmd_t *zc)
+{
+	return (lzc_ioctl_fd(hdl->libzfs_fd, request, zc));
+}
+
 /*
  * Display an out of memory error message and abort the current program.
  */

lib/libzfs/os/freebsd/libzfs_compat.c

@@ -218,12 +218,6 @@ libzfs_error_init(int error)
 	return (errbuf);
 }
 
-int
-zfs_ioctl(libzfs_handle_t *hdl, int request, zfs_cmd_t *zc)
-{
-	return (lzc_ioctl_fd(hdl->libzfs_fd, request, zc));
-}
-
 /*
  * Verify the required ZFS_DEV device is available and optionally attempt
  * to load the ZFS modules.  Under normal circumstances the modules

lib/libzfs/os/linux/libzfs_util_os.c

@@ -53,12 +53,6 @@
 
 #define	ZDIFF_SHARESDIR		"/.zfs/shares/"
 
-int
-zfs_ioctl(libzfs_handle_t *hdl, int request, zfs_cmd_t *zc)
-{
-	return (ioctl(hdl->libzfs_fd, request, zc));
-}
-
 const char *
 libzfs_error_init(int error)
 {

lib/libzfs_core/Makefile.am

@@ -1,18 +1,21 @@
 libzfs_core_la_CFLAGS  = $(AM_CFLAGS) $(LIBRARY_CFLAGS)
 libzfs_core_la_CFLAGS += -fvisibility=hidden
 
+libzfs_core_la_CPPFLAGS  = $(AM_CPPFLAGS)
+libzfs_core_la_CPPFLAGS += -I$(srcdir)/%D%
+
 lib_LTLIBRARIES += libzfs_core.la
 CPPCHECKTARGETS += libzfs_core.la
 
 libzfs_core_la_SOURCES = \
-	%D%/libzfs_core.c
+	%D%/libzfs_core.c \
+	%D%/libzfs_core_impl.h
 
 if BUILD_LINUX
 libzfs_core_la_SOURCES += \
 	%D%/os/linux/libzfs_core_ioctl.c
 endif
 
-libzfs_core_la_CPPFLAGS  = $(AM_CPPFLAGS)
 if BUILD_FREEBSD
 libzfs_core_la_CPPFLAGS += -Iinclude/os/freebsd/zfs
 

lib/libzfs_core/libzfs_core.c

@@ -96,10 +96,14 @@
 #define	BIG_PIPE_SIZE (64 * 1024) /* From sys/pipe.h */
 #endif
 
+#include "libzfs_core_impl.h"
+
 static int g_fd = -1;
 static pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER;
 static int g_refcount;
 
+static int g_ioc_trace = 0;
+
 #ifdef ZFS_DEBUG
 static zfs_ioc_t fail_ioc_cmd = ZFS_IOC_LAST;
 static zfs_errno_t fail_ioc_err;
@@ -152,6 +156,10 @@ libzfs_core_init(void)
 #ifdef ZFS_DEBUG
 	libzfs_core_debug_ioc();
 #endif
+
+	if (getenv("ZFS_IOC_TRACE"))
+		g_ioc_trace = 1;
+
 	(void) pthread_mutex_unlock(&g_lock);
 	return (0);
 }
@@ -171,6 +179,42 @@ libzfs_core_fini(void)
 	(void) pthread_mutex_unlock(&g_lock);
 }
 
+int
+lzc_ioctl_fd(int fd, unsigned long ioc, zfs_cmd_t *zc)
+{
+	if (!g_ioc_trace)
+		return (lzc_ioctl_fd_os(fd, ioc, zc));
+
+	nvlist_t *nvl;
+
+	fprintf(stderr, "=== lzc_ioctl: call: ioc=0x%lx name=%s\n",
+	    ioc, zc->zc_name[0] ? zc->zc_name : "[none]");
+	if (zc->zc_nvlist_src) {
+		nvl = fnvlist_unpack(
+		    (void *)(uintptr_t)zc->zc_nvlist_src,
+		    zc->zc_nvlist_src_size);
+		nvlist_print(stderr, nvl);
+		fnvlist_free(nvl);
+	}
+
+	int rc = lzc_ioctl_fd_os(fd, ioc, zc);
+	int err = errno;
+
+	fprintf(stderr, "=== lzc_ioctl: result: ioc=0x%lx name=%s "
+	    "rc=%d errno=%d\n", ioc, zc->zc_name[0] ? zc->zc_name : "[none]",
+	    rc, (rc < 0 ? err : 0));
+	if (rc >= 0 && zc->zc_nvlist_dst) {
+		nvl = fnvlist_unpack(
+		    (void *)(uintptr_t)zc->zc_nvlist_dst,
+		    zc->zc_nvlist_dst_size);
+		nvlist_print(stderr, nvl);
+		fnvlist_free(nvl);
+	}
+
+	errno = err;
+	return (rc);
+}
+
 static int
 lzc_ioctl(zfs_ioc_t ioc, const char *name,
     nvlist_t *source, nvlist_t **resultp)

lib/libzfs_core/libzfs_core_impl.h

@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: CDDL-1.0
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or https://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
+ * Copyright 2017 RackTop Systems.
+ * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
+ * Copyright (c) 2019 Datto Inc.
+ */
+
+#ifndef	_LIBZFS_CORE_IMPL_H
+#define	_LIBZFS_CORE_IMPL_H
+
+struct zfs_cmd;
+int lzc_ioctl_fd_os(int, unsigned long, struct zfs_cmd *);
+
+#endif	/* _LIBZFS_CORE_IMPL_H */

lib/libzfs_core/os/freebsd/libzfs_core_ioctl.c

@@ -26,6 +26,7 @@
 #include <os/freebsd/zfs/sys/zfs_ioctl_compat.h>
 #include <err.h>
 #include <libzfs_core.h>
+#include "libzfs_core_impl.h"
 
 int zfs_ioctl_version = ZFS_IOCVER_UNDEF;
 
@@ -101,7 +102,7 @@ zcmd_ioctl_compat(int fd, int request, zfs_cmd_t *zc, const int cflag)
  * error is returned zc_nvlist_dst_size won't be updated.
  */
 int
-lzc_ioctl_fd(int fd, unsigned long request, zfs_cmd_t *zc)
+lzc_ioctl_fd_os(int fd, unsigned long request, zfs_cmd_t *zc)
 {
 	size_t oldsize;
 	int ret, cflag = ZFS_CMD_COMPAT_NONE;

lib/libzfs_core/os/linux/libzfs_core_ioctl.c

@@ -23,9 +23,10 @@
 #include <sys/param.h>
 #include <sys/zfs_ioctl.h>
 #include <libzfs_core.h>
+#include "libzfs_core_impl.h"
 
 int
-lzc_ioctl_fd(int fd, unsigned long request, zfs_cmd_t *zc)
+lzc_ioctl_fd_os(int fd, unsigned long request, zfs_cmd_t *zc)
 {
 	return (ioctl(fd, request, zc));
 }