Do not pass -1 to strerror() from zfs_send_cb_impl()

`zfs_send_cb_impl()` calls `dump_filesystems()`, which calls
`dump_filesystem()`, which will return `-1` as an error when
`zfs_open()` returns `NULL`.

This will be passed to `zfs_standard_error()`, which passes it to
`zfs_standard_error_fmt()`, which passes it to `strerror()`.

To fix this, we modify zfs_open() to set `errno` whenever it returns
NULL. Most of the cases already have `errno` set (since they pass it to
`zfs_standard_error_fmt()`, which makes this easy. Then we modify
`dump_filesystem()` to pass `errno` instead of `-1`.

Reported-by: Coverity (CID-1524598)
Reviewed-by: Damian Szuberski <szuberskidamian@gmail.com>
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Richard Yao <richard.yao@alumni.stonybrook.edu>
Closes openzfs#14264

Author: ryao

lib/libzfs/libzfs_dataset.c

@@ -690,6 +690,7 @@ zfs_open(libzfs_handle_t *hdl, const char *path, int types)
 	 */
 	if (!zfs_validate_name(hdl, path, types, B_FALSE)) {
 		(void) zfs_error(hdl, EZFS_INVALIDNAME, errbuf);
+		errno = EINVAL;
 		return (NULL);
 	}
 
@@ -737,6 +738,7 @@ zfs_open(libzfs_handle_t *hdl, const char *path, int types)
 		    &cb_data) == 0) && (cb_data.zhp == NULL)) {
 			(void) zfs_error(hdl, EZFS_NOENT, errbuf);
 			zfs_close(pzhp);
+			errno = ENOENT;
 			return (NULL);
 		}
 		if (cb_data.zhp == NULL) {
@@ -755,6 +757,7 @@ zfs_open(libzfs_handle_t *hdl, const char *path, int types)
 	if (!(types & zhp->zfs_type)) {
 		(void) zfs_error(hdl, EZFS_BADTYPE, errbuf);
 		zfs_close(zhp);
+		errno = EINVAL;
 		return (NULL);
 	}
 

lib/libzfs/libzfs_sendrecv.c

@@ -1289,7 +1289,7 @@ dump_filesystem(zfs_handle_t *zhp, send_dump_data_t *sdd)
 			if (snap != NULL)
 				rv = dump_snapshot(snap, sdd);
 			else
-				rv = -1;
+				rv = errno;
 		}
 
 		/* Dump tosnap. */
@@ -1301,7 +1301,7 @@ dump_filesystem(zfs_handle_t *zhp, send_dump_data_t *sdd)
 			if (snap != NULL)
 				rv = dump_snapshot(snap, sdd);
 			else
-				rv = -1;
+				rv = errno;
 		}
 	}