Disable normalization implicitly when setting "utf8only=off"

When a parent dataset has normalization set to any value other than
"none", and a file system is created with the property "utf8only=off",
implicitly also set "normalization=none" instead of overriding the
desire for a non-UTF8 enforcing file system.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Mike Swanson <mikeonthecomputer@gmail.com>
Closes #11892
Closes #12038

Author: chungy

lib/libzfs/libzfs_dataset.c

@@ -1562,6 +1562,9 @@ zfs_valid_proplist(libzfs_handle_t *hdl, zfs_type_t type, nvlist_t *nvl,
 	 *
 	 * If normalization was chosen, but rejecting non-UTF8 names
 	 * was explicitly not chosen, it is an error.
+	 *
+	 * If utf8only was turned off, but the parent has normalization,
+	 * turn off normalization.
 	 */
 	if (chosen_normal > 0 && chosen_utf < 0) {
 		if (nvlist_add_uint64(ret,
@@ -1575,6 +1578,12 @@ zfs_valid_proplist(libzfs_handle_t *hdl, zfs_type_t type, nvlist_t *nvl,
 		    zfs_prop_to_name(ZFS_PROP_UTF8ONLY));
 		(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
 		goto error;
+	} else if (chosen_normal < 0 && chosen_utf == 0) {
+		if (nvlist_add_uint64(ret,
+		    zfs_prop_to_name(ZFS_PROP_NORMALIZE), 0) != 0) {
+			(void) no_memory(hdl);
+			goto error;
+		}
 	}
 	return (ret);
 

tests/zfs-tests/tests/functional/casenorm/norm_all_values.ksh

@@ -58,4 +58,15 @@ for form in formC formD formKC formKD; do
 	destroy_testfs
 done
 
+for form in formC formD formKC formKD; do
+	create_testfs "-o normalization=$form"
+	log_must zfs create -o utf8only=off $TESTPOOL/$TESTFS/$TESTSUBFS
+	normalization=$(zfs get -H -o value normalization $TESTPOOL/$TESTFS/$TESTSUBFS)
+	if [[ $normalization != "none" ]]; then
+		log_fail "Turning off utf8only didn't set normalization to none"
+	fi
+	log_must zfs destroy $TESTPOOL/$TESTFS/$TESTSUBFS
+	destroy_testfs
+done
+
 log_pass "Can create FS with all supported normalization forms"