common: accept UTF-16 in scripts

match, check-whitespace and check-license
should accept UTF-16 encoded files

src/common/util_windows.c

@@ -36,6 +36,7 @@
 
 #include <string.h>
 #include <tchar.h>
+#include <errno.h>
 #include "util.h"
 #include "out.h"
 #include "file.h"
@@ -111,65 +112,67 @@ util_aligned_free(void *ptr)
 }
 
 /*
- * util_toUTF8 -- XXX
+ * util_toUTF8 -- allocating conversion from wide char string to UTF8
  */
 char *
 util_toUTF8(const wchar_t *wstr)
 {
 	int size = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wstr, -1,
 		NULL, 0, NULL, NULL);
-	if (size == 0) {
-		errno = EINVAL;
-		return NULL;
-	}
+	if (size == 0)
+		goto err;
 
 	char *str = Malloc(size * sizeof(char));
-	if (str == NULL) {
-		errno = ENOMEM;
-		return NULL;
-	}
+	if (str == NULL)
+		goto out;
 
 	if (WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wstr, -1, str,
 		size, NULL, NULL) == 0) {
 		Free(str);
-		errno = EINVAL;
-		return NULL;
+		goto err;
 	}
 
+out:
 	return str;
+
+err:
+	errno = EINVAL;
+	return NULL;
 }
 
 /*
- * util_toUTF16 -- XXX
+ * util_toUTF16 -- allocating conversion from UTF8 to wide char string
  */
 wchar_t *
 util_toUTF16(const char *str)
 {
 	int size = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, str, -1,
 		NULL, 0);
-	if (size == 0) {
-		errno = EINVAL;
-		return NULL;
-	}
+	if (size == 0)
+		goto err;
 
 	wchar_t *wstr = Malloc(size * sizeof(wchar_t));
-	if (wstr == NULL) {
-		errno = ENOMEM;
-		return NULL;
-	}
+	if (wstr == NULL)
+		goto out;
 
 	if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, str, -1, wstr,
 		size) == 0) {
 		Free(wstr);
-		errno = EINVAL;
-		return NULL;
+		goto err;
 	}
 
+out:
 	return wstr;
+
+err:
+	errno = EINVAL;
+	return NULL;
 }
 
 /*
- * util_toUTF16 -- XXX user responsible for supplying a large enough out buffer.
+ * util_toUTF16 -- non-allocating conversion from UTF8 to wide char string
+ *
+ * The user responsible for supplying a large enough out buffer.
  */
 int
 util_toUTF16_buff(const char *in, wchar_t *out, size_t out_size)
@@ -193,44 +196,23 @@ util_toUTF16_buff(const char *in, wchar_t *out, size_t out_size)
 }
 
 /*
- * util_toUTF8_buff -- XXX user responsible for supplying a large enough out
- *	buffer.
+ * util_toUTF8_buff -- non-allocating conversion from wide char string to UTF8
+ *
+ * The user responsible for supplying a large enough out buffer.
  */
 int
 util_toUTF8_buff(const wchar_t *in, char *out, size_t out_size)
 {
 	if (out == NULL)
 		goto err;
 
-	int size = WideCharToMultiByte(CP_UTF8, 0, in, -1,
+	int size = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, in, -1,
 		NULL, 0, NULL, NULL);
 	if (size == 0 || out_size < size)
 		goto err;
 
-	if (WideCharToMultiByte(CP_UTF8, 0, in, -1, out, size,
-		NULL, NULL) == 0)
-		goto err;
-
-	return 0;
-err:
-	errno = EINVAL;
-	return -1;
-}
-
-/*
- * util_toUTF16 -- XXX user responsible for supplying a large enough out buffer.
- */
-int
-util_toUTF16_inplace(const char *in, wchar_t *out, size_t out_size)
-{
-	if (out == NULL)
-		goto err;
-
-	int size = MultiByteToWideChar(CP_UTF8, 0, in, -1, NULL, 0);
-	if (size == 0 || out_size < size)
-		goto err;
-
-	if (MultiByteToWideChar(CP_UTF8, 0, in, -1, out, size) == 0)
+	if (WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, in, -1,
+		out, size, NULL, NULL) == 0)
 		goto err;
 
 	return 0;

src/test/match

@@ -1,4 +1,4 @@
-#!/usr/bin/perl -w
+#!/usr/bin/env perl
 #
 # Copyright 2014-2017, Intel Corporation
 #
@@ -72,7 +72,12 @@
 
 use strict;
 use Getopt::Std;
+use Encode;
+use v5.16;
+
 select STDERR;
+binmode(STDOUT, ":utf8");
+binmode(STDERR, ":utf8");
 
 my $Me = $0;
 $Me =~ s,.*/,,;
@@ -160,20 +165,20 @@ sub match {
 	my $line_out = 0;
 	my $opt = 0;
 
-	open(F, $mfile) or die "$mfile: $!\n";
-	while (<F>) {
+	my $fstr = snarf($mfile);
+	for (split /^/, $fstr) {
 		$pat = $_;
 		$line_pat++;
 		$line_out++;
 		s/([*+?|{}.\\^\$\[()])/\\$1/g;
 		s/\\\$\\\(FP\\\)/[-+]?\\d*\\.?\\d+([eE][-+]?\\d+)?/g;
 		s/\\\$\\\(N\\\)/\\d+/g;
-		s/\\\$\\\(\\\*\\\)/.*/g;
+		s/\\\$\\\(\\\*\\\)/\\p{Print}*/g;
 		s/\\\$\\\(S\\\)/\\P{IsC}+/g;
-		s/\\\$\\\(X\\\)/\\p{IsXDigit}+/g;
-		s/\\\$\\\(XX\\\)/0x\\p{IsXDigit}+/g;
-		s/\\\$\\\(W\\\)/\\s*[^\n]/g;
-		s/\\\$\\\(nW\\\)/\\S*/g;
+		s/\\\$\\\(X\\\)/\\p{XPosixXDigit}+/g;
+		s/\\\$\\\(XX\\\)/0x\\p{XPosixXDigit}+/g;
+		s/\\\$\\\(W\\\)/\\p{Blank}*/g;
+		s/\\\$\\\(nW\\\)/\\p{Graph}*/g;
 		s/\\\$\\\(DD\\\)/\\d+\\+\\d+ records in\n\\d+\\+\\d+ records out\n\\d+ bytes \\\(\\d+ .B\\\) copied, [.0-9e-]+[^,]*, [.0-9]+ .B.s/g;
 		if (s/\\\$\\\(OPT\\\)//) {
 			$opt = 1;
@@ -226,13 +231,26 @@ sub match {
 # snarf -- slurp an entire file into memory
 #
 sub snarf {
-	my ($fname) = @_;
+	my ($file) = @_;
+	my $fh;
+	open($fh, '<', $file) or die "$file $!\n";
+
 	local $/;
-	my $contents;
+	$_ = <$fh>;
+	close $fh;
+
+	# check known encodings or die
+	my $decoded;
+	my @encodings = ("UTF-8", "UTF-16", "UTF-16LE", "UTF-16BE");
 
-	open(R, $fname) or die "$fname: $!\n";
-	$contents = <R>;
-	close(R);
+	foreach my $enc (@encodings) {
+		eval { $decoded = decode( $enc, $_, Encode::FB_CROAK ) };
+
+		if (!$@) {
+			$decoded =~ s/\R/\n/g;
+			return $decoded;
+		}
+	}
 
-	return $contents;
+	die "$Me: ERROR: Unknown file encoding";
 }

src/windows/getopt/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2012, Kim Gr�sman
+Copyright (c) 2012, Kim Gräsman
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -8,14 +8,14 @@ modification, are permitted provided that the following conditions are met:
     * Redistributions in binary form must reproduce the above copyright
       notice, this list of conditions and the following disclaimer in the
       documentation and/or other materials provided with the distribution.
-    * Neither the name of Kim Gr�sman nor the
+    * Neither the name of Kim Gräsman nor the
       names of contributors may be used to endorse or promote products
       derived from this software without specific prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL KIM GR�SMAN BE LIABLE FOR ANY
+DISCLAIMED. IN NO EVENT SHALL KIM GRÄSMAN BE LIABLE FOR ANY
 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND

src/windows/getopt/getopt.c

@@ -1,30 +1,29 @@
 /*
-* *Copyright (c) 2012, Kim Gräsman
-
-* All rights reserved.
-*
-* Redistribution and use in source and binary forms, with or without
-* modification, are permitted provided that the following conditions are met:
-*     * Redistributions of source code must retain the above copyright
-*       notice, this list of conditions and the following disclaimer.
-*     * Redistributions in binary form must reproduce the above copyright
-*       notice, this list of conditions and the following disclaimer in the
-*       documentation and/or other materials provided with the distribution.
-*     * Neither the name of Kim Gräsman nor the
-*       names of contributors may be used to endorse or promote products
-*       derived from this software without specific prior written permission.
-*
-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-* ARE DISCLAIMED. IN NO EVENT SHALL KIM GRÄSMAN BE LIABLE FOR ANY
-* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
+ * *Copyright (c) 2012, Kim Gräsman
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Kim Gräsman nor the
+ *       names of contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL KIM GRÄSMAN BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
 
 #include "getopt.h"
 

src/windows/getopt/getopt.h

@@ -1,5 +1,5 @@
 /*
- * *Copyright (c) 2012, Kim Gräsman
+ * *Copyright (c) 2012, Kim Gräsman
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -9,14 +9,14 @@
  *     * Redistributions in binary form must reproduce the above copyright
  *       notice, this list of conditions and the following disclaimer in the
  *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Kim Gräsman nor the
+ *     * Neither the name of Kim Gräsman nor the
  *       names of contributors may be used to endorse or promote products
  *       derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL KIM GRÄSMAN BE LIABLE FOR ANY
+ * ARE DISCLAIMED. IN NO EVENT SHALL KIM GRÄSMAN BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND

utils/check_license/check-headers.sh

@@ -56,6 +56,7 @@ shift
 
 PATTERN=`mktemp`
 TMP=`mktemp`
+TEMPFILE=`mktemp`
 rm -f $PATTERN $TMP
 
 function exit_if_not_exist()
@@ -140,7 +141,11 @@ $CHECK_LICENSE create $LICENSE $PATTERN
 RV=0
 for file in $FILES ; do
 	[ ! -f $file ] && continue
-	YEARS=`$CHECK_LICENSE check-pattern $PATTERN $file`
+	# ensure that file is UTF-8 encoded
+	ENCODING=`file -b --mime-encoding $file`
+	iconv -f $ENCODING -t "UTF-8" -o $TEMPFILE $file
+
+	YEARS=`$CHECK_LICENSE check-pattern $PATTERN $TEMPFILE`
 	if [ $? -ne 0 ]; then
 		echo -n $YEARS
 		RV=1
@@ -188,7 +193,7 @@ for file in $FILES ; do
 	fi
 done
 rm -f $TMP
-
+rm -f $TEMPFILE
 # check if error found
 if [ $RV -eq 0 ]; then
 	echo "Copyright headers are OK."

utils/check_license/check-license.c

@@ -412,7 +412,7 @@ verify_license(const char *path_to_check, char *pattern)
 	if (err_str)
 		/* found an error in the copyright notice */
 		ERROR2("%s:1: error: incorrect copyright notice: %s",
-			path_to_check, err_str);
+			name_to_print, err_str);
 
 	/* now check the license */
 	if (memcmp(license, pattern, strlen(pattern)) != 0) {
@@ -495,7 +495,9 @@ main(int argc, char *argv[])
 
 	} else {
 		ERROR("wrong mode: %s\n", argv[1]);
-		printf(help_str, argv[0]);
-		return -1;
 	}
+
+invalid_args:
+	printf(help_str, argv[0]);
+	return -1;
 }