Test application improvements:

- added possibility to load external zlib dll under windows and use its compression instead of builtin
- improved output information: new data provided, new layout
- added possibility to specify compression level
- added possibility to exclude particular directory (or many directories) from data input
- added option for removal of compressed file
- using different gz file names for different platforms (just to run compression tests in parallel from win32, win64 and/or linux VM)

Author: gildor2

.gitignore

@@ -4,3 +4,4 @@ zlib/
 *.exe
 *.pdb
 build/binaries/
+Test/dll/

Test/dll/readme.txt

@@ -0,0 +1 @@
+Put zlibwapi.dll from http://www.winimage.com/zLibDll/ here. Names used by test.sh script are zlibwapi32.dll and zlibwapi64.dll.

Test/test.cpp

@@ -1,12 +1,22 @@
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 #include <time.h>
 
 // includes for file enumeration
 #if _WIN32
 #	include <io.h>					// for findfirst() set
+#	ifndef _WIN64
+#		define PLATFORM				"win32"
+#	else
+#		define PLATFORM				"win64"
+#	endif
 #else
 #	include <dirent.h>				// for opendir() etc
 #	include <sys/stat.h>			// for stat()
+#	define stricmp					strcasecmp
+#	define strnicmp					strncasecmp
+#	define PLATFORM					"unix"
 #endif
 
 #include <vector>
@@ -21,7 +31,88 @@
 #define MAX_ITERATIONS	1			// number of passes to fully fill buffer, i.e. total processed data size will be up to (BUFFER_SIZE * MAX_ITERATIONS)
 #define COMPRESS_LEVEL	9
 
+#if _WIN32
+
+#include <windows.h>
+
+static HMODULE zlibDll = NULL;
+
+static gzFile gzopen_imp(const char* filename, const char* params)
+{
+	if (zlibDll)
+	{
+		typedef gzFile (WINAPI *gzopen_f)(const char* filename, const char* params);
+		static gzopen_f gzopen_ptr = NULL;
+		if (gzopen_ptr == NULL)
+		{
+			gzopen_ptr = (gzopen_f)GetProcAddress(zlibDll, "gzopen");
+//			assert(gzopen_ptr);
+		}
+		return gzopen_ptr(filename, params);
+	}
+	else
+	{
+		return gzopen(filename, params);
+	}
+}
+
+static int gzwrite_imp(gzFile file, voidpc buf, unsigned len)
+{
+	if (zlibDll)
+	{
+		typedef int (WINAPI *gzwrite_f)(gzFile file, voidpc buf, unsigned len);
+		static gzwrite_f gzwrite_ptr = NULL;
+		if (gzwrite_ptr == NULL)
+		{
+			gzwrite_ptr = (gzwrite_f)GetProcAddress(zlibDll, "gzwrite");
+//			assert(gzwrite_ptr);
+		}
+		return gzwrite_ptr(file, buf, len);
+	}
+	else
+	{
+		return gzwrite(file, buf, len);
+	}
+}
+
+static int gzclose_imp(gzFile file)
+{
+	if (zlibDll)
+	{
+		typedef int (WINAPI *gzclose_f)(gzFile file);
+		static gzclose_f gzclose_ptr = NULL;
+		if (gzclose_ptr == NULL)
+		{
+			gzclose_ptr = (gzclose_f)GetProcAddress(zlibDll, "gzclose");
+//			assert(gzclose_ptr);
+		}
+		return gzclose_ptr(file);
+	}
+	else
+	{
+		return gzclose(file);
+	}
+}
+
+// Hook gzip functions
+#define gzopen gzopen_imp
+#define gzwrite gzwrite_imp
+#define gzclose gzclose_imp
+
+#endif // _WIN32
+
 std::vector<std::string> fileList;
+std::vector<std::string> fileExclude;
+
+static bool IsFileExcluded(const char* filename)
+{
+	for (int i = 0; i < fileExclude.size(); i++)
+	{
+		if (!stricmp(filename, fileExclude[i].c_str()))
+			return true;
+	}
+	return false;
+}
 
 static bool ScanDirectory(const char *dir, bool recurse = true)
 {
@@ -37,6 +128,7 @@ static bool ScanDirectory(const char *dir, bool recurse = true)
 	{
 		if (found.name[0] == '.') continue;			// "." or ".."
 		sprintf(Path, "%s/%s", dir, found.name);
+		if (IsFileExcluded(Path)) continue;
 		// directory -> recurse
 		if (found.attrib & _A_SUBDIR)
 		{
@@ -60,6 +152,7 @@ static bool ScanDirectory(const char *dir, bool recurse = true)
 	{
 		if (ent->d_name[0] == '.') continue;			// "." or ".."
 		sprintf(Path, "%s/%s", dir, ent->d_name);
+		if (IsFileExcluded(Path)) continue;
 		// directory -> recurse
 		// note: using 'stat64' here because 'stat' ignores large files
 		struct stat64 buf;
@@ -113,19 +206,100 @@ int main(int argc, const char **argv)
 {
 	if (argc <= 1)
 	{
-		printf("Usage: test <directory>\n");
+	usage:
+		printf(
+			"Usage: test [options] <directory>\n"
+			"Options:\n"
+			"  --level=[0-9]     set compression level, default 9\n"
+			"  --exclude=<dir>   exclude specified directory from tests\n"
+#if _WIN32
+			"  --dll=<file>      use external WINAPI zlib dll\n"
+#endif
+			"  --compact         use compact output\n"
+			"  --delete          erase compressed file after completion\n"
+		);
 		return 1;
 	}
 
-	ScanDirectory(argv[1]);
+	// parse command line
+	const char* dirName = NULL;
+	char level = '9';
+	bool compactOutput = false;
+	bool eraseCompressedFile = false;
+
+	for (int i = 1; i < argc; i++)
+	{
+		const char* arg = argv[i];
+		if (arg[0] == '-' && arg[1] == '-')
+		{
+			// option
+			arg += 2; // skip "--"
+			if (!strnicmp(arg, "level=", 6))
+			{
+				if (!(level >= '0' && level <= '9')) goto usage;
+				level = arg[6];
+			}
+			else if (!strnicmp(arg, "exclude=", 8))
+			{
+				fileExclude.push_back(arg+8);
+			}
+			else if (!stricmp(arg, "compact"))
+			{
+				compactOutput = true;
+			}
+			else if (!stricmp(arg, "delete"))
+			{
+				eraseCompressedFile = true;
+			}
+#if _WIN32
+			else if (!strnicmp(arg, "dll=", 4))
+			{
+				if (zlibDll != NULL) goto usage;
+				zlibDll = LoadLibrary(arg+4);
+				if (zlibDll == NULL)
+				{
+					printf("Error: unable to load zlib.dll %s\n", arg+4);
+					exit(1);
+				}
+			}
+#endif // _WIN32
+			else
+			{
+				goto usage;
+			}
+		}
+		else
+		{
+			if (dirName) goto usage;
+			dirName = arg;
+		}
+	}
+
+	if (!dirName)
+	{
+		printf("Error: directory name was not specified\n");
+		exit(1);
+	}
+
+	// prepare data for compression
+	ScanDirectory(dirName);
+	if (fileList.size() == 0)
+	{
+		printf("Error: the specified location has no files\n");
+		exit(1);
+	}
 //	printf("%d files\n", fileList.size());
 
 	clock_t clocks = 0;
 
-	const char* compressedFile = "compressed-" STR(VERSION) ".gz";
-	gzFile gz = gzopen(compressedFile, "wb" STR(COMPRESS_LEVEL));
+	// open compressed stream
+	char initString[4] = "wb";
+	const char* compressedFile = "compressed-" STR(VERSION) "-" PLATFORM ".gz";
+	initString[2] = level;
+	gzFile gz = gzopen(compressedFile, initString);
 	int iteration = 0;
 	int totalDataSize = 0;
+	// perform compression
 	while (FillBuffer() && iteration < MAX_ITERATIONS)
 	{
 		clock_t clock_a = clock();
@@ -134,16 +308,38 @@ int main(int argc, const char **argv)
 		iteration++;
 		totalDataSize += bytesInBuffer;
 	}
+	// close compressed stream
 	gzclose(gz);
 
+	// determine size of compressed data
 	FILE* f = fopen(compressedFile, "rb");
 	fseek(f, 0, SEEK_END);
 	int compressedSize = ftell(f);
 	fclose(f);
 
-	printf("Compressed %.1f Mb of data by method %s with level %d (%s)\n", (float)totalDataSize / (1024*1024), STR(VERSION), COMPRESS_LEVEL, argv[1]);
-	printf("Time: %.1f s\n", clocks / (float)CLOCKS_PER_SEC);
-	printf("Size: %d\n", compressedSize);
+	// erase compressed file
+	if (eraseCompressedFile)
+	{
+		remove(compressedFile);
+	}
+
+	// print results
+	const char* method = STR(VERSION);
+#if _WIN32
+	if (zlibDll) method = "DLL";
+#endif
+	float time = clocks / (float)CLOCKS_PER_SEC;
+	float originalSizeMb = totalDataSize / double(1<<20);
+	if (!compactOutput)
+	{
+		printf("Compressed %.1f Mb of data by method %s with level %c (%s)\n", originalSizeMb, method, level, dirName);
+	}
+	else
+	{
+		printf("%6s:%c   Data: %.1f Mb   ", method, level, originalSizeMb);
+	}
+	printf("Time: %-5.1f s   Size: %d bytes   Speed: %5.2f Mb/s   Ratio: %.2f\n",
+		time, compressedSize, totalDataSize / double(1<<20) / time, (double)totalDataSize / compressedSize);
 
 	return 0;
 }

test.sh

@@ -2,13 +2,17 @@
 
 platform=win32
 
+noorig=0
+noasm=0
+nodll=0
+extraargs="--delete --compact"
+dllname=zlibwapi32.dll
+
 if [ "$OSTYPE" == "linux-gnu" ] || [ "$OSTYPE" == "linux" ]; then
 	platform=unix
+	nodll=1
 fi
 
-noorig=0
-noasm=0
-
 for arg in "$@"; do		# using quoted $@ will allow to correctly separate arguments like [ --path="some string with spaces" -debug ]
 #	echo "ARG=$arg"
 	case $arg in
@@ -18,14 +22,21 @@ for arg in "$@"; do		# using quoted $@ will allow to correctly separate argument
 	--noorig)
 		noorig=1
 		;;
+	--nodll)
+		nodll=1
+		;;
 	--win64)
 		platform=win64
+		dllname=zlibwapi64.dll
+		;;
+	--level=*|--exclude=*)
+		extraargs="$extraargs $arg"
 		;;
 	*)
 		if [ -d "$arg" ]; then
 			dir="$arg"
 		else
-			echo "Usage: test.sh [path] [--noasm] [--noorig] [--win64]"
+			echo "Usage: test.sh [path] [--noasm] [--noorig] [--nodll] [--win64] [--level=X]"
 			exit
 		fi
 	esac
@@ -45,6 +56,9 @@ echo "Testing for $platform"
 function DoTests
 {
 	local dir="$1"
+	shift
+
+	echo "--- Processing data at $dir ---"
 
 	if [ "$platform" == "unix" ]; then
 		# convert Windows path to Linux (VM)
@@ -54,18 +68,21 @@ function DoTests
 	fi
 
 	if [ "$platform" != "win64" ] && [ $noasm == 0 ]; then
-		obj/bin/test-Asm-$platform "$dir"
+		obj/bin/test-Asm-$platform "$dir" $extraargs $*
+	fi
+	obj/bin/test-C-$platform "$dir" $extraargs $*
+	if [ $nodll == 0 ]; then
+		obj/bin/test-Orig-$platform "$dir" $extraargs --dll=test/dll/$dllname $*
 	fi
-	obj/bin/test-C-$platform "$dir"
 	if [ $noorig == 0 ]; then
-		obj/bin/test-Orig-$platform "$dir"
+		obj/bin/test-Orig-$platform "$dir" $extraargs $*
 	fi
 }
 
 if [ "$dir" ]; then
 	DoTests "$dir"
 else
-	DoTests C:/Projects/Epic/UnrealEngine4-latest/Engine/Source/Runtime
+	DoTests C:/Projects/Epic/UnrealEngine4-latest/Engine/Source --exclude=ThirdParty
 	DoTests C:/3-UnrealEngine/4.14/Engine/Binaries/Win64
 	DoTests C:/1
 fi