fix(compress): Fix a few memory leaks on fail conditions in Compression code (TheSuperHackers#1509)

Author: vitimiti

Core/Libraries/Source/Compression/EAC/btreeencode.cpp

@@ -367,11 +367,18 @@ static void BTREE_treepack(struct BTreeEncodeContext *EC,
 
 	EC->buf1 =	(unsigned char *) galloc(buf1size);
 	if (!EC->buf1)
-        return; /* failure Insufficient memory for work buffer */
+	{
+		gfree(treebuf);
+		return; /* failure Insufficient memory for work buffer */
+	}
 
 	EC->buf2 =	(unsigned char *) galloc(buf2size);
 	if (!EC->buf2)
-        return; /* failure Insufficient memory for work buffer */
+	{
+		gfree(treebuf);
+		gfree(EC->buf1);
+		return; /* failure Insufficient memory for work buffer */
+	}
 
     memcpy(EC->buf1, EC->buffer, EC->ulen); /* copy to scratch buffer */
 

Core/Libraries/Source/Compression/EAC/refencode.cpp

@@ -79,7 +79,10 @@ static int refcompress(unsigned char *from, int len, unsigned char *dest, int ma
         return(0);
 	link = (int *) galloc(131072L*sizeof(int));
 	if (!link)
-        return(0);
+	{
+	    gfree(hashtbl);
+	    return(0);
+	}
 
     memset(hashtbl,-1,65536L*sizeof(int));
 

Core/Libraries/Source/Compression/LZHCompress/NoxCompress.cpp

@@ -69,7 +69,11 @@ Bool DecompressFile		(char *infile, char *outfile)
 		outBlock= (char *) DbgMalloc( rawSize );
 
 		if (( inBlock == NULL ) || ( outBlock == NULL ))
+		{
+			if (inBlock) DbgFree(inBlock);
+			if (outBlock) DbgFree(outBlock);
 			return FALSE;
+		}
 
 		// Read in a big chunk o file
 		NoxRead(inBlock, 1, compressedSize, inFilePtr);
@@ -97,21 +101,19 @@ Bool DecompressFile		(char *infile, char *outfile)
 
 		DEBUG_LOG(("Decompressed %s to %s, output size = %d", infile, outfile, rawSize));
 
+		Bool success = FALSE;
 		LZHLDestroyDecompressor(decompress);
 		outFilePtr = fopen(outfile, "wb");
 		if (outFilePtr)
 		{
 			fwrite (outBlock, rawSize, 1, outFilePtr);
 			fclose(outFilePtr);
+			success = TRUE;
 		}
-		else
-			return FALSE;
 
-		// Clean up this mess
 		DbgFree(inBlock);
 		DbgFree(outBlock);
-		return TRUE;
-
+		return success;
 	} // End of if fileptr
 
 	return FALSE;
@@ -148,7 +150,11 @@ Bool CompressFile			(char *infile, char *outfile)
 		outBlock= (char *) DbgMalloc( LZHLCompressorCalcMaxBuf( rawSize ));
 
 		if (( inBlock == NULL ) || ( outBlock == NULL ))
+		{
+			DbgFree(inBlock);
+			DbgFree(outBlock);
 			return FALSE;
+		}
 
 		// Read in a big chunk o file
 		NoxRead(inBlock, 1, rawSize, inFilePtr);
@@ -164,23 +170,21 @@ Bool CompressFile			(char *infile, char *outfile)
 			compressedSize += compressed;
 		}
 
+		Bool success = FALSE;
 		LZHLDestroyCompressor(compressor);
-
 		outFilePtr = fopen(outfile, "wb");
 		if (outFilePtr)
 		{
 			// write out the uncompressed size first.
 			fwrite(&rawSize, sizeof(UnsignedInt), 1, outFilePtr);
 			fwrite(outBlock, compressedSize, 1, outFilePtr);
 			fclose(outFilePtr);
+			success = TRUE;
 		}
-		else
-			return FALSE;
 
-		// Clean up
 		DbgFree(inBlock);
 		DbgFree(outBlock);
-		return TRUE;
+		return success;
 	}
 
 	return FALSE;