CHANGE: rewritten compress and decompress so it's now compatible with the recent Red language. Now it also correctly handles output buffer expansion. It does not store the uncompress length at the tail of compressed data anymore!

resolves: #31

Author: Oldes

src/core/n-strings.c

@@ -254,44 +254,53 @@ static struct digest {
 */	REBNATIVE(compress)
 /*
 //	compress: native [
-//		{Compresses data. Default is deflate with Adler32 checksum and uncompressed size in last 4 bytes.}
+//		{Compresses data.}
 //		data [binary! string!] {If string, it will be UTF8 encoded}
+//		method [word!] "zlib deflate gzip lzma"
 //		/part length {Length of source data}
-//		/zlib {Use ZLIB (Adler32 checksum) without uncompressed length appended}
-//		/gzip {Use ZLIB with GZIP envelope (using CRC32 checksum)}
-//		/lzma {Use LZMA compression}
 //		/level lvl [integer!] {Compression level 0-9}
 //	]
 ***********************************************************************/
 {
-	REBVAL *data    = D_ARG(1);
-  //REBOOL ref_part = D_REF(2);
-	REBVAL *length  = D_ARG(3);
-    REBOOL ref_zlib = D_REF(4);
-	REBOOL ref_gzip = D_REF(5);
-	REBOOL ref_lzma = D_REF(6);
-	REBOOL ref_level= D_REF(7);
-	REBVAL *level   = D_ARG(8);
+	REBVAL *data     = D_ARG(1);
+	REBINT  method   = VAL_WORD_CANON(D_ARG(2));
+//	REBOOL ref_part  = D_REF(3);
+	REBVAL *length   = D_ARG(4);
+	REBOOL ref_level = D_REF(5);
+	REBVAL *level    = D_ARG(6);
 
 	REBSER *ser;
 	REBCNT index;
 	REBCNT len;
-    
-    if ((ref_zlib && (ref_gzip || ref_lzma)) || (ref_gzip && ref_lzma)) Trap0(RE_BAD_REFINES);
+	REBINT windowBits = MAX_WBITS;
 
 	len = Partial1(data, length);
 	ser = Prep_Bin_Str(data, &index, &len); // result may be a SHARED BUFFER!
 
-	if(ref_lzma) {
+	switch (method) {
+	case SYM_ZLIB:
+	zlib_compress:
+		Set_Binary(D_RET, CompressZlib(ser, index, (REBINT)len, ref_level ? VAL_INT32(level) : -1, windowBits));
+		break;
+	
+	case SYM_DEFLATE:
+		windowBits = -windowBits;
+		goto zlib_compress;
+	
+	case SYM_GZIP:
+		windowBits |= 16;
+		goto zlib_compress;
+
+	case SYM_LZMA:
 #ifdef INCLUDE_LZMA
 		Set_Binary(D_RET, CompressLzma(ser, index, (REBINT)len, ref_level ? VAL_INT32(level) : -1));
 #else
 		Trap0(RE_FEATURE_NA);
 #endif
-	} else {
-		int windowBits = MAX_WBITS;
-		if (ref_gzip) windowBits |= 16;
-		Set_Binary(D_RET, CompressZlib(ser, index, (REBINT)len, ref_level ? VAL_INT32(level) : -1, windowBits));
+		break;
+
+	default:
+		Trap1(RE_INVALID_ARG, D_ARG(2));
 	}
 
 	return R_RET;
@@ -303,56 +312,56 @@ static struct digest {
 */	REBNATIVE(decompress)
 /*
 //	decompress: native [
-//		{Decompresses data. Result is binary.}
+//		{Decompresses data.}
 //		data [binary!] {Source data to decompress}
+//		method [word!] "zlib deflate gzip lzma" 
 //		/part "Limits source data to a given length or position"
 //			length [number! series!] {Length of compressed data (must match end marker)}
-//		/zlib {Data are in ZLIB format with Adler32 checksum}
-//		/gzip {Data are in ZLIB format with CRC32 checksum}
-//		/lzma {Data are in LZMA format}
-//		/deflate {Data are raw DEFLATE data}
 //		/size
-//			bytes [integer!] {Number of decompressed bytes. If not used, size is detected from last 4 source data bytes.}
+//			bytes [integer!] {Number of uncompressed bytes.}
 ]
 ***********************************************************************/
 {
-	REBVAL *data     = D_ARG(1);
-  //REBOOL ref_part  = D_REF(2);
-    REBVAL *length   = D_ARG(3);
-    REBOOL ref_zlib  = D_REF(4);
-	REBOOL ref_gzip  = D_REF(5);
-	REBOOL ref_lzma  = D_REF(6);
-	REBOOL ref_defl  = D_REF(7);
-	REBOOL ref_size  = D_REF(8);
-	REBVAL *size     = D_ARG(9);
+	REBVAL *data    = D_ARG(1);
+	REBINT  method  = VAL_WORD_CANON(D_ARG(2));
+//	REBOOL ref_part = D_REF(3);
+	REBVAL *length  = D_ARG(4);
+	REBOOL ref_size = D_REF(5);
+	REBVAL *size    = D_ARG(6);
 
 	REBCNT limit = 0;
 	REBCNT len;
 	REBINT windowBits = MAX_WBITS;
 
-	// test if only one compression type refinement is used 
-    if (
-		(ref_zlib && (ref_gzip || ref_lzma || ref_defl)) ||
-		(ref_gzip && (ref_zlib || ref_lzma || ref_defl)) ||
-		(ref_lzma && (ref_zlib || ref_gzip || ref_defl))
-	)	Trap0(RE_BAD_REFINES);
-    
 	len = Partial1(data, length);
 
 	if (ref_size) limit = (REBCNT)Int32s(size, 1); // /limit size
 
-	if (ref_lzma) {
+	switch (method) {
+	case SYM_ZLIB:
+	zlib_decompress:
+		Set_Binary(D_RET, DecompressZlib(VAL_SERIES(data), VAL_INDEX(data), (REBINT)len, limit, windowBits));
+		break;
+
+	case SYM_DEFLATE:
+		windowBits = -windowBits;
+		goto zlib_decompress;
+
+	case SYM_GZIP:
+		windowBits |= 16;
+		goto zlib_decompress;
+
+	case SYM_LZMA:
 #ifdef INCLUDE_LZMA
 		Set_Binary(D_RET, DecompressLzma(VAL_SERIES(data), VAL_INDEX(data), (REBINT)len, limit));
 #else
 		Trap0(RE_FEATURE_NA);
 #endif
-	} else {
-		if (ref_defl) windowBits = -windowBits;
-		else if (ref_gzip) windowBits |= 16;
-		Set_Binary(D_RET, DecompressZlib(VAL_SERIES(data), VAL_INDEX(data), (REBINT)len, limit, windowBits));
+		break;
+
+	default:
+		Trap1(RE_INVALID_ARG, D_ARG(2));
 	}
-	
 
 	return R_RET;
 }

src/core/u-compress.c

@@ -131,26 +131,29 @@ void Trap_ZStream_Error(z_stream *stream, int err, REBOOL while_compression)
 	stream.next_in = cast(const z_Bytef*, BIN_HEAD(input) + index);
 
 	output = Make_Binary(size);
-	stream.avail_out = size;
+	stream.avail_out = SERIES_AVAIL(output);
 	stream.next_out = BIN_HEAD(output);
 
-	err = deflate(&stream, Z_FINISH);
-	//printf("deflate err: %i  stream.total_out: %i .avail_out: %i\n", err, stream.total_out, stream.avail_out);
-	
-	if (err != Z_STREAM_END)
-		Trap_ZStream_Error(&stream, err, TRUE);
+	for (;;) {
+		err = deflate(&stream, Z_FINISH);
+		if (err == Z_STREAM_END)
+			break; // Finished or we have enough data.
+		//printf("deflate err: %i  stream.total_out: %i .avail_out: %i\n", err, stream.total_out, stream.avail_out);
+		if (err != Z_OK)
+			Trap_ZStream_Error(&stream, err, FALSE);
+		if (stream.avail_out == 0) {
+			// expand output buffer...
+			SERIES_TAIL(output) = stream.total_out;
+			Expand_Series(output, AT_TAIL, in_len);
+			stream.next_out = BIN_SKIP(output, stream.total_out);
+			stream.avail_out = SERIES_REST(output) - stream.total_out;
+		}
+	}
 
 	SET_STR_END(output, stream.total_out);
 	SERIES_TAIL(output) = stream.total_out;
-
-	if((windowBits & 16) != 16) { // Not GZIP
-		// Tag the size to the end. Only when not using GZIP envelope.
-		REBYTE out_size[sizeof(REBCNT)];
-		REBCNT_To_Bytes(out_size, (REBCNT)in_len);
-		Append_Series(output, (REBYTE*)out_size, sizeof(REBCNT));
-	}
 
-	if (SERIES_AVAIL(output) > 1024) // Is there wasted space?
+	if (SERIES_AVAIL(output) > 4096)  // Is there wasted space?
 		output = Copy_Series(output); // Trim it down if too big. !!! Revisit this based on mem alloc alg.
 
 	deflateEnd(&stream);
@@ -171,19 +174,7 @@ void Trap_ZStream_Error(z_stream *stream, int err, REBOOL while_compression)
 	REBINT err;
 
 	if (len < 0 || (index + len > BIN_LEN(input))) len = BIN_LEN(input) - index;
-	if (limit > 0) {
-		size = limit;
-	} else if (windowBits < 0) {
-		// limit was not specified, but data are supposed to be raw DEFLATE data
-		// max teoretic DEFLATE ration is 1032:1, but that is quite unrealistic
-		// it will be more around 3:1 or 4:1, so 10:1 could be enough for automatic setup.
-		size = 10 * (REBCNT)len; //@@ fix me, if you don't agree with above claim
-	} else {
-		// Get the uncompressed size from last 4 source data bytes.
-		if (len < 4) Trap0(RE_PAST_END); // !!! better msg needed
-		size = cast(REBU64, Bytes_To_REBCNT(BIN_SKIP(input, index + len) - sizeof(REBCNT)));
-		if (size > (uLongf)len * 14) Trap_Num(RE_SIZE_LIMIT, size); // check for a realistic limit
-	}
+	size = (limit > 0) ? limit : (uLongf)len * 3;
 
 	output = Make_Binary(size);
 
@@ -194,30 +185,40 @@ void Trap_ZStream_Error(z_stream *stream, int err, REBOOL while_compression)
 	stream.total_out = 0;
 
 	stream.avail_in = len;
-	stream.next_in = cast(const Bytef*, BIN_HEAD(input) + index);
+	stream.next_in = cast(const Bytef*, BIN_SKIP(input, index));
 
 	err = inflateInit2(&stream, windowBits);
 	if (err != Z_OK) Trap_ZStream_Error(&stream, err, FALSE);
 
-	stream.avail_out = size;
+	stream.avail_out = SERIES_AVAIL(output);
 	stream.next_out = BIN_HEAD(output);
 
 	for(;;) {
 		err = inflate(&stream, Z_NO_FLUSH);
-		if (err == Z_STREAM_END || stream.total_out == size)
-			break; // Finished. (and buffer was big enough)
+		if (err == Z_STREAM_END || (limit && stream.total_out >= limit))
+			break; // Finished or we have enough data.
 		//printf("err: %i size: %i avail_out: %i total_out: %i\n", err, size, stream.avail_out, stream.total_out);
-		if(err != Z_OK) Trap_ZStream_Error(&stream, err, FALSE);
-		//@@: may need to resize the destination buffer! But...
-		//@@: so far let's expect that size is always correct
-		//@@: and introduce self expanding buffers in compression port implementation
+		if (err != Z_OK) Trap_ZStream_Error(&stream, err, FALSE);
+		if (stream.avail_out == 0) {
+			// expand output buffer...
+			SERIES_TAIL(output) = stream.total_out;
+			Expand_Series(output, AT_TAIL, len);
+			stream.next_out = BIN_SKIP(output, stream.total_out);
+			stream.avail_out = SERIES_REST(output) - stream.total_out;
+		}
 	}
 	//printf("total_out: %i\n", stream.total_out);
 	inflateEnd(&stream);
 
+	if (limit && stream.total_out > limit) {
+		stream.total_out = limit;
+	}
 	SET_STR_END(output, stream.total_out);
 	SERIES_TAIL(output) = stream.total_out;
 
+	if (SERIES_AVAIL(output) > 4096) // Is there wasted space?
+		output = Copy_Series(output); // Trim it down if too big. !!! Revisit this based on mem alloc alg.
+
 	return output;
 }
 

src/mezz/codec-gzip.reb

@@ -13,11 +13,11 @@ register-codec [
 	suffixes: [%.gz]
 	decode: function[data [binary!]] [
 		if verbose > 0 [ identify data ]
-		decompress/gzip data
+		decompress data 'gzip
 	]
 
 	encode: function [data [binary!]][
-		compress/gzip/level data level
+		compress/level data 'gzip level
 	]
 
 	identify: function [data [binary!]][

src/mezz/codec-pdf.reb

@@ -301,7 +301,7 @@ decompress-obj: func[obj [object!] /local p][
 		switch obj/spec/Filter [
 			FlateDecode [
 				try [
-					obj/data: decompress/deflate skip obj/data 2
+					obj/data: decompress skip obj/data 2 'deflate
 					obj/spec/Filter: none
 					obj/spec/Length: length? obj/data 
 
@@ -431,7 +431,7 @@ emit-stream: func[obj [object!] /local data][
 		300 > length? data ; if data are small enough
 	][
 		obj/spec/Filter: 'FlateDecode
-		data: compress/zlib obj/data
+		data: compress obj/data 'zlib
 	]
 	unless binary? data [
 		; make sure that data are in binary, so the length is correct!

src/mezz/codec-swf.reb

@@ -1253,7 +1253,7 @@ import module [
 				; not a SWF file
 				return none
 			]
-			fileSize: binary/read bin 'UI32LE
+			fileSize: (binary/read bin 'UI32LE) - 8
 			if verbose > 0 [
 				print [
 					"SWF file version:" version
@@ -1263,17 +1263,17 @@ import module [
 						90 "compressed using LZMA"
 					] compression
 					lf
-					"Data size:" fileSize - 8 "bytes"
+					"Data size:" fileSize "bytes"
 				]
 			]
 
 			switch compression [
 				67 [
-					binary/init bin decompress/zlib/size bin/buffer fileSize - 8
+					binary/init bin decompress/size bin/buffer 'zlib fileSize
 				]
 				90 [
 					packed: binary/read bin 'UI32LE
-					binary/init bin decompress/lzma/size bin/buffer fileSize - 8
+					binary/init bin decompress/size bin/buffer 'lzma fileSize
 				]
 			]
 

src/mezz/codec-zip.reb

@@ -207,12 +207,11 @@ register-codec [
 			]
 			method: either any [
 				none? data
-				lesser-or-equal? size: length? data length? compressed-data: compress data
+				lesser-or-equal? size: length? data length? compressed-data: compress data 'deflate
 			][
 				compressed-data: data
 				0 ;store
 			][
-				compressed-data: copy/part skip compressed-data 2 skip tail compressed-data -8 ;@@ FIXME once compress/zlib will be fixed!
 				8 ;deflate
 			]
 
@@ -347,10 +346,10 @@ register-codec [
 
 		switch/default method [
 			8  [ ;- deflate
-				output: decompress/deflate/size data unc-size
+				output: decompress/size data 'deflate unc-size
 			]
 			14 [ ;- LZMA
-				output: decompress/lzma/part/size (skip data 4) cmp-size unc-size
+				output: decompress/part/size (skip data 4) 'lzma cmp-size unc-size
 			]
 			0  [ ;- store
 				output: copy/part data cmp-size

src/mezz/mezz-debug.reb

@@ -65,7 +65,7 @@ speed?: function [
 			insert/dup tmp "abcdefghij" 50000
 			loop 10 [
 				random tmp
-				decompress compress tmp
+				decompress compress tmp 'zlib 'zlib
 			]
 			calc: [(length? tmp) * 10 / secs / 1900]
 		][

src/mezz/mezz-save.reb

@@ -102,7 +102,7 @@ save: function [
 		; Checksum uncompressed data, if requested
 		tmp: find header-data 'checksum [change next tmp checksum data: to-binary data 'sha1]
 		; Compress the data if necessary
-		compress [data: lib/compress data]
+		compress [data: lib/compress data 'zlib]
 		; File content is encoded as base-64:
 		method = 'script [data: mold64 data]
 		not binary? data [data: to-binary data]

src/mezz/prot-http.reb

@@ -640,8 +640,8 @@ decode-result: func[
 		either find ["gzip" "deflate"] encoding [
 			try/except [
 				result/2: switch encoding [
-					"gzip"    [ decompress/gzip    result/2 ]
-					"deflate" [ decompress/deflate result/2 ]
+					"gzip"    [ decompress result/2 'gzip]
+					"deflate" [ decompress result/2 'deflate]
 				]
 			][
 				sys/log/info 'HTTP ["Failed to decode data using:^[[22m" encoding]