| Author |
Message |
< Erlang patches mailing list ~ Patch to add zlib:crc32_combine/4 and zlib:adler32_combine/4 |
| mdempsky |
 Posted: Tue Nov 06, 2007 8:48 pm |
 |
|
|
User
Joined: 24 Sep 2007
Posts: 118
|
I wrote an Erlang module to concatenate zlib compressed data, but to
generate proper zlib and gzip trailers, I need to compute the adler32
and crc32 checksums. adler32 has a simple algebraic structure that
makes it easy to implement in just a few lines of Erlang code (give it
a shot---it's fun  , but combining crc32 checksums involves
computing x*2^n in a Galois field (which isn't so fun in Erlang).
Anyways, I noticed the zlib C library already provides functions for
these, but they're not exposed by the Erlang interface... so I wrote
the patch below. (I'm mainly interested in crc32_combine, but I
thought it would be nice to have adler32_combine for completeness.)
--- erts/emulator/drivers/common/zlib_drv.c.orig 2007-11-06
11:44:12.000000000 -0800
+++ erts/emulator/drivers/common/zlib_drv.c 2007-11-06 11:30:11.000000000 -0800
@@ -55,6 +55,9 @@
#define ADLER32_1 21
#define ADLER32_2 22
+#define CRC32_COMBINE 23
+#define ADLER32_COMBINE 24
+
#define DEFAULT_BUFSZ 4000
static int zlib_init(void);
@@ -597,6 +600,26 @@
adler = adler32(adler, buf+4, len-4);
return zlib_value(adler, rbuf, rlen);
}
+
+ case CRC32_COMBINE: {
+ uLong crc1, crc2, len2;
+ if (len != 12) goto badarg;
+ crc1 = i32(buf);
+ crc2 = i32(buf+4);
+ len2 = i32(buf+ ;
+ crc1 = crc32_combine(crc1, crc2, len2);
+ return zlib_value(crc1, rbuf, rlen);
+ }
+
+ case ADLER32_COMBINE: {
+ uLong adler1, adler2, len2;
+ if (len != 12) goto badarg;
+ adler1 = i32(buf);
+ adler2 = i32(buf+4);
+ len2 = i32(buf+;
+ adler1 = adler32_combine(adler1, adler2, len2);
+ return zlib_value(adler1, rbuf, rlen);
+ }
}
badarg:
--- lib/kernel/src/zlib.erl.orig 2007-11-06 11:20:14.000000000 -0800
+++ lib/kernel/src/zlib.erl 2007-11-06 11:43:28.000000000 -0800
@@ -25,6 +25,7 @@
inflateSync/1,inflateReset/1,inflate/2,inflateEnd/1,
setBufSize/2,getBufSize/1,
crc32/1,crc32/2,crc32/3,adler32/2,adler32/3,getQSize/1,
+ crc32_combine/4,adler32_combine/4,
compress/1,uncompress/1,zip/1,unzip/1,
gzip/1,gunzip/1]).
@@ -108,6 +109,9 @@
-define(ADLER32_1, 21).
-define(ADLER32_2, 22).
+-define(CRC32_COMBINE, 23).
+-define(ADLER32_COMBINE, 24).
+
%% open a z_stream
open() ->
open_port({spawn, zlib_drv}, [binary]).
@@ -213,6 +217,16 @@
adler32(_Z, _Adler, _Binary) ->
erlang:error(badarg).
+crc32_combine(Z, CRC1, CRC2, Len2) when is_integer(CRC1),
is_integer(CRC2), is_integer(Len2) ->
+ call(Z, ?CRC32_COMBINE, <<CRC1:32, CRC2:32, Len2:32>>);
+crc32_combine(_Z, _CRC1, _CRC2, _Len2) ->
+ erlang:error(badarg).
+
+adler32_combine(Z, Adler1, Adler2, Len2) when is_integer(Adler1),
is_integer(Adler2), is_integer(Len2) ->
+ call(Z, ?ADLER32_COMBINE, <<Adler1:32, Adler2:32, Len2:32>>);
+adler32_combine(_Z, _Adler1, _Adler2, _Len2) ->
+ erlang:error(badarg).
+
getQSize(Z) ->
call(Z, ?GET_QSIZE, []).
_______________________________________________
erlang-patches mailing list
erlang-patches@erlang.org
http://www.erlang.org/mailman/listinfo/erlang-patches
Post recived from mailinglist |
|
|
| Back to top |
 
|
| mdempsky |
| Posted: Tue Nov 13, 2007 7:14 pm |
|
|
|
User
Joined: 24 Sep 2007
Posts: 118
|
On 11/6/07, Matthew Dempsky <matthew@dempsky.org> wrote:
> I wrote an Erlang module to concatenate zlib compressed data, but to
> generate proper zlib and gzip trailers, I need to compute the adler32
> and crc32 checksums.
Also, for what it's worth, this code has been released as part of our
eswf project on google code. You can find the module source at:
http://eswf.googlecode.com/svn/trunk/src/zipchunk.erl
Right now the public API only supports zlib encoding, but the code
internally could handle raw deflate and gzip encoding without much
effort. Support for crc32 checksum joining is currently using a slow
workaround until the otp patch I submitted is accepted.
As such, I'm interested in feedback on my patch. Will it be merged
into R12B? Is there anything else I can do to help ensure this
happens?
Thanks.
_______________________________________________
erlang-patches mailing list
erlang-patches@erlang.org
http://www.erlang.org/mailman/listinfo/erlang-patches
Post recived from mailinglist |
|
|
| Back to top |
|
| mdempsky |
| Posted: Wed Nov 14, 2007 12:56 am |
|
|
|
User
Joined: 24 Sep 2007
Posts: 118
|
Some further testing with the zipchunk module revealed that the
zlib_drv does not properly flush the zlib buffers. In particular, a
call to zlib:deflate(Z, Binary, full) can leave part of Binary still
in Z's internal state.
The patch below properly follows the advice of zlib.h that ``If
deflate returns with avail_out == 0, this function must be called
again with the same value of the flush parameter and more output space
(updated avail_out).''
Also, Z_STREAM_END is only returned when Z_FINISH is given.
--- otp_src_R11B-5/erts/emulator/drivers/common/zlib_drv.c.orig 2007-11-06
11:44:12.000000000 -0800
+++ otp_src_R11B-5/erts/emulator/drivers/common/zlib_drv.c 2007-11-13
16:39:48.000000000 -0800
@@ -323,12 +323,15 @@
}
}
} else {
- while (d->s.avail_out < d->binsz) {
+ while (d->s.avail_out == 0) {
zlib_output(d);
- if (res == Z_STREAM_END) {
- break;
+ if ((res = deflate(&d->s, flush)) < 0) {
+ return res;
}
}
+ if (d->s.avail_out < d->binsz) {
+ zlib_output(d);
+ }
}
}
return res;
_______________________________________________
erlang-patches mailing list
erlang-patches@erlang.org
http://www.erlang.org/mailman/listinfo/erlang-patches
Post recived from mailinglist |
|
|
| Back to top |
|
| Guest |
| Posted: Wed Nov 14, 2007 1:55 pm |
|
|
|
Guest
|
"Matthew Dempsky" <matthew@dempsky.org> writes:
> As such, I'm interested in feedback on my patch. Will it be merged
> into R12B?
Barred any unforseen problems, you patches should make it to the R12B release.
/Bjorn
--
Bj |
|
|
| Back to top |
|
| mdempsky |
| Posted: Wed Nov 14, 2007 5:48 pm |
|
|
|
User
Joined: 24 Sep 2007
Posts: 118
|
|
| Back to top |
|
|
|
All times are GMT
|
|
You cannot post new topics in this forum
You cannot reply to topics in this forum
You cannot edit your posts in this forum
You cannot delete your posts in this forum
You cannot vote in polls in this forum
You cannot attach files in this forum
You cannot download files in this forum
|
|
|
Digg It
Del.icio.us
Reddit
Facebook
Stumble Upon
Technorati
