From 4a0a5711b15eb792154cc58d960f2e8a880727af Mon Sep 17 00:00:00 2001 From: "Marshall T. Vandegrift" Date: Tue, 9 Dec 2008 08:54:43 -0500 Subject: [PATCH] Integrated LZX compression code. --- setup.py | 4 +- src/calibre/ebooks/lit/lzxcomp.py | 34 +- src/calibre/utils/lzx/lzc.c | 389 +++++++ src/calibre/utils/lzx/lzc.h | 60 ++ src/calibre/utils/lzx/lzxc.c | 1259 +++++++++++++++++++++++ src/calibre/utils/lzx/lzxc.h | 57 + src/calibre/utils/lzx/lzxd.c | 2 +- src/calibre/utils/lzx/{lzx.h => lzxd.h} | 0 src/calibre/utils/lzx/lzxmodule.c | 16 +- 9 files changed, 1794 insertions(+), 27 deletions(-) create mode 100644 src/calibre/utils/lzx/lzc.c create mode 100644 src/calibre/utils/lzx/lzc.h create mode 100644 src/calibre/utils/lzx/lzxc.c create mode 100644 src/calibre/utils/lzx/lzxc.h rename src/calibre/utils/lzx/{lzx.h => lzxd.h} (100%) diff --git a/setup.py b/setup.py index aa72b46f00..0465795970 100644 --- a/setup.py +++ b/setup.py @@ -374,7 +374,9 @@ if __name__ == '__main__': ext_modules = [ Extension('calibre.plugins.lzx', sources=['src/calibre/utils/lzx/lzxmodule.c', - 'src/calibre/utils/lzx/lzxd.c'], + 'src/calibre/utils/lzx/lzxd.c', + 'src/calibre/utils/lzx/lzc.c', + 'src/calibre/utils/lzx/lzxc.c'], include_dirs=['src/calibre/utils/lzx']), Extension('calibre.plugins.msdes', diff --git a/src/calibre/ebooks/lit/lzxcomp.py b/src/calibre/ebooks/lit/lzxcomp.py index 4f147a90a1..1a3f944c89 100644 --- a/src/calibre/ebooks/lit/lzxcomp.py +++ b/src/calibre/ebooks/lit/lzxcomp.py @@ -3,11 +3,11 @@ import sys import os from cStringIO import StringIO from ctypes import * +from calibre import plugins +_lzx, LzxError = plugins['lzx'] __all__ = ['Compressor'] -liblzxcomp = cdll.LoadLibrary('liblzxcomp.so') - class lzx_data(Structure): pass @@ -25,32 +25,22 @@ class lzx_results(Structure): # lzx_at_eof_t at_eof, # lzx_put_bytes_t put_bytes, void *put_bytes_arg, # lzx_mark_frame_t mark_frame, void *mark_frame_arg); -lzx_init = liblzxcomp.lzx_init -lzx_init.restype = c_int -lzx_init.argtypes = [POINTER(POINTER(lzx_data)), c_int, - lzx_get_bytes_t, c_voidp, - lzx_at_eof_t, - lzx_put_bytes_t, c_voidp, - lzx_mark_frame_t, c_voidp] +lzx_init_t = CFUNCTYPE( + c_int, POINTER(POINTER(lzx_data)), c_int, lzx_get_bytes_t, c_voidp, + lzx_at_eof_t, lzx_put_bytes_t, c_voidp, lzx_mark_frame_t, c_voidp) +lzx_init = lzx_init_t(_lzx._lzxc_init) # void lzx_reset(lzx_data *lzxd); -lzx_reset = liblzxcomp.lzx_reset -lzx_reset.restype = None -lzx_reset.argtypes = [POINTER(lzx_data)] +lzx_reset_t = CFUNCTYPE(None, POINTER(lzx_data)) +lzx_reset = lzx_reset_t(_lzx._lzxc_reset) # int lzx_compress_block(lzx_data *lzxd, int block_size, int subdivide); -lzx_compress_block = liblzxcomp.lzx_compress_block -lzx_compress_block.restype = c_int -lzx_compress_block.argtypes = [POINTER(lzx_data), c_int, c_int] +lzx_compress_block_t = CFUNCTYPE(c_int, POINTER(lzx_data), c_int, c_int) +lzx_compress_block = lzx_compress_block_t(_lzx._lzxc_compress_block) # int lzx_finish(struct lzx_data *lzxd, struct lzx_results *lzxr); -lzx_finish = liblzxcomp.lzx_finish -lzx_finish.restype = c_int -lzx_finish.argtypes = [POINTER(lzx_data), POINTER(lzx_results)] - - -class LzxError(Exception): - pass +lzx_finish_t = CFUNCTYPE(c_int, POINTER(lzx_data), POINTER(lzx_results)) +lzx_finish = lzx_finish_t(_lzx._lzxc_finish) class Compressor(object): diff --git a/src/calibre/utils/lzx/lzc.c b/src/calibre/utils/lzx/lzc.c new file mode 100644 index 0000000000..4ce6f24227 --- /dev/null +++ b/src/calibre/utils/lzx/lzc.c @@ -0,0 +1,389 @@ +/* + File lz_nonslide.c, part of lzxcomp library + Copyright (C) 2002 Matthew T. Russotto + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; version 2.1 only + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +/* + * Document here + */ +#include +#include +#include +#include +#include +#ifdef DEBUG_PERF +#include +#include +#endif +#include + +#define MAX_MATCH 253 +#define MIN_MATCH 2 + +void lz_init(lz_info *lzi, int wsize, int max_dist, + int max_match, int min_match, + int frame_size, + get_chars_t get_chars, + output_match_t output_match, + output_literal_t output_literal, void *user_data) +{ + /* the reason for the separate max_dist value is LZX can't reach the + first three characters in its nominal window. But using a smaller + window results in inefficiency when dealing with reset intervals + which are the length of the nominal window */ + + lzi->wsize = wsize; + if (max_match > wsize) + lzi->max_match = wsize; + else + lzi->max_match = max_match; + + lzi->min_match = min_match; + if (lzi->min_match < 3) lzi->min_match = 3; + + lzi->max_dist = max_dist; + lzi->block_buf_size = wsize + lzi->max_dist; + lzi->block_buf = malloc(lzi->block_buf_size); + lzi->block_bufe = lzi->block_buf + lzi->block_buf_size; + assert(lzi->block_buf != NULL); + + lzi->cur_loc = 0; + lzi->block_loc = 0; + lzi->chars_in_buf = 0; + lzi->eofcount = 0; + lzi->get_chars = get_chars; + lzi->output_match = output_match; + lzi->output_literal = output_literal; + lzi->user_data = user_data; + lzi->frame_size = frame_size; + lzi->lentab = calloc(lzi->block_buf_size + 1, sizeof(int)); + lzi->prevtab = calloc(lzi->block_buf_size + 1, sizeof(u_char *)); + lzi->analysis_valid = 0; +} + +void lz_release(lz_info *lzi) +{ + free(lzi->block_buf); + free(lzi->lentab); + free(lzi->prevtab); +} + +void lz_reset(lz_info *lzi) +{ + int residual = lzi->chars_in_buf - lzi->block_loc; + memmove(lzi->block_buf, lzi->block_buf + lzi->block_loc, residual); + lzi->chars_in_buf = residual; + lzi->block_loc = 0; + lzi->analysis_valid = 0; +} + +#ifdef LZNONSLIDE_MAIN +typedef struct lz_user_data +{ + FILE *infile; + FILE *outfile; + int R0, R1, R2; +} lz_user_data; + +int tmp_get_chars(lz_info *lzi, int n, u_char *buf) +{ + lz_user_data *lzud = (lz_user_data *)lzi->user_data; + return fread(buf, 1, n, lzud->infile); +} + +int tmp_output_match(lz_info *lzi, int match_pos, int match_len) +{ + lz_user_data *lzud = (lz_user_data *)lzi->user_data; + int mod_match_loc; + + mod_match_loc = match_pos; + + fprintf(lzud->outfile, "(%d, %d)(%d)\n", match_pos, match_len, mod_match_loc); + return 0; +} + +void tmp_output_literal(lz_info *lzi, u_char ch) +{ + lz_user_data *lzud = (lz_user_data *)lzi->user_data; + fprintf(lzud->outfile, "'%c'", ch); +} + +int main(int argc, char *argv[]) +{ + int wsize = atoi(argv[1]); + lz_info lzi; + lz_user_data lzu = {stdin, stdout, 1, 1, 1}; + + lz_init(&lzi, wsize, wsize, MAX_MATCH, MIN_MATCH, 8192, tmp_get_chars, tmp_output_match, tmp_output_literal,&lzu); + lz_compress(&lzi); + return 0; +} +#endif + +__inline__ int lz_left_to_process(lz_info *lzi) +{ + return lzi->chars_in_buf - lzi->block_loc; +} + +static void +fill_blockbuf(lz_info *lzi, int maxchars) +{ + int toread; + u_char *readhere; + int nread; + + if (lzi->eofcount) return; + maxchars -= lz_left_to_process(lzi); + toread = lzi->block_buf_size - lzi->chars_in_buf; + if (toread > maxchars) toread = maxchars; + readhere = lzi->block_buf + lzi->chars_in_buf; + nread = lzi->get_chars(lzi, toread, readhere); + lzi->chars_in_buf += nread; + if (nread != toread) + lzi->eofcount++; +} + +static void lz_analyze_block(lz_info *lzi) +{ + int *lentab, *lenp; + u_char **prevtab, **prevp; + u_char *bbp, *bbe; + u_char *chartab[256]; + u_char *cursor; + int prevlen; + int ch; + int maxlen; + long wasinc; + int max_dist = lzi->max_dist; +#ifdef DEBUG_ANALYZE_BLOCK + static short n = 0; +#endif +#ifdef DEBUG_PERF + struct rusage innerloop; + struct timeval innertime, tmptime; + struct rusage outerloop; + struct timeval outertime; + struct rusage initialloop; + struct timeval initialtime; + struct rusage totalloop; + struct timeval totaltime; +#endif + +#ifdef DEBUG_ANALYZE_BLOCK + fprintf(stderr, "Analyzing block %d, cur_loc = %06x\n", n, lzi->cur_loc); +#endif + memset(chartab, 0, sizeof(chartab)); + prevtab = prevp = lzi->prevtab; + lentab = lenp = lzi->lentab; + memset(prevtab, 0, sizeof(*prevtab) * lzi->chars_in_buf); + memset(lentab, 0, sizeof(*lentab) * lzi->chars_in_buf); +#ifdef DEBUG_PERF + memset(&innertime, 0, sizeof(innertime)); + memset(&outertime, 0, sizeof(outertime)); + getrusage(RUSAGE_SELF, &initialloop); + totalloop = initialloop; +#endif + bbp = lzi->block_buf; + bbe = bbp + lzi->chars_in_buf; + while (bbp < bbe) { + if (chartab[ch = *bbp]) { + *prevp = chartab[ch]; + *lenp = 1; + } + chartab[ch] = bbp; + bbp++; + prevp++; + lenp++; + } +#ifdef DEBUG_PERF + initialtime = initialloop.ru_utime; + getrusage(RUSAGE_SELF, &initialloop); + timersub(&initialloop.ru_utime, &initialtime, &initialtime); +#endif + wasinc = 1; + for (maxlen = 1; wasinc && (maxlen < lzi->max_match); maxlen++) { +#ifdef DEBUG_PERF + getrusage(RUSAGE_SELF, &outerloop); +#endif + bbp = bbe - maxlen - 1; + lenp = lentab + lzi->chars_in_buf - maxlen - 1; + prevp = prevtab + lzi->chars_in_buf - maxlen - 1; + wasinc = 0; + while (bbp > lzi->block_buf) { + if (*lenp == maxlen) { +#ifdef DEBUG_PERF + getrusage(RUSAGE_SELF, &innerloop); +#endif + ch = bbp[maxlen]; + cursor = *prevp; + while(cursor && ((bbp - cursor) <= max_dist)) { + prevlen = *(cursor - lzi->block_buf + lentab); + if (cursor[maxlen] == ch) { + *prevp = cursor; + (*lenp)++; + wasinc++; + break; + } + if (prevlen != maxlen) break; + cursor = *(cursor - lzi->block_buf + prevtab); + } +#ifdef DEBUG_PERF + tmptime = innerloop.ru_utime; + getrusage(RUSAGE_SELF, &innerloop); + timersub(&innerloop.ru_utime, &tmptime, &tmptime); + timeradd(&tmptime, &innertime, &innertime); +#endif + } + bbp--; + prevp--; + lenp--; + } +#ifdef DEBUG_PERF + tmptime = outerloop.ru_utime; + getrusage(RUSAGE_SELF, &outerloop); + timersub(&outerloop.ru_utime, &tmptime, &tmptime); + timeradd(&tmptime, &outertime, &outertime); +#endif + // fprintf(stderr, "maxlen = %d, wasinc = %ld\n", maxlen, wasinc); + } +#ifdef DEBUG_PERF + totaltime = totalloop.ru_utime; + getrusage(RUSAGE_SELF, &totalloop); + timersub(&totalloop.ru_utime, &totaltime, &totaltime); + fprintf(stderr, "Time spend in initial loop = %f\n", initialtime.tv_sec + initialtime.tv_usec/(double)1E6); + fprintf(stderr, "Time spend in outer loop = %f\n", outertime.tv_sec + outertime.tv_usec/(double)1E6); + fprintf(stderr, "Time spend in inner loop = %f\n", innertime.tv_sec + innertime.tv_usec/(double)1E6); + fprintf(stderr, "Time spend in all loops = %f\n", totaltime.tv_sec + totaltime.tv_usec/(double)1E6); +#endif + lzi->analysis_valid = 1; +#ifdef DEBUG_ANALYZE_BLOCK + fprintf(stderr, "Done analyzing block %d, cur_loc = %06x\n", n++, lzi->cur_loc); +#endif +} + +void lz_stop_compressing(lz_info *lzi) +{ + lzi->stop = 1; + /* fprintf(stderr, "Stopping...\n");*/ +} + +int lz_compress(lz_info *lzi, int nchars) +{ + + u_char *bbp, *bbe; + int *lentab, *lenp; + u_char **prevtab, **prevp; + int len; + int holdback; + short trimmed; + + lzi->stop = 0; + while ((lz_left_to_process(lzi) || !lzi->eofcount) && !lzi->stop && nchars > 0) { +#if 1 + if (!lzi->analysis_valid || + (!lzi->eofcount && + ((lzi->chars_in_buf- lzi->block_loc) < nchars))) { + int residual = lzi->chars_in_buf - lzi->block_loc; + int bytes_to_move = lzi->max_dist + residual; + if (bytes_to_move > lzi->chars_in_buf) + bytes_to_move = lzi->chars_in_buf; +#ifdef DEBUG_ANALYZE_BLOCK + fprintf(stderr, "Moving %06x, chars_in_buf %06x, residual = %06x, nchars= %06x block_loc = %06x\n", bytes_to_move, lzi->chars_in_buf, residual, nchars, lzi->block_loc); +#endif + memmove(lzi->block_buf, lzi->block_buf + lzi->chars_in_buf - bytes_to_move, + bytes_to_move); + + lzi->block_loc = bytes_to_move - residual; + lzi->chars_in_buf = bytes_to_move; +#ifdef DEBUG_ANALYZE_BLOCK + fprintf(stderr, "New chars_in_buf %06x, new block_loc = %06x, eof = %1d\n", lzi->chars_in_buf, lzi->block_loc, lzi->eofcount); +#endif + fill_blockbuf(lzi, nchars); +#ifdef DEBUG_ANALYZE_BLOCK + fprintf(stderr, "Really new chars_in_buf %06x, new block_loc = %06x, eof = %1d\n", lzi->chars_in_buf, lzi->block_loc, lzi->eofcount); +#endif + lz_analyze_block(lzi); + } +#else + if (!lzi->analysis_valid || + (lzi->block_loc - lzi->chars_in_buf) == 0) { + lzi->block_loc = 0; + lzi->chars_in_buf = 0; + fill_blockbuf(lzi, nchars); + lz_analyze_block(lzi); + } +#endif + prevtab = prevp = lzi->prevtab + lzi->block_loc; + lentab = lenp = lzi->lentab + lzi->block_loc; + bbp = lzi->block_buf + lzi->block_loc; + holdback = lzi->max_match; + if (lzi->eofcount) holdback = 0; + if (lzi->chars_in_buf < (nchars + lzi->block_loc)) + bbe = lzi->block_buf + lzi->chars_in_buf - holdback; + else + bbe = bbp + nchars; + while ((bbp < bbe) && (!lzi->stop)) { + trimmed = 0; + len = *lenp; + if (lzi->frame_size && (len > (lzi->frame_size - lzi->cur_loc % lzi->frame_size))) { +#ifdef DEBUG_TRIMMING + fprintf(stderr, "Trim for framing: %06x %d %d\n", lzi->cur_loc,len, (lzi->frame_size - lzi->cur_loc % lzi->frame_size)); +#endif + trimmed = 1; + len = (lzi->frame_size - lzi->cur_loc % lzi->frame_size); + } + if (len > nchars) { +#ifdef DEBUG_TRIMMING + fprintf(stderr, "Trim for blocking: %06x %d %d\n", lzi->cur_loc,len, nchars); +#endif + trimmed = 1; + len = nchars; + } + if (len >= lzi->min_match) { +#ifdef LAZY + if ((bbp < bbe -1) && !trimmed && + ((lenp[1] > (len + 1)) /* || ((lenp[1] == len) && (prevp[1] > prevp[0])) */)) { + len = 1; + /* this is the lazy eval case */ + } + else +#endif + if (lzi->output_match(lzi, (*prevp - lzi->block_buf) - lzi->block_loc, + len) < 0) { + // fprintf(stderr, "Match rejected: %06x %d\n", lzi->cur_loc, len); + len = 1; /* match rejected */ + } + } + else + len = 1; + + if (len < lzi->min_match) { + assert(len == 1); + lzi->output_literal(lzi, *bbp); + } + // fprintf(stderr, "len = %3d, *lenp = %3d, cur_loc = %06x, block_loc = %06x\n", len, *lenp, lzi->cur_loc, lzi->block_loc); + bbp += len; + prevp += len; + lenp += len; + lzi->cur_loc += len; + lzi->block_loc += len; + assert(nchars >= len); + nchars -= len; + + } + } + return 0; +} diff --git a/src/calibre/utils/lzx/lzc.h b/src/calibre/utils/lzx/lzc.h new file mode 100644 index 0000000000..a721fede60 --- /dev/null +++ b/src/calibre/utils/lzx/lzc.h @@ -0,0 +1,60 @@ +/* + File lz_nonslide.h, part of lzxcomp library + Copyright (C) 2002 Matthew T. Russotto + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; version 2.1 only + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +typedef struct lz_info lz_info; +typedef int (*get_chars_t)(lz_info *lzi, int n, u_char *buf); +typedef int (*output_match_t)(lz_info *lzi, int match_pos, int match_len); +typedef void (*output_literal_t)(lz_info *lzi, u_char ch); + +struct lz_info +{ + int wsize; /* window size in bytes */ + int max_match; /* size of longest match in bytes */ + int min_match; + u_char *block_buf; + u_char *block_bufe; + int block_buf_size; + int chars_in_buf; + int cur_loc; /* location within stream */ + int block_loc; + int frame_size; + int max_dist; + u_char **prevtab; + int *lentab; + short eofcount; + short stop; + short analysis_valid; + + get_chars_t get_chars; + output_match_t output_match; + output_literal_t output_literal; + void *user_data; +}; + +void lz_init(lz_info *lzi, int wsize, int max_dist, + int max_match, int min_match, + int frame_size, + get_chars_t get_chars, + output_match_t output_match, + output_literal_t output_literal, void *user_data); + +void lz_release(lz_info *lzi); + +void lz_reset(lz_info *lzi); +void lz_stop_compressing(lz_info *lzi); +int lz_left_to_process(lz_info *lzi); /* returns # chars read in but unprocessed */ +int lz_compress(lz_info *lzi, int nchars); diff --git a/src/calibre/utils/lzx/lzxc.c b/src/calibre/utils/lzx/lzxc.c new file mode 100644 index 0000000000..445cf92767 --- /dev/null +++ b/src/calibre/utils/lzx/lzxc.c @@ -0,0 +1,1259 @@ +/* + File lzx_layer.c, part of lzxcomp library + Copyright (C) 2002 Matthew T. Russotto + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; version 2.1 only + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +#include +#include +#include +#include /* for memset on Linux */ +#include +#include + +#include +#include + +/* Force using (actually working) non-sliding version. */ +#define NONSLIDE + +/* these named constants are from the Microsoft LZX documentation */ +#define MIN_MATCH 2 +#define MAX_MATCH 257 +#define NUM_CHARS 256 +#define NUM_PRIMARY_LENGTHS 7 +#define NUM_SECONDARY_LENGTHS 249 + +/* Debugging defines useful during development. All add diagnostic output + at various points in the system */ + +/*#define DEBUG_MATCHES *//* When matches come in from the LZ engine */ +/*#define DEBUG_MATCHES_2 *//* When matches are being output */ +/*#define DEBUG_HUFFMAN *//* When huffman trees are built */ +/*#define DEBUG_ENTROPY *//* In entropy calculation */ +/*#define DEBUG_LZ *//* Uncompressed input reconstructed from + LZ engine */ +/*#define DEBUG_BITBUF *//* Raw output to upper layer */ +/*#define DEBUG_EXTRA_BITS *//* Savings due to extra bits huffman tree */ +/*#define DEBUG_POSITION_SLOT_LOOKUP */ +/*#define DEBUG_TREE_COMPRESSION *//* During RLE compression of trees */ + +/* number of position slots given window_size-5 */ +/* as corrected by Caie */ +short num_position_slots[] = {30, 32, 34, 36, 38, 42, 50}; +unsigned long position_base[51]; +u_char extra_bits[52]; +double rloge2; + +typedef struct ih_elem { + int freq; + short sym; + short pathlength; + struct ih_elem *parent; + struct ih_elem *left; + struct ih_elem *right; +} ih_elem; + +typedef struct h_elem { + int freq; + short sym; + short pathlength; + struct ih_elem *parent; + unsigned short code; +} h_elem; + +typedef struct huff_entry { + short codelength; + unsigned short code; +} huff_entry; + +static int cmp_leaves(const void *in_a, const void *in_b) +{ + const struct h_elem *a = in_a; + const struct h_elem *b = in_b; + + if (!a->freq && b->freq) + return 1; + if (a->freq && !b->freq) + return -1; + + if (a->freq == b->freq) + return a->sym - b->sym; + + return a->freq - b->freq; +} + +static int +cmp_pathlengths(const void *in_a, const void *in_b) +{ + const struct h_elem *a = in_a; + const struct h_elem *b = in_b; + + if (a->pathlength == b->pathlength) +#if 0 + return a->sym - b->sym; +#else + /* see note on canonical pathlengths */ + return b->sym - a->sym; +#endif + return b->pathlength - a->pathlength; +} + +/* standard huffman building algorithm */ +static void +build_huffman_tree(int nelem, int max_code_length, int *freq, huff_entry *tree) +{ + h_elem *leaves = malloc(nelem * sizeof(h_elem)); + ih_elem *inodes; + ih_elem *next_inode; + ih_elem *cur_inode; + h_elem *cur_leaf; + int leaves_left; + int nleaves; + int pathlength; + unsigned short cur_code; + short codes_too_long = 0; + ih_elem *f1, *f2; + int i; + + for (i = 0; i < nelem; i++) { + leaves[i].freq = freq[i]; + leaves[i].sym = i; + leaves[i].pathlength = 0; + } + qsort(leaves, nelem, sizeof(h_elem), cmp_leaves); + for (leaves_left = 0; leaves_left < nelem; leaves_left++) { +#ifdef DEBUG_HUFFMAN + fprintf(stderr, "%3d: %3d '%c'\n", leaves_left, leaves[leaves_left].freq, + leaves[leaves_left].sym); +#endif + if (!leaves[leaves_left].freq) break; + } + nleaves = leaves_left; + + if (nleaves >= 2) { + inodes = malloc((nelem-1) * sizeof(ih_elem)); + do { + if (codes_too_long) { + for (leaves_left = 0; leaves_left < nelem; leaves_left++) { + if (!leaves[leaves_left].freq) break; + if (leaves[leaves_left].freq != 1) { + leaves[leaves_left].freq >>= 1; + codes_too_long = 0; + } + } + assert (!codes_too_long); + } + + cur_leaf = leaves; + next_inode = cur_inode = inodes; + + do { + f1 = f2 = NULL; + if (leaves_left && + ((cur_inode == next_inode) || + (cur_leaf->freq <= cur_inode->freq))) { + f1 = (ih_elem *)cur_leaf++; + leaves_left--; + } + else if (cur_inode != next_inode) { + f1 = cur_inode++; + } + + if (leaves_left && + ((cur_inode == next_inode) || + (cur_leaf->freq <= cur_inode->freq))) { + f2 = (ih_elem *)cur_leaf++; + leaves_left--; + } + else if (cur_inode != next_inode) { + f2 = cur_inode++; + } + +#ifdef DEBUG_HUFFMAN + fprintf(stderr, "%d %d\n", f1, f2); +#endif + if (f1 && f2) { + next_inode->freq = f1->freq + f2->freq; + next_inode->sym = -1; + next_inode->left = f1; + next_inode->right = f2; + next_inode->parent = NULL; + f1->parent = next_inode; + f2->parent = next_inode; + if (f1->pathlength > f2->pathlength) + next_inode->pathlength = f1->pathlength + 1; + else + next_inode->pathlength = f2->pathlength + 1; + if (next_inode->pathlength > max_code_length) { + codes_too_long = 1; + break; + } + next_inode++; + } + } + while (f1 && f2); + } + while (codes_too_long); + +#ifdef DEBUG_HUFFMAN + cur_inode = inodes; + while (cur_inode < next_inode) { + fprintf(stderr, "%d l: %3d%c r: %3d%c freq: %8d\n", + cur_inode - inodes, + (cur_inode->left->sym!=-1)?(((struct h_elem *)cur_inode->left)-leaves):(cur_inode->left-inodes), + (cur_inode->left->sym!=-1)?'l':'i', + (cur_inode->right->sym!=-1)?(((struct h_elem *)cur_inode->right)-leaves):(cur_inode->right-inodes), + (cur_inode->right->sym!=-1)?'l':'i', + (cur_inode->freq) + ); + cur_inode++; + } +#endif + + /* now traverse tree depth-first */ + cur_inode = next_inode - 1; + pathlength = 0; + cur_inode->pathlength = -1; + do { + /* precondition: at unmarked node*/ + if (cur_inode->sym == -1) /*&& (cur_inode->left)*/ { + /* left node of unmarked node is unmarked */ + cur_inode = cur_inode->left; + cur_inode->pathlength = -1; + pathlength++; + } + else { + /* mark node */ + cur_inode->pathlength = pathlength; +#if 0 + if (cur_inode->right) { + /* right node of previously unmarked node is unmarked */ + cur_inode = cur_inode->right; + cur_inode->pathlength = -1; + pathlength++; + } + else +#endif + { + + /* time to come up. Keep coming up until an unmarked node is reached */ + /* or the tree is exhausted */ + do { + cur_inode = cur_inode->parent; + pathlength--; + } + while (cur_inode && (cur_inode->pathlength != -1)); + if (cur_inode) { + /* found unmarked node; mark it and go right */ + cur_inode->pathlength = pathlength; + cur_inode = cur_inode->right; + cur_inode->pathlength = -1; + pathlength++; + /* would be complex if cur_inode could be null here. It can't */ + } + } + } + } + while (cur_inode); + +#ifdef DEBUG_HUFFMAN + cur_inode = inodes; + while (cur_inode < next_inode) { + fprintf(stderr, "%d l: %3d%c r: %3d%c freq: %8d pathlength %4d\n", + cur_inode - inodes, + (cur_inode->left->sym!=-1)?(((struct h_elem *)cur_inode->left)-leaves):(cur_inode->left-inodes), + (cur_inode->left->sym!=-1)?'l':'i', + (cur_inode->right->sym!=-1)?(((struct h_elem *)cur_inode->right)-leaves):(cur_inode->right-inodes), + (cur_inode->right->sym!=-1)?'l':'i', + (cur_inode->freq), + (cur_inode->pathlength) + ); + cur_inode++; + } +#endif + free(inodes); + + /* the pathlengths are already in order, so this sorts by symbol */ + qsort(leaves, nelem, sizeof(h_elem), cmp_pathlengths); + + /** + Microsoft's second condition on its canonical huffman codes is: + + For each level, starting at the deepest level of the tree and then + moving upwards, leaf nodes must start as far left as possible. An + alternative way of stating this constraint is that if any tree node + has children then all tree nodes to the left of it with the same path + length must also have children. + + These 'alternatives' are not equivalent. The latter alternative gives + the common canonical code where the longest code is all zeros. The former + gives an opposite code where the longest code is all ones. Microsoft uses the + former alternative. + **/ + +#if 0 + pathlength = leaves[0].pathlength; + cur_code = 0; + for (i = 0; i < nleaves; i++) { + while (leaves[i].pathlength < pathlength) { + assert(!(cur_code & 1)); + cur_code >>= 1; + pathlength--; + } + leaves[i].code = cur_code; + cur_code++; + } +#else + pathlength = leaves[nleaves-1].pathlength; + assert(leaves[0].pathlength <= 16); /* this method cannot deal with bigger codes, though + the other canonical method can in some cases + (because it starts with zeros ) */ + cur_code = 0; + for (i = nleaves - 1; i >= 0; i--) { + while (leaves[i].pathlength > pathlength) { + cur_code <<= 1; + pathlength++; + } + leaves[i].code = cur_code; + cur_code++; + } +#endif + +#ifdef DEBUG_HUFFMAN + for (i = 0; i < nleaves; i++) { + char code[18]; + int j; + + cur_code = leaves[i].code; + code[leaves[i].pathlength] = 0; + for (j = leaves[i].pathlength-1; j >= 0; j--) { + if (cur_code & 1) code[j] = '1'; + else code[j] = '0'; + cur_code >>= 1; + } + fprintf(stderr, "%3d: %3d %3d %-16.16s '%c'\n", i, leaves[i].freq, leaves[i].pathlength, code, + leaves[i].sym); + } +#endif + } + else if (nleaves == 1) { + /* 0 symbols is OK (not according to doc, but according to Caie) */ + /* but if only one symbol is present, two symbols are required */ + nleaves = 2; + leaves[0].pathlength = leaves[1].pathlength = 1; + if (leaves[1].sym > leaves[0].sym) { + leaves[1].code = 1; + leaves[0].code = 0; + } + else { + leaves[0].code = 1; + leaves[1].code = 0; + } + } + + memset(tree, 0, nelem * sizeof(huff_entry)); + for (i = 0; i < nleaves; i++) { + tree[leaves[i].sym].codelength = leaves[i].pathlength; + tree[leaves[i].sym].code = leaves[i].code; + } + + free(leaves); +} + +/* from Stuart Caie's code -- I'm hoping this code is too small to encumber + this file. If not, you could rip it out and hard-code the tables */ + +static void lzx_init_static(void) +{ + int i, j; + + if (extra_bits[49]) return; + + rloge2 = 1.0/log(2); + for (i=0, j=0; i <= 50; i += 2) { + extra_bits[i] = extra_bits[i+1] = j; /* 0,0,0,0,1,1,2,2,3,3... */ + if ((i != 0) && (j < 17)) j++; /* 0,0,1,2,3,4...15,16,17,17,17,17... */ + } + + for (i=0, j=0; i <= 50; i++) { + position_base[i] = j; /* 0,1,2,3,4,6,8,12,16,24,32,... */ + j += 1 << extra_bits[i]; /* 1,1,1,1,2,2,4,4,8,8,16,16,32,32,... */ + } +} + +struct lzx_data +{ + void *in_arg; + void *out_arg; + void *mark_frame_arg; + lzx_get_bytes_t get_bytes; + lzx_at_eof_t at_eof; + lzx_put_bytes_t put_bytes; + lzx_mark_frame_t mark_frame; + struct lz_info *lzi; + /* a 'frame' is an 0x8000 byte thing. Called that because otherwise + I'd confuse myself overloading 'block' */ + int left_in_frame; + int left_in_block; + int R0, R1, R2; + int num_position_slots; + /* this is the LZX block size */ + int block_size; + int *main_freq_table; + int length_freq_table[NUM_SECONDARY_LENGTHS]; + int aligned_freq_table[LZX_ALIGNED_SIZE]; + uint32_t *block_codes; + uint32_t *block_codesp; + huff_entry *main_tree; + huff_entry length_tree[NUM_SECONDARY_LENGTHS]; + huff_entry aligned_tree[LZX_ALIGNED_SIZE]; + int main_tree_size; + uint16_t bit_buf; + int bits_in_buf; + double main_entropy; + double last_ratio; + uint8_t *prev_main_treelengths; + uint8_t prev_length_treelengths[NUM_SECONDARY_LENGTHS]; + uint32_t len_uncompressed_input; + uint32_t len_compressed_output; + short need_1bit_header; + short subdivide; /* 0 = don't subdivide, 1 = allowed, -1 = requested */ +}; + +static int +lzx_get_chars(lz_info *lzi, int n, u_char *buf) +{ + /* force lz compression to stop after every block */ + int chars_read; + int chars_pad; + + lzx_data *lzud = (lzx_data *)lzi->user_data; +#ifdef OLDFRAMING + if (lzud->subdivide < 0) return 0; + if (n > lzud->left_in_frame) + n = lzud->left_in_frame; + if (n > lzud->left_in_block) + n = lzud->left_in_block; +#endif + chars_read = lzud->get_bytes(lzud->in_arg, n, buf); +#ifdef OLDFRAMING + lzud->left_in_frame -= chars_read; + lzud->left_in_block -= chars_read; +#else + lzud->left_in_frame -= chars_read % LZX_FRAME_SIZE; + if (lzud->left_in_frame < 0) + lzud->left_in_frame += LZX_FRAME_SIZE; +#endif + if ((chars_read < n) && (lzud->left_in_frame)) { + chars_pad = n - chars_read; + if (chars_pad > lzud->left_in_frame) chars_pad = lzud->left_in_frame; + /* never emit a full frame of padding. This prevents silliness when + lzx_compress is called when at EOF but EOF not yet detected */ + if (chars_pad == LZX_FRAME_SIZE) chars_pad = 0; +#ifdef OLDFRAMING + if (chars_pad > lzud->left_in_block) chars_pad = lzud->left_in_block; +#endif + memset(buf + chars_read, 0, chars_pad); + lzud->left_in_frame -= chars_pad; +#ifdef OLDFRAMING + lzud->left_in_block -= chars_pad; +#endif + chars_read += chars_pad; + } + return chars_read; +} + +#ifdef NONSLIDE +static int find_match_at(lz_info *lzi, int loc, int match_len, int *match_locp) +{ + u_char *matchb; + u_char *nmatchb; + u_char *c1, *c2; + int j; + + if (-*match_locp == loc) return -1; + if (loc < match_len) return -1; + + matchb = lzi->block_buf + lzi->block_loc + *match_locp; + nmatchb = lzi->block_buf + lzi->block_loc - loc; + c1 = matchb; + c2 = nmatchb; + for (j = 0; j < match_len; j++) { + if (*c1++ != *c2++) break; + } + if (j == match_len) { +#ifdef DEBUG_MATCHES + fprintf(stderr, "match found %d, old = %d new = %d len = %d\n", lzi->cur_loc, -*match_locp, loc, match_len); +#endif + *match_locp = -loc; + return 0; + } + return -1; +} +#else +static int find_match_at(lz_info *lzi, int loc, int match_len, int *match_locp) +{ + u_char *matchb; + u_char *nmatchb; + u_char *c1, *c2; + int j; + + if (-*match_locp == loc) return -1; + if (loc < match_len) return -1; + + matchb = lzi->slide_bufp + *match_locp; + if (matchb < lzi->slide_buf) matchb += lzi->slide_buf_size; + nmatchb = lzi->slide_bufp - loc; + if (nmatchb < lzi->slide_buf) nmatchb += lzi->slide_buf_size; + c1 = matchb; + c2 = nmatchb; + for (j = 0; j < match_len; j++) { + if (*c1++ != *c2++) break; + if (c1 == lzi->slide_bufe) c1 = lzi->slide_buf; + if (c2 == lzi->slide_bufe) c2 = lzi->slide_buf; + } + if (j == match_len) { +#ifdef DEBUG_MATCHES + fprintf(stderr, "match found %d, old = %d new = %d len = %d\n", lzi->cur_loc, -*match_locp, loc, match_len); +#endif + *match_locp = -loc; + return 0; + } + return -1; +} +#endif +static void check_entropy(lzx_data *lzud, int main_index) +{ + /* entropy = - sum_alphabet P(x) * log2 P(x) */ + /* entropy = - sum_alphabet f(x)/N * log2 (f(x)/N) */ + /* entropy = - 1/N sum_alphabet f(x) * (log2 f(x) - log2 N) */ + /* entropy = - 1/N (sum_alphabet f(x) * log2 f(x)) - sum_alphabet f(x) log2 N */ + /* entropy = - 1/N (sum_alphabet f(x) * log2 f(x)) - log2 N sum_alphabet f(x) */ + /* entropy = - 1/N (sum_alphabet f(x) * log2 f(x)) - N * log2 N */ + + /* entropy = - 1/N ((sum_alphabet f(x) * log2 f(x) ) - N * log2 N) */ + /* entropy = - 1/N ((sum_alphabet f(x) * ln f(x) * 1/ln 2) - N * ln N * 1/ln 2) */ + /* entropy = 1/(N ln 2) (N * ln N - (sum_alphabet f(x) * ln f(x))) */ + /* entropy = 1/(N ln 2) (N * ln N + (sum_alphabet -f(x) * ln f(x))) */ + + /* entropy = 1/(N ln 2) ( sum_alphabet ln N * f(x) + (sum_alphabet -f(x) * ln f(x))) */ + /* entropy = 1/(N ln 2) ( sum_alphabet ln N * f(x) + (-f(x) * ln f(x))) */ + /* entropy = -1/(N ln 2) ( sum_alphabet -ln N * f(x) + (f(x) * ln f(x))) */ + /* entropy = -1/(N ln 2) ( sum_alphabet f(x)(- ln N + ln f(x))) */ + /* entropy = -1/(N ln 2) ( sum_alphabet f(x)(ln f(x)/N)) */ + /* entropy = -1/N ( sum_alphabet (1/(ln 2))f(x)(ln f(x)/N)) */ + /* entropy = -1/N ( sum_alphabet f(x)(log2 f(x)/N)) */ + /* entropy = - ( sum_alphabet f(x)/N(log2 f(x)/N)) */ + /* entropy = - ( sum_alphabet P(x)(log2 P(x))) */ + + + double freq; + double n_ln_n; + double rn_ln2; + double cur_ratio; + int n; + + /* delete old entropy accumulation */ + if (lzud->main_freq_table[main_index] != 1) { + freq = (double)lzud->main_freq_table[main_index]-1; + lzud->main_entropy += freq * log(freq); + } + /* add new entropy accumulation */ + freq = (double)lzud->main_freq_table[main_index]; + lzud->main_entropy -= freq * log(freq); + n = lzud->block_codesp - lzud->block_codes; + + if (((n & 0xFFF) == 0) && (lzud->left_in_block >= 0x1000)) { + n_ln_n = (double)n * log((double)n); + rn_ln2 = rloge2 / (double)n; + cur_ratio = (n * rn_ln2 *(n_ln_n + lzud->main_entropy) + 24 + 3 * 80 + NUM_CHARS + (lzud->main_tree_size-NUM_CHARS)*3 + NUM_SECONDARY_LENGTHS ) / (double)n; +#ifdef DEBUG_ENTROPY + fprintf(stderr, "n = %d\n", n); + fprintf(stderr, "main entropy = %f\n", rn_ln2 *(n_ln_n + lzud->main_entropy) ); + fprintf(stderr, "compression ratio (raw) = %f\n", 100.0 * rn_ln2 *(n_ln_n + lzud->main_entropy) /9.0 ); + fprintf(stderr, "compression ratio (ovh) = %f\n", 100.0 * cur_ratio/9.0); +#endif + if (cur_ratio > lzud->last_ratio) { +#ifdef DEBUG_ENTROPY + fprintf(stderr, "resetting huffman tables at %d\n", n); +#endif + lzud->subdivide = -1; + lz_stop_compressing(lzud->lzi); + } + lzud->last_ratio = cur_ratio; + } +} + +static int +lzx_output_match(lz_info *lzi, int match_pos, int match_len) +{ + lzx_data *lzud = (lzx_data *)lzi->user_data; + uint32_t formatted_offset; + uint32_t position_footer; + uint8_t length_footer; + uint8_t length_header; + uint16_t len_pos_header; + int position_slot; + short btdt; + +#ifdef DEBUG_LZ + { + int i; + int pos; + for (i = 0; i < match_len; i++) { + +#ifdef NONSLIDE + pos = match_pos + lzi->block_loc + i; + fprintf(stderr, "%c", lzi->block_buf[pos]); +#else + pos = match_pos + lzi->front_offset + i; + if (pos > lzi->slide_buf_size) + pos -= lzi->slide_buf_size; + fprintf(stderr, "%c", lzi->slide_buf[pos]); +#endif + } + } +#endif + position_footer = 0; + btdt = 0; + testforr: + if (match_pos == -lzud->R0) { + match_pos = 0; + formatted_offset = 0; + position_slot = 0; + } + else if (match_pos == -lzud->R1) { + lzud->R1 = lzud->R0; + lzud->R0 = -match_pos; + match_pos = 1; + formatted_offset = 1; + position_slot = 1; + } + else if (match_pos == -lzud->R2) { + lzud->R2 = lzud->R0; + lzud->R0 = -match_pos; + match_pos = 2; + formatted_offset = 2; + position_slot = 2; + } + else { + if (!btdt) { + btdt = 1; + if (find_match_at(lzi, lzud->R0, match_len, &match_pos) == 0) + goto testforr; + if (find_match_at(lzi, lzud->R1, match_len, &match_pos) == 0) + goto testforr; + if (find_match_at(lzi, lzud->R2, match_len, &match_pos) == 0) + goto testforr; + } + + formatted_offset = -match_pos + 2; + + if ((match_len < 3) || + ((formatted_offset >= 64) && (match_len < 4)) || + ((formatted_offset >= 2048) && (match_len < 5)) || + ((formatted_offset >= 65536) && (match_len < 6))) { + /* reject matches where extra_bits will likely be bigger than just outputting + literals. The numbers are basically derived through guessing + and trial and error */ + return -1; /* reject the match */ + } + + lzud->R2 = lzud->R1; + lzud->R1 = lzud->R0; + lzud->R0 = -match_pos; + + /* calculate position base using binary search of table; if log2 can be + done in hardware, approximation might work; + trunc(log2(formatted_offset*formatted_offset)) gets either the proper + position slot or the next one, except for slots 0, 1, and 39-49 + + Slots 0-1 are handled by the R0-R1 procedures + + Slots 36-49 (formatted_offset >= 262144) can be found by + (formatted_offset/131072) + 34 == + (formatted_offset >> 17) + 34; + */ + if (formatted_offset >= 262144) { + position_slot = (formatted_offset >> 17) + 34; + } + else { + int left, right, mid; + + left = 3; + right = lzud->num_position_slots - 1; + position_slot = -1; + while (left <= right) { + mid = (left + right)/2; + if ((position_base[mid] <= formatted_offset) && + position_base[mid+1] > formatted_offset) { + position_slot = mid; + break; + } +#if 0 + fprintf(stderr, "BEFORE: %06x %06x %06x %06x\n", + position_base[left], position_base[mid], + formatted_offset, position_base[right]); +#endif + if (formatted_offset > position_base[mid]) + /* too low */ + left = mid + 1; + else /* too high */ + right = mid; +#if 0 + fprintf(stderr, "AFTER : %06x %06x %06x %06x\n", + position_base[left], position_base[mid], + formatted_offset, position_base[right]); +#endif + } +#ifdef DEBUG_POSITION_SLOT_LOOKUP + if (position_slot < 0) { + fprintf(stderr, "lmr npr: %d %d %d %d\n", left, mid, right, lzud->num_position_slots); + fprintf(stderr, "AFTER : %07d %07d %07d %07d\n", + position_base[left], position_base[mid], + formatted_offset, position_base[right]); + fprintf(stderr, "(%d, %d, %d, %d, %d)\n", match_pos, match_len, formatted_offset, position_slot, position_footer); + } +#endif + assert(position_slot >= 0); + /* FIXME precalc extra_mask table */ + } + position_footer = ((1UL << extra_bits[position_slot]) - 1) & formatted_offset; + } +#ifdef DEBUG_MATCHES +#ifdef NONSLIDE + fprintf(stderr, "(%08x, %d, %d, %d, %d, %d)\n", lzud->lzi->cur_loc , match_pos, match_len, formatted_offset, position_slot, position_footer); +#else + fprintf(stderr, "(%08x, %d, %d, %d, %d, %d)\n", lzud->lzi->cur_loc - lzud->lzi->chars_in_match , match_pos, match_len, formatted_offset, position_slot, position_footer); +#endif +#endif + /* match length = 8 bits */ + /* position_slot = 6 bits */ + /* position_footer = 17 bits */ + /* total = 31 bits */ + /* plus one to say whether it's a literal or not */ + *lzud->block_codesp++ = 0x80000000 | /* bit 31 in intelligent bit ordering */ + (position_slot << 25) | /* bits 30-25 */ + (position_footer << 8) | /* bits 8-24 */ + (match_len - MIN_MATCH); /* bits 0-7 */ + + if (match_len < (NUM_PRIMARY_LENGTHS + MIN_MATCH)) { + length_header = match_len - MIN_MATCH; + /* length_footer = 255; */ /* not necessary */ + } + else { + length_header = NUM_PRIMARY_LENGTHS; + length_footer = match_len - (NUM_PRIMARY_LENGTHS + MIN_MATCH); + lzud->length_freq_table[length_footer]++; + } + len_pos_header = (position_slot << 3) | length_header; + lzud->main_freq_table[len_pos_header + NUM_CHARS]++; + if (extra_bits[position_slot] >= 3) { + lzud->aligned_freq_table[position_footer & 7]++; + } +#ifndef OLDFRAMING + lzud->left_in_block -= match_len; +#endif + if (lzud->subdivide) + check_entropy(lzud, len_pos_header + NUM_CHARS); + return 0; /* accept the match */ +} + +static void +lzx_output_literal(lz_info *lzi, u_char ch) +{ + lzx_data *lzud = (lzx_data *)lzi->user_data; + +#ifndef OLDFRAMING + lzud->left_in_block--; +#endif + *lzud->block_codesp++ = ch; +#ifdef DEBUG_LZ + fprintf(stderr, "%c", ch); +#endif + lzud->main_freq_table[ch]++; + if (lzud->subdivide) + check_entropy(lzud, ch); +} + +static void lzx_write_bits(lzx_data *lzxd, int nbits, uint32_t bits) +{ + int cur_bits; + int shift_bits; + int rshift_bits; + uint16_t mask_bits; + +#ifdef DEBUG_BITBUF + fprintf(stderr, "WB: %2d %08x\n", nbits, bits); +#endif + cur_bits = lzxd->bits_in_buf; + while ((cur_bits + nbits) >= 16) { + shift_bits = 16 - cur_bits; + rshift_bits = nbits - shift_bits; + if (shift_bits == 16) { + lzxd->bit_buf = (bits>>rshift_bits) & 0xFFFF; + } + else { + mask_bits = (1U << shift_bits) - 1; + lzxd->bit_buf <<= shift_bits; + lzxd->bit_buf |= (bits>>rshift_bits) & mask_bits; + } +#ifdef DEBUG_BITBUF + fprintf(stderr, "WBB: %04x\n", lzxd->bit_buf); +#endif +#ifdef LZX_BIG_ENDIAN + lzxd->bit_buf = ((lzxd->bit_buf & 0xFF)<<8) | (lzxd->bit_buf >> 8); +#endif + lzxd->put_bytes(lzxd->out_arg, sizeof(lzxd->bit_buf), &lzxd->bit_buf); + lzxd->len_compressed_output += sizeof(lzxd->bit_buf); + lzxd->bit_buf = 0; + nbits -= shift_bits; + cur_bits = 0; + } + /* (cur_bits + nbits) < 16. If nbits = 0, we're done. + otherwise move bits in */ + shift_bits = nbits; + mask_bits = (1U << shift_bits) - 1; + lzxd->bit_buf <<= shift_bits; + lzxd->bit_buf |= bits & mask_bits; + cur_bits += nbits; + +#ifdef DEBUG_BITBUF + fprintf(stderr, "OBB: %2d %04x\n", cur_bits, lzxd->bit_buf); +#endif + lzxd->bits_in_buf = cur_bits; +} + +static void lzx_align_output(lzx_data *lzxd) +{ + if (lzxd->bits_in_buf) { + lzx_write_bits(lzxd, 16 - lzxd->bits_in_buf, 0); + } + if (lzxd->mark_frame) + lzxd->mark_frame(lzxd->mark_frame_arg, lzxd->len_uncompressed_input, lzxd->len_compressed_output); +} + +static void +lzx_write_compressed_literals(lzx_data *lzxd, int block_type) +{ + uint32_t *cursor = lzxd->block_codes; + uint32_t *endp = lzxd->block_codesp; + uint16_t position_slot; + uint32_t position_footer; + uint32_t match_len_m2; /* match length minus 2, which is MIN_MATCH */ + uint32_t verbatim_bits; + uint32_t block_code; + uint16_t length_header; + uint16_t length_footer; + uint16_t len_pos_header; + huff_entry *huffe; + int frame_count = (lzxd->len_uncompressed_input % LZX_FRAME_SIZE); + + lzxd->len_uncompressed_input -= frame_count; /* will be added back in later */ + while (cursor < endp) { + block_code = *cursor++; + if (block_code & 0x80000000) { + /* + * 0x80000000 | bit 31 in intelligent bit ordering + * (position_slot << 25) | bits 30-25 + * (position_footer << 8) | bits 8-24 + * (match_len - MIN_MATCH); bits 0-7 + * + */ + + match_len_m2 = block_code & 0xFF; /* 8 bits */ + position_footer = (block_code >> 8)& 0x1FFFF; /* 17 bits */ + position_slot = (block_code >> 25) & 0x3F; /* 6 bits */ + +#ifdef DEBUG_MATCHES_2 + fprintf(stderr, "%08x, %3d %2d %d\n", lzxd->len_uncompressed_input + frame_count, match_len_m2, position_slot, position_footer); +#endif + if (match_len_m2 < NUM_PRIMARY_LENGTHS) { + length_header = match_len_m2; + length_footer = 255; /* personal encoding for NULL */ + } + else { + length_header = NUM_PRIMARY_LENGTHS; + length_footer = match_len_m2 - NUM_PRIMARY_LENGTHS; + } + len_pos_header = (position_slot << 3) | length_header; + huffe = &lzxd->main_tree[len_pos_header+NUM_CHARS]; + lzx_write_bits(lzxd, huffe->codelength, huffe->code); + if (length_footer != 255) { + huffe = &lzxd->length_tree[length_footer]; + lzx_write_bits(lzxd, huffe->codelength, huffe->code); + } + if ((block_type == LZX_ALIGNED_OFFSET_BLOCK) && (extra_bits[position_slot] >= 3)) { + /* aligned offset block and code */ + verbatim_bits = position_footer >> 3; + lzx_write_bits(lzxd, extra_bits[position_slot] - 3, verbatim_bits); + huffe = &lzxd->aligned_tree[position_footer&7]; + lzx_write_bits(lzxd, huffe->codelength, huffe->code); + } + else { + verbatim_bits = position_footer; + lzx_write_bits(lzxd, extra_bits[position_slot], verbatim_bits); + } + frame_count += match_len_m2 + 2; + } + else { + /* literal */ + assert(block_code < NUM_CHARS); + huffe = &lzxd->main_tree[block_code]; + lzx_write_bits(lzxd, huffe->codelength, huffe->code); + frame_count++; + } + if (frame_count == LZX_FRAME_SIZE) { + lzxd->len_uncompressed_input += frame_count; + lzx_align_output(lzxd); + frame_count = 0; + } +#ifdef DEBUG_MATCHES_2 + if (frame_count > LZX_FRAME_SIZE) { + fprintf(stderr, "uncomp_len = %x, frame_count = %x, block_code = %08x, match_len_m2 = %d", lzxd->len_uncompressed_input, frame_count, block_code, match_len_m2); + } +#endif + assert (frame_count < LZX_FRAME_SIZE); + } + lzxd->len_uncompressed_input += frame_count; +} + +static int +lzx_write_compressed_tree(struct lzx_data *lzxd, + struct huff_entry *tree, uint8_t *prevlengths, + int treesize) +{ + u_char *codes; + u_char *runs; + int freqs[LZX_PRETREE_SIZE]; + int cur_run; + int last_len; + huff_entry pretree[20]; + u_char *codep; + u_char *codee; + u_char *runp; + int excess; + int i; + int cur_code; + + codep = codes = malloc(treesize*sizeof(char)); + runp = runs = malloc(treesize*sizeof(char)); + memset(freqs, 0, sizeof(freqs)); + cur_run = 1; + last_len = tree[0].codelength; + for (i = 1; i <= treesize; i++) { + if ((i == treesize) || (tree[i].codelength != last_len)) { + if (last_len == 0) { + while (cur_run >= 20) { + excess = cur_run - 20; + if (excess > 31) excess = 31; + *codep++ = 18; + *runp++ = excess; + cur_run -= excess + 20; + freqs[18]++; + } + while (cur_run >= 4) { + excess = cur_run - 4; + if (excess > 15) excess = 15; + *codep++ = 17; + *runp++ = excess; + cur_run -= excess + 4; + freqs[17]++; + } + while (cur_run > 0) { + *codep = prevlengths[i - cur_run]; + freqs[*codep++]++; + *runp++ = 0; /* not necessary */ + cur_run--; + } + } + else { + while (cur_run >= 4) { + if (cur_run == 4) excess = 0; + else excess = 1; + *codep++ = 19; + *runp++ = excess; + freqs[19]++; + /* right, MS lies again. Code is NOT + prev_len + len (mod 17), it's prev_len - len (mod 17)*/ + *codep = prevlengths[i-cur_run] - last_len; + if (*codep > 16) *codep += 17; + freqs[*codep++]++; + *runp++ = 0; /* not necessary */ + cur_run -= excess+4; + } + while (cur_run > 0) { + *codep = prevlengths[i-cur_run] - last_len; + if (*codep > 16) *codep += 17; + *runp++ = 0; /* not necessary */ + cur_run--; + freqs[*codep++]++; + } + } + if (i != treesize) + last_len = tree[i].codelength; + cur_run = 0; + } + cur_run++; + } + codee = codep; +#ifdef DEBUG_TREE_COMPRESSION + *codep++ = 255; + *runp++ = 255; + fprintf(stderr, "num: len code run\n"); + for (i = 0; i < treesize; i++) { + fprintf(stderr, "%3d: %2d %2d %2d\n", i, tree[i].codelength, codes[i], runs[i]); + } +#endif + /* now create the huffman table and write out the pretree */ + build_huffman_tree(LZX_PRETREE_SIZE, 16, freqs, pretree); + for (i = 0; i < LZX_PRETREE_SIZE; i++) { + lzx_write_bits(lzxd, 4, pretree[i].codelength); + } + codep = codes; + runp = runs; + cur_run = 0; + while (codep < codee) { + cur_code = *codep++; + lzx_write_bits(lzxd, pretree[cur_code].codelength, pretree[cur_code].code); + if (cur_code == 17) { + cur_run += *runp + 4; + lzx_write_bits(lzxd, 4, *runp); + } + else if (cur_code == 18) { + cur_run += *runp + 20; + lzx_write_bits(lzxd, 5, *runp); + } + else if (cur_code == 19) { + cur_run += *runp + 4; + lzx_write_bits(lzxd, 1, *runp); + cur_code = *codep++; + lzx_write_bits(lzxd, pretree[cur_code].codelength, pretree[cur_code].code); + runp++; + } + else { + cur_run++; + } + runp++; + } + free(codes); + free(runs); + return 0; +} + +void +lzx_reset(lzx_data *lzxd) +{ + lzxd->need_1bit_header = 1; + lzxd->R0 = lzxd->R1 = lzxd->R2 = 1; + memset(lzxd->prev_main_treelengths, 0, lzxd->main_tree_size * sizeof(uint8_t)); + memset(lzxd->prev_length_treelengths, 0, NUM_SECONDARY_LENGTHS * sizeof(uint8_t)); + lz_reset(lzxd->lzi); +} + +int lzx_compress_block(lzx_data *lzxd, int block_size, int subdivide) +{ + int i; + uint32_t written_sofar = 0; + int block_type; + long uncomp_bits; + long comp_bits; + long comp_bits_ovh; + long uncomp_length; + + if ((lzxd->block_size != block_size) || (lzxd->block_codes == NULL)) { + if (lzxd->block_codes != NULL) free(lzxd->block_codes); + lzxd->block_size = block_size; + lzxd->block_codes = malloc(block_size * sizeof(uint32_t)); + } + lzxd->subdivide = subdivide?1:0; + + lzxd->left_in_block = block_size; + lzxd->left_in_frame = LZX_FRAME_SIZE; + lzxd->main_entropy = 0.0; + lzxd->last_ratio = 9999999.0; + lzxd->block_codesp = lzxd->block_codes; + + memset(lzxd->length_freq_table, 0, NUM_SECONDARY_LENGTHS * sizeof(int)); + memset(lzxd->main_freq_table, 0, lzxd->main_tree_size * sizeof(int)); + memset(lzxd->aligned_freq_table, 0, LZX_ALIGNED_SIZE * sizeof(int)); + do { + lz_compress(lzxd->lzi, lzxd->left_in_block); + if (lzxd->left_in_frame == 0) + lzxd->left_in_frame = LZX_FRAME_SIZE; + + if ((lzxd->subdivide<0) || !lzxd->left_in_block || + (!lz_left_to_process(lzxd->lzi) && lzxd->at_eof(lzxd->in_arg))) { + /* now one block is LZ-analyzed. */ + /* time to write it out */ + uncomp_length = lzxd->block_size - lzxd->left_in_block - written_sofar; + /* uncomp_length will sometimes be 0 when input length is + an exact multiple of frame size */ + if (uncomp_length == 0) + continue; + if (lzxd->subdivide < 0) { +#ifdef DEBUG_ENTROPY + fprintf(stderr, "subdivided\n"); +#endif + lzxd->subdivide = 1; + } + + if (lzxd->need_1bit_header) { + /* one bit Intel preprocessing header */ + /* always 0 because this implementation doesn't do Intel preprocessing */ + lzx_write_bits(lzxd, 1, 0); + lzxd->need_1bit_header = 0; + } + + /* handle extra bits */ + uncomp_bits = comp_bits = 0; + build_huffman_tree(LZX_ALIGNED_SIZE, 7, lzxd->aligned_freq_table, lzxd->aligned_tree); + for (i = 0; i < LZX_ALIGNED_SIZE; i++) { + uncomp_bits += lzxd->aligned_freq_table[i]* 3; + comp_bits += lzxd->aligned_freq_table[i]* lzxd->aligned_tree[i].codelength; + } + comp_bits_ovh = comp_bits + LZX_ALIGNED_SIZE * 3; + if (comp_bits_ovh < uncomp_bits) + block_type = LZX_ALIGNED_OFFSET_BLOCK; + else + block_type = LZX_VERBATIM_BLOCK; + +#ifdef DEBUG_EXTRA_BITS + fprintf(stderr, "Extra bits uncompressed: %5d compressed: %5d compressed w/overhead %5d gain/loss %5d\n", uncomp_bits, comp_bits, comp_bits_ovh, uncomp_bits - comp_bits_ovh); +#endif + + /* block type */ + lzx_write_bits(lzxd, 3, block_type); + /* uncompressed length */ + lzx_write_bits(lzxd, 24, uncomp_length); + + written_sofar = lzxd->block_size - lzxd->left_in_block; + + /* now write out the aligned offset trees if present */ + if (block_type == LZX_ALIGNED_OFFSET_BLOCK) { + for (i = 0; i < LZX_ALIGNED_SIZE; i++) { + lzx_write_bits(lzxd, 3, lzxd->aligned_tree[i].codelength); + } + } + /* end extra bits */ + build_huffman_tree(lzxd->main_tree_size, LZX_MAX_CODE_LENGTH, + lzxd->main_freq_table, lzxd->main_tree); + build_huffman_tree(NUM_SECONDARY_LENGTHS, 16, + lzxd->length_freq_table, lzxd->length_tree); + + + + /* now write the pre-tree and tree for main 1 */ + lzx_write_compressed_tree(lzxd, lzxd->main_tree, lzxd->prev_main_treelengths, NUM_CHARS); + + /* now write the pre-tree and tree for main 2*/ + lzx_write_compressed_tree(lzxd, lzxd->main_tree + NUM_CHARS, + lzxd->prev_main_treelengths + NUM_CHARS, + lzxd->main_tree_size - NUM_CHARS); + + /* now write the pre tree and tree for length */ + lzx_write_compressed_tree(lzxd, lzxd->length_tree, lzxd->prev_length_treelengths, + NUM_SECONDARY_LENGTHS); + + /* now write literals */ + lzx_write_compressed_literals(lzxd, block_type); + + /* copy treelengths somewhere safe to do delta compression */ + for (i = 0; i < lzxd->main_tree_size; i++) { + lzxd->prev_main_treelengths[i] = lzxd->main_tree[i].codelength; + } + for (i = 0; i < NUM_SECONDARY_LENGTHS; i++) { + lzxd->prev_length_treelengths[i] = lzxd->length_tree[i].codelength; + } + lzxd->main_entropy = 0.0; + lzxd->last_ratio = 9999999.0; + lzxd->block_codesp = lzxd->block_codes; + + memset(lzxd->length_freq_table, 0, NUM_SECONDARY_LENGTHS * sizeof(int)); + memset(lzxd->main_freq_table, 0, lzxd->main_tree_size * sizeof(int)); + memset(lzxd->aligned_freq_table, 0, LZX_ALIGNED_SIZE * sizeof(int)); + } + } + while (lzxd->left_in_block && (lz_left_to_process(lzxd->lzi) || !lzxd->at_eof(lzxd->in_arg))); + return 0; +} + +int lzx_init(struct lzx_data **lzxdp, int wsize_code, + lzx_get_bytes_t get_bytes, void *get_bytes_arg, + lzx_at_eof_t at_eof, + lzx_put_bytes_t put_bytes, void *put_bytes_arg, + lzx_mark_frame_t mark_frame, void *mark_frame_arg) +{ + int wsize; + struct lzx_data *lzxd; + + if ((wsize_code < 15) || (wsize_code > 21)) { + return -1; + } + lzx_init_static(); + + *lzxdp = lzxd = malloc(sizeof(*lzxd)); + if (lzxd == 0) + return -2; + + lzxd->in_arg = get_bytes_arg; + lzxd->out_arg = put_bytes_arg; + lzxd->mark_frame_arg = mark_frame_arg; + lzxd->get_bytes = get_bytes; + lzxd->put_bytes = put_bytes; + lzxd->at_eof = at_eof; + lzxd->mark_frame = mark_frame; + + wsize = 1 << (wsize_code); + + lzxd->bits_in_buf = 0; + lzxd->block_size = 0; + lzxd->block_codes = NULL; + lzxd->num_position_slots = num_position_slots[wsize_code-15]; + lzxd->main_tree_size = (NUM_CHARS + 8 * lzxd->num_position_slots); + + lzxd->main_freq_table = malloc(sizeof(int) * lzxd->main_tree_size); + lzxd->main_tree = malloc(sizeof(huff_entry)* lzxd->main_tree_size); + lzxd->prev_main_treelengths = malloc(sizeof(uint8_t)*lzxd->main_tree_size); + + lzxd->lzi = malloc(sizeof (*lzxd->lzi)); + /* the -3 prevents matches at wsize, wsize-1, wsize-2, all of which are illegal */ + lz_init(lzxd->lzi, wsize, wsize - 3, MAX_MATCH, MIN_MATCH, LZX_FRAME_SIZE, + lzx_get_chars, lzx_output_match, lzx_output_literal,lzxd); + lzxd->len_uncompressed_input = 0; + lzxd->len_compressed_output = 0; + lzx_reset(lzxd); + return 0; +} + +int lzx_finish(struct lzx_data *lzxd, struct lzx_results *lzxr) +{ + /* lzx_align_output(lzxd); Not needed as long as frame padding is in place */ + if (lzxr) { + lzxr->len_compressed_output = lzxd->len_compressed_output; + lzxr->len_uncompressed_input = lzxd->len_uncompressed_input; + } + lz_release(lzxd->lzi); + free(lzxd->lzi); + free(lzxd->prev_main_treelengths); + free(lzxd->main_tree); + free(lzxd->main_freq_table); + if (lzxd->block_codes) { + free(lzxd->block_codes); + } + free(lzxd); + return 0; +} + diff --git a/src/calibre/utils/lzx/lzxc.h b/src/calibre/utils/lzx/lzxc.h new file mode 100644 index 0000000000..32cb1f721a --- /dev/null +++ b/src/calibre/utils/lzx/lzxc.h @@ -0,0 +1,57 @@ +/* + File lzx_compress.h, part of lzxcomp library + Copyright (C) 2002 Matthew T. Russotto + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; version 2.1 only + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if BYTE_ORDER == BIG_ENDIAN +# define LZX_BIG_ENDIAN +#endif + +/* the names of these constants are specific to this library */ +#define LZX_MAX_CODE_LENGTH 16 +#define LZX_FRAME_SIZE 32768 +#define LZX_PRETREE_SIZE 20 +#define LZX_ALIGNED_BITS 3 +#define LZX_ALIGNED_SIZE 8 + +#define LZX_VERBATIM_BLOCK 1 +#define LZX_ALIGNED_OFFSET_BLOCK 2 + +typedef struct lzx_data lzx_data; +typedef int (*lzx_get_bytes_t)(void *arg, int n, void *buf); +typedef int (*lzx_put_bytes_t)(void *arg, int n, void *buf); +typedef void (*lzx_mark_frame_t)(void *arg, uint32_t uncomp, uint32_t comp); +typedef int (*lzx_at_eof_t)(void *arg); + +typedef struct lzx_results +{ + /* add more here? Error codes, # blocks, # frames, etc? */ + long len_compressed_output; + long len_uncompressed_input; +} lzx_results; + +int lzx_init(struct lzx_data **lzxdp, int wsize_code, + lzx_get_bytes_t get_bytes, void *get_bytes_arg, + lzx_at_eof_t at_eof, + lzx_put_bytes_t put_bytes, void *put_bytes_arg, + lzx_mark_frame_t mark_frame, void *mark_frame_arg); + +void lzx_reset(lzx_data *lzxd); + +int lzx_compress_block(lzx_data *lzxd, int block_size, int subdivide); + +int lzx_finish(struct lzx_data *lzxd, struct lzx_results *lzxr); + diff --git a/src/calibre/utils/lzx/lzxd.c b/src/calibre/utils/lzx/lzxd.c index 337af441fd..e683a9ec23 100644 --- a/src/calibre/utils/lzx/lzxd.c +++ b/src/calibre/utils/lzx/lzxd.c @@ -18,7 +18,7 @@ #include #include -#include +#include /* Microsoft's LZX document and their implementation of the * com.ms.util.cab Java package do not concur. diff --git a/src/calibre/utils/lzx/lzx.h b/src/calibre/utils/lzx/lzxd.h similarity index 100% rename from src/calibre/utils/lzx/lzx.h rename to src/calibre/utils/lzx/lzxd.h diff --git a/src/calibre/utils/lzx/lzxmodule.c b/src/calibre/utils/lzx/lzxmodule.c index c45bb22c95..2f72b58ae7 100644 --- a/src/calibre/utils/lzx/lzxmodule.c +++ b/src/calibre/utils/lzx/lzxmodule.c @@ -4,14 +4,15 @@ * Python module C glue code. */ - #include #include -#include +#include +#include static char lzx_doc[] = -"Provide basic LZX decompression using the code from libmspack."; + "Provide basic LZX compression and decompression using the code from\n" + "liblzxcomp and libmspack respectively."; static PyObject *LzxError = NULL; @@ -214,6 +215,15 @@ initlzx(void) LzxError = PyErr_NewException("lzx.LzxError", NULL, NULL); Py_INCREF(LzxError); PyModule_AddObject(m, "LzxError", LzxError); + + PyModule_AddObject(m, "_lzxc_init", + Py_BuildValue("k", (unsigned long)lzx_init)); + PyModule_AddObject(m, "_lzxc_reset", + Py_BuildValue("k", (unsigned long)lzx_reset)); + PyModule_AddObject(m, "_lzxc_compress_block", + Py_BuildValue("k", (unsigned long)lzx_compress_block)); + PyModule_AddObject(m, "_lzxc_finish", + Py_BuildValue("k", (unsigned long)lzx_finish)); return; }