Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-08 18:54:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

Drop the custom lzma module

Browse Source 
Use the builtin one from the python stdlib
This commit is contained in:
Kovid Goyal 2019-11-18 20:26:29 +05:30
parent ceb558f447
commit 113b14e376
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
23 changed files with 13 additions and 7381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
setup/extensions.json
View File

@ -144,12 +144,6 @@
"sip_files": "calibre/utils/imageops/imageops.sip",
"inc_dirs": "calibre/utils/imageops"
},
{
"name": "lzma_binding",
"sources": "calibre_lzma/*.c",
"headers": "calibre_lzma/*.h",
"defines": "_7ZIP_ST"
},
{
"name": "winutil",
"only": "windows",

7
src/calibre/gui2/icon_theme.py
View File

@ -37,7 +37,6 @@ from calibre.utils.icu import numeric_sort_key as sort_key
from calibre.utils.img import image_from_data, Canvas, optimize_png, optimize_jpeg
from calibre.utils.zipfile import ZipFile, ZIP_STORED
from calibre.utils.filenames import atomic_rename
from calibre_lzma.xz import compress, decompress
from polyglot.builtins import iteritems, map, range, reraise, filter, as_bytes, unicode_type
from polyglot import http_client
from polyglot.queue import Queue, Empty
@ -390,7 +389,8 @@ def create_themeball(report, progress=None, abort=None):
return None, None
if progress is not None:
progress(next(num), _('Compressing theme file'))
compress(buf, out, level=9)
import lzma
lzma.compress(buf.getvalue(), format=lzma.FORMAT_XZ, preset=9)
buf = BytesIO()
prefix = report.name
if abort is not None and abort.is_set():
@ -794,8 +794,9 @@ class ChooseTheme(Dialog):
dt = self.downloaded_theme
def commit_changes():
import lzma
dt.seek(0)
f = decompress(dt)
f = BytesIO(lzma.decompress(dt.getvalue()))
f.seek(0)
remove_icon_theme()
install_icon_theme(theme, f)

4
src/calibre/test_build.py
View File

@ -65,8 +65,8 @@ class BuildTest(unittest.TestCase):
self.assertEqual(detector.result['encoding'], 'utf-8')
def test_lzma(self):
from calibre_lzma.xz import test_lzma2
test_lzma2()
import lzma
lzma.open
def test_html5lib(self):
import html5lib.html5parser # noqa

15
src/calibre/utils/rapydscript.py
View File

@ -9,7 +9,6 @@ import os
import re
import subprocess
import sys
from io import BytesIO
from calibre import force_unicode
from calibre.constants import (
@ -30,7 +29,7 @@ def abspath(x):
def update_rapydscript():
from calibre_lzma.xz import compress
import lzma
d = os.path.dirname
base = d(d(d(d(d(abspath(__file__))))))
base = os.path.join(base, 'rapydscript')
@ -39,8 +38,8 @@ def update_rapydscript():
with open(os.path.join(tdir, 'rapydscript.js'), 'rb') as f:
raw = f.read()
path = P(COMPILER_PATH, allow_user_override=False)
with open(path, 'wb') as f:
compress(raw, f, 9)
with lzma.open(path, 'wb', format=lzma.FORMAT_XZ) as f:
f.write(raw)
# }}}
# Compiler {{{
@ -51,7 +50,7 @@ def to_dict(obj):
def compiler():
from calibre_lzma.xz import decompress
import lzma
ans = getattr(compiler, 'ans', None)
if ans is not None:
return ans
@ -62,8 +61,8 @@ def compiler():
from PyQt5.Qt import QApplication, QEventLoop
must_use_qt()
buf = BytesIO()
decompress(P(COMPILER_PATH, data=True, allow_user_override=False), buf)
with lzma.open(P(COMPILER_PATH, allow_user_override=False)) as lzf:
compiler_script = lzf.read().decode('utf-8')
base = base_dir()
rapydscript_dir = os.path.join(base, 'src', 'pyj')
@ -127,7 +126,7 @@ document.title = 'compiler initialized';
QWebEnginePage.__init__(self)
self.errors = []
secure_webengine(self)
script = buf.getvalue().decode('utf-8')
script = compiler_script
script += '\n\n;;\n\n' + vfs_script()
self.scripts().insert(create_script(script, 'rapydscript.js'))
self.setHtml('<p>initialize')

256
src/calibre_lzma/7zTypes.h
View File

@ -1,256 +0,0 @@
/* 7zTypes.h -- Basic types
2013-11-12 : Igor Pavlov : Public domain */
#ifndef __7Z_TYPES_H
#define __7Z_TYPES_H
#ifdef _WIN32
/* #include <windows.h> */
#endif
#include <stddef.h>
#ifndef EXTERN_C_BEGIN
#ifdef __cplusplus
#define EXTERN_C_BEGIN extern "C" {
#define EXTERN_C_END }
#else
#define EXTERN_C_BEGIN
#define EXTERN_C_END
#endif
#endif
EXTERN_C_BEGIN
#define SZ_OK 0
#define SZ_ERROR_DATA 1
#define SZ_ERROR_MEM 2
#define SZ_ERROR_CRC 3
#define SZ_ERROR_UNSUPPORTED 4
#define SZ_ERROR_PARAM 5
#define SZ_ERROR_INPUT_EOF 6
#define SZ_ERROR_OUTPUT_EOF 7
#define SZ_ERROR_READ 8
#define SZ_ERROR_WRITE 9
#define SZ_ERROR_PROGRESS 10
#define SZ_ERROR_FAIL 11
#define SZ_ERROR_THREAD 12
#define SZ_ERROR_ARCHIVE 16
#define SZ_ERROR_NO_ARCHIVE 17
typedef int SRes;
#ifdef _WIN32
/* typedef DWORD WRes; */
typedef unsigned WRes;
#else
typedef int WRes;
#endif
#ifndef RINOK
#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; }
#endif
typedef unsigned char Byte;
typedef short Int16;
typedef unsigned short UInt16;
#ifdef _LZMA_UINT32_IS_ULONG
typedef long Int32;
typedef unsigned long UInt32;
#else
typedef int Int32;
typedef unsigned int UInt32;
#endif
#ifdef _SZ_NO_INT_64
/* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers.
NOTES: Some code will work incorrectly in that case! */
typedef long Int64;
typedef unsigned long UInt64;
#else
#if defined(_MSC_VER) || defined(__BORLANDC__)
typedef __int64 Int64;
typedef unsigned __int64 UInt64;
#define UINT64_CONST(n) n
#else
typedef long long int Int64;
typedef unsigned long long int UInt64;
#define UINT64_CONST(n) n ## ULL
#endif
#endif
#ifdef _LZMA_NO_SYSTEM_SIZE_T
typedef UInt32 SizeT;
#else
typedef size_t SizeT;
#endif
typedef int Bool;
#define True 1
#define False 0
#ifdef _WIN32
#define MY_STD_CALL __stdcall
#else
#define MY_STD_CALL
#endif
#ifdef _MSC_VER
#if _MSC_VER >= 1300
#define MY_NO_INLINE __declspec(noinline)
#else
#define MY_NO_INLINE
#endif
#define MY_CDECL __cdecl
#define MY_FAST_CALL __fastcall
#else
#define MY_NO_INLINE
#define MY_CDECL
#define MY_FAST_CALL
#endif
/* The following interfaces use first parameter as pointer to structure */
typedef struct
{
Byte (*Read)(void *p); /* reads one byte, returns 0 in case of EOF or error */
} IByteIn;
typedef struct
{
void (*Write)(void *p, Byte b);
} IByteOut;
typedef struct
{
SRes (*Read)(void *p, void *buf, size_t *size);
/* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
(output(*size) < input(*size)) is allowed */
} ISeqInStream;
/* it can return SZ_ERROR_INPUT_EOF */
SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size);
SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType);
SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf);
typedef struct
{
size_t (*Write)(void *p, const void *buf, size_t size);
/* Returns: result - the number of actually written bytes.
(result < size) means error */
} ISeqOutStream;
typedef enum
{
SZ_SEEK_SET = 0,
SZ_SEEK_CUR = 1,
SZ_SEEK_END = 2
} ESzSeek;
typedef struct
{
SRes (*Read)(void *p, void *buf, size_t *size); /* same as ISeqInStream::Read */
SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin);
} ISeekInStream;
typedef struct
{
SRes (*Look)(void *p, const void **buf, size_t *size);
/* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
(output(*size) > input(*size)) is not allowed
(output(*size) < input(*size)) is allowed */
SRes (*Skip)(void *p, size_t offset);
/* offset must be <= output(*size) of Look */
SRes (*Read)(void *p, void *buf, size_t *size);
/* reads directly (without buffer). It's same as ISeqInStream::Read */
SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin);
} ILookInStream;
SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size);
SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset);
/* reads via ILookInStream::Read */
SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType);
SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size);
#define LookToRead_BUF_SIZE (1 << 14)
typedef struct
{
ILookInStream s;
ISeekInStream *realStream;
size_t pos;
size_t size;
Byte buf[LookToRead_BUF_SIZE];
} CLookToRead;
void LookToRead_CreateVTable(CLookToRead *p, int lookahead);
void LookToRead_Init(CLookToRead *p);
typedef struct
{
ISeqInStream s;
ILookInStream *realStream;
} CSecToLook;
void SecToLook_CreateVTable(CSecToLook *p);
typedef struct
{
ISeqInStream s;
ILookInStream *realStream;
} CSecToRead;
void SecToRead_CreateVTable(CSecToRead *p);
typedef struct
{
SRes (*Progress)(void *p, UInt64 inSize, UInt64 outSize);
/* Returns: result. (result != SZ_OK) means break.
Value (UInt64)(Int64)-1 for size means unknown value. */
} ICompressProgress;
typedef struct
{
void *(*Alloc)(void *p, size_t size);
void (*Free)(void *p, void *address); /* address can be 0 */
} ISzAlloc;
#define IAlloc_Alloc(p, size) (p)->Alloc((p), size)
#define IAlloc_Free(p, a) (p)->Free((p), a)
#ifdef _WIN32
#define CHAR_PATH_SEPARATOR '\\'
#define WCHAR_PATH_SEPARATOR L'\\'
#define STRING_PATH_SEPARATOR "\\"
#define WSTRING_PATH_SEPARATOR L"\\"
#else
#define CHAR_PATH_SEPARATOR '/'
#define WCHAR_PATH_SEPARATOR L'/'
#define STRING_PATH_SEPARATOR "/"
#define WSTRING_PATH_SEPARATOR L"/"
#endif
EXTERN_C_END
#endif

32
src/calibre_lzma/Compiler.h
View File

@ -1,32 +0,0 @@
/* Compiler.h
2015-08-02 : Igor Pavlov : Public domain */
#ifndef __7Z_COMPILER_H
#define __7Z_COMPILER_H
#ifdef _MSC_VER
#ifdef UNDER_CE
#define RPC_NO_WINDOWS_H
/* #pragma warning(disable : 4115) // '_RPC_ASYNC_STATE' : named type definition in parentheses */
#pragma warning(disable : 4201) // nonstandard extension used : nameless struct/union
#pragma warning(disable : 4214) // nonstandard extension used : bit field types other than int
#endif
#if _MSC_VER >= 1300
#pragma warning(disable : 4996) // This function or variable may be unsafe
#else
#pragma warning(disable : 4511) // copy constructor could not be generated
#pragma warning(disable : 4512) // assignment operator could not be generated
#pragma warning(disable : 4514) // unreferenced inline function has been removed
#pragma warning(disable : 4702) // unreachable code
#pragma warning(disable : 4710) // not inlined
#pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information
#endif
#endif
#define UNUSED_VAR(x) (void)x;
/* #define UNUSED_VAR(x) x=x; */
#endif

1044
src/calibre_lzma/LzFind.c
View File

File diff suppressed because it is too large Load Diff

117
src/calibre_lzma/LzFind.h
View File

@ -1,117 +0,0 @@
/* LzFind.h -- Match finder for LZ algorithms
2015-10-15 : Igor Pavlov : Public domain */
#ifndef __LZ_FIND_H
#define __LZ_FIND_H
#include "7zTypes.h"
EXTERN_C_BEGIN
typedef UInt32 CLzRef;
typedef struct _CMatchFinder
{
Byte *buffer;
UInt32 pos;
UInt32 posLimit;
UInt32 streamPos;
UInt32 lenLimit;
UInt32 cyclicBufferPos;
UInt32 cyclicBufferSize; /* it must be = (historySize + 1) */
Byte streamEndWasReached;
Byte btMode;
Byte bigHash;
Byte directInput;
UInt32 matchMaxLen;
CLzRef *hash;
CLzRef *son;
UInt32 hashMask;
UInt32 cutValue;
Byte *bufferBase;
ISeqInStream *stream;
UInt32 blockSize;
UInt32 keepSizeBefore;
UInt32 keepSizeAfter;
UInt32 numHashBytes;
size_t directInputRem;
UInt32 historySize;
UInt32 fixedHashSize;
UInt32 hashSizeSum;
SRes result;
UInt32 crc[256];
size_t numRefs;
} CMatchFinder;
#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer)
#define Inline_MatchFinder_GetNumAvailableBytes(p) ((p)->streamPos - (p)->pos)
#define Inline_MatchFinder_IsFinishedOK(p) \
((p)->streamEndWasReached \
&& (p)->streamPos == (p)->pos \
&& (!(p)->directInput || (p)->directInputRem == 0))
int MatchFinder_NeedMove(CMatchFinder *p);
Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p);
void MatchFinder_MoveBlock(CMatchFinder *p);
void MatchFinder_ReadIfRequired(CMatchFinder *p);
void MatchFinder_Construct(CMatchFinder *p);
/* Conditions:
historySize <= 3 GB
keepAddBufferBefore + matchMaxLen + keepAddBufferAfter < 511MB
*/
int MatchFinder_Create(CMatchFinder *p, UInt32 historySize,
UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter,
ISzAlloc *alloc);
void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc);
void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems);
void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue);
UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *buffer, CLzRef *son,
UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue,
UInt32 *distances, UInt32 maxLen);
/*
Conditions:
Mf_GetNumAvailableBytes_Func must be called before each Mf_GetMatchLen_Func.
Mf_GetPointerToCurrentPos_Func's result must be used only before any other function
*/
typedef void (*Mf_Init_Func)(void *object);
typedef UInt32 (*Mf_GetNumAvailableBytes_Func)(void *object);
typedef const Byte * (*Mf_GetPointerToCurrentPos_Func)(void *object);
typedef UInt32 (*Mf_GetMatches_Func)(void *object, UInt32 *distances);
typedef void (*Mf_Skip_Func)(void *object, UInt32);
typedef struct _IMatchFinder
{
Mf_Init_Func Init;
Mf_GetNumAvailableBytes_Func GetNumAvailableBytes;
Mf_GetPointerToCurrentPos_Func GetPointerToCurrentPos;
Mf_GetMatches_Func GetMatches;
Mf_Skip_Func Skip;
} IMatchFinder;
void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable);
void MatchFinder_Init_2(CMatchFinder *p, int readData);
void MatchFinder_Init(CMatchFinder *p);
UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances);
UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances);
void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num);
void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num);
EXTERN_C_END
#endif

57
src/calibre_lzma/LzHash.h
View File

@ -1,57 +0,0 @@
/* LzHash.h -- HASH functions for LZ algorithms
2015-04-12 : Igor Pavlov : Public domain */
#ifndef __LZ_HASH_H
#define __LZ_HASH_H
#define kHash2Size (1 << 10)
#define kHash3Size (1 << 16)
#define kHash4Size (1 << 20)
#define kFix3HashSize (kHash2Size)
#define kFix4HashSize (kHash2Size + kHash3Size)
#define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size)
#define HASH2_CALC hv = cur[0] | ((UInt32)cur[1] << 8);
#define HASH3_CALC { \
UInt32 temp = p->crc[cur[0]] ^ cur[1]; \
h2 = temp & (kHash2Size - 1); \
hv = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; }
#define HASH4_CALC { \
UInt32 temp = p->crc[cur[0]] ^ cur[1]; \
h2 = temp & (kHash2Size - 1); \
temp ^= ((UInt32)cur[2] << 8); \
h3 = temp & (kHash3Size - 1); \
hv = (temp ^ (p->crc[cur[3]] << 5)) & p->hashMask; }
#define HASH5_CALC { \
UInt32 temp = p->crc[cur[0]] ^ cur[1]; \
h2 = temp & (kHash2Size - 1); \
temp ^= ((UInt32)cur[2] << 8); \
h3 = temp & (kHash3Size - 1); \
temp ^= (p->crc[cur[3]] << 5); \
h4 = temp & (kHash4Size - 1); \
hv = (temp ^ (p->crc[cur[4]] << 3)) & p->hashMask; }
/* #define HASH_ZIP_CALC hv = ((cur[0] | ((UInt32)cur[1] << 8)) ^ p->crc[cur[2]]) & 0xFFFF; */
#define HASH_ZIP_CALC hv = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF;
#define MT_HASH2_CALC \
h2 = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1);
#define MT_HASH3_CALC { \
UInt32 temp = p->crc[cur[0]] ^ cur[1]; \
h2 = temp & (kHash2Size - 1); \
h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); }
#define MT_HASH4_CALC { \
UInt32 temp = p->crc[cur[0]] ^ cur[1]; \
h2 = temp & (kHash2Size - 1); \
temp ^= ((UInt32)cur[2] << 8); \
h3 = temp & (kHash3Size - 1); \
h4 = (temp ^ (p->crc[cur[3]] << 5)) & (kHash4Size - 1); }
#endif

378
src/calibre_lzma/Lzma2Dec.c
View File

@ -1,378 +0,0 @@
/* Lzma2Dec.c -- LZMA2 Decoder
2015-11-09 : Igor Pavlov : Public domain */
/* #define SHOW_DEBUG_INFO */
#include "Precomp.h"
#ifdef SHOW_DEBUG_INFO
#include <stdio.h>
#endif
#include <string.h>
#include "Lzma2Dec.h"
/*
00000000 - EOS
00000001 U U - Uncompressed Reset Dic
00000010 U U - Uncompressed No Reset
100uuuuu U U P P - LZMA no reset
101uuuuu U U P P - LZMA reset state
110uuuuu U U P P S - LZMA reset state + new prop
111uuuuu U U P P S - LZMA reset state + new prop + reset dic
u, U - Unpack Size
P - Pack Size
S - Props
*/
#define LZMA2_CONTROL_LZMA (1 << 7)
#define LZMA2_CONTROL_COPY_NO_RESET 2
#define LZMA2_CONTROL_COPY_RESET_DIC 1
#define LZMA2_CONTROL_EOF 0
#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & LZMA2_CONTROL_LZMA) == 0)
#define LZMA2_GET_LZMA_MODE(p) (((p)->control >> 5) & 3)
#define LZMA2_IS_THERE_PROP(mode) ((mode) >= 2)
#define LZMA2_LCLP_MAX 4
#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
#ifdef SHOW_DEBUG_INFO
#define PRF(x) x
#else
#define PRF(x)
#endif
typedef enum
{
LZMA2_STATE_CONTROL,
LZMA2_STATE_UNPACK0,
LZMA2_STATE_UNPACK1,
LZMA2_STATE_PACK0,
LZMA2_STATE_PACK1,
LZMA2_STATE_PROP,
LZMA2_STATE_DATA,
LZMA2_STATE_DATA_CONT,
LZMA2_STATE_FINISHED,
LZMA2_STATE_ERROR
} ELzma2State;
static SRes Lzma2Dec_GetOldProps(Byte prop, Byte *props)
{
UInt32 dicSize;
if (prop > 40)
return SZ_ERROR_UNSUPPORTED;
dicSize = (prop == 40) ? 0xFFFFFFFF : LZMA2_DIC_SIZE_FROM_PROP(prop);
props[0] = (Byte)LZMA2_LCLP_MAX;
props[1] = (Byte)(dicSize);
props[2] = (Byte)(dicSize >> 8);
props[3] = (Byte)(dicSize >> 16);
props[4] = (Byte)(dicSize >> 24);
return SZ_OK;
}
SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAlloc *alloc)
{
Byte props[LZMA_PROPS_SIZE];
RINOK(Lzma2Dec_GetOldProps(prop, props));
return LzmaDec_AllocateProbs(&p->decoder, props, LZMA_PROPS_SIZE, alloc);
}
SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAlloc *alloc)
{
Byte props[LZMA_PROPS_SIZE];
RINOK(Lzma2Dec_GetOldProps(prop, props));
return LzmaDec_Allocate(&p->decoder, props, LZMA_PROPS_SIZE, alloc);
}
void Lzma2Dec_Init(CLzma2Dec *p)
{
p->state = LZMA2_STATE_CONTROL;
p->needInitDic = True;
p->needInitState = True;
p->needInitProp = True;
LzmaDec_Init(&p->decoder);
}
static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b)
{
switch (p->state)
{
case LZMA2_STATE_CONTROL:
p->control = b;
PRF(printf("\n %4X ", (unsigned)p->decoder.dicPos));
PRF(printf(" %2X", (unsigned)b));
if (p->control == 0)
return LZMA2_STATE_FINISHED;
if (LZMA2_IS_UNCOMPRESSED_STATE(p))
{
if ((p->control & 0x7F) > 2)
return LZMA2_STATE_ERROR;
p->unpackSize = 0;
}
else
p->unpackSize = (UInt32)(p->control & 0x1F) << 16;
return LZMA2_STATE_UNPACK0;
case LZMA2_STATE_UNPACK0:
p->unpackSize |= (UInt32)b << 8;
return LZMA2_STATE_UNPACK1;
case LZMA2_STATE_UNPACK1:
p->unpackSize |= (UInt32)b;
p->unpackSize++;
PRF(printf(" %8u", (unsigned)p->unpackSize));
return (LZMA2_IS_UNCOMPRESSED_STATE(p)) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0;
case LZMA2_STATE_PACK0:
p->packSize = (UInt32)b << 8;
return LZMA2_STATE_PACK1;
case LZMA2_STATE_PACK1:
p->packSize |= (UInt32)b;
p->packSize++;
PRF(printf(" %8u", (unsigned)p->packSize));
return LZMA2_IS_THERE_PROP(LZMA2_GET_LZMA_MODE(p)) ? LZMA2_STATE_PROP:
(p->needInitProp ? LZMA2_STATE_ERROR : LZMA2_STATE_DATA);
case LZMA2_STATE_PROP:
{
unsigned lc, lp;
if (b >= (9 * 5 * 5))
return LZMA2_STATE_ERROR;
lc = b % 9;
b /= 9;
p->decoder.prop.pb = b / 5;
lp = b % 5;
if (lc + lp > LZMA2_LCLP_MAX)
return LZMA2_STATE_ERROR;
p->decoder.prop.lc = lc;
p->decoder.prop.lp = lp;
p->needInitProp = False;
return LZMA2_STATE_DATA;
}
}
return LZMA2_STATE_ERROR;
}
static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT size)
{
memcpy(p->dic + p->dicPos, src, size);
p->dicPos += size;
if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= size)
p->checkDicSize = p->prop.dicSize;
p->processedPos += (UInt32)size;
}
void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState);
SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit,
const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status)
{
SizeT inSize = *srcLen;
*srcLen = 0;
*status = LZMA_STATUS_NOT_SPECIFIED;
while (p->state != LZMA2_STATE_FINISHED)
{
SizeT dicPos = p->decoder.dicPos;
if (p->state == LZMA2_STATE_ERROR)
return SZ_ERROR_DATA;
if (dicPos == dicLimit && finishMode == LZMA_FINISH_ANY)
{
*status = LZMA_STATUS_NOT_FINISHED;
return SZ_OK;
}
if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT)
{
if (*srcLen == inSize)
{
*status = LZMA_STATUS_NEEDS_MORE_INPUT;
return SZ_OK;
}
(*srcLen)++;
p->state = Lzma2Dec_UpdateState(p, *src++);
if (dicPos == dicLimit && p->state != LZMA2_STATE_FINISHED)
{
p->state = LZMA2_STATE_ERROR;
return SZ_ERROR_DATA;
}
continue;
}
{
SizeT destSizeCur = dicLimit - dicPos;
SizeT srcSizeCur = inSize - *srcLen;
ELzmaFinishMode curFinishMode = LZMA_FINISH_ANY;
if (p->unpackSize <= destSizeCur)
{
destSizeCur = (SizeT)p->unpackSize;
curFinishMode = LZMA_FINISH_END;
}
if (LZMA2_IS_UNCOMPRESSED_STATE(p))
{
if (*srcLen == inSize)
{
*status = LZMA_STATUS_NEEDS_MORE_INPUT;
return SZ_OK;
}
if (p->state == LZMA2_STATE_DATA)
{
Bool initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC);
if (initDic)
p->needInitProp = p->needInitState = True;
else if (p->needInitDic)
{
p->state = LZMA2_STATE_ERROR;
return SZ_ERROR_DATA;
}
p->needInitDic = False;
LzmaDec_InitDicAndState(&p->decoder, initDic, False);
}
if (srcSizeCur > destSizeCur)
srcSizeCur = destSizeCur;
if (srcSizeCur == 0)
{
p->state = LZMA2_STATE_ERROR;
return SZ_ERROR_DATA;
}
LzmaDec_UpdateWithUncompressed(&p->decoder, src, srcSizeCur);
src += srcSizeCur;
*srcLen += srcSizeCur;
p->unpackSize -= (UInt32)srcSizeCur;
p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT;
}
else
{
SizeT outSizeProcessed;
SRes res;
if (p->state == LZMA2_STATE_DATA)
{
unsigned mode = LZMA2_GET_LZMA_MODE(p);
Bool initDic = (mode == 3);
Bool initState = (mode != 0);
if ((!initDic && p->needInitDic) || (!initState && p->needInitState))
{
p->state = LZMA2_STATE_ERROR;
return SZ_ERROR_DATA;
}
LzmaDec_InitDicAndState(&p->decoder, initDic, initState);
p->needInitDic = False;
p->needInitState = False;
p->state = LZMA2_STATE_DATA_CONT;
}
if (srcSizeCur > p->packSize)
srcSizeCur = (SizeT)p->packSize;
res = LzmaDec_DecodeToDic(&p->decoder, dicPos + destSizeCur, src, &srcSizeCur, curFinishMode, status);
src += srcSizeCur;
*srcLen += srcSizeCur;
p->packSize -= (UInt32)srcSizeCur;
outSizeProcessed = p->decoder.dicPos - dicPos;
p->unpackSize -= (UInt32)outSizeProcessed;
RINOK(res);
if (*status == LZMA_STATUS_NEEDS_MORE_INPUT)
return res;
if (srcSizeCur == 0 && outSizeProcessed == 0)
{
if (*status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK
|| p->unpackSize != 0
|| p->packSize != 0)
{
p->state = LZMA2_STATE_ERROR;
return SZ_ERROR_DATA;
}
p->state = LZMA2_STATE_CONTROL;
}
if (*status == LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK)
*status = LZMA_STATUS_NOT_FINISHED;
}
}
}
*status = LZMA_STATUS_FINISHED_WITH_MARK;
return SZ_OK;
}
SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status)
{
SizeT outSize = *destLen, inSize = *srcLen;
*srcLen = *destLen = 0;
for (;;)
{
SizeT srcSizeCur = inSize, outSizeCur, dicPos;
ELzmaFinishMode curFinishMode;
SRes res;
if (p->decoder.dicPos == p->decoder.dicBufSize)
p->decoder.dicPos = 0;
dicPos = p->decoder.dicPos;
if (outSize > p->decoder.dicBufSize - dicPos)
{
outSizeCur = p->decoder.dicBufSize;
curFinishMode = LZMA_FINISH_ANY;
}
else
{
outSizeCur = dicPos + outSize;
curFinishMode = finishMode;
}
res = Lzma2Dec_DecodeToDic(p, outSizeCur, src, &srcSizeCur, curFinishMode, status);
src += srcSizeCur;
inSize -= srcSizeCur;
*srcLen += srcSizeCur;
outSizeCur = p->decoder.dicPos - dicPos;
memcpy(dest, p->decoder.dic + dicPos, outSizeCur);
dest += outSizeCur;
outSize -= outSizeCur;
*destLen += outSizeCur;
if (res != 0)
return res;
if (outSizeCur == 0 || outSize == 0)
return SZ_OK;
}
}
SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen,
Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAlloc *alloc)
{
CLzma2Dec p;
SRes res;
SizeT outSize = *destLen, inSize = *srcLen;
*destLen = *srcLen = 0;
*status = LZMA_STATUS_NOT_SPECIFIED;
Lzma2Dec_Construct(&p);
RINOK(Lzma2Dec_AllocateProbs(&p, prop, alloc));
p.decoder.dic = dest;
p.decoder.dicBufSize = outSize;
Lzma2Dec_Init(&p);
*srcLen = inSize;
res = Lzma2Dec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status);
*destLen = p.decoder.dicPos;
if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT)
res = SZ_ERROR_INPUT_EOF;
Lzma2Dec_FreeProbs(&p, alloc);
return res;
}

80
src/calibre_lzma/Lzma2Dec.h
View File

@ -1,80 +0,0 @@
/* Lzma2Dec.h -- LZMA2 Decoder
2015-05-13 : Igor Pavlov : Public domain */
#ifndef __LZMA2_DEC_H
#define __LZMA2_DEC_H
#include "LzmaDec.h"
EXTERN_C_BEGIN
/* ---------- State Interface ---------- */
typedef struct
{
CLzmaDec decoder;
UInt32 packSize;
UInt32 unpackSize;
unsigned state;
Byte control;
Bool needInitDic;
Bool needInitState;
Bool needInitProp;
} CLzma2Dec;
#define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder)
#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc);
#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc);
SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAlloc *alloc);
SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAlloc *alloc);
void Lzma2Dec_Init(CLzma2Dec *p);
/*
finishMode:
It has meaning only if the decoding reaches output limit (*destLen or dicLimit).
LZMA_FINISH_ANY - use smallest number of input bytes
LZMA_FINISH_END - read EndOfStream marker after decoding
Returns:
SZ_OK
status:
LZMA_STATUS_FINISHED_WITH_MARK
LZMA_STATUS_NOT_FINISHED
LZMA_STATUS_NEEDS_MORE_INPUT
SZ_ERROR_DATA - Data error
*/
SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit,
const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);
SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);
/* ---------- One Call Interface ---------- */
/*
finishMode:
It has meaning only if the decoding reaches output limit (*destLen).
LZMA_FINISH_ANY - use smallest number of input bytes
LZMA_FINISH_END - read EndOfStream marker after decoding
Returns:
SZ_OK
status:
LZMA_STATUS_FINISHED_WITH_MARK
LZMA_STATUS_NOT_FINISHED
SZ_ERROR_DATA - Data error
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_UNSUPPORTED - Unsupported properties
SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src).
*/
SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen,
Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAlloc *alloc);
EXTERN_C_END
#endif

520
src/calibre_lzma/Lzma2Enc.c
View File

@ -1,520 +0,0 @@
/* Lzma2Enc.c -- LZMA2 Encoder
2015-10-04 : Igor Pavlov : Public domain */
#include "Precomp.h"
/* #include <stdio.h> */
#include <string.h>
/* #define _7ZIP_ST */
#include "Lzma2Enc.h"
#ifndef _7ZIP_ST
#include "MtCoder.h"
#else
#define NUM_MT_CODER_THREADS_MAX 1
#endif
#define LZMA2_CONTROL_LZMA (1 << 7)
#define LZMA2_CONTROL_COPY_NO_RESET 2
#define LZMA2_CONTROL_COPY_RESET_DIC 1
#define LZMA2_CONTROL_EOF 0
#define LZMA2_LCLP_MAX 4
#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
#define LZMA2_PACK_SIZE_MAX (1 << 16)
#define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX
#define LZMA2_UNPACK_SIZE_MAX (1 << 21)
#define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX
#define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16)
#define PRF(x) /* x */
/* ---------- CLzma2EncInt ---------- */
typedef struct
{
CLzmaEncHandle enc;
UInt64 srcPos;
Byte props;
Bool needInitState;
Bool needInitProp;
} CLzma2EncInt;
static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props)
{
Byte propsEncoded[LZMA_PROPS_SIZE];
SizeT propsSize = LZMA_PROPS_SIZE;
RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
p->srcPos = 0;
p->props = propsEncoded[0];
p->needInitState = True;
p->needInitProp = True;
return SZ_OK;
}
SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize,
ISzAlloc *alloc, ISzAlloc *allocBig);
SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig);
SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize);
const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp);
void LzmaEnc_Finish(CLzmaEncHandle pp);
void LzmaEnc_SaveState(CLzmaEncHandle pp);
void LzmaEnc_RestoreState(CLzmaEncHandle pp);
static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
size_t *packSizeRes, ISeqOutStream *outStream)
{
size_t packSizeLimit = *packSizeRes;
size_t packSize = packSizeLimit;
UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX;
unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0);
Bool useCopyBlock;
SRes res;
*packSizeRes = 0;
if (packSize < lzHeaderSize)
return SZ_ERROR_OUTPUT_EOF;
packSize -= lzHeaderSize;
LzmaEnc_SaveState(p->enc);
res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState,
outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize);
PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize));
if (unpackSize == 0)
return res;
if (res == SZ_OK)
useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16));
else
{
if (res != SZ_ERROR_OUTPUT_EOF)
return res;
res = SZ_OK;
useCopyBlock = True;
}
if (useCopyBlock)
{
size_t destPos = 0;
PRF(printf("################# COPY "));
while (unpackSize > 0)
{
UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE;
if (packSizeLimit - destPos < u + 3)
return SZ_ERROR_OUTPUT_EOF;
outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET);
outBuf[destPos++] = (Byte)((u - 1) >> 8);
outBuf[destPos++] = (Byte)(u - 1);
memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u);
unpackSize -= u;
destPos += u;
p->srcPos += u;
if (outStream)
{
*packSizeRes += destPos;
if (outStream->Write(outStream, outBuf, destPos) != destPos)
return SZ_ERROR_WRITE;
destPos = 0;
}
else
*packSizeRes = destPos;
/* needInitState = True; */
}
LzmaEnc_RestoreState(p->enc);
return SZ_OK;
}
{
size_t destPos = 0;
UInt32 u = unpackSize - 1;
UInt32 pm = (UInt32)(packSize - 1);
unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0);
PRF(printf(" "));
outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F));
outBuf[destPos++] = (Byte)(u >> 8);
outBuf[destPos++] = (Byte)u;
outBuf[destPos++] = (Byte)(pm >> 8);
outBuf[destPos++] = (Byte)pm;
if (p->needInitProp)
outBuf[destPos++] = p->props;
p->needInitProp = False;
p->needInitState = False;
destPos += packSize;
p->srcPos += unpackSize;
if (outStream)
if (outStream->Write(outStream, outBuf, destPos) != destPos)
return SZ_ERROR_WRITE;
*packSizeRes = destPos;
return SZ_OK;
}
}
/* ---------- Lzma2 Props ---------- */
void Lzma2EncProps_Init(CLzma2EncProps *p)
{
LzmaEncProps_Init(&p->lzmaProps);
p->numTotalThreads = -1;
p->numBlockThreads = -1;
p->blockSize = 0;
}
void Lzma2EncProps_Normalize(CLzma2EncProps *p)
{
int t1, t1n, t2, t3;
{
CLzmaEncProps lzmaProps = p->lzmaProps;
LzmaEncProps_Normalize(&lzmaProps);
t1n = lzmaProps.numThreads;
}
t1 = p->lzmaProps.numThreads;
t2 = p->numBlockThreads;
t3 = p->numTotalThreads;
if (t2 > NUM_MT_CODER_THREADS_MAX)
t2 = NUM_MT_CODER_THREADS_MAX;
if (t3 <= 0)
{
if (t2 <= 0)
t2 = 1;
t3 = t1n * t2;
}
else if (t2 <= 0)
{
t2 = t3 / t1n;
if (t2 == 0)
{
t1 = 1;
t2 = t3;
}
if (t2 > NUM_MT_CODER_THREADS_MAX)
t2 = NUM_MT_CODER_THREADS_MAX;
}
else if (t1 <= 0)
{
t1 = t3 / t2;
if (t1 == 0)
t1 = 1;
}
else
t3 = t1n * t2;
p->lzmaProps.numThreads = t1;
LzmaEncProps_Normalize(&p->lzmaProps);
t1 = p->lzmaProps.numThreads;
if (p->blockSize == 0)
{
UInt32 dictSize = p->lzmaProps.dictSize;
UInt64 blockSize = (UInt64)dictSize << 2;
const UInt32 kMinSize = (UInt32)1 << 20;
const UInt32 kMaxSize = (UInt32)1 << 28;
if (blockSize < kMinSize) blockSize = kMinSize;
if (blockSize > kMaxSize) blockSize = kMaxSize;
if (blockSize < dictSize) blockSize = dictSize;
p->blockSize = (size_t)blockSize;
}
if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1)
{
UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1;
if (temp > p->lzmaProps.reduceSize)
{
UInt64 numBlocks = temp / p->blockSize;
if (numBlocks < (unsigned)t2)
{
t2 = (unsigned)numBlocks;
if (t2 == 0)
t2 = 1;
t3 = t1 * t2;
}
}
}
p->numBlockThreads = t2;
p->numTotalThreads = t3;
}
static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)
{
return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;
}
/* ---------- Lzma2 ---------- */
typedef struct
{
Byte propEncoded;
CLzma2EncProps props;
Byte *outBuf;
ISzAlloc *alloc;
ISzAlloc *allocBig;
CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX];
#ifndef _7ZIP_ST
CMtCoder mtCoder;
#endif
} CLzma2Enc;
/* ---------- Lzma2EncThread ---------- */
static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder,
ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
{
UInt64 packTotal = 0;
SRes res = SZ_OK;
if (!mainEncoder->outBuf)
{
mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
if (!mainEncoder->outBuf)
return SZ_ERROR_MEM;
}
RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE,
mainEncoder->alloc, mainEncoder->allocBig));
for (;;)
{
size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream);
if (res != SZ_OK)
break;
packTotal += packSize;
res = Progress(progress, p->srcPos, packTotal);
if (res != SZ_OK)
break;
if (packSize == 0)
break;
}
LzmaEnc_Finish(p->enc);
if (res == SZ_OK)
{
Byte b = 0;
if (outStream->Write(outStream, &b, 1) != 1)
return SZ_ERROR_WRITE;
}
return res;
}
#ifndef _7ZIP_ST
typedef struct
{
IMtCoderCallback funcTable;
CLzma2Enc *lzma2Enc;
} CMtCallbackImp;
static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize,
const Byte *src, size_t srcSize, int finished)
{
CMtCallbackImp *imp = (CMtCallbackImp *)pp;
CLzma2Enc *mainEncoder = imp->lzma2Enc;
CLzma2EncInt *p = &mainEncoder->coders[index];
SRes res = SZ_OK;
{
size_t destLim = *destSize;
*destSize = 0;
if (srcSize != 0)
{
RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE,
mainEncoder->alloc, mainEncoder->allocBig));
while (p->srcPos < srcSize)
{
size_t packSize = destLim - *destSize;
res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL);
if (res != SZ_OK)
break;
*destSize += packSize;
if (packSize == 0)
{
res = SZ_ERROR_FAIL;
break;
}
if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK)
{
res = SZ_ERROR_PROGRESS;
break;
}
}
LzmaEnc_Finish(p->enc);
if (res != SZ_OK)
return res;
}
if (finished)
{
if (*destSize == destLim)
return SZ_ERROR_OUTPUT_EOF;
dest[(*destSize)++] = 0;
}
}
return res;
}
#endif
/* ---------- Lzma2Enc ---------- */
CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig)
{
CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc));
if (!p)
return NULL;
Lzma2EncProps_Init(&p->props);
Lzma2EncProps_Normalize(&p->props);
p->outBuf = 0;
p->alloc = alloc;
p->allocBig = allocBig;
{
unsigned i;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
p->coders[i].enc = 0;
}
#ifndef _7ZIP_ST
MtCoder_Construct(&p->mtCoder);
#endif
return p;
}
void Lzma2Enc_Destroy(CLzma2EncHandle pp)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
unsigned i;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
{
CLzma2EncInt *t = &p->coders[i];
if (t->enc)
{
LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig);
t->enc = 0;
}
}
#ifndef _7ZIP_ST
MtCoder_Destruct(&p->mtCoder);
#endif
IAlloc_Free(p->alloc, p->outBuf);
IAlloc_Free(p->alloc, pp);
}
SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
CLzmaEncProps lzmaProps = props->lzmaProps;
LzmaEncProps_Normalize(&lzmaProps);
if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX)
return SZ_ERROR_PARAM;
p->props = *props;
Lzma2EncProps_Normalize(&p->props);
return SZ_OK;
}
Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
unsigned i;
UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps);
for (i = 0; i < 40; i++)
if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i))
break;
return (Byte)i;
}
SRes Lzma2Enc_Encode(CLzma2EncHandle pp,
ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
int i;
for (i = 0; i < p->props.numBlockThreads; i++)
{
CLzma2EncInt *t = &p->coders[(unsigned)i];
if (!t->enc)
{
t->enc = LzmaEnc_Create(p->alloc);
if (!t->enc)
return SZ_ERROR_MEM;
}
}
#ifndef _7ZIP_ST
if (p->props.numBlockThreads > 1)
{
CMtCallbackImp mtCallback;
mtCallback.funcTable.Code = MtCallbackImp_Code;
mtCallback.lzma2Enc = p;
p->mtCoder.progress = progress;
p->mtCoder.inStream = inStream;
p->mtCoder.outStream = outStream;
p->mtCoder.alloc = p->alloc;
p->mtCoder.mtCallback = &mtCallback.funcTable;
p->mtCoder.blockSize = p->props.blockSize;
p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16;
if (p->mtCoder.destBlockSize < p->props.blockSize)
{
p->mtCoder.destBlockSize = (size_t)0 - 1;
if (p->mtCoder.destBlockSize < p->props.blockSize)
return SZ_ERROR_FAIL;
}
p->mtCoder.numThreads = p->props.numBlockThreads;
return MtCoder_Code(&p->mtCoder);
}
#endif
return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress);
}

62
src/calibre_lzma/Lzma2Enc.h
View File

@ -1,62 +0,0 @@
/* Lzma2Enc.h -- LZMA2 Encoder
2013-01-18 : Igor Pavlov : Public domain */
#ifndef __LZMA2_ENC_H
#define __LZMA2_ENC_H
#include "LzmaEnc.h"
EXTERN_C_BEGIN
typedef struct
{
CLzmaEncProps lzmaProps;
size_t blockSize;
int numBlockThreads;
int numTotalThreads;
} CLzma2EncProps;
void Lzma2EncProps_Init(CLzma2EncProps *p);
void Lzma2EncProps_Normalize(CLzma2EncProps *p);
/* ---------- CLzmaEnc2Handle Interface ---------- */
/* Lzma2Enc_* functions can return the following exit codes:
Returns:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater in props
SZ_ERROR_WRITE - Write callback error
SZ_ERROR_PROGRESS - some break from progress callback
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
*/
typedef void * CLzma2EncHandle;
CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig);
void Lzma2Enc_Destroy(CLzma2EncHandle p);
SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props);
Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p);
SRes Lzma2Enc_Encode(CLzma2EncHandle p,
ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress);
/* ---------- One Call Interface ---------- */
/* Lzma2Encode
Return code:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater
SZ_ERROR_OUTPUT_EOF - output buffer overflow
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
*/
/*
SRes Lzma2Encode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
const CLzmaEncProps *props, Byte *propsEncoded, int writeEndMark,
ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig);
*/
EXTERN_C_END
#endif

1100
src/calibre_lzma/LzmaDec.c
View File

File diff suppressed because it is too large Load Diff

227
src/calibre_lzma/LzmaDec.h
View File

@ -1,227 +0,0 @@
/* LzmaDec.h -- LZMA Decoder
2013-01-18 : Igor Pavlov : Public domain */
#ifndef __LZMA_DEC_H
#define __LZMA_DEC_H
#include "7zTypes.h"
EXTERN_C_BEGIN
/* #define _LZMA_PROB32 */
/* _LZMA_PROB32 can increase the speed on some CPUs,
but memory usage for CLzmaDec::probs will be doubled in that case */
#ifdef _LZMA_PROB32
#define CLzmaProb UInt32
#else
#define CLzmaProb UInt16
#endif
/* ---------- LZMA Properties ---------- */
#define LZMA_PROPS_SIZE 5
typedef struct _CLzmaProps
{
unsigned lc, lp, pb;
UInt32 dicSize;
} CLzmaProps;
/* LzmaProps_Decode - decodes properties
Returns:
SZ_OK
SZ_ERROR_UNSUPPORTED - Unsupported properties
*/
SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size);
/* ---------- LZMA Decoder state ---------- */
/* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case.
Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */
#define LZMA_REQUIRED_INPUT_MAX 20
typedef struct
{
CLzmaProps prop;
CLzmaProb *probs;
Byte *dic;
const Byte *buf;
UInt32 range, code;
SizeT dicPos;
SizeT dicBufSize;
UInt32 processedPos;
UInt32 checkDicSize;
unsigned state;
UInt32 reps[4];
unsigned remainLen;
int needFlush;
int needInitState;
UInt32 numProbs;
unsigned tempBufSize;
Byte tempBuf[LZMA_REQUIRED_INPUT_MAX];
} CLzmaDec;
#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; }
void LzmaDec_Init(CLzmaDec *p);
/* There are two types of LZMA streams:
0) Stream with end mark. That end mark adds about 6 bytes to compressed size.
1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */
typedef enum
{
LZMA_FINISH_ANY, /* finish at any point */
LZMA_FINISH_END /* block must be finished at the end */
} ELzmaFinishMode;
/* ELzmaFinishMode has meaning only if the decoding reaches output limit !!!
You must use LZMA_FINISH_END, when you know that current output buffer
covers last bytes of block. In other cases you must use LZMA_FINISH_ANY.
If LZMA decoder sees end marker before reaching output limit, it returns SZ_OK,
and output value of destLen will be less than output buffer size limit.
You can check status result also.
You can use multiple checks to test data integrity after full decompression:
1) Check Result and "status" variable.
2) Check that output(destLen) = uncompressedSize, if you know real uncompressedSize.
3) Check that output(srcLen) = compressedSize, if you know real compressedSize.
You must use correct finish mode in that case. */
typedef enum
{
LZMA_STATUS_NOT_SPECIFIED, /* use main error code instead */
LZMA_STATUS_FINISHED_WITH_MARK, /* stream was finished with end mark. */
LZMA_STATUS_NOT_FINISHED, /* stream was not finished */
LZMA_STATUS_NEEDS_MORE_INPUT, /* you must provide more input bytes */
LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK /* there is probability that stream was finished without end mark */
} ELzmaStatus;
/* ELzmaStatus is used only as output value for function call */
/* ---------- Interfaces ---------- */
/* There are 3 levels of interfaces:
1) Dictionary Interface
2) Buffer Interface
3) One Call Interface
You can select any of these interfaces, but don't mix functions from different
groups for same object. */
/* There are two variants to allocate state for Dictionary Interface:
1) LzmaDec_Allocate / LzmaDec_Free
2) LzmaDec_AllocateProbs / LzmaDec_FreeProbs
You can use variant 2, if you set dictionary buffer manually.
For Buffer Interface you must always use variant 1.
LzmaDec_Allocate* can return:
SZ_OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_UNSUPPORTED - Unsupported properties
*/
SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc);
void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc);
SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAlloc *alloc);
void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc);
/* ---------- Dictionary Interface ---------- */
/* You can use it, if you want to eliminate the overhead for data copying from
dictionary to some other external buffer.
You must work with CLzmaDec variables directly in this interface.
STEPS:
LzmaDec_Constr()
LzmaDec_Allocate()
for (each new stream)
{
LzmaDec_Init()
while (it needs more decompression)
{
LzmaDec_DecodeToDic()
use data from CLzmaDec::dic and update CLzmaDec::dicPos
}
}
LzmaDec_Free()
*/
/* LzmaDec_DecodeToDic
The decoding to internal dictionary buffer (CLzmaDec::dic).
You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!!
finishMode:
It has meaning only if the decoding reaches output limit (dicLimit).
LZMA_FINISH_ANY - Decode just dicLimit bytes.
LZMA_FINISH_END - Stream must be finished after dicLimit.
Returns:
SZ_OK
status:
LZMA_STATUS_FINISHED_WITH_MARK
LZMA_STATUS_NOT_FINISHED
LZMA_STATUS_NEEDS_MORE_INPUT
LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK
SZ_ERROR_DATA - Data error
*/
SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit,
const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);
/* ---------- Buffer Interface ---------- */
/* It's zlib-like interface.
See LzmaDec_DecodeToDic description for information about STEPS and return results,
but you must use LzmaDec_DecodeToBuf instead of LzmaDec_DecodeToDic and you don't need
to work with CLzmaDec variables manually.
finishMode:
It has meaning only if the decoding reaches output limit (*destLen).
LZMA_FINISH_ANY - Decode just destLen bytes.
LZMA_FINISH_END - Stream must be finished after (*destLen).
*/
SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);
/* ---------- One Call Interface ---------- */
/* LzmaDecode
finishMode:
It has meaning only if the decoding reaches output limit (*destLen).
LZMA_FINISH_ANY - Decode just destLen bytes.
LZMA_FINISH_END - Stream must be finished after (*destLen).
Returns:
SZ_OK
status:
LZMA_STATUS_FINISHED_WITH_MARK
LZMA_STATUS_NOT_FINISHED
LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK
SZ_ERROR_DATA - Data error
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_UNSUPPORTED - Unsupported properties
SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src).
*/
SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen,
const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode,
ELzmaStatus *status, ISzAlloc *alloc);
EXTERN_C_END
#endif

2351
src/calibre_lzma/LzmaEnc.c
View File

File diff suppressed because it is too large Load Diff

78
src/calibre_lzma/LzmaEnc.h
View File

@ -1,78 +0,0 @@
/* LzmaEnc.h -- LZMA Encoder
2013-01-18 : Igor Pavlov : Public domain */
#ifndef __LZMA_ENC_H
#define __LZMA_ENC_H
#include "7zTypes.h"
EXTERN_C_BEGIN
#define LZMA_PROPS_SIZE 5
typedef struct _CLzmaEncProps
{
int level; /* 0 <= level <= 9 */
UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version
(1 << 12) <= dictSize <= (1 << 30) for 64-bit version
default = (1 << 24) */
UInt64 reduceSize; /* estimated size of data that will be compressed. default = 0xFFFFFFFF.
Encoder uses this value to reduce dictionary size */
int lc; /* 0 <= lc <= 8, default = 3 */
int lp; /* 0 <= lp <= 4, default = 0 */
int pb; /* 0 <= pb <= 4, default = 2 */
int algo; /* 0 - fast, 1 - normal, default = 1 */
int fb; /* 5 <= fb <= 273, default = 32 */
int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */
int numHashBytes; /* 2, 3 or 4, default = 4 */
UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */
unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */
int numThreads; /* 1 or 2, default = 2 */
} CLzmaEncProps;
void LzmaEncProps_Init(CLzmaEncProps *p);
void LzmaEncProps_Normalize(CLzmaEncProps *p);
UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2);
/* ---------- CLzmaEncHandle Interface ---------- */
/* LzmaEnc_* functions can return the following exit codes:
Returns:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater in props
SZ_ERROR_WRITE - Write callback error.
SZ_ERROR_PROGRESS - some break from progress callback
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
*/
typedef void * CLzmaEncHandle;
CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc);
void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig);
SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props);
SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size);
SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream,
ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig);
SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig);
/* ---------- One Call Interface ---------- */
/* LzmaEncode
Return code:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater
SZ_ERROR_OUTPUT_EOF - output buffer overflow
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
*/
SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark,
ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig);
EXTERN_C_END
#endif

10
src/calibre_lzma/Precomp.h
View File

@ -1,10 +0,0 @@
/* Precomp.h -- StdAfx
2013-11-12 : Igor Pavlov : Public domain */
#ifndef __7Z_PRECOMP_H
#define __7Z_PRECOMP_H
#include "Compiler.h"
/* #include "7zTypes.h" */
#endif

9
src/calibre_lzma/__init__.py
View File

@ -1,9 +0,0 @@
#!/usr/bin/env python
# vim:fileencoding=utf-8
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'

24
src/calibre_lzma/errors.py
View File

@ -1,24 +0,0 @@
#!/usr/bin/env python
# vim:fileencoding=utf-8
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
from calibre.constants import plugins
lzma = plugins['lzma_binding'][0]
if not lzma:
raise RuntimeError('Failed to load lzma_binding module with error: %s' % plugins['lzma_binding'][1])
LzmaError = lzma.error
class NotXZ(LzmaError):
pass
class InvalidXZ(LzmaError):
pass
class NotLzma(LzmaError):
pass

42
src/calibre_lzma/lzma1.py
View File

@ -1,42 +0,0 @@
#!/usr/bin/env python
# vim:fileencoding=utf-8
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
from io import BytesIO
from struct import unpack
from calibre.ptempfile import SpooledTemporaryFile
from .errors import NotLzma, lzma
def read_header(f):
raw = f.read(13)
try:
props, dict_size, uncompressed_size = unpack(b'<BIQ', raw)
except Exception:
raise NotLzma('Not a LZMA file')
if props > (4 * 5 + 4) * 9 + 8:
raise NotLzma('Not a LZMA file')
return uncompressed_size, raw
def decompress(raw, outfile=None, bufsize=10 * 1024 * 1024):
if isinstance(raw, bytes):
raw = BytesIO(raw)
uncompressed_size, header = read_header(raw)
outfile = outfile or SpooledTemporaryFile(50 * 1024 * 1024, '_lzma_decompress')
lzma.decompress(
raw.read, raw.seek, outfile.write, uncompressed_size, header, bufsize
)
if uncompressed_size < outfile.tell():
outfile.seek(uncompressed_size)
outfile.truncate()
return outfile
if __name__ == '__main__':
import sys
decompress(open(sys.argv[-1], 'rb'))

472
src/calibre_lzma/lzma_binding.c
View File

@ -1,472 +0,0 @@
/*
* lzma_binding.c
* Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
*
* Distributed under terms of the GPL3 license.
*/
#define PY_SSIZE_T_CLEAN
#define UNICODE
#include "Python.h"
#include "Lzma2Dec.h"
#include "Lzma2Enc.h"
#define UNUSED_VAR(x) (void)x;
static void *Alloc(void *p, size_t size) { UNUSED_VAR(p); return PyMem_Malloc(size); }
static void Free(void *p, void *address) { UNUSED_VAR(p); PyMem_Free(address); }
static ISzAlloc allocator = { Alloc, Free };
static const char* error_codes[18] = {
"OK",
"SZ_ERROR_DATA",
"SZ_ERROR_MEM",
"SZ_ERROR_CRC",
"SZ_ERROR_UNSUPPORTED",
"SZ_ERROR_PARAM",
"SZ_ERROR_INPUT_EOF",
"SZ_ERROR_OUTPUT_EOF",
"SZ_ERROR_READ",
"SZ_ERROR_WRITE",
"SZ_ERROR_PROGRESS",
"SZ_ERROR_FAIL",
"SZ_ERROR_THREAD",
"UNKNOWN", "UNKNOWN", "UNKNOWN",
"SZ_ERROR_ARCHIVE",
"SZ_ERROR_NO_ARCHIVE",
};
#define SET_ERROR(x) PyErr_SetString(LZMAError, ((x) > 0 && (x) < 17) ? error_codes[(x)] : "UNKNOWN")
typedef struct {
ISeqInStream stream;
PyObject *read;
PyThreadState **thread_state;
} InStream;
typedef struct {
ISeqOutStream stream;
PyObject *write;
PyThreadState **thread_state;
} OutStream;
typedef struct {
ICompressProgress progress;
PyObject *callback;
PyThreadState **thread_state;
} Progress;
static PyObject *LZMAError = NULL;
// Utils {{{
static UInt64 crc64_table[256];
#if PY_MAJOR_VERSION >= 3
#define BYTES_FMT "y#"
#else
#define BYTES_FMT "s#"
#endif
static void init_crc_table() {
static const UInt64 poly64 = (UInt64)(0xC96C5795D7870F42);
size_t i, j;
for (i = 0; i < 256; ++i) {
UInt64 crc64 = i;
for (j = 0; j < 8; ++j) {
if (crc64 & 1)
crc64 = (crc64 >> 1) ^ poly64;
else
crc64 >>= 1;
}
crc64_table[i] = crc64;
}
}
static PyObject *
crc64(PyObject *self, PyObject *args) {
unsigned char *data = NULL;
Py_ssize_t size = 0;
UInt64 crc = 0;
size_t i;
if (!PyArg_ParseTuple(args, BYTES_FMT "|K", &data, &size, &crc)) return NULL;
crc = ~crc;
for (i = 0; i < (size_t)size; ++i)
crc = crc64_table[data[i] ^ (crc & 0xFF)] ^ (crc >> 8);
return Py_BuildValue("K", ~crc);
}
static PyObject*
delta_decode(PyObject *self, PyObject *args) {
PyObject *array = NULL, *histarray = NULL;
unsigned char *data = NULL, pos = 0, *history = NULL;
unsigned int distance = 0;
Py_ssize_t datalen = 0, i;
if (!PyArg_ParseTuple(args, "O!O!BB", &PyByteArray_Type, &array, &PyByteArray_Type, &histarray, &pos, &distance)) return NULL;
if (PyByteArray_GET_SIZE(histarray) != 256) {
PyErr_SetString(PyExc_TypeError, "histarray must be 256 bytes long");
return NULL;
}
data = (unsigned char*)PyByteArray_AS_STRING(array); history = (unsigned char*)PyByteArray_AS_STRING(histarray);
datalen = PyBytes_GET_SIZE(array);
for (i = 0; i < datalen; i++) {
data[i] += history[(unsigned char)(pos + distance)];
history[pos--] = data[i];
}
return Py_BuildValue("B", pos);
}
// }}}
// LZMA2 decompress {{{
static PyObject *
decompress2(PyObject *self, PyObject *args) {
PyObject *read = NULL, *seek = NULL, *write = NULL, *rres = NULL;
SizeT bufsize = 0, bytes_written = 0, bytes_read = 0, inbuf_pos = 0, inbuf_len = 0;
Py_ssize_t leftover = 0;
unsigned char props = 0;
char *inbuf = NULL, *outbuf = NULL;
CLzma2Dec state;
SRes res = SZ_OK;
ELzmaStatus status = LZMA_STATUS_NOT_FINISHED;
if (!PyArg_ParseTuple(args, "OOOBk", &read, &seek, &write, &props, &bufsize)) return NULL;
Lzma2Dec_Construct(&state);
res = Lzma2Dec_Allocate(&state, (Byte)props, &allocator);
if (res == SZ_ERROR_MEM) { PyErr_NoMemory(); return NULL; }
if (res != SZ_OK) { PyErr_SetString(PyExc_TypeError, "Incorrect stream properties"); goto exit; }
inbuf = (char*)PyMem_Malloc(bufsize);
outbuf = (char*)PyMem_Malloc(bufsize);
if (!inbuf || !outbuf) {PyErr_NoMemory(); goto exit;}
Lzma2Dec_Init(&state);
while (status != LZMA_STATUS_FINISHED_WITH_MARK) {
bytes_written = bufsize; bytes_read = inbuf_len - inbuf_pos;
if (bytes_read) {
Py_BEGIN_ALLOW_THREADS;
res = Lzma2Dec_DecodeToBuf(&state, (Byte*)outbuf, &bytes_written, (Byte*)(inbuf) + inbuf_pos, &bytes_read, LZMA_FINISH_ANY, &status);
Py_END_ALLOW_THREADS;
} else { res = SZ_OK; bytes_written = 0; status = LZMA_STATUS_NEEDS_MORE_INPUT; }
if (res != SZ_OK) { SET_ERROR(res); goto exit; }
if (bytes_written > 0) {
if(!PyObject_CallFunction(write, BYTES_FMT, outbuf, bytes_written)) goto exit;
}
if (inbuf_len > inbuf_pos && !bytes_read && !bytes_written && status != LZMA_STATUS_NEEDS_MORE_INPUT && status != LZMA_STATUS_FINISHED_WITH_MARK) {
SET_ERROR(SZ_ERROR_DATA); goto exit;
}
if (bytes_read > 0) inbuf_pos += bytes_read;
if (status == LZMA_STATUS_NEEDS_MORE_INPUT) {
leftover = inbuf_len - inbuf_pos;
inbuf_pos = 0;
if (!PyObject_CallFunction(seek, "ii", -leftover, SEEK_CUR)) goto exit;
rres = PyObject_CallFunction(read, "n", bufsize);
if (rres == NULL) goto exit;
inbuf_len = PyBytes_GET_SIZE(rres);
if (inbuf_len == 0) { PyErr_SetString(LZMAError, "LZMA2 block was truncated"); goto exit; }
memcpy(inbuf, PyBytes_AS_STRING(rres), inbuf_len);
Py_DECREF(rres); rres = NULL;
}
}
leftover = inbuf_len - inbuf_pos;
if (leftover > 0) {
if (!PyObject_CallFunction(seek, "ii", -leftover, SEEK_CUR)) goto exit;
}
exit:
Lzma2Dec_Free(&state, &allocator);
PyMem_Free(inbuf); PyMem_Free(outbuf);
if (PyErr_Occurred()) return NULL;
Py_RETURN_NONE;
}
// }}}
// LZMA1 decompress {{{
static PyObject*
decompress(PyObject *self, PyObject *args) {
PyObject *read = NULL, *seek = NULL, *write = NULL, *rres = NULL;
UInt64 decompressed_size = 0;
int size_known = 0;
Py_ssize_t header_size = 0, leftover = 0;
unsigned char *header = NULL, *inbuf = NULL, *outbuf = NULL;
CLzmaDec state;
SRes res = 0;
SizeT bufsize = 0, bytes_written = 0, bytes_read = 0, inbuf_pos = 0, inbuf_len = 0, total_written = 0;
ELzmaStatus status = LZMA_STATUS_NOT_FINISHED;
ELzmaFinishMode finish_mode = LZMA_FINISH_ANY;
if(!PyArg_ParseTuple(args, "OOOK" BYTES_FMT "k", &read, &seek, &write, &decompressed_size, &header, &header_size, &bufsize)) return NULL;
size_known = (decompressed_size != (UInt64)(Int64)-1);
if (header_size != 13) { PyErr_SetString(LZMAError, "Header must be exactly 13 bytes long"); return NULL; }
if (!decompressed_size) { PyErr_SetString(LZMAError, "Cannot decompress empty file"); return NULL; }
LzmaDec_Construct(&state);
res = LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &allocator);
if (res == SZ_ERROR_MEM) { PyErr_NoMemory(); return NULL; }
if (res != SZ_OK) { PyErr_SetString(PyExc_TypeError, "Incorrect stream properties"); goto exit; }
inbuf = (unsigned char*)PyMem_Malloc(bufsize);
outbuf = (unsigned char*)PyMem_Malloc(bufsize);
if (!inbuf || !outbuf) {PyErr_NoMemory(); goto exit;}
LzmaDec_Init(&state);
while (status != LZMA_STATUS_FINISHED_WITH_MARK) {
bytes_written = bufsize; bytes_read = inbuf_len - inbuf_pos;
if (bytes_read) {
Py_BEGIN_ALLOW_THREADS;
finish_mode = LZMA_FINISH_ANY;
if (size_known && total_written + bufsize > decompressed_size) finish_mode = LZMA_FINISH_END;
res = LzmaDec_DecodeToBuf(&state, (Byte*)outbuf, &bytes_written, (Byte*)(inbuf) + inbuf_pos, &bytes_read, finish_mode, &status);
Py_END_ALLOW_THREADS;
} else { res = SZ_OK; bytes_written = 0; status = LZMA_STATUS_NEEDS_MORE_INPUT; }
if (res != SZ_OK) { SET_ERROR(res); goto exit; }
if (bytes_written > 0) {
if(!PyObject_CallFunction(write, BYTES_FMT, outbuf, bytes_written)) goto exit;
total_written += bytes_written;
}
if (inbuf_len > inbuf_pos && !bytes_read && !bytes_written && status != LZMA_STATUS_NEEDS_MORE_INPUT && status != LZMA_STATUS_FINISHED_WITH_MARK) {
SET_ERROR(SZ_ERROR_DATA); goto exit;
}
if (bytes_read > 0) inbuf_pos += bytes_read;
if (size_known && total_written >= decompressed_size) break;
if (status == LZMA_STATUS_NEEDS_MORE_INPUT) {
leftover = inbuf_len - inbuf_pos;
inbuf_pos = 0;
if (!PyObject_CallFunction(seek, "ii", -leftover, SEEK_CUR)) goto exit;
rres = PyObject_CallFunction(read, "n", bufsize);
if (rres == NULL) goto exit;
inbuf_len = PyBytes_GET_SIZE(rres);
if (inbuf_len == 0) { PyErr_SetString(LZMAError, "LZMA block was truncated"); goto exit; }
memcpy(inbuf, PyBytes_AS_STRING(rres), inbuf_len);
Py_DECREF(rres); rres = NULL;
}
}
leftover = inbuf_len - inbuf_pos;
if (leftover > 0) {
if (!PyObject_CallFunction(seek, "ii", -leftover, SEEK_CUR)) goto exit;
}
exit:
LzmaDec_Free(&state, &allocator);
PyMem_Free(inbuf); PyMem_Free(outbuf);
if (PyErr_Occurred()) return NULL;
Py_RETURN_NONE;
}
// }}}
// LZMA2 Compress {{{
static void
init_props(CLzma2EncProps *props, int preset) {
int level = (preset < 0) ? 0 : ((preset > 9) ? 9 : preset);
props->blockSize = 0;
props->numBlockThreads = 1;
props->numTotalThreads = 1;
props->lzmaProps.numThreads = 1;
props->lzmaProps.writeEndMark = 1;
props->lzmaProps.level = level;
props->lzmaProps.dictSize = 0;
props->lzmaProps.reduceSize = 0xFFFFFFFF;
props->lzmaProps.lc = -1;
props->lzmaProps.lp = -1;
props->lzmaProps.pb = -1;
props->lzmaProps.algo = -1;
props->lzmaProps.fb = -1;
props->lzmaProps.btMode = -1;
props->lzmaProps.numHashBytes = -1;
props->lzmaProps.mc = 0;
}
#define ACQUIRE_GIL PyEval_RestoreThread(*(self->thread_state)); *(self->thread_state) = NULL;
#define RELEASE_GIL *(self->thread_state) = PyEval_SaveThread();
static SRes iread(void *p, void *buf, size_t *size) {
InStream *self = (InStream*)p;
PyObject *res = NULL;
char *str = NULL;
if (*size == 0) return SZ_OK;
ACQUIRE_GIL
res = PyObject_CallFunction(self->read, "n", size);
if (res == NULL) return SZ_ERROR_READ;
str = PyBytes_AsString(res);
if (str == NULL) { Py_DECREF(res); return SZ_ERROR_READ; }
*size = PyBytes_Size(res);
if(*size) memcpy(buf, str, *size);
Py_DECREF(res);
RELEASE_GIL
return SZ_OK;
}
static size_t owrite(void *p, const void *buf, size_t size) {
OutStream *self = (OutStream*)p;
PyObject *res = NULL;
if (!size) return 0;
ACQUIRE_GIL
res = PyObject_CallFunction(self->write, BYTES_FMT, (char*)buf, size);
if (res == NULL) return 0;
Py_DECREF(res);
RELEASE_GIL
return size;
}
static SRes report_progress(void *p, UInt64 in_size, UInt64 out_size) {
Progress *self = (Progress*)p;
PyObject *res = NULL;
if (!self->callback) return SZ_OK;
ACQUIRE_GIL
res = PyObject_CallFunction(self->callback, "KK", in_size, out_size);
if (!res || !PyObject_IsTrue(res)) { Py_DECREF(res); return SZ_ERROR_PROGRESS; }
Py_DECREF(res);
RELEASE_GIL
return SZ_OK;
}
static PyObject*
get_lzma2_properties(int preset) {
CLzma2EncHandle lzma2 = NULL;
CLzma2EncProps props;
Byte props_out = 0;
SRes res = SZ_OK;
lzma2 = Lzma2Enc_Create(&allocator, &allocator);
if (lzma2 == NULL) { PyErr_NoMemory(); goto exit; }
// Initialize parameters based on the preset
init_props(&props, preset);
res = Lzma2Enc_SetProps(lzma2, &props);
if (res != SZ_OK) { SET_ERROR(res); goto exit; }
props_out = Lzma2Enc_WriteProperties(lzma2);
exit:
if (lzma2) Lzma2Enc_Destroy(lzma2);
if (PyErr_Occurred()) return NULL;
return Py_BuildValue(BYTES_FMT, &props_out, 1);
}
static PyObject*
compress(PyObject *self, PyObject *args) {
PyObject *read = NULL, *write = NULL, *progress_callback = NULL;
CLzma2EncHandle lzma2 = NULL;
CLzma2EncProps props;
int preset = 5;
InStream in_stream;
OutStream out_stream;
Progress progress;
SRes res = SZ_OK;
Byte props_out = 0;
PyThreadState *ts = NULL;
if (!PyArg_ParseTuple(args, "OO|Oi", &read, &write, &progress_callback, &preset)) return NULL;
if (progress_callback && !PyCallable_Check(progress_callback)) progress_callback = NULL;
lzma2 = Lzma2Enc_Create(&allocator, &allocator);
if (lzma2 == NULL) { PyErr_NoMemory(); goto exit; }
// Initialize parameters based on the preset
init_props(&props, preset);
res = Lzma2Enc_SetProps(lzma2, &props);
if (res != SZ_OK) { SET_ERROR(res); goto exit; }
// Write the dict size to the output stream
props_out = Lzma2Enc_WriteProperties(lzma2);
// Create the streams and progress callback
in_stream.stream.Read = iread;
in_stream.read = read;
out_stream.stream.Write = owrite;
out_stream.write = write;
progress.progress.Progress = report_progress;
progress.callback = progress_callback;
// Run the compressor
ts = PyEval_SaveThread();
in_stream.thread_state = &ts;
out_stream.thread_state = &ts;
progress.thread_state = &ts;
res = Lzma2Enc_Encode(lzma2, (ISeqOutStream*)&out_stream, (ISeqInStream*)&in_stream, (ICompressProgress*)&progress);
if (res != SZ_OK && !PyErr_Occurred()) SET_ERROR(res);
if (ts) PyEval_RestoreThread(ts);
exit:
if (lzma2) Lzma2Enc_Destroy(lzma2);
if (PyErr_Occurred()) return NULL;
return Py_BuildValue(BYTES_FMT, &props_out, 1);
}
// }}}
static char lzma_binding_doc[] = "Bindings to the LZMA (de)compression C code";
static PyMethodDef lzma_binding_methods[] = {
{"decompress2", decompress2, METH_VARARGS,
"Decompress an LZMA2 encoded block, of unknown compressed size (reads till LZMA2 EOS marker)"
},
{"compress", compress, METH_VARARGS,
"Compress data into an LZMA2 block, writing it to outfile. Returns the LZMA2 properties as a bytestring."
},
{"decompress", decompress, METH_VARARGS,
"Decompress an LZMA encoded block, of (un)known size (reads till LZMA EOS marker when size unknown)"
},
{"crc64", crc64, METH_VARARGS,
"crc64(bytes) -> CRC 64 for the provided python bytes object"
},
{"delta_decode", delta_decode, METH_VARARGS,
"delta_decode(rawarray, histarray, pos, distance) -> Apply the delta decode filter to the bytearray rawarray"
},
{NULL, NULL, 0, NULL}
};
#if PY_MAJOR_VERSION >= 3
#define INITERROR return NULL
#define INITMODULE PyModule_Create(&lzma_binding_module)
static struct PyModuleDef lzma_binding_module = {
/* m_base */ PyModuleDef_HEAD_INIT,
/* m_name */ "lzma_binding",
/* m_doc */ lzma_binding_doc,
/* m_size */ -1,
/* m_methods */ lzma_binding_methods,
/* m_slots */ 0,
/* m_traverse */ 0,
/* m_clear */ 0,
/* m_free */ 0,
};
CALIBRE_MODINIT_FUNC PyInit_lzma_binding(void) {
#else
#define INITERROR return
#define INITMODULE Py_InitModule3("lzma_binding", lzma_binding_methods, lzma_binding_doc)
CALIBRE_MODINIT_FUNC initlzma_binding(void) {
#endif
PyObject *m = NULL, *preset_map = NULL, *temp = NULL;
int i = 0;
init_crc_table();
LZMAError = PyErr_NewException("lzma_binding.error", NULL, NULL);
if (!LZMAError) {
INITERROR;
}
m = INITMODULE;
if (m == NULL) {
INITERROR;
}
preset_map = PyTuple_New(10);
if (preset_map == NULL) {
INITERROR;
}
for (i = 0; i < 10; i++) {
temp = get_lzma2_properties(i);
if (temp == NULL) {
INITERROR;
}
PyTuple_SET_ITEM(preset_map, i, temp);
}
PyModule_AddObject(m, "preset_map", preset_map);
Py_INCREF(LZMAError);
PyModule_AddObject(m, "error", LZMAError);
#if PY_MAJOR_VERSION >= 3
return m;
#endif
}

503
src/calibre_lzma/xz.py
View File

@ -1,503 +0,0 @@
#!/usr/bin/env python
# vim:fileencoding=utf-8
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
# See http://tukaani.org/xz/xz-file-format.txt for file format details
from collections import namedtuple
from io import BytesIO
from hashlib import sha256
from struct import unpack, error as struct_error, pack
from binascii import crc32 as _crc32
from calibre.ptempfile import SpooledTemporaryFile
from .errors import NotXZ, InvalidXZ, lzma
from polyglot.builtins import error_message
HEADER_MAGIC = b'\xfd7zXZ\0'
FOOTER_MAGIC = b'YZ'
DELTA_FILTER_ID = 0x03
LZMA2_FILTER_ID = 0x21
def align(raw):
extra = len(raw) % 4
if extra:
raw += b'\0' * (4 - extra)
return raw
def as_bytes(*args):
return bytes(bytearray(args))
def crc32(raw, start=0):
return 0xFFFFFFFF & _crc32(raw, start)
def decode_var_int(f):
ans, i, ch = 0, -1, 0x80
while ch >= 0x80:
ch = ord(f.read(1))
i += 1
if ch == 0:
return 0
ans |= (ch & 0x7f) << (i * 7)
return ans
def decode_var_int2(raw, pos):
ans, ch, opos = 0, 0x80, pos
while ch >= 0x80:
ch = ord(raw[pos:pos+1])
if ch == 0:
return 0, pos
ans |= (ch & 0x7f) << ((pos - opos) * 7)
pos += 1
return ans, pos
def encode_var_int(num):
if num == 0:
return b'\0'
buf = bytearray()
a = buf.append
while num != 0:
a(0x80 | (num & 0x7F))
num >>= 7
buf[-1] &= 0x7F
return bytes(buf)
def read_stream_header(f):
try:
magic, stream_flags1, stream_flags2, crc = unpack(b'<6s2BI', f.read(12))
except struct_error as e:
raise NotXZ('Not an XZ file. Invalid stream header: ' % e)
if magic != HEADER_MAGIC:
raise NotXZ('Not an XZ file. Header Magic is: %r' % magic)
if stream_flags1 != 0:
raise InvalidXZ('Stream flags first byte is not null')
check_type, reserved = 0x0f & stream_flags2, 0xf0 & stream_flags2
if reserved != 0:
raise InvalidXZ('Stream flags reserved bits not null')
if crc32(bytes(bytearray([stream_flags1, stream_flags2]))) != crc:
raise InvalidXZ('Stream flags header CRC incorrect')
return check_type
class CRCChecker(object):
def __init__(self, check_type):
self.code = 0
if check_type == 0x1:
self.func = crc32
self.size = 4
self.fmt = b'<I'
else:
self.func = lzma.crc64
self.size = 8
self.fmt = b'<Q'
def __call__(self, raw):
self.code = self.func(raw, self.code)
def finish(self):
if self.func is not crc32:
self.code = 0xFFFFFFFFFFFFFFFF & self.code
@property
def code_as_bytes(self):
return pack(self.fmt, self.code)
def check(self, raw):
return self.code == unpack(self.fmt, raw)[0]
class Sha256Checker(object):
def __init__(self, *args):
self.h = sha256()
self.func = self.h.update
self.code = None
self.size = 32
def __call__(self, raw):
self.func(raw)
def finish(self):
self.code = self.code_as_bytes = self.h.digest()
self.h = self.func = None
def check(self, raw):
return self.code == raw
class DummyChecker(object):
size = 0
code_as_bytes = None
def __init__(self, *args):
pass
def __call__(self, raw):
pass
def finish(self):
pass
class LZMA2Filter(object):
BUFSIZE = 10 # MB
def __init__(self, props, check_type, bufsize=None):
if len(props) != 1:
raise InvalidXZ('Invalid properties length for LZMA2 filter')
props = ord(props)
self.dictionary_size = props & 0x3F
if props & 0xC0 != 0:
raise InvalidXZ('Invalid high bytes for LZMA2 filter properties')
self.props = props
if check_type in (0x1, 0x4):
self.crc = CRCChecker(check_type)
elif check_type == 0x0A:
self.crc = Sha256Checker()
else:
if check_type:
raise InvalidXZ('Unsupported CRC check type: %s' % check_type)
self.crc = DummyChecker()
if bufsize is None:
bufsize = self.BUFSIZE
self.bufsize = int(bufsize * 1024 * 1024)
def __call__(self, f, outfile, filters=()):
w = outfile.write
c = self.crc
def write(raw):
if filters:
raw = bytearray(raw)
for flt in filters:
raw = flt(raw)
raw = bytes(raw)
w(raw), c(raw)
try:
lzma.decompress2(f.read, f.seek, write, self.props, self.bufsize)
except lzma.error as e:
raise InvalidXZ(
'Failed to decode LZMA2 block with error code: %s' % error_message(e)
)
self.crc.finish()
class DeltaFilter(object):
def __init__(self, props, *args):
if len(props) != 1:
raise InvalidXZ('Invalid properties length for Delta filter')
self.distance = ord(props) + 1
self.pos = 0
self.history = bytearray(256)
def __call__(self, raw):
self.pos = lzma.delta_decode(raw, self.history, self.pos, self.distance)
return raw
def test_delta_filter():
raw = b'\xA1\xB1\x01\x02\x01\x02\x01\x02'
draw = b'\xA1\xB1\xA2\xB3\xA3\xB5\xA4\xB7'
def eq(s, d):
if s != d:
raise ValueError('%r != %r' % (s, d))
eq(draw, bytes(DeltaFilter(b'\x01')(bytearray(raw))))
f = DeltaFilter(b'\x01')
for ch, dch in zip(raw, draw):
eq(dch, bytes(f(bytearray(ch))))
Block = namedtuple('Block', 'unpadded_size uncompressed_size')
def read_block_header(f, block_header_size_, check_type):
block_header_size = 4 * (ord(block_header_size_) + 1)
if block_header_size < 8:
raise InvalidXZ('Invalid block header size: %d' % block_header_size)
header, crc = unpack(
b'<%dsI' % (block_header_size - 5), f.read(block_header_size - 1)
)
if crc != crc32(block_header_size_ + header):
raise InvalidXZ('Block header CRC mismatch')
block_flags = ord(header[0:1])
number_of_filters = (0x03 & block_flags) + 1
if not (0 < number_of_filters <= 4):
raise InvalidXZ('Invalid number of filters: %d' % number_of_filters)
if block_flags & 0x3c != 0:
raise InvalidXZ('Non-zero reserved bits in block flags')
has_compressed_size = block_flags & 0x40
has_uncompressed_size = block_flags & 0x80
compressed_size = uncompressed_size = None
pos = 1
if has_compressed_size:
compressed_size, pos = decode_var_int2(header, pos)
if has_uncompressed_size:
uncompressed_size, pos = decode_var_int2(header, pos)
filters = []
while number_of_filters:
number_of_filters -= 1
filter_id, pos = decode_var_int2(header, pos)
size_of_properties, pos = decode_var_int2(header, pos)
if filter_id >= 0x4000000000000000:
raise InvalidXZ('Invalid filter id: %d' % filter_id)
if filter_id not in (LZMA2_FILTER_ID, DELTA_FILTER_ID):
raise InvalidXZ('Unsupported filter ID: 0x%x' % filter_id)
props = header[pos:pos + size_of_properties]
pos += size_of_properties
if len(props) != size_of_properties:
raise InvalidXZ('Incomplete filter properties')
if filter_id == LZMA2_FILTER_ID and number_of_filters:
raise InvalidXZ('LZMA2 filter must be the last filter')
elif filter_id == DELTA_FILTER_ID and not number_of_filters:
raise InvalidXZ('Delta filter cannot be the last filter')
filters.append(
(LZMA2Filter
if filter_id == LZMA2_FILTER_ID else DeltaFilter)(props, check_type)
)
padding = header[pos:]
if padding.lstrip(b'\0'):
raise InvalidXZ('Non-null block header padding: %r' % padding)
filters.reverse()
return filters, compressed_size, uncompressed_size
def read_block(f, block_header_size_, check_type, outfile):
start_pos = f.tell() - 1
filters, compressed_size, uncompressed_size = read_block_header(
f, block_header_size_, check_type
)
fpos, opos = f.tell(), outfile.tell()
filters[0](f, outfile, filters[1:])
actual_compressed_size = f.tell() - fpos
uncompressed_actual_size = outfile.tell() - opos
if uncompressed_size is not None and uncompressed_size != uncompressed_actual_size:
raise InvalidXZ('Uncompressed size for block does not match')
if compressed_size is not None and compressed_size != actual_compressed_size:
raise InvalidXZ('Compressed size for block does not match')
padding_count = f.tell() % 4
if padding_count:
padding_count = 4 - padding_count
padding = f.read(padding_count)
if len(padding) != padding_count:
raise InvalidXZ('Block is not aligned')
if padding.lstrip(b'\0'):
raise InvalidXZ('Block padding has non null bytes')
if check_type:
q = f.read(filters[0].crc.size)
if not filters[0].crc.check(q):
raise InvalidXZ('CRC for data does not match')
return Block(f.tell() - padding_count - start_pos, uncompressed_actual_size)
def read_index(f):
pos = f.tell() - 1
number_of_records = decode_var_int(f)
while number_of_records:
number_of_records -= 1
unpadded_size = decode_var_int(f)
if unpadded_size < 1:
raise InvalidXZ('Invalid unpadded size in index: %d' % unpadded_size)
yield Block(unpadded_size, decode_var_int(f))
if f.tell() % 4:
padding_count = 4 - f.tell() % 4
padding = f.read(padding_count)
if len(padding) != padding_count or padding.lstrip(b'\0'):
raise InvalidXZ('Incorrect Index padding')
epos = f.tell()
f.seek(pos)
raw = f.read(epos - pos)
crc, = unpack(b'<I', f.read(4))
if crc != crc32(raw):
raise InvalidXZ('Index field CRC mismatch')
def read_stream_footer(f, check_type, index_size):
crc, = unpack(b'<I', f.read(4))
raw = f.read(6)
backward_size, stream_flags1, stream_flags2 = unpack(b'<I2B', raw)
if stream_flags1 != 0 or stream_flags2 & 0xf0 != 0 or stream_flags2 & 0xf != check_type:
raise InvalidXZ('Footer stream flags != header stream flags')
backward_size = 4 * (1 + backward_size)
if backward_size != index_size:
raise InvalidXZ('Footer backward size != actual index size')
if f.read(2) != FOOTER_MAGIC:
raise InvalidXZ('Stream footer has incorrect magic bytes')
if crc != crc32(raw):
raise InvalidXZ('Stream footer CRC mismatch')
def read_stream(f, outfile):
check_type = read_stream_header(f)
blocks, index = [], None
index_size = 0
while True:
sz = f.read(1)
if sz == b'\0':
pos = f.tell() - 1
index = tuple(read_index(f))
index_size = f.tell() - pos
break
else:
blocks.append(read_block(f, sz, check_type, outfile))
if index != tuple(blocks):
raise InvalidXZ('Index does not match actual blocks in file')
read_stream_footer(f, check_type, index_size)
def decompress(raw, outfile=None):
'''
Decompress the specified data.
:param raw: A bytestring or a file-like object open for reading
:outfile: A file like object open for writing.
The decompressed data is written into it. If not specified then a SpooledTemporaryFile
is created and returned by this function.
'''
if isinstance(raw, bytes):
raw = BytesIO(raw)
outfile = outfile or SpooledTemporaryFile(50 * 1024 * 1024, '_xz_decompress')
while True:
read_stream(raw, outfile)
pos = raw.tell()
trail = raw.read(1024)
if len(trail) < 20:
break
idx = trail.find(HEADER_MAGIC)
if idx == -1:
break
if idx > -1:
# Found another stream
raw.seek(pos)
if idx:
padding = raw.read(idx)
if padding.lstrip(b'\0') or len(padding) % 4:
raise InvalidXZ('Found trailing garbage between streams')
return outfile
def compress(raw, outfile=None, level=5, check_type='crc64'):
'''
Compress the specified data into a .xz stream (which can be written directly as
an .xz file.
:param raw: A bytestring or a file-like object open for reading
:outfile: A file like object open for writing.
The .xz stream is written into it. If not specified then a SpooledTemporaryFile
is created and returned by this function.
:level: An integer between 0 and 9 with 0 being fastest/worst compression and 9 being
slowest/best compression
:check_type: The type of data integrity check to write into the output .xz stream.
Should be one of: 'crc32', 'crc64', 'sha256', or None
'''
if isinstance(raw, bytes):
raw = BytesIO(raw)
outfile = outfile or SpooledTemporaryFile(50 * 1024 * 1024, '_xz_decompress')
# Write stream header
outfile.write(HEADER_MAGIC)
check_type = {
'crc': 1,
'crc32': 1,
'sha256': 0xa,
None: 0,
'': 0,
'none': 0,
'None': 0
}.get(check_type, 4)
stream_flags = as_bytes(0, check_type)
outfile.write(stream_flags)
outfile.write(pack(b'<I', crc32(stream_flags)))
# Write block header
filter_flags = encode_var_int(LZMA2_FILTER_ID
) + encode_var_int(1) + lzma.preset_map[level]
block_header = align(b'\0\0' + filter_flags)
bhs = ((4 + len(block_header)) // 4) - 1
block_header = as_bytes(bhs) + block_header[1:]
block_header += pack(b'<I', crc32(block_header))
start = outfile.tell()
outfile.write(block_header)
# Write compressed data and check
checker = {
0: DummyChecker,
1: CRCChecker,
4: CRCChecker,
0xa: Sha256Checker
}[check_type](check_type)
uncompressed_size = [0]
def read(n):
ans = raw.read(n)
if ans:
uncompressed_size[0] += len(ans)
checker(ans)
return ans
lzma.compress(read, outfile.write, None, level)
unpadded_size = outfile.tell() - start
pos = outfile.tell()
if pos % 4:
outfile.write(b'\0' * (4 - (pos % 4)))
checker.finish()
if check_type:
cc = checker.code_as_bytes
outfile.write(cc)
unpadded_size += len(cc)
# Write index
index = b'\0' + encode_var_int(1)
index += encode_var_int(unpadded_size) + encode_var_int(uncompressed_size[0])
if len(index) % 4:
index += b'\0' * (4 - len(index) % 4)
outfile.write(index), outfile.write(pack(b'<I', crc32(index)))
# Write stream footer
backwards_size = pack(b'<I', ((len(index) + 4) // 4) - 1)
outfile.write(pack(b'<I', crc32(backwards_size + stream_flags)))
outfile.write(backwards_size), outfile.write(stream_flags
), outfile.write(FOOTER_MAGIC)
def test_lzma2():
raw = P('template-functions.json', allow_user_override=False, data=True)
ibuf, obuf = BytesIO(raw), BytesIO()
props = lzma.compress(ibuf.read, obuf.write, False)
cc = obuf.getvalue()
ibuf, obuf = BytesIO(cc), BytesIO()
LZMA2Filter(props, 0, 1)(ibuf, obuf)
if obuf.getvalue() != raw:
raise ValueError('Roundtripping via LZMA2 failed')
def test_xz():
raw = P('template-functions.json', allow_user_override=False, data=True)
ibuf, obuf = BytesIO(raw), BytesIO()
compress(ibuf, obuf, check_type='sha256')
cc = obuf.getvalue()
ibuf, obuf = BytesIO(cc), BytesIO()
decompress(ibuf, obuf)
if obuf.getvalue() != raw:
raise ValueError('Roundtripping via XZ failed')
if __name__ == '__main__':
import sys
decompress(open(sys.argv[-1], 'rb'))