summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCarsten Haitzler (Rasterman) <raster@rasterman.com>2020-01-28 11:17:51 +0000
committerCarsten Haitzler (Rasterman) <raster@rasterman.com>2020-01-28 11:17:51 +0000
commit897f27eff407638f8b6979951eed32284737e6a0 (patch)
tree76fc3ef3259b97642007a8e8b2bd91fb2f663229
parentade8a2b780f0b953a2f251af0bdb5203d7f4770d (diff)
lz4 - sync to latest current lz4 upstream source
-rw-r--r--src/static_libs/lz4/lz4.c445
-rw-r--r--src/static_libs/lz4/lz4.h142
-rw-r--r--src/static_libs/lz4/lz4frame.c28
-rw-r--r--src/static_libs/lz4/lz4frame.h9
-rw-r--r--src/static_libs/lz4/lz4hc.c122
-rw-r--r--src/static_libs/lz4/lz4hc.h3
6 files changed, 515 insertions, 234 deletions
diff --git a/src/static_libs/lz4/lz4.c b/src/static_libs/lz4/lz4.c
index e614c4577f..82ab490811 100644
--- a/src/static_libs/lz4/lz4.c
+++ b/src/static_libs/lz4/lz4.c
@@ -106,6 +106,7 @@
106#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ 106#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */
107#endif 107#endif
108 108
109#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */
109#include "lz4.h" 110#include "lz4.h"
110/* see also "memory routines" below */ 111/* see also "memory routines" below */
111 112
@@ -183,6 +184,60 @@
183 184
184 185
185/*-************************************ 186/*-************************************
187* Common Constants
188**************************************/
189#define MINMATCH 4
190
191#define WILDCOPYLENGTH 8
192#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
193#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
194#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */
195#define FASTLOOP_SAFE_DISTANCE 64
196static const int LZ4_minLength = (MFLIMIT+1);
197
198#define KB *(1 <<10)
199#define MB *(1 <<20)
200#define GB *(1U<<30)
201
202#define LZ4_DISTANCE_ABSOLUTE_MAX 65535
203#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */
204# error "LZ4_DISTANCE_MAX is too big : must be <= 65535"
205#endif
206
207#define ML_BITS 4
208#define ML_MASK ((1U<<ML_BITS)-1)
209#define RUN_BITS (8-ML_BITS)
210#define RUN_MASK ((1U<<RUN_BITS)-1)
211
212
213/*-************************************
214* Error detection
215**************************************/
216#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)
217# include <assert.h>
218#else
219# ifndef assert
220# define assert(condition) ((void)0)
221# endif
222#endif
223
224#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
225
226#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
227# include <stdio.h>
228 static int g_debuglog_enable = 1;
229# define DEBUGLOG(l, ...) { \
230 if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \
231 fprintf(stderr, __FILE__ ": "); \
232 fprintf(stderr, __VA_ARGS__); \
233 fprintf(stderr, " \n"); \
234 } }
235#else
236# define DEBUGLOG(l, ...) {} /* disabled */
237#endif
238
239
240/*-************************************
186* Types 241* Types
187**************************************/ 242**************************************/
188#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 243#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@@ -194,6 +249,10 @@
194 typedef uint64_t U64; 249 typedef uint64_t U64;
195 typedef uintptr_t uptrval; 250 typedef uintptr_t uptrval;
196#else 251#else
252# include <limits.h>
253# if UINT_MAX != 4294967295UL
254# error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4"
255# endif
197 typedef unsigned char BYTE; 256 typedef unsigned char BYTE;
198 typedef unsigned short U16; 257 typedef unsigned short U16;
199 typedef unsigned int U32; 258 typedef unsigned int U32;
@@ -315,7 +374,12 @@ static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3};
315 374
316 375
317#ifndef LZ4_FAST_DEC_LOOP 376#ifndef LZ4_FAST_DEC_LOOP
318# if defined(__i386__) || defined(__x86_64__) 377# if defined __i386__ || defined _M_IX86 || defined __x86_64__ || defined _M_X64
378# define LZ4_FAST_DEC_LOOP 1
379# elif defined(__aarch64__) && !defined(__clang__)
380 /* On aarch64, we disable this optimization for clang because on certain
381 * mobile chipsets, performance is reduced with clang. For information
382 * refer to https://github.com/lz4/lz4/pull/707 */
319# define LZ4_FAST_DEC_LOOP 1 383# define LZ4_FAST_DEC_LOOP 1
320# else 384# else
321# define LZ4_FAST_DEC_LOOP 0 385# define LZ4_FAST_DEC_LOOP 0
@@ -358,29 +422,35 @@ LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd)
358 do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d<e); 422 do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d<e);
359} 423}
360 424
425/* LZ4_memcpy_using_offset() presumes :
426 * - dstEnd >= dstPtr + MINMATCH
427 * - there is at least 8 bytes available to write after dstEnd */
361LZ4_FORCE_O2_INLINE_GCC_PPC64LE void 428LZ4_FORCE_O2_INLINE_GCC_PPC64LE void
362LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) 429LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset)
363{ 430{
364 BYTE v[8]; 431 BYTE v[8];
432
433 assert(dstEnd >= dstPtr + MINMATCH);
434 LZ4_write32(dstPtr, 0); /* silence an msan warning when offset==0 */
435
365 switch(offset) { 436 switch(offset) {
366 case 1: 437 case 1:
367 memset(v, *srcPtr, 8); 438 memset(v, *srcPtr, 8);
368 goto copy_loop; 439 break;
369 case 2: 440 case 2:
370 memcpy(v, srcPtr, 2); 441 memcpy(v, srcPtr, 2);
371 memcpy(&v[2], srcPtr, 2); 442 memcpy(&v[2], srcPtr, 2);
372 memcpy(&v[4], &v[0], 4); 443 memcpy(&v[4], &v[0], 4);
373 goto copy_loop; 444 break;
374 case 4: 445 case 4:
375 memcpy(v, srcPtr, 4); 446 memcpy(v, srcPtr, 4);
376 memcpy(&v[4], srcPtr, 4); 447 memcpy(&v[4], srcPtr, 4);
377 goto copy_loop; 448 break;
378 default: 449 default:
379 LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); 450 LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset);
380 return; 451 return;
381 } 452 }
382 453
383 copy_loop:
384 memcpy(dstPtr, v, 8); 454 memcpy(dstPtr, v, 8);
385 dstPtr += 8; 455 dstPtr += 8;
386 while (dstPtr < dstEnd) { 456 while (dstPtr < dstEnd) {
@@ -392,63 +462,6 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si
392 462
393 463
394/*-************************************ 464/*-************************************
395* Common Constants
396**************************************/
397#define MINMATCH 4
398
399#define WILDCOPYLENGTH 8
400#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
401#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
402#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */
403#define FASTLOOP_SAFE_DISTANCE 64
404static const int LZ4_minLength = (MFLIMIT+1);
405
406#define KB *(1 <<10)
407#define MB *(1 <<20)
408#define GB *(1U<<30)
409
410#ifndef LZ4_DISTANCE_MAX /* can be user - defined at compile time */
411# define LZ4_DISTANCE_MAX 65535
412#endif
413
414#if (LZ4_DISTANCE_MAX > 65535) /* max supported by LZ4 format */
415# error "LZ4_DISTANCE_MAX is too big : must be <= 65535"
416#endif
417
418#define ML_BITS 4
419#define ML_MASK ((1U<<ML_BITS)-1)
420#define RUN_BITS (8-ML_BITS)
421#define RUN_MASK ((1U<<RUN_BITS)-1)
422
423
424/*-************************************
425* Error detection
426**************************************/
427#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)
428# include <assert.h>
429#else
430# ifndef assert
431# define assert(condition) ((void)0)
432# endif
433#endif
434
435#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
436
437#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
438# include <stdio.h>
439static int g_debuglog_enable = 1;
440# define DEBUGLOG(l, ...) { \
441 if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \
442 fprintf(stderr, __FILE__ ": "); \
443 fprintf(stderr, __VA_ARGS__); \
444 fprintf(stderr, " \n"); \
445 } }
446#else
447# define DEBUGLOG(l, ...) {} /* disabled */
448#endif
449
450
451/*-************************************
452* Common functions 465* Common functions
453**************************************/ 466**************************************/
454static unsigned LZ4_NbCommonBytes (reg_t val) 467static unsigned LZ4_NbCommonBytes (reg_t val)
@@ -460,7 +473,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
460 _BitScanForward64( &r, (U64)val ); 473 _BitScanForward64( &r, (U64)val );
461 return (int)(r>>3); 474 return (int)(r>>3);
462# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 475# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
463 return (__builtin_ctzll((U64)val) >> 3); 476 return (unsigned)__builtin_ctzll((U64)val) >> 3;
464# else 477# else
465 static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 478 static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
466 0, 3, 1, 3, 1, 4, 2, 7, 479 0, 3, 1, 3, 1, 4, 2, 7,
@@ -478,7 +491,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
478 _BitScanForward( &r, (U32)val ); 491 _BitScanForward( &r, (U32)val );
479 return (int)(r>>3); 492 return (int)(r>>3);
480# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 493# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
481 return (__builtin_ctz((U32)val) >> 3); 494 return (unsigned)__builtin_ctz((U32)val) >> 3;
482# else 495# else
483 static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 496 static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
484 3, 2, 2, 1, 3, 2, 0, 1, 497 3, 2, 2, 1, 3, 2, 0, 1,
@@ -494,7 +507,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
494 _BitScanReverse64( &r, val ); 507 _BitScanReverse64( &r, val );
495 return (unsigned)(r>>3); 508 return (unsigned)(r>>3);
496# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 509# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
497 return (__builtin_clzll((U64)val) >> 3); 510 return (unsigned)__builtin_clzll((U64)val) >> 3;
498# else 511# else
499 static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. 512 static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits.
500 Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. 513 Just to avoid some static analyzer complaining about shift by 32 on 32-bits target.
@@ -511,7 +524,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
511 _BitScanReverse( &r, (unsigned long)val ); 524 _BitScanReverse( &r, (unsigned long)val );
512 return (unsigned)(r>>3); 525 return (unsigned)(r>>3);
513# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 526# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
514 return (__builtin_clz((U32)val) >> 3); 527 return (unsigned)__builtin_clz((U32)val) >> 3;
515# else 528# else
516 unsigned r; 529 unsigned r;
517 if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } 530 if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
@@ -606,9 +619,11 @@ int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
606extern "C" { 619extern "C" {
607#endif 620#endif
608 621
609int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); 622int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize);
610 623
611int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize); 624int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,
625 int compressedSize, int maxOutputSize,
626 const void* dictStart, size_t dictSize);
612 627
613#if defined (__cplusplus) 628#if defined (__cplusplus)
614} 629}
@@ -617,7 +632,7 @@ int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int
617/*-****************************** 632/*-******************************
618* Compression functions 633* Compression functions
619********************************/ 634********************************/
620static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) 635LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType)
621{ 636{
622 if (tableType == byU16) 637 if (tableType == byU16)
623 return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); 638 return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
@@ -625,7 +640,7 @@ static U32 LZ4_hash4(U32 sequence, tableType_t const tableType)
625 return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); 640 return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
626} 641}
627 642
628static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) 643LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
629{ 644{
630 const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; 645 const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG;
631 if (LZ4_isLittleEndian()) { 646 if (LZ4_isLittleEndian()) {
@@ -643,7 +658,19 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab
643 return LZ4_hash4(LZ4_read32(p), tableType); 658 return LZ4_hash4(LZ4_read32(p), tableType);
644} 659}
645 660
646static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) 661LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType)
662{
663 switch (tableType)
664 {
665 default: /* fallthrough */
666 case clearedTable: { /* illegal! */ assert(0); return; }
667 case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; }
668 case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; }
669 case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; }
670 }
671}
672
673LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType)
647{ 674{
648 switch (tableType) 675 switch (tableType)
649 { 676 {
@@ -655,7 +682,7 @@ static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t cons
655 } 682 }
656} 683}
657 684
658static void LZ4_putPositionOnHash(const BYTE* p, U32 h, 685LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h,
659 void* tableBase, tableType_t const tableType, 686 void* tableBase, tableType_t const tableType,
660 const BYTE* srcBase) 687 const BYTE* srcBase)
661{ 688{
@@ -680,7 +707,7 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
680 * Assumption 1 : only valid if tableType == byU32 or byU16. 707 * Assumption 1 : only valid if tableType == byU32 or byU16.
681 * Assumption 2 : h is presumed valid (within limits of hash table) 708 * Assumption 2 : h is presumed valid (within limits of hash table)
682 */ 709 */
683static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) 710LZ4_FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType)
684{ 711{
685 LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); 712 LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2);
686 if (tableType == byU32) { 713 if (tableType == byU32) {
@@ -703,18 +730,19 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType
703 { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ 730 { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
704} 731}
705 732
706LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, 733LZ4_FORCE_INLINE const BYTE*
707 const void* tableBase, tableType_t tableType, 734LZ4_getPosition(const BYTE* p,
708 const BYTE* srcBase) 735 const void* tableBase, tableType_t tableType,
736 const BYTE* srcBase)
709{ 737{
710 U32 const h = LZ4_hashPosition(p, tableType); 738 U32 const h = LZ4_hashPosition(p, tableType);
711 return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); 739 return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
712} 740}
713 741
714LZ4_FORCE_INLINE void LZ4_prepareTable( 742LZ4_FORCE_INLINE void
715 LZ4_stream_t_internal* const cctx, 743LZ4_prepareTable(LZ4_stream_t_internal* const cctx,
716 const int inputSize, 744 const int inputSize,
717 const tableType_t tableType) { 745 const tableType_t tableType) {
718 /* If compression failed during the previous step, then the context 746 /* If compression failed during the previous step, then the context
719 * is marked as dirty, therefore, it has to be fully reset. 747 * is marked as dirty, therefore, it has to be fully reset.
720 */ 748 */
@@ -729,9 +757,10 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
729 * out if it's safe to leave as is or whether it needs to be reset. 757 * out if it's safe to leave as is or whether it needs to be reset.
730 */ 758 */
731 if (cctx->tableType != clearedTable) { 759 if (cctx->tableType != clearedTable) {
760 assert(inputSize >= 0);
732 if (cctx->tableType != tableType 761 if (cctx->tableType != tableType
733 || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU) 762 || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU)
734 || (tableType == byU32 && cctx->currentOffset > 1 GB) 763 || ((tableType == byU32) && cctx->currentOffset > 1 GB)
735 || tableType == byPtr 764 || tableType == byPtr
736 || inputSize >= 4 KB) 765 || inputSize >= 4 KB)
737 { 766 {
@@ -811,9 +840,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
811 DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); 840 DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType);
812 /* If init conditions are not met, we don't have to mark stream 841 /* If init conditions are not met, we don't have to mark stream
813 * as having dirty context, since no action was taken yet */ 842 * as having dirty context, since no action was taken yet */
814 if (outputDirective == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */ 843 if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */
815 if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ 844 if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported inputSize, too large (or negative) */
816 if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ 845 if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */
817 if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ 846 if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */
818 assert(acceleration >= 1); 847 assert(acceleration >= 1);
819 848
@@ -841,6 +870,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
841 for ( ; ; ) { 870 for ( ; ; ) {
842 const BYTE* match; 871 const BYTE* match;
843 BYTE* token; 872 BYTE* token;
873 const BYTE* filledIp;
844 874
845 /* Find a match */ 875 /* Find a match */
846 if (tableType == byPtr) { 876 if (tableType == byPtr) {
@@ -909,10 +939,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
909 forwardH = LZ4_hashPosition(forwardIp, tableType); 939 forwardH = LZ4_hashPosition(forwardIp, tableType);
910 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); 940 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
911 941
912 if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */ 942 DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex);
943 if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */
913 assert(matchIndex < current); 944 assert(matchIndex < current);
914 if ((tableType != byU16) && (matchIndex+LZ4_DISTANCE_MAX < current)) continue; /* too far */ 945 if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX))
915 if (tableType == byU16) assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* too_far presumed impossible with byU16 */ 946 && (matchIndex+LZ4_DISTANCE_MAX < current)) {
947 continue;
948 } /* too far */
949 assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */
916 950
917 if (LZ4_read32(match) == LZ4_read32(ip)) { 951 if (LZ4_read32(match) == LZ4_read32(ip)) {
918 if (maybe_extMem) offset = current - matchIndex; 952 if (maybe_extMem) offset = current - matchIndex;
@@ -923,15 +957,16 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
923 } 957 }
924 958
925 /* Catch up */ 959 /* Catch up */
960 filledIp = ip;
926 while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } 961 while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
927 962
928 /* Encode Literals */ 963 /* Encode Literals */
929 { unsigned const litLength = (unsigned)(ip - anchor); 964 { unsigned const litLength = (unsigned)(ip - anchor);
930 token = op++; 965 token = op++;
931 if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ 966 if ((outputDirective == limitedOutput) && /* Check output buffer overflow */
932 (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) 967 (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) {
933 return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ 968 return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */
934 969 }
935 if ((outputDirective == fillOutput) && 970 if ((outputDirective == fillOutput) &&
936 (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { 971 (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) {
937 op--; 972 op--;
@@ -1002,12 +1037,26 @@ _next_match:
1002 } 1037 }
1003 1038
1004 if ((outputDirective) && /* Check output buffer overflow */ 1039 if ((outputDirective) && /* Check output buffer overflow */
1005 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) { 1040 (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) {
1006 if (outputDirective == fillOutput) { 1041 if (outputDirective == fillOutput) {
1007 /* Match description too long : reduce it */ 1042 /* Match description too long : reduce it */
1008 U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255; 1043 U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255;
1009 ip -= matchCode - newMatchCode; 1044 ip -= matchCode - newMatchCode;
1045 assert(newMatchCode < matchCode);
1010 matchCode = newMatchCode; 1046 matchCode = newMatchCode;
1047 if (unlikely(ip <= filledIp)) {
1048 /* We have already filled up to filledIp so if ip ends up less than filledIp
1049 * we have positions in the hash table beyond the current position. This is
1050 * a problem if we reuse the hash table. So we have to remove these positions
1051 * from the hash table.
1052 */
1053 const BYTE* ptr;
1054 DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip));
1055 for (ptr = ip; ptr <= filledIp; ++ptr) {
1056 U32 const h = LZ4_hashPosition(ptr, tableType);
1057 LZ4_clearHash(h, cctx->hashTable, tableType);
1058 }
1059 }
1011 } else { 1060 } else {
1012 assert(outputDirective == limitedOutput); 1061 assert(outputDirective == limitedOutput);
1013 return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ 1062 return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */
@@ -1027,6 +1076,8 @@ _next_match:
1027 } else 1076 } else
1028 *token += (BYTE)(matchCode); 1077 *token += (BYTE)(matchCode);
1029 } 1078 }
1079 /* Ensure we have enough space for the last literals. */
1080 assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit));
1030 1081
1031 anchor = ip; 1082 anchor = ip;
1032 1083
@@ -1076,7 +1127,7 @@ _next_match:
1076 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); 1127 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
1077 assert(matchIndex < current); 1128 assert(matchIndex < current);
1078 if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) 1129 if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
1079 && ((tableType==byU16) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) 1130 && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current))
1080 && (LZ4_read32(match) == LZ4_read32(ip)) ) { 1131 && (LZ4_read32(match) == LZ4_read32(ip)) ) {
1081 token=op++; 1132 token=op++;
1082 *token=0; 1133 *token=0;
@@ -1143,7 +1194,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
1143 return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); 1194 return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
1144 } 1195 }
1145 } else { 1196 } else {
1146 if (inputSize < LZ4_64Klimit) {; 1197 if (inputSize < LZ4_64Klimit) {
1147 return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); 1198 return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
1148 } else { 1199 } else {
1149 const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; 1200 const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
@@ -1306,12 +1357,12 @@ static size_t LZ4_stream_t_alignment(void)
1306LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) 1357LZ4_stream_t* LZ4_initStream (void* buffer, size_t size)
1307{ 1358{
1308 DEBUGLOG(5, "LZ4_initStream"); 1359 DEBUGLOG(5, "LZ4_initStream");
1309 if (buffer == NULL) return NULL; 1360 if (buffer == NULL) { return NULL; }
1310 if (size < sizeof(LZ4_stream_t)) return NULL; 1361 if (size < sizeof(LZ4_stream_t)) { return NULL; }
1311#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : 1362#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :
1312 it reports an aligment of 8-bytes, 1363 it reports an aligment of 8-bytes,
1313 while actually aligning LZ4_stream_t on 4 bytes. */ 1364 while actually aligning LZ4_stream_t on 4 bytes. */
1314 if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) return NULL; /* alignment check */ 1365 if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) { return NULL; } /* alignment check */
1315#endif 1366#endif
1316 MEM_INIT(buffer, 0, sizeof(LZ4_stream_t)); 1367 MEM_INIT(buffer, 0, sizeof(LZ4_stream_t));
1317 return (LZ4_stream_t*)buffer; 1368 return (LZ4_stream_t*)buffer;
@@ -1361,18 +1412,18 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
1361 * there are only valid offsets in the window, which allows an optimization 1412 * there are only valid offsets in the window, which allows an optimization
1362 * in LZ4_compress_fast_continue() where it uses noDictIssue even when the 1413 * in LZ4_compress_fast_continue() where it uses noDictIssue even when the
1363 * dictionary isn't a full 64k. */ 1414 * dictionary isn't a full 64k. */
1364
1365 if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
1366 base = dictEnd - 64 KB - dict->currentOffset;
1367 dict->dictionary = p;
1368 dict->dictSize = (U32)(dictEnd - p);
1369 dict->currentOffset += 64 KB; 1415 dict->currentOffset += 64 KB;
1370 dict->tableType = tableType;
1371 1416
1372 if (dictSize < (int)HASH_UNIT) { 1417 if (dictSize < (int)HASH_UNIT) {
1373 return 0; 1418 return 0;
1374 } 1419 }
1375 1420
1421 if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
1422 base = dictEnd - dict->currentOffset;
1423 dict->dictionary = p;
1424 dict->dictSize = (U32)(dictEnd - p);
1425 dict->tableType = tableType;
1426
1376 while (p <= dictEnd-HASH_UNIT) { 1427 while (p <= dictEnd-HASH_UNIT) {
1377 LZ4_putPosition(p, dict->hashTable, tableType, base); 1428 LZ4_putPosition(p, dict->hashTable, tableType, base);
1378 p+=3; 1429 p+=3;
@@ -1381,26 +1432,37 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
1381 return (int)dict->dictSize; 1432 return (int)dict->dictSize;
1382} 1433}
1383 1434
1384void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) { 1435void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) {
1436 const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL :
1437 &(dictionaryStream->internal_donotuse);
1438
1439 DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)",
1440 workingStream, dictionaryStream,
1441 dictCtx != NULL ? dictCtx->dictSize : 0);
1442
1385 /* Calling LZ4_resetStream_fast() here makes sure that changes will not be 1443 /* Calling LZ4_resetStream_fast() here makes sure that changes will not be
1386 * erased by subsequent calls to LZ4_resetStream_fast() in case stream was 1444 * erased by subsequent calls to LZ4_resetStream_fast() in case stream was
1387 * marked as having dirty context, e.g. requiring full reset. 1445 * marked as having dirty context, e.g. requiring full reset.
1388 */ 1446 */
1389 LZ4_resetStream_fast(working_stream); 1447 LZ4_resetStream_fast(workingStream);
1390 1448
1391 if (dictionary_stream != NULL) { 1449 if (dictCtx != NULL) {
1392 /* If the current offset is zero, we will never look in the 1450 /* If the current offset is zero, we will never look in the
1393 * external dictionary context, since there is no value a table 1451 * external dictionary context, since there is no value a table
1394 * entry can take that indicate a miss. In that case, we need 1452 * entry can take that indicate a miss. In that case, we need
1395 * to bump the offset to something non-zero. 1453 * to bump the offset to something non-zero.
1396 */ 1454 */
1397 if (working_stream->internal_donotuse.currentOffset == 0) { 1455 if (workingStream->internal_donotuse.currentOffset == 0) {
1398 working_stream->internal_donotuse.currentOffset = 64 KB; 1456 workingStream->internal_donotuse.currentOffset = 64 KB;
1457 }
1458
1459 /* Don't actually attach an empty dictionary.
1460 */
1461 if (dictCtx->dictSize == 0) {
1462 dictCtx = NULL;
1399 } 1463 }
1400 working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse);
1401 } else {
1402 working_stream->internal_donotuse.dictCtx = NULL;
1403 } 1464 }
1465 workingStream->internal_donotuse.dictCtx = dictCtx;
1404} 1466}
1405 1467
1406 1468
@@ -1435,7 +1497,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream,
1435 1497
1436 DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); 1498 DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize);
1437 1499
1438 if (streamPtr->dirty) return 0; /* Uninitialized structure detected */ 1500 if (streamPtr->dirty) { return 0; } /* Uninitialized structure detected */
1439 LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ 1501 LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */
1440 if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; 1502 if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
1441 1503
@@ -1532,8 +1594,8 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
1532 LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; 1594 LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
1533 const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; 1595 const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
1534 1596
1535 if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ 1597 if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
1536 if ((U32)dictSize > dict->dictSize) dictSize = (int)dict->dictSize; 1598 if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
1537 1599
1538 memmove(safeBuffer, previousDictEnd - dictSize, dictSize); 1600 memmove(safeBuffer, previousDictEnd - dictSize, dictSize);
1539 1601
@@ -1567,8 +1629,8 @@ typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_err
1567LZ4_FORCE_INLINE unsigned 1629LZ4_FORCE_INLINE unsigned
1568read_variable_length(const BYTE**ip, const BYTE* lencheck, int loop_check, int initial_check, variable_length_error* error) 1630read_variable_length(const BYTE**ip, const BYTE* lencheck, int loop_check, int initial_check, variable_length_error* error)
1569{ 1631{
1570 unsigned length = 0; 1632 U32 length = 0;
1571 unsigned s; 1633 U32 s;
1572 if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ 1634 if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */
1573 *error = initial_error; 1635 *error = initial_error;
1574 return length; 1636 return length;
@@ -1607,7 +1669,7 @@ LZ4_decompress_generic(
1607 const size_t dictSize /* note : = 0 if noDict */ 1669 const size_t dictSize /* note : = 0 if noDict */
1608 ) 1670 )
1609{ 1671{
1610 if (src == NULL) return -1; 1672 if (src == NULL) { return -1; }
1611 1673
1612 { const BYTE* ip = (const BYTE*) src; 1674 { const BYTE* ip = (const BYTE*) src;
1613 const BYTE* const iend = ip + srcSize; 1675 const BYTE* const iend = ip + srcSize;
@@ -1636,9 +1698,13 @@ LZ4_decompress_generic(
1636 1698
1637 /* Special cases */ 1699 /* Special cases */
1638 assert(lowPrefix <= op); 1700 assert(lowPrefix <= op);
1639 if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ 1701 if ((endOnInput) && (unlikely(outputSize==0))) {
1640 if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); 1702 /* Empty output buffer */
1641 if ((endOnInput) && unlikely(srcSize==0)) return -1; 1703 if (partialDecoding) return 0;
1704 return ((srcSize==1) && (*ip==0)) ? 0 : -1;
1705 }
1706 if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); }
1707 if ((endOnInput) && unlikely(srcSize==0)) { return -1; }
1642 1708
1643 /* Currently the fast loop shows a regression on qualcomm arm chips. */ 1709 /* Currently the fast loop shows a regression on qualcomm arm chips. */
1644#if LZ4_FAST_DEC_LOOP 1710#if LZ4_FAST_DEC_LOOP
@@ -1651,7 +1717,7 @@ LZ4_decompress_generic(
1651 while (1) { 1717 while (1) {
1652 /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ 1718 /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */
1653 assert(oend - op >= FASTLOOP_SAFE_DISTANCE); 1719 assert(oend - op >= FASTLOOP_SAFE_DISTANCE);
1654 if (endOnInput) assert(ip < iend); 1720 if (endOnInput) { assert(ip < iend); }
1655 token = *ip++; 1721 token = *ip++;
1656 length = token >> ML_BITS; /* literal length */ 1722 length = token >> ML_BITS; /* literal length */
1657 1723
@@ -1661,18 +1727,18 @@ LZ4_decompress_generic(
1661 if (length == RUN_MASK) { 1727 if (length == RUN_MASK) {
1662 variable_length_error error = ok; 1728 variable_length_error error = ok;
1663 length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); 1729 length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error);
1664 if (error == initial_error) goto _output_error; 1730 if (error == initial_error) { goto _output_error; }
1665 if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ 1731 if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
1666 if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ 1732 if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
1667 1733
1668 /* copy literals */ 1734 /* copy literals */
1669 cpy = op+length; 1735 cpy = op+length;
1670 LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); 1736 LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
1671 if (endOnInput) { /* LZ4_decompress_safe() */ 1737 if (endOnInput) { /* LZ4_decompress_safe() */
1672 if ((cpy>oend-32) || (ip+length>iend-32)) goto safe_literal_copy; 1738 if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; }
1673 LZ4_wildCopy32(op, ip, cpy); 1739 LZ4_wildCopy32(op, ip, cpy);
1674 } else { /* LZ4_decompress_fast() */ 1740 } else { /* LZ4_decompress_fast() */
1675 if (cpy>oend-8) goto safe_literal_copy; 1741 if (cpy>oend-8) { goto safe_literal_copy; }
1676 LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : 1742 LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
1677 * it doesn't know input length, and only relies on end-of-block properties */ 1743 * it doesn't know input length, and only relies on end-of-block properties */
1678 } 1744 }
@@ -1682,14 +1748,14 @@ LZ4_decompress_generic(
1682 if (endOnInput) { /* LZ4_decompress_safe() */ 1748 if (endOnInput) { /* LZ4_decompress_safe() */
1683 DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); 1749 DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);
1684 /* We don't need to check oend, since we check it once for each loop below */ 1750 /* We don't need to check oend, since we check it once for each loop below */
1685 if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) goto safe_literal_copy; 1751 if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; }
1686 /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ 1752 /* Literals can only be 14, but hope compilers optimize if we copy by a register size */
1687 memcpy(op, ip, 16); 1753 memcpy(op, ip, 16);
1688 } else { /* LZ4_decompress_fast() */ 1754 } else { /* LZ4_decompress_fast() */
1689 /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : 1755 /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
1690 * it doesn't know input length, and relies on end-of-block properties */ 1756 * it doesn't know input length, and relies on end-of-block properties */
1691 memcpy(op, ip, 8); 1757 memcpy(op, ip, 8);
1692 if (length > 8) memcpy(op+8, ip+8, 8); 1758 if (length > 8) { memcpy(op+8, ip+8, 8); }
1693 } 1759 }
1694 ip += length; op = cpy; 1760 ip += length; op = cpy;
1695 } 1761 }
@@ -1697,17 +1763,17 @@ LZ4_decompress_generic(
1697 /* get offset */ 1763 /* get offset */
1698 offset = LZ4_readLE16(ip); ip+=2; 1764 offset = LZ4_readLE16(ip); ip+=2;
1699 match = op - offset; 1765 match = op - offset;
1766 assert(match <= op);
1700 1767
1701 /* get matchlength */ 1768 /* get matchlength */
1702 length = token & ML_MASK; 1769 length = token & ML_MASK;
1703 1770
1704 if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
1705
1706 if (length == ML_MASK) { 1771 if (length == ML_MASK) {
1707 variable_length_error error = ok; 1772 variable_length_error error = ok;
1773 if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
1708 length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); 1774 length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error);
1709 if (error != ok) goto _output_error; 1775 if (error != ok) { goto _output_error; }
1710 if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ 1776 if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */
1711 length += MINMATCH; 1777 length += MINMATCH;
1712 if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { 1778 if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {
1713 goto safe_match_copy; 1779 goto safe_match_copy;
@@ -1719,8 +1785,12 @@ LZ4_decompress_generic(
1719 } 1785 }
1720 1786
1721 /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ 1787 /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */
1722 if (!(dict == usingExtDict) || (match >= lowPrefix)) { 1788 if ((dict == withPrefix64k) || (match >= lowPrefix)) {
1723 if (offset >= 8) { 1789 if (offset >= 8) {
1790 assert(match >= lowPrefix);
1791 assert(match <= op);
1792 assert(op + 18 <= oend);
1793
1724 memcpy(op, match, 8); 1794 memcpy(op, match, 8);
1725 memcpy(op+8, match+8, 8); 1795 memcpy(op+8, match+8, 8);
1726 memcpy(op+16, match+16, 2); 1796 memcpy(op+16, match+16, 2);
@@ -1728,12 +1798,15 @@ LZ4_decompress_generic(
1728 continue; 1798 continue;
1729 } } } 1799 } } }
1730 1800
1801 if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
1731 /* match starting within external dictionary */ 1802 /* match starting within external dictionary */
1732 if ((dict==usingExtDict) && (match < lowPrefix)) { 1803 if ((dict==usingExtDict) && (match < lowPrefix)) {
1733 if (unlikely(op+length > oend-LASTLITERALS)) { 1804 if (unlikely(op+length > oend-LASTLITERALS)) {
1734 if (partialDecoding) length = MIN(length, (size_t)(oend-op)); 1805 if (partialDecoding) {
1735 else goto _output_error; /* doesn't respect parsing restriction */ 1806 length = MIN(length, (size_t)(oend-op)); /* reach end of buffer */
1736 } 1807 } else {
1808 goto _output_error; /* end-of-block condition violated */
1809 } }
1737 1810
1738 if (length <= (size_t)(lowPrefix-match)) { 1811 if (length <= (size_t)(lowPrefix-match)) {
1739 /* match fits entirely within external dictionary : just copy */ 1812 /* match fits entirely within external dictionary : just copy */
@@ -1748,7 +1821,7 @@ LZ4_decompress_generic(
1748 if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ 1821 if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */
1749 BYTE* const endOfMatch = op + restSize; 1822 BYTE* const endOfMatch = op + restSize;
1750 const BYTE* copyFrom = lowPrefix; 1823 const BYTE* copyFrom = lowPrefix;
1751 while (op < endOfMatch) *op++ = *copyFrom++; 1824 while (op < endOfMatch) { *op++ = *copyFrom++; }
1752 } else { 1825 } else {
1753 memcpy(op, lowPrefix, restSize); 1826 memcpy(op, lowPrefix, restSize);
1754 op += restSize; 1827 op += restSize;
@@ -1821,11 +1894,11 @@ LZ4_decompress_generic(
1821 1894
1822 /* decode literal length */ 1895 /* decode literal length */
1823 if (length == RUN_MASK) { 1896 if (length == RUN_MASK) {
1824 variable_length_error error = ok; 1897 variable_length_error error = ok;
1825 length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); 1898 length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error);
1826 if (error == initial_error) goto _output_error; 1899 if (error == initial_error) { goto _output_error; }
1827 if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ 1900 if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
1828 if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ 1901 if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
1829 } 1902 }
1830 1903
1831 /* copy literals */ 1904 /* copy literals */
@@ -1837,21 +1910,50 @@ LZ4_decompress_generic(
1837 if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) 1910 if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) )
1838 || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) 1911 || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
1839 { 1912 {
1913 /* We've either hit the input parsing restriction or the output parsing restriction.
1914 * If we've hit the input parsing condition then this must be the last sequence.
1915 * If we've hit the output parsing condition then we are either using partialDecoding
1916 * or we've hit the output parsing condition.
1917 */
1840 if (partialDecoding) { 1918 if (partialDecoding) {
1841 if (cpy > oend) { cpy = oend; assert(op<=oend); length = (size_t)(oend-op); } /* Partial decoding : stop in the middle of literal segment */ 1919 /* Since we are partial decoding we may be in this block because of the output parsing
1842 if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ 1920 * restriction, which is not valid since the output buffer is allowed to be undersized.
1921 */
1922 assert(endOnInput);
1923 /* If we're in this block because of the input parsing condition, then we must be on the
1924 * last sequence (or invalid), so we must check that we exactly consume the input.
1925 */
1926 if ((ip+length>iend-(2+1+LASTLITERALS)) && (ip+length != iend)) { goto _output_error; }
1927 assert(ip+length <= iend);
1928 /* We are finishing in the middle of a literals segment.
1929 * Break after the copy.
1930 */
1931 if (cpy > oend) {
1932 cpy = oend;
1933 assert(op<=oend);
1934 length = (size_t)(oend-op);
1935 }
1936 assert(ip+length <= iend);
1843 } else { 1937 } else {
1844 if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ 1938 /* We must be on the last sequence because of the parsing limitations so check
1845 if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ 1939 * that we exactly regenerate the original size (must be exact when !endOnInput).
1940 */
1941 if ((!endOnInput) && (cpy != oend)) { goto _output_error; }
1942 /* We must be on the last sequence (or invalid) because of the parsing limitations
1943 * so check that we exactly consume the input and don't overrun the output buffer.
1944 */
1945 if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; }
1846 } 1946 }
1847 memcpy(op, ip, length); 1947 memmove(op, ip, length); /* supports overlapping memory regions, which only matters for in-place decompression scenarios */
1848 ip += length; 1948 ip += length;
1849 op += length; 1949 op += length;
1850 if (!partialDecoding || (cpy == oend)) { 1950 /* Necessarily EOF when !partialDecoding. When partialDecoding
1851 /* Necessarily EOF, due to parsing restrictions */ 1951 * it is EOF if we've either filled the output buffer or hit
1952 * the input parsing restriction.
1953 */
1954 if (!partialDecoding || (cpy == oend) || (ip == iend)) {
1852 break; 1955 break;
1853 } 1956 }
1854
1855 } else { 1957 } else {
1856 LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ 1958 LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
1857 ip += length; op = cpy; 1959 ip += length; op = cpy;
@@ -1865,13 +1967,6 @@ LZ4_decompress_generic(
1865 length = token & ML_MASK; 1967 length = token & ML_MASK;
1866 1968
1867 _copy_match: 1969 _copy_match:
1868 if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
1869 if (!partialDecoding) {
1870 assert(oend > op);
1871 assert(oend - op >= 4);
1872 LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */
1873 } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */
1874
1875 if (length == ML_MASK) { 1970 if (length == ML_MASK) {
1876 variable_length_error error = ok; 1971 variable_length_error error = ok;
1877 length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); 1972 length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error);
@@ -1883,6 +1978,7 @@ LZ4_decompress_generic(
1883#if LZ4_FAST_DEC_LOOP 1978#if LZ4_FAST_DEC_LOOP
1884 safe_match_copy: 1979 safe_match_copy:
1885#endif 1980#endif
1981 if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
1886 /* match starting within external dictionary */ 1982 /* match starting within external dictionary */
1887 if ((dict==usingExtDict) && (match < lowPrefix)) { 1983 if ((dict==usingExtDict) && (match < lowPrefix)) {
1888 if (unlikely(op+length > oend-LASTLITERALS)) { 1984 if (unlikely(op+length > oend-LASTLITERALS)) {
@@ -1910,6 +2006,7 @@ LZ4_decompress_generic(
1910 } } 2006 } }
1911 continue; 2007 continue;
1912 } 2008 }
2009 assert(match >= lowPrefix);
1913 2010
1914 /* copy match within block */ 2011 /* copy match within block */
1915 cpy = op + length; 2012 cpy = op + length;
@@ -1921,16 +2018,17 @@ LZ4_decompress_generic(
1921 const BYTE* const matchEnd = match + mlen; 2018 const BYTE* const matchEnd = match + mlen;
1922 BYTE* const copyEnd = op + mlen; 2019 BYTE* const copyEnd = op + mlen;
1923 if (matchEnd > op) { /* overlap copy */ 2020 if (matchEnd > op) { /* overlap copy */
1924 while (op < copyEnd) *op++ = *match++; 2021 while (op < copyEnd) { *op++ = *match++; }
1925 } else { 2022 } else {
1926 memcpy(op, match, mlen); 2023 memcpy(op, match, mlen);
1927 } 2024 }
1928 op = copyEnd; 2025 op = copyEnd;
1929 if (op==oend) break; 2026 if (op == oend) { break; }
1930 continue; 2027 continue;
1931 } 2028 }
1932 2029
1933 if (unlikely(offset<8)) { 2030 if (unlikely(offset<8)) {
2031 LZ4_write32(op, 0); /* silence msan warning when offset==0 */
1934 op[0] = match[0]; 2032 op[0] = match[0];
1935 op[1] = match[1]; 2033 op[1] = match[1];
1936 op[2] = match[2]; 2034 op[2] = match[2];
@@ -1946,25 +2044,26 @@ LZ4_decompress_generic(
1946 2044
1947 if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { 2045 if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {
1948 BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); 2046 BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1);
1949 if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ 2047 if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */
1950 if (op < oCopyLimit) { 2048 if (op < oCopyLimit) {
1951 LZ4_wildCopy8(op, match, oCopyLimit); 2049 LZ4_wildCopy8(op, match, oCopyLimit);
1952 match += oCopyLimit - op; 2050 match += oCopyLimit - op;
1953 op = oCopyLimit; 2051 op = oCopyLimit;
1954 } 2052 }
1955 while (op < cpy) *op++ = *match++; 2053 while (op < cpy) { *op++ = *match++; }
1956 } else { 2054 } else {
1957 memcpy(op, match, 8); 2055 memcpy(op, match, 8);
1958 if (length > 16) LZ4_wildCopy8(op+8, match+8, cpy); 2056 if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); }
1959 } 2057 }
1960 op = cpy; /* wildcopy correction */ 2058 op = cpy; /* wildcopy correction */
1961 } 2059 }
1962 2060
1963 /* end of decoding */ 2061 /* end of decoding */
1964 if (endOnInput) 2062 if (endOnInput) {
1965 return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ 2063 return (int) (((char*)op)-dst); /* Nb of output bytes decoded */
1966 else 2064 } else {
1967 return (int) (((const char*)ip)-src); /* Nb of input bytes read */ 2065 return (int) (((const char*)ip)-src); /* Nb of input bytes read */
2066 }
1968 2067
1969 /* Overflow error detected */ 2068 /* Overflow error detected */
1970 _output_error: 2069 _output_error:
@@ -2079,7 +2178,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void)
2079 2178
2080int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) 2179int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
2081{ 2180{
2082 if (LZ4_stream == NULL) return 0; /* support free on NULL */ 2181 if (LZ4_stream == NULL) { return 0; } /* support free on NULL */
2083 FREEMEM(LZ4_stream); 2182 FREEMEM(LZ4_stream);
2084 return 0; 2183 return 0;
2085} 2184}
@@ -2214,18 +2313,22 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed
2214 if (dictSize==0) 2313 if (dictSize==0)
2215 return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); 2314 return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
2216 if (dictStart+dictSize == dest) { 2315 if (dictStart+dictSize == dest) {
2217 if (dictSize >= 64 KB - 1) 2316 if (dictSize >= 64 KB - 1) {
2218 return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); 2317 return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
2219 return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize); 2318 }
2319 assert(dictSize >= 0);
2320 return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize);
2220 } 2321 }
2221 return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize); 2322 assert(dictSize >= 0);
2323 return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize);
2222} 2324}
2223 2325
2224int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) 2326int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
2225{ 2327{
2226 if (dictSize==0 || dictStart+dictSize == dest) 2328 if (dictSize==0 || dictStart+dictSize == dest)
2227 return LZ4_decompress_fast(source, dest, originalSize); 2329 return LZ4_decompress_fast(source, dest, originalSize);
2228 return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, dictSize); 2330 assert(dictSize >= 0);
2331 return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize);
2229} 2332}
2230 2333
2231 2334
@@ -2237,9 +2340,9 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in
2237{ 2340{
2238 return LZ4_compress_default(source, dest, inputSize, maxOutputSize); 2341 return LZ4_compress_default(source, dest, inputSize, maxOutputSize);
2239} 2342}
2240int LZ4_compress(const char* source, char* dest, int inputSize) 2343int LZ4_compress(const char* src, char* dest, int srcSize)
2241{ 2344{
2242 return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); 2345 return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize));
2243} 2346}
2244int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) 2347int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize)
2245{ 2348{
diff --git a/src/static_libs/lz4/lz4.h b/src/static_libs/lz4/lz4.h
index a9c932ccb1..32108e2329 100644
--- a/src/static_libs/lz4/lz4.h
+++ b/src/static_libs/lz4/lz4.h
@@ -46,7 +46,7 @@ extern "C" {
46/** 46/**
47 Introduction 47 Introduction
48 48
49 LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core, 49 LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core,
50 scalable with multi-cores CPU. It features an extremely fast decoder, with speed in 50 scalable with multi-cores CPU. It features an extremely fast decoder, with speed in
51 multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. 51 multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
52 52
@@ -58,16 +58,19 @@ extern "C" {
58 - unbounded multiple steps (described as Streaming compression) 58 - unbounded multiple steps (described as Streaming compression)
59 59
60 lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). 60 lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md).
61 Decompressing a block requires additional metadata, such as its compressed size. 61 Decompressing such a compressed block requires additional metadata.
62 Exact metadata depends on exact decompression function.
63 For the typical case of LZ4_decompress_safe(),
64 metadata includes block's compressed size, and maximum bound of decompressed size.
62 Each application is free to encode and pass such metadata in whichever way it wants. 65 Each application is free to encode and pass such metadata in whichever way it wants.
63 66
64 lz4.h only handle blocks, it can not generate Frames. 67 lz4.h only handle blocks, it can not generate Frames.
65 68
66 Blocks are different from Frames (doc/lz4_Frame_format.md). 69 Blocks are different from Frames (doc/lz4_Frame_format.md).
67 Frames bundle both blocks and metadata in a specified manner. 70 Frames bundle both blocks and metadata in a specified manner.
68 This are required for compressed data to be self-contained and portable. 71 Embedding metadata is required for compressed data to be self-contained and portable.
69 Frame format is delivered through a companion API, declared in lz4frame.h. 72 Frame format is delivered through a companion API, declared in lz4frame.h.
70 Note that the `lz4` CLI can only manage frames. 73 The `lz4` CLI can only manage frames.
71*/ 74*/
72 75
73/*^*************************************************************** 76/*^***************************************************************
@@ -97,7 +100,7 @@ extern "C" {
97/*------ Version ------*/ 100/*------ Version ------*/
98#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ 101#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
99#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ 102#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */
100#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ 103#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
101 104
102#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) 105#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
103 106
@@ -129,29 +132,35 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string;
129* Simple Functions 132* Simple Functions
130**************************************/ 133**************************************/
131/*! LZ4_compress_default() : 134/*! LZ4_compress_default() :
132 Compresses 'srcSize' bytes from buffer 'src' 135 * Compresses 'srcSize' bytes from buffer 'src'
133 into already allocated 'dst' buffer of size 'dstCapacity'. 136 * into already allocated 'dst' buffer of size 'dstCapacity'.
134 Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). 137 * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
135 It also runs faster, so it's a recommended setting. 138 * It also runs faster, so it's a recommended setting.
136 If the function cannot compress 'src' into a more limited 'dst' budget, 139 * If the function cannot compress 'src' into a more limited 'dst' budget,
137 compression stops *immediately*, and the function result is zero. 140 * compression stops *immediately*, and the function result is zero.
138 In which case, 'dst' content is undefined (invalid). 141 * In which case, 'dst' content is undefined (invalid).
139 srcSize : max supported value is LZ4_MAX_INPUT_SIZE. 142 * srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
140 dstCapacity : size of buffer 'dst' (which must be already allocated) 143 * dstCapacity : size of buffer 'dst' (which must be already allocated)
141 @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) 144 * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
142 or 0 if compression fails 145 * or 0 if compression fails
143 Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). 146 * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
144*/ 147 */
145LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); 148LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
146 149
147/*! LZ4_decompress_safe() : 150/*! LZ4_decompress_safe() :
148 compressedSize : is the exact complete size of the compressed block. 151 * compressedSize : is the exact complete size of the compressed block.
149 dstCapacity : is the size of destination buffer, which must be already allocated. 152 * dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size.
150 @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) 153 * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
151 If destination buffer is not large enough, decoding will stop and output an error code (negative value). 154 * If destination buffer is not large enough, decoding will stop and output an error code (negative value).
152 If the source stream is detected malformed, the function will stop decoding and return a negative result. 155 * If the source stream is detected malformed, the function will stop decoding and return a negative result.
153 Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer). 156 * Note 1 : This function is protected against malicious data packets :
154*/ 157 * it will never writes outside 'dst' buffer, nor read outside 'source' buffer,
158 * even if the compressed block is maliciously modified to order the decoder to do these actions.
159 * In such case, the decoder stops immediately, and considers the compressed block malformed.
160 * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them.
161 * The implementation is free to send / store / derive this information in whichever way is most beneficial.
162 * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead.
163 */
155LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); 164LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
156 165
157 166
@@ -388,6 +397,8 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod
388 */ 397 */
389LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); 398LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
390 399
400#endif /* LZ4_H_2983827168210 */
401
391 402
392/*^************************************* 403/*^*************************************
393 * !!!!!! STATIC LINKING ONLY !!!!!! 404 * !!!!!! STATIC LINKING ONLY !!!!!!
@@ -413,14 +424,17 @@ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int sr
413 * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. 424 * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library.
414 ******************************************************************************/ 425 ******************************************************************************/
415 426
427#ifdef LZ4_STATIC_LINKING_ONLY
428
429#ifndef LZ4_STATIC_3504398509
430#define LZ4_STATIC_3504398509
431
416#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS 432#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS
417#define LZ4LIB_STATIC_API LZ4LIB_API 433#define LZ4LIB_STATIC_API LZ4LIB_API
418#else 434#else
419#define LZ4LIB_STATIC_API 435#define LZ4LIB_STATIC_API
420#endif 436#endif
421 437
422#ifdef LZ4_STATIC_LINKING_ONLY
423
424 438
425/*! LZ4_compress_fast_extState_fastReset() : 439/*! LZ4_compress_fast_extState_fastReset() :
426 * A variant of LZ4_compress_fast_extState(). 440 * A variant of LZ4_compress_fast_extState().
@@ -462,8 +476,75 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c
462 */ 476 */
463LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); 477LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);
464 478
479
480/*! In-place compression and decompression
481 *
482 * It's possible to have input and output sharing the same buffer,
483 * for highly contrained memory environments.
484 * In both cases, it requires input to lay at the end of the buffer,
485 * and decompression to start at beginning of the buffer.
486 * Buffer size must feature some margin, hence be larger than final size.
487 *
488 * |<------------------------buffer--------------------------------->|
489 * |<-----------compressed data--------->|
490 * |<-----------decompressed size------------------>|
491 * |<----margin---->|
492 *
493 * This technique is more useful for decompression,
494 * since decompressed size is typically larger,
495 * and margin is short.
496 *
497 * In-place decompression will work inside any buffer
498 * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize).
499 * This presumes that decompressedSize > compressedSize.
500 * Otherwise, it means compression actually expanded data,
501 * and it would be more efficient to store such data with a flag indicating it's not compressed.
502 * This can happen when data is not compressible (already compressed, or encrypted).
503 *
504 * For in-place compression, margin is larger, as it must be able to cope with both
505 * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
506 * and data expansion, which can happen when input is not compressible.
507 * As a consequence, buffer size requirements are much higher,
508 * and memory savings offered by in-place compression are more limited.
509 *
510 * There are ways to limit this cost for compression :
511 * - Reduce history size, by modifying LZ4_DISTANCE_MAX.
512 * Note that it is a compile-time constant, so all compressions will apply this limit.
513 * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
514 * so it's a reasonable trick when inputs are known to be small.
515 * - Require the compressor to deliver a "maximum compressed size".
516 * This is the `dstCapacity` parameter in `LZ4_compress*()`.
517 * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
518 * in which case, the return code will be 0 (zero).
519 * The caller must be ready for these cases to happen,
520 * and typically design a backup scheme to send data uncompressed.
521 * The combination of both techniques can significantly reduce
522 * the amount of margin required for in-place compression.
523 *
524 * In-place compression can work in any buffer
525 * which size is >= (maxCompressedSize)
526 * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success.
527 * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
528 * so it's possible to reduce memory requirements by playing with them.
529 */
530
531#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32)
532#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */
533
534#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */
535# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */
465#endif 536#endif
466 537
538#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */
539#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
540
541#endif /* LZ4_STATIC_3504398509 */
542#endif /* LZ4_STATIC_LINKING_ONLY */
543
544
545
546#ifndef LZ4_H_98237428734687
547#define LZ4_H_98237428734687
467 548
468/*-************************************************************ 549/*-************************************************************
469 * PRIVATE DEFINITIONS 550 * PRIVATE DEFINITIONS
@@ -567,6 +648,7 @@ union LZ4_streamDecode_u {
567} ; /* previously typedef'd to LZ4_streamDecode_t */ 648} ; /* previously typedef'd to LZ4_streamDecode_t */
568 649
569 650
651
570/*-************************************ 652/*-************************************
571* Obsolete Functions 653* Obsolete Functions
572**************************************/ 654**************************************/
@@ -601,8 +683,8 @@ union LZ4_streamDecode_u {
601#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ 683#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
602 684
603/* Obsolete compression functions */ 685/* Obsolete compression functions */
604LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); 686LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize);
605LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); 687LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize);
606LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); 688LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
607LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); 689LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
608LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); 690LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
@@ -674,7 +756,7 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
674LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); 756LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
675 757
676 758
677#endif /* LZ4_H_2983827168210 */ 759#endif /* LZ4_H_98237428734687 */
678 760
679 761
680#if defined (__cplusplus) 762#if defined (__cplusplus)
diff --git a/src/static_libs/lz4/lz4frame.c b/src/static_libs/lz4/lz4frame.c
index 95da80ec32..71ee73d8a8 100644
--- a/src/static_libs/lz4/lz4frame.c
+++ b/src/static_libs/lz4/lz4frame.c
@@ -213,8 +213,8 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
213 213
214static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */ 214static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */
215static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ 215static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */
216static const size_t BHSize = 4; /* block header : size, and compress flag */ 216static const size_t BHSize = LZ4F_BLOCK_HEADER_SIZE; /* block header : size, and compress flag */
217static const size_t BFSize = 4; /* block footer : checksum (optional) */ 217static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checksum (optional) */
218 218
219 219
220/*-************************************ 220/*-************************************
@@ -327,6 +327,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
327{ 327{
328 LZ4F_preferences_t prefsNull = LZ4F_INIT_PREFERENCES; 328 LZ4F_preferences_t prefsNull = LZ4F_INIT_PREFERENCES;
329 prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ 329 prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */
330 prefsNull.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; /* worst case */
330 { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; 331 { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
331 U32 const flush = prefsPtr->autoFlush | (srcSize==0); 332 U32 const flush = prefsPtr->autoFlush | (srcSize==0);
332 LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID; 333 LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID;
@@ -1130,8 +1131,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
1130 } 1131 }
1131 1132
1132 /* control magic number */ 1133 /* control magic number */
1134#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
1133 if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) 1135 if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)
1134 return err0r(LZ4F_ERROR_frameType_unknown); 1136 return err0r(LZ4F_ERROR_frameType_unknown);
1137#endif
1135 dctx->frameInfo.frameType = LZ4F_frame; 1138 dctx->frameInfo.frameType = LZ4F_frame;
1136 1139
1137 /* Flags */ 1140 /* Flags */
@@ -1170,10 +1173,12 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
1170 1173
1171 /* check header */ 1174 /* check header */
1172 assert(frameHeaderSize > 5); 1175 assert(frameHeaderSize > 5);
1176#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
1173 { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); 1177 { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
1174 if (HC != srcPtr[frameHeaderSize-1]) 1178 if (HC != srcPtr[frameHeaderSize-1])
1175 return err0r(LZ4F_ERROR_headerChecksum_invalid); 1179 return err0r(LZ4F_ERROR_headerChecksum_invalid);
1176 } 1180 }
1181#endif
1177 1182
1178 /* save */ 1183 /* save */
1179 dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; 1184 dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
@@ -1210,8 +1215,10 @@ size_t LZ4F_headerSize(const void* src, size_t srcSize)
1210 return 8; 1215 return 8;
1211 1216
1212 /* control magic number */ 1217 /* control magic number */
1218#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
1213 if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) 1219 if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
1214 return err0r(LZ4F_ERROR_frameType_unknown); 1220 return err0r(LZ4F_ERROR_frameType_unknown);
1221#endif
1215 1222
1216 /* Frame Header Size */ 1223 /* Frame Header Size */
1217 { BYTE const FLG = ((const BYTE*)src)[4]; 1224 { BYTE const FLG = ((const BYTE*)src)[4];
@@ -1493,7 +1500,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
1493 /* next block is a compressed block */ 1500 /* next block is a compressed block */
1494 dctx->tmpInTarget = nextCBlockSize + crcSize; 1501 dctx->tmpInTarget = nextCBlockSize + crcSize;
1495 dctx->dStage = dstage_getCBlock; 1502 dctx->dStage = dstage_getCBlock;
1496 if (dstPtr==dstEnd) { 1503 if (dstPtr==dstEnd || srcPtr==srcEnd) {
1497 nextSrcSizeHint = BHSize + nextCBlockSize + crcSize; 1504 nextSrcSizeHint = BHSize + nextCBlockSize + crcSize;
1498 doAnotherStage = 0; 1505 doAnotherStage = 0;
1499 } 1506 }
@@ -1554,8 +1561,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
1554 } 1561 }
1555 { U32 const readCRC = LZ4F_readLE32(crcSrc); 1562 { U32 const readCRC = LZ4F_readLE32(crcSrc);
1556 U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); 1563 U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
1564#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
1557 if (readCRC != calcCRC) 1565 if (readCRC != calcCRC)
1558 return err0r(LZ4F_ERROR_blockChecksum_invalid); 1566 return err0r(LZ4F_ERROR_blockChecksum_invalid);
1567#else
1568 (void)readCRC;
1569 (void)calcCRC;
1570#endif
1559 } } 1571 } }
1560 dctx->dStage = dstage_getBlockHeader; /* new block */ 1572 dctx->dStage = dstage_getBlockHeader; /* new block */
1561 break; 1573 break;
@@ -1594,8 +1606,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
1594 assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */ 1606 assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */
1595 { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget); 1607 { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
1596 U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0); 1608 U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
1609#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
1597 if (readBlockCrc != calcBlockCrc) 1610 if (readBlockCrc != calcBlockCrc)
1598 return err0r(LZ4F_ERROR_blockChecksum_invalid); 1611 return err0r(LZ4F_ERROR_blockChecksum_invalid);
1612#else
1613 (void)readBlockCrc;
1614 (void)calcBlockCrc;
1615#endif
1599 } } 1616 } }
1600 1617
1601 if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) { 1618 if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
@@ -1723,8 +1740,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
1723 /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */ 1740 /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */
1724 { U32 const readCRC = LZ4F_readLE32(selectedIn); 1741 { U32 const readCRC = LZ4F_readLE32(selectedIn);
1725 U32 const resultCRC = XXH32_digest(&(dctx->xxh)); 1742 U32 const resultCRC = XXH32_digest(&(dctx->xxh));
1743#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
1726 if (readCRC != resultCRC) 1744 if (readCRC != resultCRC)
1727 return err0r(LZ4F_ERROR_contentChecksum_invalid); 1745 return err0r(LZ4F_ERROR_contentChecksum_invalid);
1746#else
1747 (void)readCRC;
1748 (void)resultCRC;
1749#endif
1728 nextSrcSizeHint = 0; 1750 nextSrcSizeHint = 0;
1729 LZ4F_resetDecompressionContext(dctx); 1751 LZ4F_resetDecompressionContext(dctx);
1730 doAnotherStage = 0; 1752 doAnotherStage = 0;
diff --git a/src/static_libs/lz4/lz4frame.h b/src/static_libs/lz4/lz4frame.h
index 742c2528e0..391e484011 100644
--- a/src/static_libs/lz4/lz4frame.h
+++ b/src/static_libs/lz4/lz4frame.h
@@ -253,6 +253,15 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
253#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */ 253#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */
254#define LZ4F_HEADER_SIZE_MAX 19 254#define LZ4F_HEADER_SIZE_MAX 19
255 255
256/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */
257#define LZ4F_BLOCK_HEADER_SIZE 4
258
259/* Size in bytes of a block checksum footer in little-endian format. */
260#define LZ4F_BLOCK_CHECKSUM_SIZE 4
261
262/* Size in bytes of the content checksum. */
263#define LZ4F_CONTENT_CHECKSUM_SIZE 4
264
256/*! LZ4F_compressBegin() : 265/*! LZ4F_compressBegin() :
257 * will write the frame header into dstBuffer. 266 * will write the frame header into dstBuffer.
258 * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. 267 * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
diff --git a/src/static_libs/lz4/lz4hc.c b/src/static_libs/lz4/lz4hc.c
index 936f73969e..afadb3f169 100644
--- a/src/static_libs/lz4/lz4hc.c
+++ b/src/static_libs/lz4/lz4hc.c
@@ -151,6 +151,21 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
151 return back; 151 return back;
152} 152}
153 153
154#if defined(_MSC_VER)
155# define LZ4HC_rotl32(x,r) _rotl(x,r)
156#else
157# define LZ4HC_rotl32(x,r) ((x << r) | (x >> (32 - r)))
158#endif
159
160
161static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern)
162{
163 size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3;
164 if (bitsToRotate == 0)
165 return pattern;
166 return LZ4HC_rotl32(pattern, (int)bitsToRotate);
167}
168
154/* LZ4HC_countPattern() : 169/* LZ4HC_countPattern() :
155 * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ 170 * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */
156static unsigned 171static unsigned
@@ -203,6 +218,16 @@ LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
203 return (unsigned)(iStart - ip); 218 return (unsigned)(iStart - ip);
204} 219}
205 220
221/* LZ4HC_protectDictEnd() :
222 * Checks if the match is in the last 3 bytes of the dictionary, so reading the
223 * 4 byte MINMATCH would overflow.
224 * @returns true if the match index is okay.
225 */
226static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex)
227{
228 return ((U32)((dictLimit - 1) - matchIndex) >= 3);
229}
230
206typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; 231typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e;
207typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; 232typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e;
208 233
@@ -228,7 +253,7 @@ LZ4HC_InsertAndGetWiderMatch (
228 const U32 dictLimit = hc4->dictLimit; 253 const U32 dictLimit = hc4->dictLimit;
229 const BYTE* const lowPrefixPtr = base + dictLimit; 254 const BYTE* const lowPrefixPtr = base + dictLimit;
230 const U32 ipIndex = (U32)(ip - base); 255 const U32 ipIndex = (U32)(ip - base);
231 const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; 256 const U32 lowestMatchIndex = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX;
232 const BYTE* const dictBase = hc4->dictBase; 257 const BYTE* const dictBase = hc4->dictBase;
233 int const lookBackLength = (int)(ip-iLowLimit); 258 int const lookBackLength = (int)(ip-iLowLimit);
234 int nbAttempts = maxNbAttempts; 259 int nbAttempts = maxNbAttempts;
@@ -287,14 +312,21 @@ LZ4HC_InsertAndGetWiderMatch (
287 if (chainSwap && matchLength==longest) { /* better match => select a better chain */ 312 if (chainSwap && matchLength==longest) { /* better match => select a better chain */
288 assert(lookBackLength==0); /* search forward only */ 313 assert(lookBackLength==0); /* search forward only */
289 if (matchIndex + (U32)longest <= ipIndex) { 314 if (matchIndex + (U32)longest <= ipIndex) {
315 int const kTrigger = 4;
290 U32 distanceToNextMatch = 1; 316 U32 distanceToNextMatch = 1;
317 int const end = longest - MINMATCH + 1;
318 int step = 1;
319 int accel = 1 << kTrigger;
291 int pos; 320 int pos;
292 for (pos = 0; pos <= longest - MINMATCH; pos++) { 321 for (pos = 0; pos < end; pos += step) {
293 U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos); 322 U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos);
323 step = (accel++ >> kTrigger);
294 if (candidateDist > distanceToNextMatch) { 324 if (candidateDist > distanceToNextMatch) {
295 distanceToNextMatch = candidateDist; 325 distanceToNextMatch = candidateDist;
296 matchChainPos = (U32)pos; 326 matchChainPos = (U32)pos;
297 } } 327 accel = 1 << kTrigger;
328 }
329 }
298 if (distanceToNextMatch > 1) { 330 if (distanceToNextMatch > 1) {
299 if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ 331 if (distanceToNextMatch > matchIndex) break; /* avoid overflow */
300 matchIndex -= distanceToNextMatch; 332 matchIndex -= distanceToNextMatch;
@@ -313,34 +345,61 @@ LZ4HC_InsertAndGetWiderMatch (
313 } else { 345 } else {
314 repeat = rep_not; 346 repeat = rep_not;
315 } } 347 } }
316 if ( (repeat == rep_confirmed) 348 if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex)
317 && (matchCandidateIdx >= dictLimit) ) { /* same segment only */ 349 && LZ4HC_protectDictEnd(dictLimit, matchCandidateIdx) ) {
318 const BYTE* const matchPtr = base + matchCandidateIdx; 350 const int extDict = matchCandidateIdx < dictLimit;
351 const BYTE* const matchPtr = (extDict ? dictBase : base) + matchCandidateIdx;
319 if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ 352 if (LZ4_read32(matchPtr) == pattern) { /* good candidate */
320 size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); 353 const BYTE* const dictStart = dictBase + hc4->lowLimit;
321 const BYTE* const lowestMatchPtr = (lowPrefixPtr + LZ4_DISTANCE_MAX >= ip) ? lowPrefixPtr : ip - LZ4_DISTANCE_MAX; 354 const BYTE* const iLimit = extDict ? dictBase + dictLimit : iHighLimit;
322 size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); 355 size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern);
323 size_t const currentSegmentLength = backLength + forwardPatternLength; 356 if (extDict && matchPtr + forwardPatternLength == iLimit) {
324 357 U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern);
325 if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ 358 forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern);
326 && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ 359 }
327 matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ 360 { const BYTE* const lowestMatchPtr = extDict ? dictStart : lowPrefixPtr;
328 } else { 361 size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);
329 matchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ 362 size_t currentSegmentLength;
330 if (lookBackLength==0) { /* no back possible */ 363 if (!extDict && matchPtr - backLength == lowPrefixPtr && hc4->lowLimit < dictLimit) {
331 size_t const maxML = MIN(currentSegmentLength, srcPatternLength); 364 U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern);
332 if ((size_t)longest < maxML) { 365 backLength += LZ4HC_reverseCountPattern(dictBase + dictLimit, dictStart, rotatedPattern);
333 assert(base + matchIndex < ip); 366 }
334 if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break; 367 /* Limit backLength not go further than lowestMatchIndex */
335 assert(maxML < 2 GB); 368 backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex);
336 longest = (int)maxML; 369 assert(matchCandidateIdx - backLength >= lowestMatchIndex);
337 *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ 370 currentSegmentLength = backLength + forwardPatternLength;
338 *startpos = ip; 371 /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */
372 if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */
373 && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
374 U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */
375 if (LZ4HC_protectDictEnd(dictLimit, newMatchIndex))
376 matchIndex = newMatchIndex;
377 else {
378 /* Can only happen if started in the prefix */
379 assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict);
380 matchIndex = dictLimit;
339 } 381 }
340 { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); 382 } else {
341 if (distToNextPattern > matchIndex) break; /* avoid overflow */ 383 U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */
342 matchIndex -= distToNextPattern; 384 if (!LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) {
343 } } } 385 assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict);
386 matchIndex = dictLimit;
387 } else {
388 matchIndex = newMatchIndex;
389 if (lookBackLength==0) { /* no back possible */
390 size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
391 if ((size_t)longest < maxML) {
392 assert(base + matchIndex != ip);
393 if ((size_t)(ip - base) - matchIndex > LZ4_DISTANCE_MAX) break;
394 assert(maxML < 2 GB);
395 longest = (int)maxML;
396 *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */
397 *startpos = ip;
398 }
399 { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex);
400 if (distToNextPattern > matchIndex) break; /* avoid overflow */
401 matchIndex -= distToNextPattern;
402 } } } } }
344 continue; 403 continue;
345 } } 404 } }
346 } } /* PA optimization */ 405 } } /* PA optimization */
@@ -969,7 +1028,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr,
969 const char* dictionary, int dictSize) 1028 const char* dictionary, int dictSize)
970{ 1029{
971 LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; 1030 LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
972 DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize); 1031 DEBUGLOG(4, "LZ4_loadDictHC(ctx:%p, dict:%p, dictSize:%d)", LZ4_streamHCPtr, dictionary, dictSize);
973 assert(LZ4_streamHCPtr != NULL); 1032 assert(LZ4_streamHCPtr != NULL);
974 if (dictSize > 64 KB) { 1033 if (dictSize > 64 KB) {
975 dictionary += (size_t)dictSize - 64 KB; 1034 dictionary += (size_t)dictSize - 64 KB;
@@ -1005,6 +1064,9 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl
1005 ctxPtr->base = newBlock - ctxPtr->dictLimit; 1064 ctxPtr->base = newBlock - ctxPtr->dictLimit;
1006 ctxPtr->end = newBlock; 1065 ctxPtr->end = newBlock;
1007 ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ 1066 ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
1067
1068 /* cannot reference an extDict and a dictCtx at the same time */
1069 ctxPtr->dictCtx = NULL;
1008} 1070}
1009 1071
1010static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, 1072static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
diff --git a/src/static_libs/lz4/lz4hc.h b/src/static_libs/lz4/lz4hc.h
index cdc6d895ef..44e35bbf6b 100644
--- a/src/static_libs/lz4/lz4hc.h
+++ b/src/static_libs/lz4/lz4hc.h
@@ -336,6 +336,9 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL
336#ifndef LZ4_HC_SLO_098092834 336#ifndef LZ4_HC_SLO_098092834
337#define LZ4_HC_SLO_098092834 337#define LZ4_HC_SLO_098092834
338 338
339#define LZ4_STATIC_LINKING_ONLY /* LZ4LIB_STATIC_API */
340#include "lz4.h"
341
339#if defined (__cplusplus) 342#if defined (__cplusplus)
340extern "C" { 343extern "C" {
341#endif 344#endif