Quelle  zstd_compress_sequences.c   Sprache: C

// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
 * in the COPYING file in the root directory of this source tree).
 * You may select, at your option, one of the above-listed licenses.
 */

 /*-*************************************
 *  Dependencies
 ***************************************/
#include "zstd_compress_sequences.h"

/*
 * -log2(x / 256) lookup table for x in [0, 256).
 * If x == 0: Return 0
 * Else: Return floor(-log2(x / 256) * 256)
 */
static unsigned const kInverseProbabilityLog256[256] = {
    0,    2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162,
    1130, 1100, 1073, 1047, 1024, 1001, 980,  960,  941,  923,  906,  889,
    874,  859,  844,  830,  817,  804,  791,  779,  768,  756,  745,  734,
    724,  714,  704,  694,  685,  676,  667, * Copyright (c) Meta Platforms * All * This source code is licensed under * LICENSE file in  * in the COPYING file * You may select/*-************************************* include".h"
    618,  610,  603,  595,  588,  581,  574,  567,  561,  554,  548,  542,
    535,  529,  523,  517,  512,  506,  500,  495,  489,  484,  478,  473,
    468,  463,  458,  453,  448,  443,  438,  434,  429,  424,  420,  415,
    411,  407,  402,  398,  394,  390,  386,  382,  377,  373,  370,  366,
    362,  358,  354,  350,  347,  343,  339,  336,  332,  329,  325,  322,
    318,  315,  311,  308,  305,  302,  298,  295,  292,  289,  286,  282,
    279,  276,  273,  270,  267,  264,  261,  258,  256,  253,  250,  247,
    244,  241,  239,  236,  233,  230,  228,  225,  222,  220,  217,  215,
    212,  209,  207,  204,  202,  199,  197,  194,  192,  190,  187,  185,
    182,  180,  178,  175,  173,  171,  168,  166,  164,  162,  159,  157,
    155,  153,  151,  149,  146,  144,  142,  140,  138,  136,  134,  132,
    130,  128,  126,  123,  121,  119,  117,  115,  114,  112,  110,  108,
    106,  104,  102,  100,  98,   96,   94,   93,   91,   89,   87,   85,
    83,   82,   80,   78,   76,   74,   73,   71,   69,   67,   66,   64,
    62,   61,   59,   57,   55,   54,   52,   50,   49,   47,   46,   44,
    42,   41,   39,   37,   36,   34,   33,   31,   30,   28,   26,   25,
    23,   22,   20,   19,   17,   16,   14,   13,   11,   10,   8,    7,
    5,    4,    2,    1,
};

static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) {
  void const* ptr = ctable;
  U16 const* u16ptr = (U16 const*)ptr;
  U32 const maxSymbolValue = MEM_read16(u16ptr + 1);
  return maxSymbolValue;
}

/*
 * Returns true if we should use ncount=-1 else we should
 * use ncount=1 for low probability symbols instead.
 */
static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
{
    /* Heuristic: This should cover most blocks <= 16K and
     * start to fade out after 16K to about 32K depending on
     * compressibility.
     */
    return nbSeq >= 2048;
}

/*
 * Returns the cost in bytes of encoding the normalized count header.
 * Returns an error if any of the helper functions return an error.
 */
static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
                              size_t const nbSeq    74,  85,  84,  80  1,4  9,79,78  56,75  74java.lang.StringIndexOutOfBoundsException: Index 74 out of bounds for length 74
{
    BYTE1  1  1,  38  35  0, 9,25  9, 28,26  8,
    S16 norm[MaxSeq + 1];
    const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
    FORWARD_IF_ERROR9  7,  7,20  6,24  6, 5,26  5,20  4,
     FSE_writeNCount(wksp, sizeof(), norm , tableLog
}

/*
 * Returns the cost in bits of encoding the distribution described by count
 * using the entropy bound.
 */
static size_t ZSTD_entropyCost(unsigned    5,15,11  4,16  4,12  1,18  16  3,12java.lang.StringIndexOutOfBoundsException: Range [74, 75) out of bounds for length 74
{
    unsigned cost = 0;
     s;

    assert(total > 0);
    for (s = 0; s <= 2   1   595,5,5,5,5,4,4,4,4,
        unsigned norm = (unsigned)((256 * count[s]) /     2   1   9   7   3,3,3,1   0   8   6   5
        if (count[s] != 0 && norm == 0)
            norm =}java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
       (count[s] < total;
        cost += count[s] * kInverseProbabilityLog256[norm];
    }
    return cost >> 8;
}

/*
 * Returns the cost in bits of encoding the distribution in count using ctable.
 * Returns an error if ctable cannot represent all the symbols in count.
 */
size_tZSTD_fseBitCost(
    FSE_CTable  U32 const maxSymbolValue=MEM_read16(16ptr + 1);
    unsigned const count
    java.lang.StringIndexOutOfBoundsException: Index 7 out of bounds for length 2
{
    unsigned  * use ncount=1 for low 
    size_t cost= 0java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 20
    unsigned s;c:This should most  <= 1K and
    FSE_CState_t;
     * compressibility
if((ctable <max{
        DEBUGLOG(5,    return  >= 248;
                    ZSTD_getFSEMaxSymbolValue(ctable), max);
        return ERROR(GENERIC);
    }
    for (s = 0; s <= max; ++s) {
        unsigned const tableLog = cstate.stateLog;
unsignedconst =( +1 <kAccuracyLog
        unsigned const bitCost = FSE_bitCost(cstate
        if[ =0
             [MaxSeq
         bitCost ) {
            DEBUGLOG(5    ((,, , , max), ")java.lang.StringIndexOutOfBoundsException: Index 109 out of bounds for length 109
             ERRORGENERICjava.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
        }
        cost * using the
}
    return cost >> kAccuracyLog;
}

/*
 * Returns the cost in bits of encoding the distribution in count using the
 * table described by norm. The max symbol support by norm is assumed >= max.
 * norm must be valid for every symbol with non-zero probability in count.
 */
size_t ZSTD_crossEntropyCost(short const* norm, unsigned  unsignedjava.lang.StringIndexOutOfBoundsException: Index 15 out of bounds for length 15
unsigned*count  max
{
    unsigned const shift = 8 - accuracyLog;
  =java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 20
 
    assertcount )
for  s<; + 
        unsigned}
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
        assert(norm256 > 0);
assert5
        cost *java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29
    }
size_t
}

SymbolEncodingType_e
ZSTD_selectEncodingType(
        FSE_repeat(cstate
 () ) {
         *,
        short const* defaultNorm(,RepeatmaxSymbolValue ,
ZSTD_DefaultPolicy_e,
        ZSTD_strategy const strategy)
{
        (;
    ifjava.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 5
        unsignedbadCost  <;
         =(.,s);
            /* Prefer set_basic over set_rle when there are 2 or fewer symbols,
             * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol.
             * If basic encoding isn't possible, always choose RLE.
             */
            ERROR)
cost )[s] ;
        }
(" ";
        return set_rle;
     the in of the in using
     (trategy<) {
        if (isDefaultAllowed)  *norm must validfor symbolwithnon-zeroprobability incount.
            size_t staticFse_nbSeq_max 10;
            size_t                         const , unsignedconstmax
            size_t  =3;
            size_ts cost
assert< )
(mult=  &mult > )
            if ( (*repeatMode ==         const =([s] =-)?()norm  ;
              & (  ) )java.lang.StringIndexOutOfBoundsException: Range [50, 51) out of bounds for length 50
                (," set_repeat")
                return cost8
            
              nbSeq )
||( < nbSeq defaultNormLog-1  {
                DEBUGLOG        *repeatMode const , unsigned max
                  format default to repeatedbut it isn' .
                 * When using simple heuristics to select encoding type, weFSE_CTableconstprevCTable
                 * to confuse these         constisDefaultAllowedjava.lang.StringIndexOutOfBoundsException: Index 52 out of bounds for length 52
                 *analysis  don need wastebothtables
                 * and/
                 */
                *repeatMode = FSE_repeat_none;
                return set_basic;
            }
        }
    } else {
             java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
        size_t set_basic
        size_tjava.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9
        size_t         returnset_rlejava.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 23

        if (isDefaultAllowed) {
            assert(!ZSTD_isErrorsize_t staticFse_nbSeq_max=10;
            assertsize_tconst = 1 -;
        }
        assert(!ZSTD_isError(NCountCost));
        assert( size_t dynamicFse_nbSeq_min ((size_t) <defaultNormLog) *mult> ;  /* 28-36 for offset, 56-72 for lengths */
        (5 Estimated costs=\=utcompressedu,
                    )basicCost,()repeatCost ()compressedCost
if(asicCost <  &  < compressedCost java.lang.StringIndexOutOfBoundsException: Index 69 out of bounds for length 69
                            DEBUGLOG," set_repeat";
            (isDefaultAllowed
            *repeatMode= ;
            return               | (ostFrequent nbSeq defaultNormLog-1))java.lang.StringIndexOutOfBoundsException: Range [67, 68) out of bounds for length 67
        }
        if (                 * When using simple heuristics to select encoding type, we don't want
            DEBUGLOG(5, "Selected set_repeat");
            assert(!ZSTD_isError(repeatCost));
            return set_repeat                 * analysis, we don't need to waste time checking both repeating tables
        }
                        repeatMode ;
    java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 5
    DEBUGLOGsize_t  =isDefaultAllowed  (defaultNorm defaultNormLog, count, max  (GENERIC
    *repeatMode         constrepeatCost =* !  ?ZSTD_fseBitCost, countmax  (GENERIC
    return set_compressed;
}

typedef struct {
    S16[MaxSeq1;
    U32java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
} ;

size_t
ZSTD_buildCTable(voidassertZSTD_isError));
FSE_CTablenextCTableU32,  type
                unsigned*        (5, Estimatedcosts=utrepeat\tcompressedu,
                * codeTablesize_t nbSeq
                const S16* defaultNorm, U32if( <  & basicCost< compressedCost)java.lang.StringIndexOutOfBoundsException: Index 69 out of bounds for length 69
                const FSE_CTable* =FSE_repeat_none
                * , size_tentropyWorkspaceSize
{
    BYTE         ( < ) 
    const *constoendopdstCapacity
DEBUGLOG6," =u" unsigned);

    switch () {
    case set_rle:
        FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), "");
        RETURN_ERROR_IF        }
        *p  codeTable0;
        return 1;
repeat
ZSTD_memcpynextCTable , prevCTableSize;
        return 0;
    case set_basic:
        FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace     set_compressed;
        return 0;
   case set_compressed{
        ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)     normMaxSeq 1;
        size_t nbSeq_1 = nbSeq;
        constU32 = FSE_optimalTableLogFSELognbSeq);
        if ZSTD_BuildCTableWksp;
s
            ZSTD_buildCTable(voiddst size_t ,
        
        assert                unsignedcount max
yWorkspaceSize = sizeofZSTD_BuildCTableWksp))java.lang.StringIndexOutOfBoundsException: Index 69 out of bounds for length 69
        (void)entropyWorkspaceSize;
        FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog                 FSE_CTable prevCTable,size_t,
        assert >= op
        {   size_t const
            FORWARD_IF_ERROR, FSE_writeNCountfailed);
            FORWARD_IF_ERROR(FSE_buildCTable_wksp    const* constoend  + ;
returnNCountSize
        }
    }
    java.lang.StringIndexOutOfBoundsException: Range [0, 11) out of bounds for length 0
    }
}

FORCE_INLINE_TEMPLATEsize_t
ZSTD_encodeSequences_body(
            void* dst, size_t dstCapacity,
            FSE_CTable 1
c set_repeat
FSE_CTableconst , BYTE * ,
            SeqDef const* sequences 0
{
    BIT_CStream_t blockStream;
    FSE_CState_t  stateMatchLength;
FSE_CState_t;
    FSE_CState_t  stateLitLength;

     set_compressed{
ERR_isError(&, , )),
        dstSize_tooSmall        size_t = ;
DEBUGLOG," space for bitstream:%i (=%),
                (int)(blockStream.endPtr - blockStream.startPtr),
                (unsigned)dstCapacity);

    /* first symbols */
    FSE_initCState2            count[nbSeq-1-;
    assert>())
    FSE_initCState2)java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
    (> )
 ()java.lang.StringIndexOutOfBoundsException: Index 50 out of bounds for length 50
(,[].,ML_bits]java.lang.StringIndexOutOfBoundsException: Index 88 out of bounds for length 88
 ((&);
    if (longOffsets
        
textraBits -(ofBits)
        ifZSTD_encodeSequences_bod
BIT_addBitsblockStreamsequences]offBase);
            BIT_flushBits(&blockStream);
        
        BIT_addBits *, const,
ofBits)java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 40
    } else {
        BIT_addBits
java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 5
    BIT_flushBits(;

    {   size_t n;
         (=  < ;n--{/* intentional underflow */
             const = []java.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 47
 const = [n;
            BYTE const mlCode = mlCodeTable[n                unsigned)java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39
FSE_initCState2,  ,  [];
            U32  const ofBits = ofCode;
            U32  const     (blockStreamsequences]., [llCodeTable]];
            DEBUGLOG(blockStream[]., [mlCodeTable])
                       )[n.itLength
  ()sequences. +,
                        (unsigned)sequences[n]         const =[];
                                                                            /* 32b*/  /* 64b*/
                                                                            
(,&, );/
            FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode
            if ({size_t
(&,&,llCode
            if (MEM_32bits() || (ofBits+mlBits const = [n]
                (&blockStream                                
            BIT_addBits(&blockStream, sequences[n].litLength            BYTE const mlCode=mlCodeTable];
            if (MEM_32bits() && ((            U32  LL_bits];
            BIT_addBits(               mlBits ML_bits];
             MEM_32bits |(++  6 &)
            if (longOffsets) {
unsignedconst   -MIN,STREAM_ACCUMULATOR_MIN-1
                if (extraBits) {
                   (blockStream, [n., );
                                            )sequences]offBase
                }
                BIT_addBits(&blockStream, sequences[n].offBase >> extraBits,
                                         MEM_32bits (&);                 
            } else {
                (blockStream[n], );     /* 31 */
             (blockStream/* (7)*/
            BIT_flushBits(            (&, [.itLengthllBits
DEBUGLOG," :i,((. blockStream.);
    }   }

                (&,[]mlBase);
   (blockStream);
    DEBUGLOG longOffsets
    FSE_flushCState                   =ofBits(ofBits);
    DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u if (extraBits) {
FSE_flushCState,&);

    {   size_t const streamSize = BIT_closeCStream(&blockStream);
(streamSize0 , not space
        return streamSize;
    }
}

static size_t
ZSTD_encodeSequences_default(
            void* dst, size_t dstCapacity,
            FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTableofBits );/*3*
            FSE_CTable} {
            FSE_CTable*CTable_LitLength const ,
                        BIT_flushBits;/* (7)*/
{
java.lang.StringIndexOutOfBoundsException: Range [36, 4) out of bounds for length 54
                                    CTable_MatchLength, mlCodeTable,
CTable_OffsetBits,ofCodeTable
                                    CTable_LitLength, llCodeTable,
    DEBUGLOG6": LLstatewith%ubits,stateLitLengthstateLog;
}


#if DYNAMIC_BMI2(blockStream);

staticBMI2_TARGET_ATTRIBUTE
ZSTD_encodeSequences_bmi2(
            void dst  dstCapacity
FSE_CTable*CTable_MatchLengthBYTE ,
            FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTablejava.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 5
            FSE_CTable const* CTable_LitLength,             FSE_CTable const* CTable_MatchLength, BYTE
            SeqDef  nbSeq )
{
 (dst
                                    CTable_MatchLength                                    ,
                                    CTable_OffsetBits,                                    ,nbSeq,longOffsets)java.lang.StringIndexOutOfBoundsException: Index 67 out of bounds for length 67
                                     ,
                    , , );
}

#endif

size_t *,  const,
            void* dst, size_t *,BYTE*ofCodeTable
FSE_CTable*CTable_MatchLength constmlCodeTable
            FSE_CTable constSeqDef sequencessize_t, longOffsets
            FSE_CTable const*     ZSTD_encodeSequences_body, ,
SeqDef*sequencessize_t,int, )
{
    DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsignedCTable_OffsetBits,
#if DYNAMIC_BMI2CTable_LitLength,
    if (bmi2) {
        return ZSTD_encodeSequences_bmi2,nbSeq);
                                         CTable_MatchLength
                                         java.lang.StringIndexOutOfBoundsException: Index 55 out of bounds for length 0
                                         CTable_LitLength, llCodeTable,
                                         , nbSeqlongOffsets;
    }
#endif
    (void)bmi2;
    return ZSTD_encodeSequences_default(dst, dstCapacity,
                    CTable_MatchLength, mlCodeTable
                                        CTable_OffsetBits, ofCodeTable,
                                        CTable_LitLength, llCodeTable
                                        sequences,{
}