#ifdefined(BROTLI_EXPERIMENTAL) /* Word length must be at least 4 bytes */ static uint32_t Hash(const uint8_t* data, int bits) {
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32; /* The higher bits contain more mixture from the multiplication,
so we take our results from there. */ return h >> (32 - bits);
}
/* Theoretical max possible word size after transform */ #define kTransformedBufferSize \
(256 + 256 + SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH)
/* To be safe buffer must have at least kTransformedBufferSize */ staticvoid TransformedDictionaryWord(uint32_t word_idx, int len, int transform, const BrotliTransforms* transforms, const BrotliEncoderDictionary* dict,
uint8_t* buffer, size_t* size) { const uint8_t* dict_word = &dict->words->data[
dict->words->offsets_by_length[len] + (uint32_t)len * word_idx];
*size = (size_t)BrotliTransformDictionaryWord(buffer, dict_word, len,
transforms, transform);
}
static uint32_t BrotliTrieAlloc(MemoryManager* m, size_t num, BrotliTrie* trie,
BrotliTrieNode** keep) {
uint32_t result;
uint32_t keep_index = 0; if (keep && *keep != &trie->root) { /* Optional node to keep, since address may change after re-allocating */
keep_index = (uint32_t)(*keep - trie->pool);
} if (trie->pool_size == 0) { /* Have a placeholder node in the front. We do not want the result to be 0,
it must be at least 1, 0 represents "null pointer" */
trie->pool_size = 1;
}
BROTLI_ENSURE_CAPACITY(m, BrotliTrieNode, trie->pool, trie->pool_capacity,
trie->pool_size + num); if (BROTLI_IS_OOM(m)) return0; /* Init the new nodes to empty */
memset(trie->pool + trie->pool_size, 0, sizeof(*trie->pool) * num);
result = (uint32_t)trie->pool_size;
trie->pool_size += num; if (keep && *keep != &trie->root) {
*keep = trie->pool + keep_index;
} return result;
}
if (transforms->num_transforms > 0) { for (l = SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH;
l <= SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH; ++l) {
uint16_t n = dict->words->size_bits_by_length[l] ?
(uint16_t)(1 << dict->words->size_bits_by_length[l]) : 0u; for (idx = 0; idx < n; ++idx) {
uint32_t key; /* First transform (usually identity) */
TransformedDictionaryWord(idx, l, 0, transforms, dict, word,
&word_size); /* Cannot hash words smaller than 4 bytes */ if (word_size < 4) { /* Break instead of continue, all next words of this length will have
same length after transform */ break;
} if (!BrotliTrieAdd(m, 0, idx, word, word_size, &dedup)) { return BROTLI_FALSE;
}
key = Hash(word, NUM_HASH_BITS);
BROTLI_ENSURE_CAPACITY_APPEND(m, DictWord, words_by_hash[key],
words_by_hash_capacity[key], words_by_hash_size[key],
MakeDictWord(l, 0, idx));
++total;
}
}
}
/* These LUT transforms only supported if no custom transforms. This is
ok, we will use the heavy trie instead. */ if (transforms == BrotliGetTransforms()) { for (l = SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH;
l <= SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH; ++l) {
uint16_t n = dict->words->size_bits_by_length[l] ?
(uint16_t)(1 << dict->words->size_bits_by_length[l]) : 0u; for (idx = 0; idx < n; ++idx) { int k;
BROTLI_BOOL is_ascii = BROTLI_TRUE;
size_t offset = dict->words->offsets_by_length[l] + (size_t)l * idx; const uint8_t* data = &dict->words->data[offset]; for (k = 0; k < l; ++k) { if (data[k] >= 128) is_ascii = BROTLI_FALSE;
} if (data[0] < 128) { int transform = 9; /* {empty, uppercase first, empty} */
uint32_t ix = idx + (uint32_t)transform * n; const BrotliTrieNode* it;
TransformedDictionaryWord(idx, l, transform, transforms,
dict, word, &word_size);
it = BrotliTrieFind(&dedup, word, word_size); if (!it || it->idx_ > ix) {
uint32_t key = Hash(word, NUM_HASH_BITS); if (!BrotliTrieAdd(m, 0, ix, word, word_size, &dedup)) { return BROTLI_FALSE;
}
BROTLI_ENSURE_CAPACITY_APPEND(m, DictWord, words_by_hash[key],
words_by_hash_capacity[key], words_by_hash_size[key],
MakeDictWord(l, BROTLI_TRANSFORM_UPPERCASE_FIRST, idx));
++total;
}
} if (is_ascii) { int transform = 44; /* {empty, uppercase all, empty} */
uint32_t ix = idx + (uint32_t)transform * n; const BrotliTrieNode* it;
TransformedDictionaryWord(idx, l, transform, transforms,
dict, word, &word_size);
it = BrotliTrieFind(&dedup, word, word_size); if (!it || it->idx_ > ix) {
uint32_t key = Hash(word, NUM_HASH_BITS); if (!BrotliTrieAdd(m, 0, ix, word, word_size, &dedup)) { return BROTLI_FALSE;
}
BROTLI_ENSURE_CAPACITY_APPEND(m, DictWord, words_by_hash[key],
words_by_hash_capacity[key], words_by_hash_size[key],
MakeDictWord(l, BROTLI_TRANSFORM_UPPERCASE_ALL, idx));
++total;
}
}
}
}
}
/* Unused; makes offsets start from 1. */
dict_words[0] = MakeDictWord(0, 0, 0);
total = 1; for (i = 0; i < NUM_HASH_BUCKETS; ++i) {
size_t num_words = words_by_hash_size[i]; if (num_words > 0) {
buckets[i] = (uint16_t)(total);
memcpy(&dict_words[total], &words_by_hash[i][0], sizeof(dict_words[0]) * num_words);
total += num_words;
dict_words[total - 1].len |= 0x80;
} else {
buckets[i] = 0;
}
}
for (i = 0; i < NUM_HASH_BUCKETS; ++i) {
BrotliFree(m, words_by_hash[i]);
}
BrotliFree(m, words_by_hash);
BrotliFree(m, words_by_hash_size);
BrotliFree(m, words_by_hash_capacity);
BrotliTrieFree(m, &dedup);
return BROTLI_TRUE;
}
staticvoid BuildDictionaryHashTable(uint16_t* hash_table_words,
uint8_t* hash_table_lengths, const BrotliDictionary* dict) { int j, len; /* The order of the loops is such that in case of collision, words with shorterlengtharepreferred,andincaseofsamelength,wordswith
smaller index. There is only a single word per bucket. */ /* TODO(lode): consider adding optional user-supplied frequency_map to use forpreferredwordsinstead,thiscanmaketheencoderbetterfor
quality 9 and below without affecting the decoder */
memset(hash_table_words, 0, sizeof(kStaticDictionaryHashWords));
memset(hash_table_lengths, 0, sizeof(kStaticDictionaryHashLengths)); for (len = SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH;
len >= SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH; --len) { const size_t num_words = dict->size_bits_by_length[len] ?
(1u << dict->size_bits_by_length[len]) : 0; for (j = (int)num_words - 1; j >= 0; --j) {
size_t offset = dict->offsets_by_length[len] +
(size_t)len * (size_t)j; const uint8_t* word = &dict->data[offset]; const uint32_t key = Hash(word, 14); int idx = (int)(key << 1) + (len < 8 ? 1 : 0);
BROTLI_DCHECK(idx < (int)NUM_HASH_BUCKETS);
hash_table_words[idx] = (uint16_t)j;
hash_table_lengths[idx] = (uint8_t)len;
}
}
}
static BROTLI_BOOL GenerateWordsHeavy(MemoryManager* m, const BrotliTransforms* transforms,
BrotliEncoderDictionary* dict) { int i, j, l; for (j = (int)transforms->num_transforms - 1; j >= 0 ; --j) { for (l = 0; l < 32; l++) { int num = (int)((1u << dict->words->size_bits_by_length[l]) & ~1u); for (i = 0; i < num; i++) {
uint8_t transformed[kTransformedBufferSize];
size_t size;
TransformedDictionaryWord(
(uint32_t)i, l, j, transforms, dict, transformed, &size); if (size < 4) continue; if (!BrotliTrieAdd(m, (uint8_t)l, (uint32_t)(i + num * j),
transformed, size, &dict->trie)) { return BROTLI_FALSE;
}
}
}
} return BROTLI_TRUE;
}
/* Computes cutoffTransformsCount (in count) and cutoffTransforms (in data) for thecustomtransforms,wherepossiblewithinthelimitsofthe cutoffTransformsencoding.Thefastencoderusesthistodofastlookupfor
transforms that remove the N last characters (OmitLast). */ staticvoid ComputeCutoffTransforms( const BrotliTransforms* transforms,
uint32_t* count, uint64_t* data) { int i; /* The encoding in a 64-bit integer of transform N in the data is: (N << 2) + ((cutoffTransforms>>(N*6))&0x3F),soforexampletheidentity transformcodemustbe0-63,forN=1thetransformcodemustbe4-67,..., forN=9itmustbe36-99. TODO(lode):considerasimpleflexibleuint8_t[10]insteadoftheuint64_t forthecutofftransforms,sothatshareddictionariescanhavethe
OmitLast transforms anywhere without loss. */
*count = 0;
*data = 0; for (i = 0; i < BROTLI_TRANSFORMS_MAX_CUT_OFF + 1; i++) { int idx = transforms->cutOffTransforms[i]; if (idx == -1) break; /* Not found */ if (idx < (i << 2)) break; /* Too small for the encoding */ if (idx >= (i << 2) + 64) break; /* Too large for the encoding */
(*count)++;
*data |= (uint64_t)(((uint64_t)idx -
((uint64_t)i << 2u)) << ((uint64_t)i * 6u));
}
}
static BROTLI_BOOL ComputeDictionary(MemoryManager* m, int quality, const BrotliTransforms* transforms,
BrotliEncoderDictionary* current) { int default_words = current->words == BrotliGetDictionary(); int default_transforms = transforms == BrotliGetTransforms();
if (default_words && default_transforms) { /* hashes are already set to Brotli defaults */ return BROTLI_TRUE;
}
current->hash_table_data_words_ = (uint16_t*)BrotliAllocate(
m, sizeof(kStaticDictionaryHashWords));
current->hash_table_data_lengths_ = (uint8_t*)BrotliAllocate(
m, sizeof(kStaticDictionaryHashLengths)); if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
current->hash_table_words = current->hash_table_data_words_;
current->hash_table_lengths = current->hash_table_data_lengths_;
/* Only compute the data for slow encoder if the requested quality is high
enough to need it */ if (quality >= ZOPFLIFICATION_QUALITY) { if (!BuildDictionaryLut(m, transforms, current)) return BROTLI_FALSE;
/* For the built-in Brotli transforms, there is a hard-coded function to handlealltransforms,butforcustomtransforms,weusethefollowing
large hammer instead */
current->has_words_heavy = !default_transforms; if (current->has_words_heavy) { if (!GenerateWordsHeavy(m, transforms, current)) return BROTLI_FALSE;
}
}
#ifdefined(BROTLI_EXPERIMENTAL) /* TODO(eustas): make sure that tooling will warn user if not all the cutoff
transforms are available (for low-quality encoder). */ static BROTLI_BOOL InitCustomSharedEncoderDictionary(
MemoryManager* m, const BrotliSharedDictionary* decoded_dict, int quality, SharedEncoderDictionary* dict) {
ContextualEncoderDictionary* contextual;
CompoundDictionary* compound;
BrotliEncoderDictionary* instances; int i;
BrotliInitSharedEncoderDictionary(dict);
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.