Spracherkennung für: .patch vermutete Sprache: Unknown {[0] [0] [0]} [Methode: Schwerpunktbildung, einfache Gewichte, sechs Dimensionen]
--- a/src/hashmgr.cxx
+++ b/src/hashmgr.cxx
@@ -68,12 +68,15 @@
* SUCH DAMAGE.
*/
+#include <algorithm>
+#include <cassert>
#include <cstdlib>
#include <cstring>
#include <cstdio>
#include <cctype>
#include <limits>
#include <sstream>
+#include <type_traits>
#if __cplusplus >= 202002L || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)
#include <bit>
#endif
@@ -118,7 +121,7 @@
while (ptr) {
nt = ptr->next;
release_flags(ptr->astr, ptr->var & H_OPT_OWNFLAGS);
- free(ptr);
+ arena_free(ptr);
ptr = nt;
}
}
@@ -128,12 +131,16 @@
HashMgr::~HashMgr() {
free_table();
+ static_assert(std::is_trivially_destructible<unsigned short>::value,
+ "arena_free replaces delete[]; aliasf elements must have trivial destructors");
for (auto& j : aliasf)
- delete[] j;
+ arena_free(j);
aliasf.clear();
+ static_assert(std::is_trivially_destructible<char>::value,
+ "arena_free replaces delete[]; aliasm elements must have trivial destructors");
for (auto& j : aliasm)
- delete[] j;
+ arena_free(j);
aliasm.clear();
#ifdef MOZILLA_CLIENT
@@ -221,7 +228,8 @@
int descl = desc ? (!aliasm.empty() ? sizeof(char*) : desc->size() + 1) : 0;
// variable-length hash record with word and optional fields
auto hp =
- (struct hentry*)malloc(sizeof(struct hentry) + word->size() + descl);
+ (struct hentry*)arena_alloc(sizeof(struct hentry) + word->size() + descl,
+ alignof(struct hentry));
if (!hp) {
delete desc_copy;
delete word_copy;
@@ -374,7 +382,7 @@
dp->alen = hp->alen;
dp->var &= ~H_OPT_OWNFLAGS;
dp->var |= (hp->var & H_OPT_OWNFLAGS);
- free(hp);
+ arena_free(hp);
delete desc_copy;
delete word_copy;
return 0;
@@ -382,7 +390,7 @@
!hp->astr && hp->alen == 0) {
// word already exists with no flags, skip duplicate
release_flags(hp->astr, hp->var & H_OPT_OWNFLAGS);
- free(hp);
+ arena_free(hp);
delete desc_copy;
delete word_copy;
return 0;
@@ -404,7 +412,7 @@
dp->alen = hp->alen;
dp->var &= ~H_OPT_OWNFLAGS;
dp->var |= (hp->var & H_OPT_OWNFLAGS);
- free(hp);
+ arena_free(hp);
delete desc_copy;
delete word_copy;
return 0;
@@ -412,7 +420,7 @@
!hp->astr && hp->alen == 0) {
// word already exists with no flags, skip duplicate
release_flags(hp->astr, hp->var & H_OPT_OWNFLAGS);
- free(hp);
+ arena_free(hp);
delete desc_copy;
delete word_copy;
return 0;
@@ -428,7 +436,7 @@
} else {
// remove hidden onlyupcase homonym
release_flags(hp->astr, hp->var & H_OPT_OWNFLAGS);
- free(hp);
+ arena_free(hp);
}
delete desc_copy;
@@ -451,7 +459,8 @@
if (((captype == HUHCAP) || (captype == HUHINITCAP) ||
((captype == ALLCAP) && (flagslen != 0))) &&
!((flagslen != 0) && TESTAFF(flags, forbiddenword, flagslen))) {
- unsigned short* flags2 = new unsigned short[flagslen + 1];
+ auto flags2 = (unsigned short*)arena_alloc((flagslen + 1) * sizeof(unsigned short),
+ alignof(unsigned short));
flags2[flagslen] = ONLYUPCASEFLAG;
if (flagslen) {
memcpy(flags2, flags, flagslen * sizeof(unsigned short));
@@ -464,12 +473,12 @@
mkallsmall_utf(w, langnum);
mkinitcap_utf(w, langnum);
u16_u8(st, w);
- return add_word(st, wcl, flags2, flagslen + 1, dp, true, INITCAP, true);
+ return add_word(st, wcl, flags2, flagslen + 1, dp, true, INITCAP, false);
} else {
std::string new_word(word);
mkallsmall(new_word, csconv);
mkinitcap(new_word, csconv);
- int ret = add_word(new_word, wcl, flags2, flagslen + 1, dp, true, INITCAP, true);
+ int ret = add_word(new_word, wcl, flags2, flagslen + 1, dp, true, INITCAP, false);
return ret;
}
}
@@ -707,7 +716,7 @@
dict->getlinenum());
}
} else {
- al = decode_flags(&flags, ap, dict);
+ al = decode_flags(&flags, ap, dict, /* arena = */ true);
if (al == -1) {
HUNSPELL_WARNING(stderr, "Can't allocate memory.\n");
delete dict;
@@ -724,7 +733,8 @@
int wcl = get_clen_and_captype(ts, &captype, workbuf);
const std::string* dp_str = dp.empty() ? nullptr : &dp;
// add the word and its index plus its capitalized form optionally
- bool own = aliasf.empty();
+ // flags are arena-allocated, so own_aff must be false
+ bool own = false;
if (add_word(ts, wcl, flags, al, dp_str, false, captype, own) ||
add_hidden_capitalized_word(ts, wcl, flags, al, dp_str, captype)) {
delete dict;
@@ -759,6 +769,15 @@
}
int HashMgr::decode_flags(unsigned short** result, const std::string& flags, FileMgr* af) const {
+ return decode_flags(result, flags, af, /* arena = */ false);
+}
+
+int HashMgr::decode_flags(unsigned short** result, const std::string& flags, FileMgr* af, bool use_arena) const {
+ auto alloc = [&](int n) -> unsigned short* {
+ return use_arena ? (unsigned short*)this->arena_alloc(n * sizeof(unsigned short),
+ alignof(unsigned short))
+ : new unsigned short[n];
+ };
int len;
if (flags.empty()) {
*result = nullptr;
@@ -771,7 +790,7 @@
HUNSPELL_WARNING(stderr, "error: line %d: bad flagvector\n",
af->getlinenum());
len >>= 1;
- *result = new unsigned short[len];
+ *result = alloc(len);
for (int i = 0; i < len; i++) {
unsigned short flag = ((unsigned short)((unsigned char)flags[i << 1]) << 8) |
((unsigned short)((unsigned char)flags[(i << 1) | 1]));
@@ -783,7 +802,7 @@
case FLAG_NUM: { // decimal numbers separated by comma (4521,23,233 -> 4521
// 23 233)
len = int(1 + std::count_if(flags.begin(), flags.end(), [](char c) { return c == ','; }));
- *result = new unsigned short[len];
+ *result = alloc(len);
unsigned short* dest = *result;
const char* src = flags.c_str();
for (size_t p = 0; p < flags.size(); ++p) {
@@ -820,7 +839,7 @@
std::vector<w_char> w;
u8_u16(w, flags);
len = w.size();
- *result = new unsigned short[len];
+ *result = alloc(len);
#if defined(_WIN32) || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__)) || defined(__LITTLE_ENDIAN__)
memcpy(*result, w.data(), len * sizeof(unsigned short));
#else
@@ -834,7 +853,7 @@
}
default: { // Ispell's one-character flags (erfg -> e r f g)
len = flags.size();
- *result = new unsigned short[len];
+ *result = alloc(len);
unsigned short* dest = *result;
for (const char flag : flags) {
*dest = (unsigned char)flag;
@@ -1169,7 +1188,7 @@
case 1: {
std::string piece(start_piece, iter);
aliaslen =
- (unsigned short)decode_flags(&alias, piece, af);
+ (unsigned short)decode_flags(&alias, piece, af, /* arena = */ true);
std::sort(alias, alias + aliaslen);
break;
}
@@ -1182,7 +1201,7 @@
}
if (!alias) {
for (int k = 0; k < j; ++k) {
- delete[] aliasf[k];
+ arena_free(aliasf[k]);
}
aliasf.clear();
aliasflen.clear();
@@ -1281,8 +1300,10 @@
reverseword(chunk);
}
size_t sl = chunk.size() + 1;
- alias = new char[sl];
- memcpy(alias, chunk.c_str(), sl);
+ alias = (char*)arena_alloc(sl, alignof(char));
+ if (alias) {
+ memcpy(alias, chunk.c_str(), sl);
+ }
break;
}
default:
@@ -1294,7 +1315,7 @@
}
if (!alias) {
for (int k = 0; k < j; ++k) {
- delete[] aliasm[k];
+ arena_free(aliasm[k]);
}
aliasm.clear();
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n",
@@ -1412,3 +1433,46 @@
const std::vector<replentry>& HashMgr::get_reptable() const {
return reptable;
}
+
+void* HashMgr::arena_alloc(size_t num_bytes, size_t alignment) const {
+ // Fixed-size 64KB chunks: small enough to avoid significant waste on small
+ // dictionaries, large enough to amortize per-chunk malloc overhead on large
+ // ones. make_unique throws std::bad_alloc on OOM.
+ static const size_t MIN_CHUNK_SIZE = 65536;
+ static const size_t MAX_ALIGNMENT = alignof(std::max_align_t);
+ // Chunk sizes are rounded up to MAX_ALIGNMENT below; with this invariant,
+ // any alignment that divides MAX_ALIGNMENT keeps aligned_offset within bounds.
+ assert(alignment > 0 && alignment <= MAX_ALIGNMENT);
+ // Pad the offset up to the requested alignment before placing this allocation.
+ // make_unique returns memory aligned for any scalar, so chunk-start is fine.
+ size_t aligned_offset = (current_chunk_offset + alignment - 1) & ~(alignment - 1);
+ if (arena.empty() || current_chunk_size - aligned_offset < num_bytes) {
+ // Round the new chunk's size up to a multiple of MAX_ALIGNMENT so that an
+ // oversized num_bytes (>= MIN_CHUNK_SIZE) cannot leave a non-aligned
+ // current_chunk_size that would later cause aligned_offset to overshoot.
+ // Allocate before mutating current_chunk_size so a throwing make_unique
+ // leaves the HashMgr in a consistent state.
+ size_t new_size = std::max(MIN_CHUNK_SIZE, num_bytes);
+ new_size = (new_size + MAX_ALIGNMENT - 1) & ~(MAX_ALIGNMENT - 1);
+ arena.push_back(std::make_unique<uint8_t[]>(new_size));
+ current_chunk_size = new_size;
+ aligned_offset = 0;
+ }
+
+ uint8_t* ptr = &arena.back()[aligned_offset];
+ current_chunk_offset = aligned_offset + num_bytes;
+ ++outstanding_arena_allocations;
+ return ptr;
+}
+
+void HashMgr::arena_free(void*) const {
+ // The arena vector owns all allocations and frees them in bulk at HashMgr
+ // destruction, so this is a no-op for the memory itself. The counter is a
+ // memory-safety check: more arena_free calls than arena_alloc calls would
+ // indicate a double-free or use-after-free in Hunspell. Abort hard rather
+ // than silently desynchronize tracking, even in release builds.
+ if (outstanding_arena_allocations == 0) {
+ std::abort();
+ }
+ --outstanding_arena_allocations;
+}
--- a/src/hashmgr.hxx
+++ b/src/hashmgr.hxx
@@ -72,6 +72,8 @@
#define HASHMGR_HXX_
#include <cstdio>
+#include <cstdint>
+#include <memory>
#include <string>
#include <vector>
@@ -154,6 +156,24 @@
void remove_forbidden_flag(const std::string& word);
void free_table();
void release_flags(unsigned short* astr, bool owned);
+
+ // Only internal consumers are allowed to arena-allocate.
+ int decode_flags(unsigned short** result, const std::string& flags, FileMgr* af, bool use_arena) const;
+
+ // Our Mozilla fork uses a simple arena allocator for strings and hentry structs
+ // which persist for the lifetime of the HashMgr, to avoid heap fragmentation.
+ // It's a bump-allocator. arena_free does not release memory (the arena
+ // vector frees everything in bulk at HashMgr destruction); it only updates
+ // a counter and aborts on underflow as a memory-safety check.
+ // Declared const (with arena members mutable) because decode_flags is
+ // part of the public const API and calls this internally.
+ void* arena_alloc(size_t num_bytes, size_t alignment) const;
+ void arena_free(void* ptr) const;
+
+ mutable std::vector<std::unique_ptr<uint8_t[]>> arena;
+ mutable size_t current_chunk_size = 0;
+ mutable size_t current_chunk_offset = 0;
+ mutable size_t outstanding_arena_allocations = 0;
};
#endif