/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v.
2.
0. If a copy of the MPL was not distributed with this
* file, You can obtain one at
https://mozilla.org/MPL/2.
0/. */
/**
* Represents a single message exchanged in a chat with a language model.
*
* This dictionary is used to define the role and content of each message in the
* prompt passed to a llama.cpp-based LLM (e.g., user or assistant messages).
*/
dictionary LlamaChatMessage {
/**
* The role of the speaker in the chat conversation.
* Common values include "system", "user", or "assistant".
*/
required UTF8String role;
/**
* The textual content of the message associated with the given role.
*/
required UTF8String content;
};
enum LlamaChatPhase {
"prompt",
"generation"
};
[GenerateConversionToJS]
dictionary LlamaChatResponse {
required UTF8String piece;
required sequence<long> tokens;
required LlamaChatPhase phase;
required boolean isPhaseCompleted;
};
dictionary LlamaFormatChatOptions {
/**
* List of role and their content to be formatted into a string prompt.
*/
required sequence<LlamaChatMessage> messages;
/**
* If true, the assistant role will be added to the generated prompt
*/
boolean addAssistant = true;
};
enum LlamaKVCacheDtype {
"f32",
"f16",
"q8_0",
"q5_1",
"q5_0",
"q4_1",
"q4_0"
};
/**
* Configuration options for creating a LLaMA context.
* Name are chosen to map directly to the ones from llama.cpp.
*/
dictionary LlamaContextOptions {
/**
* Maximum combined length of input & generate tokens.
*/
unsigned long nCtx =
2048;
/**
* Logical max batch size submitted to llama_decode.
*/
unsigned long nBatch =
2048;
/**
* Physical max batch size for actual processing.
*/
unsigned long nUbatch =
512;
/**
* Maximum number of concurrent sequences (distinct KV states).
*/
unsigned long nSeqMax =
1;
/**
* Number of threads for generation.
*/
required long nThreads;
/**
* Number of threads for batch/prompt processing.
*/
required long nThreadsBatch;
/**
* Data type for the K (key) cache [EXPERIMENTAL].
*/
LlamaKVCacheDtype kCacheDtype = "f16";
/**
* Data type for the V (value) cache [EXPERIMENTAL].
*/
LlamaKVCacheDtype vCacheDtype = "f16";
/**
* If true, offload K, Q, V ops (including KV cache) to the GPU.
* Only active if GPU is used with NGpULayers !=
0 in LlamaModelOptions
*/
boolean offloadKqv = true;
/**
* If true, use FlashAttention (experimental).
*/
boolean flashAttn = false;
/**
* If true, disable performance measurement (no timing output).
*/
boolean noPerf = false;
/**
* If true, offload host-side tensor operations to the device.
* Only active if GPU is used with NGpULayers !=
0 in LlamaModelOptions
*/
boolean opOffload = true;
/**
* If true, use a full-size SWA (sliding window attention) cache.
*/
boolean swaFull = true;
};
/**
* Configuration options for loading a LLaMA model.
* See comments here
https://github.com/ggml-org/llama.cpp/blob/b5774/include/llama.h#L298
* for more description on each field.
*/
dictionary LlamaModelOptions {
/**
* If true, use `mmap` for loading the model, if supported.
*/
boolean useMmap = true;
/**
* If true, attempt to lock the model in RAM using `mlock`.
*/
boolean useMlock = false;
/**
* If true, perform extra validation on model tensor data.
*/
boolean checkTensors = false;
/**
* Number of model layers to offload to GPU.
* A value of
0 disables GPU offloading.
*/
long nGpuLayers =
0;
/**
* Context configuration (e.g. nCtx, threads).
*/
LlamaContextOptions context = {};
};
enum LlamaSamplerType {
"logit-bias",
"dry",
"top-k",
"top-p",
"top-n-sigma",
"min-p",
"xtc",
"typical",
"temperature",
"temperature-ext",
"infill",
"penalties",
"mirostat",
"dist",
};
dictionary LlamaLogitBias {
required long token;
required float bias;
};
/**
* Represents a configured sampler.
*/
dictionary LlamaSamplerConfig {
/**
* The sampler algorithm to use.
*/
required LlamaSamplerType type;
/**
* Minimum number of tokens to keep (
0 = disabled).
*/
long minKeep =
0;
/**
* Top-K cutoff. If <=
0, uses full vocabulary.
*/
long topK =
40;
/**
* Top-P (nucleus) sampling threshold.
*/
float topP =
0.
95;
/**
* Minimum P cutoff.
*/
float minP =
0.
05;
/**
* XTC sampling probability (
0.
0 = disabled).
*/
float xtcProbability =
0.
0;
/**
* XTC threshold (values >
0.
5 disable XTC).
*/
float xtcThreshold =
0.
10;
/**
* Typical sampling cutoff (
1.
0 = disabled).
*/
float typP =
1.
0;
/**
* Sampling temperature (
0.
0 or below = greedy decoding).
*/
float temp =
0.
80;
/**
* Dynamic temperature range (
0.
0 = disabled).
*/
float dynatempRange =
0.
0;
/**
* Dynamic temperature exponent (entropy-to-temp mapping).
*/
float dynatempExponent =
1.
0;
/**
* Repetition penalty: number of tokens to track (-
1 = context size).
*/
long penaltyLastN =
64;
/**
* Repetition penalty multiplier (
1.
0 = disabled).
*/
float penaltyRepeat =
1.
0;
/**
* Frequency penalty (
0.
0 = disabled).
*/
float penaltyFreq =
0.
0;
/**
* Presence penalty (
0.
0 = disabled).
*/
float penaltyPresent =
0.
0;
/**
* DRY multiplier (
0.
0 = disabled).
*/
float dryMultiplier =
0.
0;
/**
* DRY base exponent (
0.
0 = disabled).
*/
float dryBase =
1.
75;
/**
* DRY allowed repetition length before penalization starts.
*/
long dryAllowedLength =
2;
/**
* DRY lookback window (
0 = disable, -
1 = context size).
*/
long dryPenaltyLastN = -
1;
/**
* Mirostat mode (
0 = disabled,
1 = v1,
2 = v2).
*/
long mirostat =
0;
/**
* Top-n sigma sampling cutoff (-
1.
0 = disabled).
*/
float topNSigma = -
1.
0;
/**
* Mirostat target entropy (tau).
*/
float mirostatTau =
5.
0;
/**
* Mirostat learning rate (eta).
*/
float mirostatEta =
0.
1;
/**
* List of token-specific logit biases.
*/
sequence<LlamaLogitBias> logitBias = [];
/**
* If true, disables performance metrics.
*/
boolean noPerf = false;
/**
* Random number seed for sampling.
*/
unsigned long seed;
};
dictionary LlamaDeTokenizationOptions {
/**
* Maximum number of UTF-
8 characters that may be contained in a single model token.
* This is used to reserve enough space during detokenization.
*/
long maxCharsPerToken =
256;
/**
* Whether to render special tokens such as <BOS>, <EOS>, or <UNK> in the output.
* If false, these tokens will be omitted from the detokenized result.
*/
boolean renderSpecialTokens = true;
};
dictionary LlamaTokenizationOptions {
/**
* Allow to add BOS and EOS tokens if model is configured to do so.
*/
boolean addBosAndEos = true;
/**
* Allow tokenizing special and/or control tokens which otherwise are not exposed and treated
* as plaintext. Does not insert a leading space.
*/
boolean parseSpecilControlTokens = true;
};
dictionary LlamaChatOptions {
/**
* Sampler stack to apply during decoding.
*/
sequence<LlamaSamplerConfig> samplers = [];
/**
* Input prompt text to process.
*/
required UTF8String prompt;
/**
* Optional output buffer size (
0 = no preallocation).
*/
long minOutputBufferSize =
1;
/**
* Maximum number of generation steps (tokens).
*/
long maxGeneratedTokens =
512;
/**
* If true, stop when encountering known model end-of-generation tokens.
*/
boolean stopOnEndOfGenerationTokens = true;
/**
* List of token IDs that should stop generation.
*/
sequence<long> stopTokens = [];
LlamaTokenizationOptions tokenizationOptions = {};
LlamaDeTokenizationOptions deTokenizationOptions = {};
};
[Func="LlamaRunner::InInferenceProcess", Exposed=(DedicatedWorker,Window)]
interface LlamaRunner {
[Throws] constructor();
[Throws] Promise<undefined> initialize(LlamaModelOptions options, Blob modelBlob);
[Throws] Promise<UTF8String> formatChat(LlamaFormatChatOptions options);
[NewObject, Throws] ReadableStream createGenerationStream(LlamaChatOptions options);
};
