config_sample.yml

# Sample YAML file for configuration.
# Comment and uncomment values as needed.
# Every value has a default within the application.
# This file serves to be a drop in for config.yml

# Unless specified in the comments, DO NOT put these options in quotes!
# You can use https://www.yamllint.com/ if you want to check your YAML formatting.

# Options for networking
network:
  # The IP to host on (default: 127.0.0.1).
  # Use 0.0.0.0 to expose on all network adapters.
  host: 127.0.0.1

  # The port to host on (default: 5000).
  # TabbyAPI exits if the port is taken rather than moving to another one.
  port: 5000

  # Disable HTTP token authentication with requests.
  # WARNING: This will make your instance vulnerable!
  # Only turn this on if nothing but trusted local clients can reach the API.
  # Note that web pages open in a browser on this machine also count as local
  # callers; restrict allowed_origins below if you disable auth.
  disable_auth: false

  # Origins allowed to call the API from a browser (default: ["*"]).
  # This is a CORS allowlist, not authentication: "*" lets any site open in your
  # browser send requests to this instance, which matters most when disable_auth
  # is on. Restrict to your own frontends (e.g. ["http://localhost:8000"]) or use
  # an empty list [] to block all browser (cross-origin) callers.
  allowed_origins: ["*"]

  # Disable fetching external content in response to requests,such as images from URLs.
  disable_fetch_requests: false

  # Send tracebacks over the API (default: False).
  # NOTE: Only enable this for debug purposes.
  send_tracebacks: false

  # Select API servers to enable (default: ["OAI"]).
  # Possible values: OAI, Kobold.
  api_servers: ["OAI"]

  # Seconds between SSE keep-alive pings on streaming responses (default: 15).
  # Pings are SSE comments, ignored by compliant clients, and prevent
  # connections from dropping during long prefills. Set to 0 to disable.
  sse_ping_interval: 15

  # Log every HTTP request with client address, method, path and status (default: False).
  # Generation requests are already logged in detail; this adds the rest, such as model list and health polls.
  access_log: false

  # Serve the built-in chat client at /ui (default: True).
  # A static page for testing models and settings; it only uses the
  # regular API, so it does nothing until a browser opens it.
  web_ui: true

# Options for logging
logging:
  # Enable prompt logging (default: False).
  log_prompt: false

  # Enable generation parameter logging (default: False).
  log_generation_params: false

  # Enable request logging (default: False).
  # NOTE: Only use this for debugging!
  log_requests: false

  # Show a live status line below the log with cache usage and in-flight jobs (default: True).
  # Only shown on an interactive terminal.
  log_live_status: true

  # Prefix console log lines with the time of day (default: True).
  # The log files under logs/ always carry full timestamps.
  log_timestamps: true

  # Write every /v1/chat/completions request to logs/debug/ as JSON (default: False).
  # Also saves the fully templated prompt (the exact text sent to the tokenizer)
  # as a .txt file with the same basename.
  # PRIVACY WARNING: Enabling this creates a comprehensive request log, including the
  # full message history and generation parameters. API keys are redacted, but prompts
  # and user-provided content are preserved for bug-report reproduction.
  log_chat_completion_requests: false

# Options for model overrides and loading
# Please read the comments to understand how arguments are handled
# between initial and API loads
model:
  # Directory to look for models (default: models).
  # Windows users, do NOT put this path in quotes!
  model_dir: models

  # Allow direct loading of models from a completion or chat completion request (default: False).
  # This method of loading is strict: a request naming a model that doesn't exist or fails to load
  # is rejected instead of running on the loaded model. Enable dummy models to add exceptions for
  # model names that clients send without meaning a specific model.
  inline_model_loading: false

  # Sends dummy model names when the models endpoint is queried. (default: False)
  # Enable this if the client is looking for specific OAI models.
  use_dummy_models: false

  # A list of fake model names that are sent via the /v1/models endpoint. (default: ["gpt-3.5-turbo"])
  # Also used as bypasses for strict mode if inline_model_loading is true.
  dummy_model_names: ["gpt-3.5-turbo"]

  # An initial model to load.
  # Make sure the model is located in the model directory!
  # REQUIRED: This must be filled out to load a model on startup.
  model_name:

  # Names of args to use as a fallback for API load requests (default: []).
  # For example, if you always want cache_mode to be Q4 instead of on the inital model load, add "cache_mode" to this array.
  # Example: ['max_seq_len', 'cache_mode'].
  use_as_default: []

  # Backend to use for this model (auto-detect if not specified)
  # Options: exllamav3
  backend:

  # Max sequence length (default: min(max_position_embeddings, cache_size)).
  # Set to -1 to fetch from the model's config.json
  max_seq_len:

  # Size of the key/value cache to allocate, in tokens (default: 4096).
  # Must be a multiple of 256.
  cache_size:

  # Enable different cache modes for VRAM savings (default: FP16).
  # Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8).
  # The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted.
  cache_mode: FP16

  # Store GatedDeltaNet/KDA recurrent state in FP32 instead of FP16 (default: False).
  # Uses twice the state and checkpoint memory; scans compute in FP32 either way.
  recurrent_state_fp32: false

  # Load model with tensor parallelism.
  # Falls back to autosplit if GPU split isn't provided.
  # This ignores the gpu_split_auto value.
  tensor_parallel: false

  # Sets a backend type for tensor parallelism. (default: native).
  # Options: native, nccl
  # Native is recommended for PCIe GPUs
  # NCCL is recommended for NVLink.
  tensor_parallel_backend: native

  # Automatically allocate resources to GPUs (default: True).
  # Not parsed for single GPU users.
  gpu_split_auto: true

  # Reserve VRAM used when loading a model (default: 96 MB on GPU 0).
  # Represented as an array of MB per GPU.
  # A negative value excludes that GPU from the model split, so
  # excluding every GPU will fail to load.
  # Ignored for a model whose placement is already set by gpu_split
  # or draft_gpu_split.
  autosplit_reserve: [96]

  # Array of VRAM sizes to split between GPUs, in GB (default: []).
  # Used both with and without tensor parallelism.
  gpu_split: []

  # Number of mixture-of-expert layers to offload to CPU inference (default: 0)
  # Only affects MoE models. Set a large value such as 999 to offload all layers
  # Mutually exclusive with cpu_moe_split_experts.
  cpu_moe_offload_layers:

  # Number of routed experts per MoE layer to offload to CPU inference (default: 0).
  # Unlike cpu_moe_offload_layers, this splits every MoE layer instead of offloading whole
  # layers: the coldest experts are kept in system RAM and computed on the CPU, overlapping
  # each layer's own GPU compute, with dynamic placement keeping hot experts in VRAM.
  # Mutually exclusive with cpu_moe_offload_layers; not supported with tensor parallelism.
  cpu_moe_split_experts:

  # Worker thread count for CPU MoE inference (default: None).
  # Applies to both cpu_moe_offload_layers and cpu_moe_split_experts. When unset,
  # defers to the EXL3_MOE_CPU_THREADS environment variable, then half the CPU core count.
  cpu_moe_threads:

  # Keep the CPU MoE worker's expert weights in shared, page-locked memory (default: None).
  # Streamed prefill then copies experts straight from it to the GPU, and cpu_moe_split_experts
  # can promote frequently used experts into spare VRAM slots on the fly. Costs about 0.2 s per
  # GiB of offloaded experts at load. Only has an effect with cpu_moe_offload_layers or
  # cpu_moe_split_experts. When unset, defers to the EXL3_MOE_PINNED_ARENA environment variable.
  cpu_moe_pinned_arena:

  # RYS layer map for the exllamav3 backend (default: unset).
  # Either a list of layer indices or a string spec of comma-separated ints and
  # inclusive ranges, e.g. "0..6,4..6". Forward passes traverse the model in this
  # order; repeated layers get individual cache tensors. Weights still load in the
  # original layer order.
  layer_map:

  # Load a model's n-gram embedding table fully into system RAM (default: False).
  # Only affects PLE models with n-gram embeddings (e.g. Qwen3.8-Flash-Next). By default
  # the table is streamed from disk during inference; loading it into RAM avoids
  # per-token disk reads at the cost of tens of GB of system memory.
  ngram_ram: false

  # Stream token embeddings from disk on demand (default: False).
  # Rows of the token embedding table are read per forward pass instead of holding the
  # whole table in system RAM. Saves memory on models with large vocabularies at the cost
  # of small disk reads during inference. Works for quantized and unquantized tables.
  embed_stream_from_disk: false

  # NOTE: If a model has YaRN rope scaling, it will automatically be enabled by ExLlama.
  # rope_scale and rope_alpha settings won't apply in this case.

  # Rope scale (default: 1.0).
  # Same as compress_pos_emb.
  # Use if the model was trained on long context with rope.
  # Leave blank to pull the value from the model.
  rope_scale: 1.0

  # Rope alpha (default: None).
  # Same as alpha_value. Set to "auto" to auto-calculate.
  # Leaving this value blank will either pull from the model or auto-calculate.
  rope_alpha:

  # Chunk size for prompt ingestion (default: 2048).
  # A lower value reduces VRAM usage but decreases ingestion speed.
  # NOTE: Effects vary depending on the model.
  # An ideal value is between 512 and 4096.
  chunk_size: 2048

  # Use output chunking (default: True)
  # Instead of allocating cache space for the entire completion at once, allocate in chunks as needed.
  # Used by EXL3 models only.
  output_chunking: true

  # Set the maximum number of generation jobs that can run concurrently
  # The default maximum batch size for transformer architectures is 32. Recurrent
  # models with linear or sliding attention use more VRAM to support larger batches,
  # so the default value is reduced to 4. If you do not require concurrency at all, you
  # can reduce it further to minimize VRAM overhead.
  max_batch_size:

  # Tokens between recurrent state checkpoints near the end of the prompt and
  # during generation (default: None, the engine's per-architecture default,
  # 2048 for most models). Only used by models with recurrent (linear or sliding
  # attention) layers. Must be a multiple of 256.
  recurrent_checkpoint_interval:

  # Tokens between recurrent state checkpoints during prompt ingestion, further
  # than 2 * chunk_size from the end of the prompt (default: None, the engine
  # default of 32768). Only used by models with recurrent layers. Must be a
  # multiple of 256 and is rounded up to a multiple of chunk_size.
  # Recurrent states cannot be rolled back, so a request that edits an earlier
  # part of a cached prompt replays from the last checkpoint before the edit.
  # With the default, a long prompt is only checkpointed near its end and an
  # early edit costs a full re-prefill; with a denser grid the replay cost becomes
  # proportional to the distance from the edit to the end of the prompt.
  # Each checkpoint costs one recurrent state of system RAM (148 MiB for a 27B
  # hybrid with 48 recurrent layers), bounded by memory.sysmem_recurrent_cache,
  # and cold prefill is 2-3% slower at 2048 or 1024.
  recurrent_checkpoint_interval_pp:

  # Set the prompt template for this model. (default: None)
  # If empty, attempts to look for the model's chat template.
  # If a model contains multiple templates in its tokenizer_config.json,
  # set prompt_template to the name of the template you want to use.
  # NOTE: Only works with chat completion message lists!
  prompt_template:

  # Enables vision support if the model supports it. (default: False)
  vision: false

  # Sampler overrides for this model (default: None).
  # Same syntax as the top-level sampling section: an optional override_preset plus any
  # sampler written inline. Applied on top of the global sampling section while this model
  # is loaded; meant for the model folder's tabby_config.yml. For example:
  # sampling:
  #   override_preset: my_model_preset
  #   temperature:
  #     override: 0.5
  #     force: false
  sampling:

  # Warm up the model after loading (default: False).
  # Runs a short schedule of forward passes so kernel compilation, autotuning and CUDA graph
  # capture happen at load time instead of on the first requests. Adds some seconds to loading;
  # sized from the cache, batch and chunk settings in effect.
  warmup: false

  # Keep the vision model's weights in system RAM instead of VRAM (default: False).
  # Weights are stored in pinned host memory and streamed to the GPU during inference,
  # trading vision speed for VRAM. Only applies when vision is enabled.
  vision_offload: false

  # Default chat template variables (default: {}).
  # Merged into the template variables of every chat completion request; values
  # sent by the client (template_vars / chat_template_kwargs, or the top-level
  # reasoning_effort field) take precedence. Use for model-specific reasoning
  # knobs, e.g. {enable_thinking: true} or {reasoning_effort: high}.
  template_vars_default: {}

  # Forced chat template variables (default: {}).
  # Like template_vars_default, but these override any values sent by the client.
  # Replaces the deprecated force_enable_thinking option, which is still accepted
  # as an alias for {enable_thinking: true}.
  template_vars_force: {}

  # Enable the reasoning parser (default: True).
  # Splits the response into reasoning_content and content fields. With the tokens below
  # left at auto, reasoning is only parsed when the model's template or tokenizer shows
  # which tags it uses.
  reasoning: true

  # Start token for the reasoning parser (default: auto).
  # auto takes the tags from the detected tool format or the chat template; set both
  # tokens explicitly (e.g. "<think>" and "</think>") to override.
  reasoning_start_token: auto

  # End token for the reasoning parser (default: auto).
  reasoning_end_token: auto

  # Whether generation starts inside a reasoning block (default: auto).
  # Options: auto, always, never
  # auto guesses by scanning the end of the templated prompt for an unclosed reasoning start token.
  start_in_reasoning: auto

  # Parse tool calls that occur inside reasoning content (default: True).
  # If False, tool call tags inside a reasoning block are treated as plain reasoning text.
  tool_calls_in_reasoning: true

  # Default reasoning token budget (default: None).
  # When a request's reasoning content exceeds the budget, the server forces the end of
  # the reasoning phase by injecting reasoning_budget_message followed by the model's
  # end-of-reasoning tokens. 0 ends reasoning as soon as it starts; None or a negative
  # value disables the budget. Overridable per request via reasoning_budget_tokens
  # (aliases: reasoning_budget, thinking_budget, thinking_token_budget) or
  # reasoning.max_tokens. Requires a reasoning format: reasoning tags, Harmony or Muse Glimmer.
  reasoning_budget_tokens:

  # Text injected before the end-of-reasoning tokens when the reasoning budget is
  # exhausted (default: no text, only the end-of-reasoning tokens are forced). Also overridable
  # per request via reasoning_budget_message.
  reasoning_budget_message:

  # Tool call format (default: auto).
  # auto picks a parser from the model's chat template, tokenizer and architecture, and warns
  # if none matches. Set a format name, e.g. 'qwen3_coder', to override; see the Tool Calling
  # docs for the supported formats. Leave blank to disable tool call parsing.
  tool_format: auto

  # Parse responses in the Harmony message format (gpt-oss models).
  # Auto-detected from the model's special tokens by default; set to true or false
  # to override. Setting 'tool_format: harmony' is equivalent to setting this to true.
  # When active, supersedes the reasoning and tool format settings.
  harmony:

  # Parse responses in the Muse Glimmer message format.
  # Auto-detected from the model's special tokens by default; set to true or false
  # to override. Setting 'tool_format: muse_glimmer' is equivalent to setting this to true.
  # When active, supersedes the reasoning and tool format settings.
  muse_glimmer:

# Options for draft models (speculative decoding)
# This will use more VRAM!
draft_model:
  # Drafting mode for exllamav3 (default: model).
  # Options: model, disabled, mtp, ngram.
  # In `model` mode, drafting is disabled if no draft_model_name is provided.
  draft_mode: model

  # Directory to look for draft models (default: models)
  draft_model_dir: models

  # An initial draft model to load.
  # Ensure the model is in the model directory.
  draft_model_name:

  # Rope scale for draft models (default: 1.0).
  # Same as compress_pos_emb.
  # Use if the draft model was trained on long context with rope.
  draft_rope_scale: 1.0

  # Rope alpha for draft models (default: None).
  # Same as alpha_value. Set to "auto" to auto-calculate.
  # Leaving this value blank will either pull from the model or auto-calculate.
  draft_rope_alpha:

  # Cache mode for draft models to save VRAM (default: FP16).
  # Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8).
  # The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted.
  draft_cache_mode: FP16

  # Array of VRAM sizes to split between GPUs, in GB (default: []).
  # If this isn't filled in, the draft model is autosplit.
  draft_gpu_split: []

  # Number of tokens to draft per iteration (default: draft model default)
  draft_num_tokens:

  # Adjust number of draft tokens dynamically based on observed acceptance rates (default: False)
  # Ceiling is given by num_draft_tokens.
  dynamic_draft:

  # Target acceptance probability for drafted positions when dynamic_draft is enabled
  # (default: 0.4). Evaluated against an online mapping from drafter confidence scores
  # to observed acceptance rates. Lower values keep longer drafts; higher values
  # truncate more aggressively. Must be between 0 and 1. Ignored for n-gram drafting.
  draft_confidence: 0.4

  # Minimum match length for exllamav3 n-gram drafting (default: 2).
  # Only used when draft_mode is ngram.
  ngram_match_min: 2

  # Hybrid drafting: when the model is repeating a passage of its context verbatim (reciting
  # a drafted reply, editing source), draft this many tokens of the earlier continuation in
  # one verification pass (default: 0, off). Works on top of any draft mode, or on its own.
  # Longer values advance more tokens at prefill speed on a hit, and cost a wider
  # verification pass.
  hybrid_draft_tokens: 0

# Options for Sampling
sampling:
  # Select a sampler override preset (default: None).
  # Find this in the sampler-overrides folder.
  # This overrides default fallbacks for sampler values that are passed to the API.
  # NOTE: safe_defaults is noob friendly and provides fallbacks for frontends that don't send sampling parameters.
  # for frontends that don't send sampling parameters. Leaving this blank means no fallbacks at all.
  override_preset: safe_defaults

  # Individual overrides can also be written directly in this section, using the same
  # syntax as a preset file. They apply on top of override_preset, or on their own if
  # no preset is given. For example:
  # temperature:
  #   override: 0.5
  #   force: false
  # min_p:
  #   override: 0.1
  #   force: true

  # Settings nested under reasoning_override apply only while a model is reasoning;
  # the response after the reasoning block uses the regular ones. Anything not listed
  # is the same in both. Covers the sampler settings and banned_strings. For example:
  # reasoning_override:
  #   temperature:
  #     override: 1.5
  #     force: true
  #   banned_strings:
  #     override: ["but wait,"]

# Options for Loras
lora:
  # Directory to look for LoRAs (default: loras).
  lora_dir: loras

  # List of LoRAs to load and associated scaling factors (default scale: 1.0).
  # For the YAML file, add each entry as a YAML list:
  # - name: lora1
  #   scaling: 1.0
  loras:

# Options for embedding models and loading.
# NOTE: Embeddings requires the "extras" feature to be installed
# Install it via "pip install .[extras]"
embeddings:
  # Directory to look for embedding models (default: models).
  embedding_model_dir: models

  # Device to load embedding models on (default: cpu).
  # Possible values: cpu, auto, cuda.
  # NOTE: It's recommended to load embedding models on the CPU.
  # If using an AMD GPU, set this value to 'cuda'.
  embeddings_device: cpu

  # An initial embedding model to load on the infinity backend.
  embedding_model_name:

# Global memory settings
memory:

  # Max size of recurrent cache in system memory, in MB (default: 4096)
  sysmem_recurrent_cache: 4096

  # Size of system memory second-tier K/V cache, in MB (default: 0)
  sysmem_kv_cache: 0

  # Directory for a persistent third-tier cache on disk (default: None).
  # K/V pages entering the system memory tier and recurrent checkpoints are written through
  # to disk and read back when a prompt's pages are in no other tier, also in a later server
  # process. Requires sysmem_kv_cache > 0 and ExLlamaV3 1.6.2 or later (ignored with a warning
  # otherwise). Each model and cache configuration gets its own subdirectory, so one directory
  # serves every model. Experimental.
  disk_cache_dir:

  # Size budget of the whole disk cache directory, in MB (default: 51200).
  # Shared by every model and cache configuration stored under disk_cache_dir; the least
  # recently used entries are removed to stay within it.
  disk_cache_size: 51200

  # Also write every complete page still in VRAM to the disk cache whenever the generator
  # goes idle, so a context is persisted without having to be evicted first (default: True).
  disk_cache_flush_on_idle: true

  # Minimum spacing, in tokens, between the recurrent checkpoints the disk cache keeps along
  # a chain of cached pages (default: 4096). A resumed prompt replays at most this many
  # tokens. Checkpoints take far more space than pages on hybrid models. 0 keeps every
  # checkpoint.
  disk_cache_checkpoint_interval: 4096

  # Size of the image embedding cache in system memory, in MB (default: 1024).
  # Encoded images are kept so repeated turns of a conversation don't re-run the vision model.
  # Images already in use by a request are never evicted; a context whose images exceed the
  # budget is cached only partially, with a warning. Only applies when vision is enabled.
  sysmem_multimodal_cache: 1024

  # Use the cudaMallocAsync allocator backend in Torch (default: False).
  # When False, the allocator is left to the environment: unless PYTORCH_CUDA_ALLOC_CONF is set,
  # ExLlamaV3 enables expandable segments in Torch's native allocator, which performs better
  # than cudaMallocAsync. Enable this to force the cudaMallocAsync backend instead.
  cuda_malloc_async: false

# Options for development and experimentation
developer:
  # Skip Exllamav3 version check (default: False).
  # WARNING: It's highly recommended to update your dependencies rather than enabling this flag.
  unsafe_launch: false

  # Disable API request streaming (default: False).
  disable_request_streaming: false

  # Set process to use a higher priority.
  # For realtime process priority, run as administrator or sudo.
  # Otherwise, the priority will be set to high.
  realtime_process_priority: false

  # Enable extremely verbose seqlog logging, requires a running Seq server
  seqlog: false

  # Seq server url:port
  seqlog_server_url: http://localhost:5341

  # Seq server API key (default: None)
  seqlog_api_key: