llama

Chat over llama-cpp-python GGUF models — prefix KV cache, GPU offload, and mtmd multimodal (image + audio via patched MTMDChatHandler).

from fastcore.test import test_eq, test_fail, test_close

Loading GGUF models

Any GGUF repo on the HuggingFace Hub works; the Qwen3 and Gemma 3 ids below are handy defaults (Qwen3 gives you thinking and Hermes-style tool calls out of the box). get_model resolves a .gguf file with the same cache-first ladder as rishi.core.get_model: explicit model_path, then the local HF cache (no network), then a download - preferring the requested quant and skipping mmproj projector files. get_mmproj is the complement: it resolves the projector itself the same way, for vision models.

---

source

get_mmproj

def get_mmproj(
    model_id, mmproj_path:NoneType=None
):

Return a local mmproj projector path: mmproj_path, else HF cache, else download.

fs = ['README.md', 'model-Q8_0.gguf', 'model-Q4_K_M.gguf', 'mmproj-model.gguf']
test_eq(_gguf(fs), 'model-Q4_K_M.gguf')
test_eq(_gguf(fs, 'Q8_0'), 'model-Q8_0.gguf')
test_eq(_gguf(fs, 'IQ2_XS'), 'model-Q8_0.gguf')  # falls back to first non-mmproj gguf
test_eq(_gguf(['README.md']), None)
test_eq(_mmproj(fs), 'mmproj-model.gguf')        # the complement of _gguf
test_eq(_mmproj(['model-Q4_K_M.gguf']), None)
test_eq(_mmproj(['mmproj-dir/model.gguf']), None)  # matches the filename, not the directory

---

source

gpu_offload_supported

def gpu_offload_supported():

True if the installed llama-cpp-python build has GPU (Metal/CUDA) offload compiled in.

# build-level GPU check: True on the Metal (macOS/arm64) or CUDA wheel, False on a CPU wheel
res = gpu_offload_supported()
assert isinstance(res, bool)
print('gpu offload supported:', res)

ggml_metal_device_init: tensor API disabled for pre-M5 and pre-A19 devices
ggml_metal_library_init: using embedded metal library
ggml_metal_library_init: loaded in 0.032 sec
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
ggml_metal_device_init: GPU name:   MTL0 (Apple M3 Max)
ggml_metal_device_init: GPU family: MTLGPUFamilyApple9  (1009)
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4  (5002)
ggml_metal_device_init: simdgroup reduction   = true
ggml_metal_device_init: simdgroup matrix mul. = true
ggml_metal_device_init: has unified memory    = true
ggml_metal_device_init: has bfloat            = true
ggml_metal_device_init: has tensor            = false
ggml_metal_device_init: use residency sets    = true
ggml_metal_device_init: use shared buffers    = true
ggml_metal_device_init: recommendedMaxWorkingSetSize  = 30150.67 MB

gpu offload supported: True

Audio

llama-cpp-python’s MTMD handler only wired images upstream. rishi @patches four MTMDChatHandler methods so audio follows the same path: collect input_audio parts, collapse string templates for Llama/Voxtral, decode with read_audio (libsndfile), and feed mtmd_bitmap_init_from_audio.

---

source

read_audio

def read_audio(
    o, sr:int=16000
):

Decode audio bytes/Path (any libsndfile format) to contiguous mono float32 samples at sr Hz, ready for mtmd.

---

source

MTMDChatHandler.get_image_urls

def get_image_urls(
    messages
):

Media data-URIs in document order - audio as well as images (upstream collects images only).

import io
from fastcore.all import Path
from contextlib import ExitStack
from llama_cpp import mtmd_cpp
import tempfile

# read_audio decodes the real 24s clip and resamples it, preserving duration
wav = Path('speech.wav').read_bytes()
a = read_audio(wav, 16000)
test_eq(a.dtype, np.dtype('float32'))
assert a.flags['C_CONTIGUOUS'] and -1.0 <= a.min() and a.max() <= 1.0
test_eq(round(len(a) / 16000), 24)                       # 24s in, 24s out
test_eq(round(len(read_audio(wav, 8000)) / 8000), 24)    # and at a different target rate

# soundfile handles compressed containers too, not just WAV
sig = (0.2 * np.sin(2 * np.pi * 220 * np.arange(8000) / 8000)).astype(np.float32)
for fmt in ('FLAC', 'OGG', 'MP3'):
    b = io.BytesIO(); sf.write(b, sig, 8000, format=fmt)
    d = read_audio(b.getvalue(), 8000)
    test_eq(d.dtype, np.dtype('float32')); assert d.flags['C_CONTIGUOUS'], fmt
    assert abs(len(d) - 8000) <= 2400, (fmt, len(d))     # ~1s back; lossy codecs pad a little

# stereo is downmixed to mono
st = np.stack([sig, np.zeros_like(sig)], axis=1)
b = io.BytesIO(); sf.write(b, st, 8000, format='WAV', subtype='FLOAT')
test_eq(read_audio(b.getvalue(), 8000).ndim, 1)

test_fail(lambda: read_audio(b'not audio at all'), contains='could not decode audio')

# a template that renders `content` directly (Llama/Voxtral-style) must get a STRING, not a list repr:
# _get_template_messages collapses media+text to a clean string with the marker inline
h = object.__new__(MTMDChatHandler)
msg = {'role': 'user', 'content': [{'type': 'text', 'text': 'Transcribe this clip.'},
                                   {'type': 'input_audio', 'input_audio': {'data': 'AAA', 'format': 'wav'}}]}
conv = h._get_template_messages([msg], '<M>')[0]['content']
test_eq(conv, 'Transcribe this clip.\n<M>')
assert isinstance(conv, str)   # never a Python list repr
# a plain-string message is passed through untouched
test_eq(h._get_template_messages([{'role': 'user', 'content': 'hi'}], '<M>')[0]['content'], 'hi')

# the patched handler collects image *and* audio media, in document order
with tempfile.NamedTemporaryFile(suffix='.gguf') as f:
    h = MTMDChatHandler(f.name, verbose=False)
    img = Path('images.jpeg').read_bytes()
    msgs = [_mk_msg([img, 'and this?', wav])]
    urls = h.get_image_urls(msgs)
    test_eq(len(urls), 2)
    assert urls[0].startswith('data:image/jpeg;base64,') and urls[1].startswith('data:audio/wav;base64,')

    # every media part becomes the marker; text is left alone
    # media parts become the marker and the all-text content collapses to a clean string (marker inline)
    test_eq(h._get_template_messages(msgs, '<__media__>')[0]['content'], '<__media__>\nand this?\n<__media__>')

    # and the data-URI round-trips back to the exact bytes the model will decode
    test_eq(h.load_image(urls[1]), wav)
    test_eq(h.load_image(urls[0]), img)

# the decoded buffer is accepted by real llama.cpp: `mtmd_bitmap_init_from_audio` needs no
# context or model, so the actual C boundary is exercised here rather than mocked
bm = mtmd_cpp.mtmd_bitmap_init_from_audio(len(a), a.ctypes.data_as(ctypes.POINTER(ctypes.c_float)))
assert bm is not None, 'llama.cpp rejected the audio buffer'
try:
    assert mtmd_cpp.mtmd_bitmap_is_audio(bm)
    test_eq(mtmd_cpp.mtmd_bitmap_get_n_bytes(bm), len(a) * 4)     # float32 per sample
    back = np.ctypeslib.as_array(ctypes.cast(mtmd_cpp.mtmd_bitmap_get_data(bm),
                                             ctypes.POINTER(ctypes.c_float)), shape=(len(a),))
    assert np.array_equal(back, a), 'samples corrupted crossing the C boundary'
finally: mtmd_cpp.mtmd_bitmap_free(bm)

LlamaChat

create_engine() builds a Llama (+ optional MTMDChatHandler when mmproj= is set). Llama-specific knobs: quant, n_gpu_layers, mmproj, think (/think//no_think in the system prompt), and KV-cache tracking below.

---

source

LlamaChat

def LlamaChat(
    model:NoneType=None, runtime:NoneType=None, model_path:NoneType=None, engine:NoneType=None, quant:str='Q4_K_M',
    n_ctx:int=8192, n_gpu_layers:int=0, mmproj:NoneType=None, eng_kw:NoneType=None, sp:str='',
    messages:NoneType=None, tools:NoneType=None, ctx_limit:NoneType=None, approve:NoneType=None,
    tool_max_len:NoneType=None, max_steps:int=10, parallel_tools:bool=False, max_parallel_tools:NoneType=None,
    final_prompt:str="You've reached the tool-call budget for this turn. Stop calling tools and answer with what you already have.",
    think:NoneType=None, temp:NoneType=None, top_k:NoneType=None, top_p:NoneType=None, seed:NoneType=None,
    preserve_cache:bool=False, max_output_tokens:NoneType=None, comp_kw:NoneType=None, cbs:NoneType=None,
    default_cbs:bool=True
):

Sync chat over a local llama.cpp model - the rishi.core.Chat API over ToolLoopMixin’s tool loop.

The KV cache

llama.cpp manages this itself, and rishi gets it for free by holding one Llama across turns: Llama.generate compares the incoming prompt against the tokens already in the context, trims the KV cache to the common prefix, and evaluates only the tail. Because history grows by appending, each turn is an extension of the last, so an ordinary conversation re-prefills nothing.

Three things break that prefix, and rishi knows about all three, which is what cached_tokens reports:

• A one-shot on the same engine. A Llama has a single context, so classify, structured, check and grades overwrite the conversation - the next real turn re-prefills from scratch. Same for two Chats sharing one engine=, and for a judge= chat on the same engine. preserve_cache=True saves and restores the llama state around one-shots instead; it copies the whole KV cache, so it only pays off when a re-prefill would cost more than the copy.
• A rewritten history. SlidingWindowCallback eviction and context recovery both change the middle of the conversation, so everything after the first changed message is re-evaluated. Keep keep_first small.
• Media. The live turn carries the real image or audio, but on later turns it is replaced with an [image]/[audio] placeholder so it isn’t re-encoded. That is deliberate, and it means a turn containing media forces a re-prefill on the next one, from that message onward.

save_cache/load_cache persist the whole context to disk, the same API rishi.mlx exposes, so a long system prompt can be prefilled once and reused in later sessions.

Tests

Unit tests above cover GGUF resolution, GPU probe, audio decode, MTMD patches, message formatting, and offline KV-cache bookkeeping. #| eval: false cells below need a downloaded GGUF (+ projector for multimodal).

from fastcore.test import test_eq
# _strip_media collapses past-turn media to a text placeholder; plain strings pass through untouched (pure fn, no model)
test_eq(_strip_media({'role': 'user', 'content': 'plain'}), {'role': 'user', 'content': 'plain'})
past = {'role': 'user', 'content': [{'type': 'input_audio', 'input_audio': {'data': 'x', 'format': 'wav'}},
                                    {'type': 'text', 'text': 'clip 1'}]}
test_eq(_strip_media(past)['content'], '[audio]\nclip 1')
assert _is_media({'type': 'image_url', 'image_url': {'url': 'data:img'}})
assert not _is_media({'type': 'text', 'text': 'hi'})

# hist2fmt / fmt2hist across backends (model-free): a litert-origin canonical tool round feeds llama.cpp
_h = [
    {'role': 'user', 'content': 'What is 2+3?'},
    {'role': 'assistant', 'content': '', 'tool_calls': [{'id': 'call_1', 'type': 'function',
                                                         'function': {'name': 'add', 'arguments': {'a': 2, 'b': 3}}}]},
    {'role': 'tool', 'tool_call_id': 'call_1', 'name': 'add', 'content': '5'},
    {'role': 'assistant', 'content': 'It is 5.'},
]
_oai = LlamaChat.hist2fmt(_h)
test_eq(_oai[2], {'role': 'tool', 'tool_call_id': 'call_1', 'name': 'add', 'content': '5'})
test_eq(_oai[1]['tool_calls'][0]['function']['arguments'], '{"a": 2, "b": 3}')   # args JSON-encoded for llama.cpp
test_eq(LlamaChat.fmt2hist(_h), _h)                                               # no-op on already-canonical dicts

from fastcore.test import test_eq
import rishi.core, rishi.llama
test_eq(rishi.core.get_runtime('llama'), rishi.llama.LlamaChat)

# dispatch really happens in `Chat.__new__`, so it can be checked without loading a model
test_eq(type(rishi.core.Chat.__new__(rishi.core.Chat, 'Qwen/Qwen3-0.6B-GGUF')), rishi.llama.LlamaChat)
test_eq(type(rishi.core.Chat.__new__(rishi.core.Chat, '/models/x.gguf')), rishi.llama.LlamaChat)
test_eq(rishi.llama.LlamaChat._runtime, 'llama')

# the /think switch is just text in the system prompt, so it needs no model to check
test_eq(rishi.llama.LlamaChat._think_sp('Be terse.', False), 'Be terse.\n/no_think')
test_eq(rishi.llama.LlamaChat._think_sp('', True), '/think')
test_eq(rishi.llama.LlamaChat._think_sp('Be terse.', None), 'Be terse.')

Examples (manual)

Load real weights here. For generic chat, tools, streaming, classify, and structured output, use the same calls as in index with a -GGUF id or LlamaChat(...).

Below: llama-only behavior — KV prefix reuse and mmproj multimodal.

# Real llama.cpp KV-prefix test: instrument `eval` to see how many tokens are actually evaluated.
# On turn two the first eval batch should contain only the new tail, not the cached conversation.
kvchat = LlamaChat(qwen3_06b, think=False, temp=0.0, max_output_tokens=16)
seen, orig_eval = [], kvchat.engine.eval
def counted_eval(tokens):
    seen.append(len(tokens))
    return orig_eval(tokens)
kvchat.engine.eval = counted_eval
try:
    kvchat(('cache verification context ' * 96) + '\nReply with exactly: stored')
    context_before_turn2 = kvchat.engine.n_tokens
    seen.clear()
    r = kvchat('What exact word did I ask you to reply with? Reply with only that word.')
    assert seen, 'llama.cpp never evaluated the second prompt'
    second_tail = seen[0]
    print(dict(context_before_turn2=context_before_turn2, second_tail_evaluated=second_tail,
               reported_cached=kvchat.use.cached_tokens, answer=resp_text(r)))
    assert second_tail < context_before_turn2, 'turn two re-evaluated the full prior context; KV prefix reuse missed'
    assert kvchat.use.cached_tokens > 0, kvchat.use
    assert 'stored' in resp_text(r).lower(), resp_text(r)
finally:
    kvchat.engine.eval = orig_eval
    kvchat.close()

llama_context: n_ctx_seq (8192) < n_ctx_train (40960) -- the full capacity of the model will not be utilized

{'context_before_turn2': 315, 'second_tail_evaluated': 28, 'reported_cached': 315, 'answer': 'stored'}

Streaming and thinking

stream=True yields markdown chunks (thinking streams as a blockquote, tool calls as ⏳ lines) - render live with display_stream, exactly like the litert backend. With a Qwen3 model, think=True turns the thinking channel on.

Multimodal (mmproj)

Vision/audio GGUFs ship as model + mmproj projector. Pass mmproj=True to resolve from the same repo. Mix bytes or Path with text; rishi emits image_url / input_audio parts and the patched handler builds mtmd bitmaps in document order.

# downloads the model *and* its mmproj projector once, then loads both from cache
vchat = LlamaChat('unsloth/gemma-4-E2B-it-GGUF', mmproj=True)
r = vchat(['Describe this image.', Path('images.jpeg')])   # raw bytes work too
print(resp_text(r))
vchat.close()

llama_context: n_ctx_seq (8192) < n_ctx_train (131072) -- the full capacity of the model will not be utilized
llama_kv_cache_iswa: using full-size SWA cache (ref: https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
llama_kv_cache: the V embeddings have different sizes across layers and FA is not enabled - padding V cache to 512
llama_kv_cache: the V embeddings have different sizes across layers and FA is not enabled - padding V cache to 512
add_text: <|turn>user
Describe this image.

add_text: <|image>
image_tokens->nx = 81
image_tokens->ny = 1
batch_f32 size = 1
add_text: <image|>
add_text: <turn|>
<|turn>model

encoding image slice...
clip_image_batch_encode: copying image 1/1 to input buffer (nx=432, ny=432)
clip_image_batch_encode: output embedding shape [1536, 81, 1]
image slice encoded in 123 ms
decoding image batch 1/1, n_tokens_batch = 81
image decoded (batch 1/1) in 336 ms

This is a photograph of a **German Shepherd dog** outdoors.

Here are some details about the image:

*   **Subject:** The main subject is a medium-to-large German Shepherd.
*   **Appearance:** The dog has rich, reddish-brown and black fur. It appears alert and focused, looking slightly off-camera. Its mouth is open, and its tongue is visible, suggesting it might be panting or happy.
*   **Setting:** The dog is in a natural, outdoor environment. The background is soft and blurred (shallow depth of field), suggesting foliage, trees, or a grassy area, indicating a park, forest edge, or rural setting. The lighting appears soft, possibly indicating an overcast day or diffused sunlight.
*   **Mood:** The image conveys a sense of alertness, health, and companionship.

Overall, it is a high-quality portrait of a beautiful German Shepherd in a natural setting.

achat = LlamaChat('unsloth/gemma-4-E2B-it-GGUF', mmproj=True)
print(resp_text(achat(['Transcribe this clip.', Path('speech.wav')])))
achat.close()

llama_context: n_ctx_seq (8192) < n_ctx_train (131072) -- the full capacity of the model will not be utilized
llama_kv_cache_iswa: using full-size SWA cache (ref: https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
llama_kv_cache: the V embeddings have different sizes across layers and FA is not enabled - padding V cache to 512
llama_kv_cache: the V embeddings have different sizes across layers and FA is not enabled - padding V cache to 512
add_text: <|turn>user
Transcribe this clip.

add_text: <|audio>
audio_tokens->n_tokens = 600
add_text: <audio|>
add_text: <turn|>
<|turn>model

encoding audio slice...
clip_image_batch_encode: output embedding shape [1536, 600, 1]
audio slice encoded in 118 ms
decoding audio batch 1/2, n_tokens_batch = 512
audio decoded (batch 1/2) in 2306 ms
decoding audio batch 2/2, n_tokens_batch = 88
audio decoded (batch 2/2) in 586 ms

Dancing in the masquerade idol truth in plain sight jaded pop roll click shot who will I be today or not but such a tide as moving seems asleep too full for sound and foam when that which drew from out the boundless deep turns again home twilight and evening bell and after that

# audio-only projector recovery: ultravox has no vision encoder, so upstream's `_init_mtmd_context` would
# hard-fail. A clean load proves the patch recovered (ctx built + audio supported -> keep it), and a non-empty
# reply proves the audio bitmap path runs end-to-end. Downloads the ~1B model + its mmproj once, then caches.
uvchat = LlamaChat('ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF', mmproj=True)
assert resp_text(uvchat(['Transcribe this clip.', Path('speech.wav')])).strip()
uvchat.close()

llama_context: n_ctx_seq (8192) < n_ctx_train (131072) -- the full capacity of the model will not be utilized
add_text: <|start_header_id|>system<|end_header_id|>

Cutting Knowledge Date: December 2023
Today Date: 13 Aug 2026

<|eot_id|><|start_header_id|>user<|end_header_id|>

Transcribe this clip.

audio_tokens->n_tokens = 187
audio_tokens->n_tokens = 187
add_text: <|eot_id|><|start_header_id|>assistant<|end_header_id|>


encoding audio slice...
clip_image_batch_encode: output embedding shape [2048, 187, 1]
audio slice encoded in 436 ms
decoding audio batch 1/1, n_tokens_batch = 187
audio decoded (batch 1/1) in 465 ms
encoding audio slice...
clip_image_batch_encode: output embedding shape [2048, 187, 1]
audio slice encoded in 386 ms
decoding audio batch 1/1, n_tokens_batch = 187
audio decoded (batch 1/1) in 383 ms

img = Path('images.jpeg').read_bytes()
m = _mk_msg([img, 'what is this?'])
with tempfile.NamedTemporaryFile(suffix='.gguf') as f:
    test_eq(MTMDChatHandler(f.name, verbose=False).get_image_urls([m]), [m['content'][0]['image_url']['url']])
test_eq(m['content'][1], {'type': 'text', 'text': 'what is this?'})
test_eq(_mk_content(Path('images.jpeg')), _mk_content(img))   # a Path is read and sniffed like bytes

# audio is no longer refused - it becomes an OpenAI `input_audio` part
p = _mk_content(Path('speech.wav'))
test_eq((p['type'], p['input_audio']['format']), ('input_audio', 'wav'))

Stopping with an LLM judge

done is any chat -> bool, so instead of output_matches (a substring check on the last code output) you can let a model decide when the loop is finished. The built-in task_complete judges - via chat.classify, isolated - whether the conversation has satisfied the request. Or write your own done that closes over a judge Chat (pass a stronger model as the judge if you like), mirroring output_matches’ factory shape. This is backend-agnostic - the same done works with the litert Chat.