KVQuant / BitForge: side-by-side proof of the same local model
This is the version that is easy to read: the prompt is on the left, the answer is on the left, the after version is on the right, and the metrics sit underneath. Same local model. Same question. Different stack.
model={esc(data['model'])}
question={esc(data['question_short'])}
before tokens={before['prompt_tokens']}
after tokens={after['prompt_tokens']}
memory saved={data['memory_saved_pct']:.1f}%
{render_card('Before latency', f"{before['latency_ms']:.1f} ms")}
{render_card('After latency', f"{after['latency_ms']:.1f} ms")}
{render_card('Before score', f"{before['score']['overall']:.3f}")}
{render_card('After score', f"{after['score']['overall']:.3f}")}
Before
Raw prompt, no compression, no semantic retrieval.
{render_panel('Prompt', before['prompt'])}
{render_panel('Answer', before['output'])}
After
Compressed context, memory notes, same question.
{render_panel('Prompt', after['prompt'])}
{render_panel('Answer', after['output'])}
Scores
| Version | Prompt tokens | Latency | Accuracy-like score | Bullets | Term hits |
|---|
| Before | {before['prompt_tokens']} | {before['latency_ms']:.1f} ms | {before['score']['overall']:.3f} | {before['score']['bullet_count']} | {before['score']['term_hits']} |
| After | {after['prompt_tokens']} | {after['latency_ms']:.1f} ms | {after['score']['overall']:.3f} | {after['score']['bullet_count']} | {after['score']['term_hits']} |
Memory and retrieval
{esc(data['memory_block'])}
Terminal transcript
{esc(data['terminal_transcript'])}
Repo retrieval hits
{esc(data['retrieval_text'])}
"""
def main() -> None:
OUT.mkdir(parents=True, exist_ok=True)
vault = ObsidianMemoryVault(OUT / 'memory-vault')
vault.add_note('Priority rule', 'If a task blocks money, trust, or correctness, fix it before polish.', tags=['priority', 'rule'])
vault.add_note('Before note', 'Before: the model sees a raw prompt, more clutter, and no compressed memory.', tags=['before', 'kvquant'])
vault.add_note('After note', 'After: the model sees compressed context, relevant retrieval, and a smaller working set.', tags=['after', 'bitforge'])
backend = HuggingFacePipelineBackend(MODEL, max_new_tokens=128)
compressor = CompressionEngine(vault=vault)
retriever = LocalRetriever(ROOT)
before_prompt = QUESTION + '\n\n' + '\n'.join(NOISY_CONTEXT)
before_output, before_latency = run_prompt(backend, before_prompt)
before_score = score_output(before_output)
before_prompt_tokens = estimate_token_count(before_prompt)
compressed_context = compressor.compress_transcript(
task='Answer the launch-priority question',
transcript=NOISY_CONTEXT,
memory_query='priority rule blocks money trust correctness checkout dashboard',
target_tokens=120,
)
after_prompt = QUESTION + '\n\n' + compressed_context.to_prompt()
after_output, after_latency = run_prompt(backend, after_prompt)
after_score = score_output(after_output)
after_prompt_tokens = estimate_token_count(after_prompt)
hits = retriever.search('priority rule checkout dashboard launch trust correctness', top_k=4)
retrieval_text = '\n'.join(f'{hit.path} | score={hit.score:.3f} | {hit.text[:180]}' for hit in hits) or 'no retrieval hits'
memory_saved_pct = 100.0 * (1 - (after_prompt_tokens / before_prompt_tokens)) if before_prompt_tokens else 0.0
peak_rss = rss_mb()
terminal_transcript = textwrap.dedent(f'''\
== KVQuant / BitForge side-by-side proof ==
model={MODEL}
before_prompt_tokens={before_prompt_tokens}
after_prompt_tokens={after_prompt_tokens}
memory_saved_pct={memory_saved_pct:.1f}%
peak_rss_mb={peak_rss:.1f}
QUESTION:
{QUESTION}
BEFORE PROMPT:
{before_prompt}
BEFORE ANSWER:
{before_output}
AFTER PROMPT:
{after_prompt}
AFTER ANSWER:
{after_output}
DELTA:
latency_delta_ms={after_latency - before_latency:.1f}
prompt_tokens_saved={before_prompt_tokens - after_prompt_tokens}
accuracy_delta={after_score['overall'] - before_score['overall']:.3f}
memory_saved_pct={memory_saved_pct:.1f}%
''').strip()
memory_block = textwrap.dedent(f'''\
BEFORE memory notes
- raw prompt
- clutter stays in context
- no compressed retrieval
AFTER memory notes
- compressed context
- relevant retrieval
- smaller working set
compressed context used after:
{compressed_context.to_prompt()}
''').strip()
data = {
'model': MODEL,
'question_short': QUESTION,
'before': {
'prompt': before_prompt,
'prompt_tokens': before_prompt_tokens,
'latency_ms': before_latency,
'output': before_output,
'score': before_score,
},
'after': {
'prompt': after_prompt,
'prompt_tokens': after_prompt_tokens,
'latency_ms': after_latency,
'output': after_output,
'score': after_score,
},
'memory_saved_pct': memory_saved_pct,
'memory_block': memory_block,
'terminal_transcript': terminal_transcript,
'retrieval_text': retrieval_text,
}
(OUT / 'comparison.json').write_text(json.dumps(data, indent=2, ensure_ascii=False))
(OUT / 'terminal_transcript.txt').write_text(terminal_transcript)
(OUT / 'report.html').write_text(build_html(data), encoding='utf-8')
print(OUT / 'report.html')
print(OUT / 'terminal_transcript.txt')
print(f"before_latency_ms={before_latency:.1f}")
print(f"after_latency_ms={after_latency:.1f}")
print(f"before_accuracy={before_score['overall']:.3f}")
print(f"after_accuracy={after_score['overall']:.3f}")
print(f"before_prompt_tokens={before_prompt_tokens}")
print(f"after_prompt_tokens={after_prompt_tokens}")
print(f"memory_saved_pct={memory_saved_pct:.1f}")
print(f"peak_rss_mb={peak_rss:.1f}")
if __name__ == '__main__':
main()