Same local model. Same question. Before vs after stack.
Real proof: raw prompt versus compressed/retrieved prompt, run on the same local model, then scored for latency, memory footprint, and output quality.
model={html.escape(data['model'])}
before tokens={data['before']['prompt_tokens']}
after tokens={data['after']['prompt_tokens']}
memory saved={data['memory_saved_pct']:.1f}%
{data['before']['latency_ms']:.1f} ms
before latency
{data['after']['latency_ms']:.1f} ms
after latency
{data['before']['accuracy']:.3f}
before accuracy
{data['after']['accuracy']:.3f}
after accuracy
{block('Before prompt', data['before']['prompt'])}
{block('After prompt', data['after']['prompt'])}
{block('Before answer', data['before']['output'])}
{block('After answer', data['after']['output'])}
Scores
{render_table(data['score_rows'])}
Memory snapshot
{html.escape(data['memory_block'])}
Terminal transcript
{html.escape(data['terminal_transcript'])}
Repo retrieval hits
{html.escape(data['retrieval_text'])}
"""
def main() -> None:
OUT.mkdir(parents=True, exist_ok=True)
backend = HuggingFacePipelineBackend(MODEL, max_new_tokens=128)
vault = ObsidianMemoryVault(OUT / 'memory-vault')
compressor = CompressionEngine(vault=vault)
retriever = LocalRetriever(ROOT)
vault.add_note('KVQuant before', 'Before: raw prompt, more clutter, no compression, no semantic retrieval, no memory vault.', tags=['before', 'kvquant'])
vault.add_note('BitForge after', 'After: compressed context, semantic retrieval, smaller prompt, same task with less clutter.', tags=['after', 'bitforge'])
before_prompt = QUESTION + '\n\n' + '\n'.join(BEFORE_CONTEXT)
before_output, before_latency = run_prompt(backend, before_prompt)
before_accuracy, before_hits, before_bullets = score_answer(before_output)
before_tokens = estimate_token_count(before_prompt)
compressed = compressor.compress_transcript(
task='Compare before vs after for KVQuant and BitForge',
transcript=AFTER_CONTEXT,
memory_query='KVQuant BitForge before after compression retrieval latency memory accuracy',
target_tokens=40,
)
compact_memory = compressed.summary
after_prompt = (
'MEMORY SUMMARY:\n'
f'{compact_memory}\n\n'
'SALIENT FACTS:\n'
'- after is compressed\n'
'- retrieval brings in only relevant memory\n\n'
f'{QUESTION}'
)
after_output, after_latency = run_prompt(backend, after_prompt)
after_accuracy, after_hits, after_bullets = score_answer(after_output)
after_tokens = estimate_token_count(after_prompt)
memory_saved_pct = 100 * (1 - after_tokens / before_tokens) if before_tokens else 0.0
peak_rss = rss_mb()
retrieval_hits = retriever.search('KVQuant BitForge before after compression retrieval', top_k=3)
retrieval_text = '\n'.join(f'{hit.path} | score={hit.score:.3f} | {hit.text[:180]}' for hit in retrieval_hits) or 'no retrieval hits'
score_rows = [
('Before', 'Latency', f'{before_latency:.1f} ms'),
('After', 'Latency', f'{after_latency:.1f} ms'),
('Before', 'Accuracy', f'{before_accuracy:.3f}'),
('After', 'Accuracy', f'{after_accuracy:.3f}'),
('Before', 'Memory', f'{before_tokens} prompt tokens'),
('After', 'Memory', f'{after_tokens} prompt tokens'),
('Delta', 'Memory saved', f'{memory_saved_pct:.1f}%'),
('System', 'Peak RSS', f'{peak_rss:.1f} MB'),
]
terminal_transcript = textwrap.dedent(f'''\
== KVQuant / BitForge before-vs-after proof ==
model={MODEL}
backend=HuggingFacePipelineBackend
before_prompt_tokens={before_tokens}
after_prompt_tokens={after_tokens}
memory_saved_pct={memory_saved_pct:.1f}%
peak_rss_mb={peak_rss:.1f}
$ python -m llm_foundry demo --backend hf --model {MODEL} --prompt "{QUESTION}"
BEFORE
latency_ms={before_latency:.1f}
accuracy_score={before_accuracy:.3f}
bullets={before_bullets}
memory={before_tokens} prompt tokens
hits={', '.join(word for word, ok in before_hits.items() if ok)}
answer:
{before_output}
AFTER
latency_ms={after_latency:.1f}
accuracy_score={after_accuracy:.3f}
bullets={after_bullets}
memory={after_tokens} prompt tokens
hits={', '.join(word for word, ok in after_hits.items() if ok)}
answer:
{after_output}
DELTA
latency_delta_ms={after_latency - before_latency:.1f}
prompt_tokens_saved={before_tokens - after_tokens}
memory_saved_pct={memory_saved_pct:.1f}%
''').strip()
memory_block = textwrap.dedent(f'''\
BEFORE NOTE
- raw prompt
- no compression
- no semantic retrieval
- more clutter
AFTER NOTE
- compressed context
- semantic retrieval
- fewer prompt tokens
- more focused task
compressed_context:
{compressed.to_prompt()}
''').strip()
data = {
'model': MODEL,
'before': {
'prompt': before_prompt,
'prompt_tokens': before_tokens,
'latency_ms': before_latency,
'accuracy': before_accuracy,
'output': before_output,
},
'after': {
'prompt': after_prompt,
'prompt_tokens': after_tokens,
'latency_ms': after_latency,
'accuracy': after_accuracy,
'output': after_output,
},
'memory_saved_pct': memory_saved_pct,
'score_rows': score_rows,
'memory_block': memory_block,
'terminal_transcript': terminal_transcript,
'retrieval_text': retrieval_text,
}
(OUT / 'comparison.json').write_text(json.dumps(data, indent=2, ensure_ascii=False))
(OUT / 'terminal_transcript.txt').write_text(terminal_transcript)
(OUT / 'report.html').write_text(render_html(data), encoding='utf-8')
print(OUT / 'report.html')
print(OUT / 'terminal_transcript.txt')
print(f"before_latency_ms={before_latency:.1f}")
print(f"after_latency_ms={after_latency:.1f}")
print(f"before_accuracy={before_accuracy:.3f}")
print(f"after_accuracy={after_accuracy:.3f}")
print(f"before_prompt_tokens={before_tokens}")
print(f"after_prompt_tokens={after_tokens}")
print(f"memory_saved_pct={memory_saved_pct:.1f}")
print(f"peak_rss_mb={peak_rss:.1f}")
if __name__ == '__main__':
main()