LLM / VLM Quantization
& Inference Handbook

from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import torch

# 4-bit NF4 (QLoRA style — for fine-tuning or quick experiments)
bnb_config_4bit = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",          # NF4 for normally-distributed weights
    bnb_4bit_compute_dtype=torch.bfloat16, # Compute in BF16 during forward pass
    bnb_4bit_use_double_quant=True,      # Quantize scale factors too (~0.4 bits extra saving)
)

# 8-bit INT8 (better for inference accuracy)
bnb_config_8bit = BitsAndBytesConfig(
    load_in_8bit=True,
    llm_int8_threshold=6.0,  # Outlier threshold — weights above go to FP16
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3-8b-hf",
    quantization_config=bnb_config_4bit,
    device_map="auto",  # Automatically distributes across available GPUs
)
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3-8b-hf")

from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig

# Step 1: Quantize (run once offline, save result)
quantize_config = BaseQuantizeConfig(
    bits=4,         # 4-bit quantization
    group_size=128,  # Larger = better quality, slightly more VRAM
    desc_act=False,  # False = faster inference; True = better accuracy
)

model = AutoGPTQForCausalLM.from_pretrained("meta-llama/Llama-3-8b-hf", quantize_config)
model.quantize(calibration_examples)  # List of tokenized text samples (~128)
model.save_quantized("./llama3-8b-gptq-4bit")

# Step 2: Load the quantized model for inference
model = AutoGPTQForCausalLM.from_quantized(
    "./llama3-8b-gptq-4bit",
    device_map="auto",
    use_triton=True,  # Faster Triton kernels if available
)

from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

# Quantize offline
model = AutoAWQForCausalLM.from_pretrained("meta-llama/Llama-3-8b-hf")
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3-8b-hf")

quant_config = {"zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM"}
model.quantize(tokenizer, quant_config=quant_config)
model.save_quantized("./llama3-8b-awq-4bit")

# Load in vLLM (preferred serving path)
# vllm serve ./llama3-8b-awq-4bit --quantization awq

# Step 1: Clone llama.cpp
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp && make -j8  # or: cmake -B build && cmake --build build --config Release

# Step 2: Convert HF model to GGUF F16
python convert_hf_to_gguf.py /path/to/llama3-8b-hf --outtype f16 --outfile llama3-8b-f16.gguf

# Step 3: Quantize to Q4_K_M (recommended)
./llama-quantize llama3-8b-f16.gguf llama3-8b-q4km.gguf Q4_K_M

# Step 4: Run inference (CPU)
./llama-cli -m llama3-8b-q4km.gguf -p "Explain quantization in one paragraph:" -n 256

# With partial GPU offload (e.g. 20 layers to GPU)
./llama-cli -m llama3-8b-q4km.gguf -ngl 20 -p "Hello"

# Serve as OpenAI-compatible API
./llama-server -m llama3-8b-q4km.gguf --port 8080

# Static FP8 quantization via vLLM's quantization toolkit
python -m llmcompressor.transformers.compression.compress \
  --model meta-llama/Llama-3-70b-hf \
  --recipe fp8_recipe.yaml \
  --save_dir ./llama3-70b-fp8

# Or: dynamic FP8 (no calibration needed, slightly lower quality)
vllm serve meta-llama/Llama-3-70b-hf \
  --dtype float16 \
  --quantization fp8 \
  --tensor-parallel-size 4

from vllm import LLM, SamplingParams

# Load AWQ 4-bit model
llm = LLM(
    model="./llama3-8b-awq-4bit",
    quantization="awq",
    tensor_parallel_size=1,      # Number of GPUs
    gpu_memory_utilization=0.90,   # Reserve 10% for safety
    enable_prefix_caching=True,   # APC - huge win for RAG
    kv_cache_dtype="fp8_e5m2",    # INT8 KV cache
    max_model_len=8192,
)

params = SamplingParams(temperature=0.7, top_p=0.9, max_tokens=512)

# Batched inference (always batch for throughput)
prompts = ["Tell me about LLM quantization", "What is AWQ?"]
outputs = llm.generate(prompts, params)

for output in outputs:
    print(output.outputs[0].text)

# Start server (AWQ model, 2 GPUs, prefix caching enabled)
vllm serve ./llama3-70b-awq \
  --quantization awq \
  --tensor-parallel-size 2 \
  --enable-prefix-caching \
  --kv-cache-dtype fp8_e5m2 \
  --max-model-len 16384 \
  --host 0.0.0.0 \
  --port 8000

# Query with standard OpenAI client
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model": "llama3-70b-awq", "messages": [{"role": "user", "content": "Hello"}]}'

def estimate_vram(param_billions, dtype_bytes=2, kv_overhead=1.3):
    """
    param_billions: model size (e.g. 7 for 7B)
    dtype_bytes: 2=FP16, 1=INT8, 0.5=INT4
    kv_overhead: 1.2-1.4x for KV cache + activations
    """
    weights_gb = ((param_billions * 1e9) * dtype_bytes) / (1024**3)
    return weights_gb * kv_overhead

# Examples
print(f"70B FP16: {estimate_vram(70, 2):.1f} GB")    # ~182 GB
print(f"70B INT8: {estimate_vram(70, 1):.1f} GB")    # ~91 GB
print(f"70B INT4: {estimate_vram(70, 0.5):.1f} GB")  # ~45 GB
print(f"8B INT4:  {estimate_vram(8, 0.5):.1f} GB")   # ~5.2 GB