How to Self-Host
Gemma 4

# docker-compose.yml
services:
  gemma:
    image: vllm/vllm-openai:latest
    ipc: host
    ports: ["8000:8000"]
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    environment:
      - HF_TOKEN=${HF_TOKEN}
    command: >
      --model google/gemma-4-26B-A4B-it
      --max-model-len 32768
      --gpu-memory-utilization 0.90
      --enable-auto-tool-choice
      --tool-call-parser gemma
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      start_period: 300s   # model loading takes minutes

# Deployment - the parts that are Gemma-specific
spec:
  containers:
  - name: vllm
    image: vllm/vllm-openai:latest
    args:
      - --model=google/gemma-4-26B-A4B-it
      - --max-model-len=32768
      - --gpu-memory-utilization=0.90
      - --enable-auto-tool-choice
      - --tool-call-parser=gemma
    resources:
      limits:
        nvidia.com/gpu: 1
    volumeMounts:
      - name: model-cache
        mountPath: /root/.cache/huggingface
      - name: dshm                    # the --ipc=host equivalent
        mountPath: /dev/shm
    readinessProbe:
      httpGet: { path: /health, port: 8000 }
      initialDelaySeconds: 300    # do not send traffic during load
      periodSeconds: 10
  volumes:
    - name: dshm
      emptyDir: { medium: Memory, sizeLimit: 16Gi }
    - name: model-cache
      persistentVolumeClaim: { claimName: hf-cache }

# Use a shared PVC for the cache so replicas don't each pull 58 GB.
# Scale with replicas, not bigger GPUs - see the sizing note above.

# Flags that actually change behaviour for Gemma 4

--enable-auto-tool-choice        # required, with the parser below
--tool-call-parser gemma         # without both, tool calls silently no-op

--max-model-len 32768            # NOT 262144 unless you need it -
                                 # vLLM pre-allocates KV cache from this

--gpu-memory-utilization 0.90    # lower if you hit OOM under load

--quantization "fp8"              # halves weights, near-identical quality

--speculative-model google/gemma-4-26B-A4B-it-assistant
                                 # the MTP drafter - up to 3.11x on the 31B

--tensor-parallel-size 2         # only when one GPU cannot hold it

# And the dependency that stops most first deployments:
pip install -U "transformers>=5.5.0"