On-Premises Deployment with vLLM

pip install vllm

# Using huggingface-cli
huggingface-cli download google/gemma-4-27b-it --local-dir ./models/gemma-4-27b-it

# Or for a pre-quantised AWQ version (smaller, faster to download)
huggingface-cli download casperhansen/gemma-4-27b-it-awq --local-dir ./models/gemma-4-27b-it-awq

# Serve the AWQ-quantised model
python -m vllm.entrypoints.openai.api_server \
  --model ./models/gemma-4-27b-it-awq \
  --quantization awq \
  --host 0.0.0.0 \
  --port 8000 \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.90 \
  --dtype float16

curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gemma-4-27b-it-awq",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "Explain the concept of data sovereignty in two sentences."}
    ],
    "temperature": 0.3,
    "max_tokens": 200
  }'

from openai import OpenAI

# Point to your vLLM server instead of OpenAI
client = OpenAI(
    base_url="http://your-vllm-server:8000/v1",
    api_key="not-needed"  # vLLM does not require an API key by default
)

response = client.chat.completions.create(
    model="gemma-4-27b-it-awq",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Summarise this contract clause..."}
    ],
    temperature=0.3,
    max_tokens=500,
    stream=True
)

for chunk in response:
    print(chunk.choices[0].delta.content, end="")

Load Balancer (nginx, HAProxy, or K8s Ingress)
├── vLLM Pod 1 (GPU node 1)
│   ├── vLLM server container
│   ├── Model volume (PVC or hostPath)
│   └── Health check sidecar
├── vLLM Pod 2 (GPU node 2)
│   ├── vLLM server container
│   ├── Model volume (PVC or hostPath)
│   └── Health check sidecar
└── Monitoring stack
    ├── Prometheus (metrics collection)
    ├── Grafana (dashboards)
    └── AlertManager (on-call notifications)

apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-gemma-27b
  namespace: ai-inference
spec:
  replicas: 2
  selector:
    matchLabels:
      app: vllm-gemma-27b
  template:
    metadata:
      labels:
        app: vllm-gemma-27b
    spec:
      containers:
      - name: vllm
        image: vllm/vllm-openai:latest
        args:
          - "--model=/models/gemma-4-27b-it-awq"
          - "--quantization=awq"
          - "--host=0.0.0.0"
          - "--port=8000"
          - "--max-model-len=8192"
          - "--gpu-memory-utilization=0.90"
        ports:
        - containerPort: 8000
        resources:
          limits:
            nvidia.com/gpu: 1
          requests:
            memory: "32Gi"
            cpu: "8"
        volumeMounts:
        - name: model-storage
          mountPath: /models
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 120
          periodSeconds: 30
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 60
          periodSeconds: 10
      volumes:
      - name: model-storage
        persistentVolumeClaim:
          claimName: model-weights-pvc
      nodeSelector:
        gpu-type: a100
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-service
  namespace: ai-inference
spec:
  selector:
    app: vllm-gemma-27b
  ports:
  - port: 8000
    targetPort: 8000
  type: ClusterIP

# Start vLLM with metrics enabled
python -m vllm.entrypoints.openai.api_server \
  --model ./models/gemma-4-27b-it-awq \
  --quantization awq \
  --host 0.0.0.0 \
  --port 8000 \
  --served-model-name gemma-4-27b \
  --enable-metrics