LLMs Open Source
Self-Hosted na AWS

from langchain_openai import ChatOpenAI

# Aponta para o NLB interno — zero mudança de interface
llm = ChatOpenAI(
    base_url="http://vllm-nlb.internal:8000/v1",
    api_key="EMPTY",                # qualquer string
    model="meta-llama/Meta-Llama-3.1-70B-Instruct",
    temperature=0,
)

# Com LoRA específico — mesmo padrão
finance_llm = ChatOpenAI(
    base_url="http://vllm-nlb.internal:8000/v1",
    api_key="EMPTY",
    model="finance-specialist",  # nome do LoRA registrado
    temperature=0,
)

from langgraph.graph import StateGraph
from langchain_openai import ChatOpenAI
from typing import Literal

VLLM_BASE = "http://vllm-nlb.internal:8000/v1"

def make_agent_llm(lora_name: str | None = None) -> ChatOpenAI:
    model = lora_name or "meta-llama/Meta-Llama-3.1-70B-Instruct"
    return ChatOpenAI(base_url=VLLM_BASE, api_key="EMPTY", model=model)

def finance_node(state):
    llm = make_agent_llm("finance-specialist")
    # lógica do agent finance...

def route_to_specialist(state) -> Literal["finance", "legal", "support"]:
    return state["specialist"]

graph = StateGraph(AgentState)
graph.add_node("finance", finance_node)
graph.add_node("legal", legal_node)
graph.add_conditional_edges("supervisor", route_to_specialist, {
    "finance": "finance",
    "legal": "legal",
})

{
  "family": "vllm-inference",
  "networkMode": "awsvpc",
  "requiresCompatibilities": ["EC2"],   // não Fargate
  "containerDefinitions": [{
    "name": "vllm-server",
    "image": "vllm/vllm-openai:latest",
    "command": [
      "--model", "meta-llama/Meta-Llama-3.1-70B-Instruct",
      "--tensor-parallel-size", "4",       // = nº GPUs
      "--max-model-len", "32768",
      "--gpu-memory-utilization", "0.90",
      "--enable-auto-tool-choice",           // para tool calling
      "--tool-call-parser", "llama3_json"
    ],
    "resourceRequirements": [{
      "type": "GPU",
      "value": "4"                           // deve bater com tensor-parallel-size
    }],
    "healthCheck": {
      "command": ["CMD-SHELL", "curl -f http://localhost:8000/health"],
      "startPeriod": 300                     // 5 min p/ modelo carregar
    },
    "mountPoints": [{
      "sourceVolume": "model-cache",
      "containerPath": "/root/.cache/huggingface"
    }]
  }],
  "placementConstraints": [{
    "type": "memberOf",
    "expression": "attribute:ecs.instance-type =~ g5.*"
  }]
}

// NLB interno — só acessível dentro do VPC
const nlb = new elbv2.NetworkLoadBalancer(this, 'VllmNlb', {
  vpc,
  internetFacing: false,        // INTERNO — crítico
});

const listener = nlb.addListener('VllmListener', { port: 8000 });
listener.addTargets('VllmTargets', {
  port: 8000,
  protocol: elbv2.Protocol.TCP,
  targets: [service],
  healthCheck: { path: '/health', port: '8080' }, // porta do proxy
  deregistrationDelay: Duration.seconds(30),
});

python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4 \
  --enable-lora \
  --max-loras 4          # simultâneos na GPU \
  --max-cpu-loras 8      # swap para CPU quando necessário \
  --max-lora-rank 64     # deve bater com o rank do fine-tuning \
  --enable-auto-tool-choice \
  --tool-call-parser llama3_json

# Registrar LoRA
curl -X POST http://vllm:8000/v1/load_lora_adapter \
  -H "Content-Type: application/json" \
  -d '{"lora_name": "finance-specialist", "lora_path": "/loras/finance"}'

# Listar LoRAs carregados
curl http://vllm:8000/v1/models

# Remover LoRA
curl -X POST http://vllm:8000/v1/unload_lora_adapter \
  -d '{"lora_name": "finance-specialist"}'

// Warm Pool — instâncias paradas (paga só EBS)
const cfnAsg = asg.node.defaultChild as autoscaling.CfnAutoScalingGroup;
cfnAsg.warmPool = {
  minSize: 1,
  maxGroupPreparedCapacity: 2,
  poolState: 'Stopped',              // economiza GPU $$$
  instanceReusePolicy: {
    reuseOnScaleIn: true,             // devolve ao pool, não termina
  },
};

// Lifecycle Hook — pausa até modelo estar pronto
asg.addLifecycleHook('WaitForModelLoad', {
  lifecycleTransition: autoscaling.LifecycleTransition.INSTANCE_LAUNCHING,
  heartbeatTimeout: Duration.minutes(20),
  defaultResult: autoscaling.DefaultResult.ABANDON,  // descarta se falhar
  notificationTarget: new hooks.TopicHook(warmupTopic),
});

blockDevices: [
  { deviceName: '/dev/xvda', volume: ec2.BlockDeviceVolume.ebs(100) },
  {
    deviceName: '/dev/sdb',
    volume: ec2.BlockDeviceVolume.ebsFromSnapshot('snap-0abc123def', {
      volumeSize: 500,                    // GB — modelo + LoRAs
      volumeType: ec2.EbsDeviceVolumeType.GP3,
      iops: 16000,                         // máximo GP3
      throughput: 1000,                     // MB/s — máximo GP3
    }),
  },
],

#!/bin/bash
set -e

# 1. Inicia vLLM em background
python -m vllm.entrypoints.openai.api_server \
  --model "${MODEL_NAME}" \
  --enable-lora --max-loras "${MAX_LORAS:-4}" \
  --port "${VLLM_PORT:-8000}" &
VLLM_PID=$!

# 2. Aguarda vLLM base estar healthy
until curl -sf "http://localhost:${VLLM_PORT}/health" > /dev/null; do
  sleep 5
  kill -0 $VLLM_PID 2>/dev/null || { echo "vLLM morreu!"; exit 1; }
done

# 3. Registra e aquece cada LoRA
for lora_path in "${LORAS_DIR}"/*/; do
  lora_name=$(basename "$lora_path")

  # Registra
  curl -sf -X POST "http://localhost:${VLLM_PORT}/v1/load_lora_adapter" \
    -H "Content-Type: application/json" \
    -d "{\"lora_name\": \"${lora_name}\", \"lora_path\": \"${lora_path}\"}"

  # Warmup (aquece CUDA kernels)
  curl -sf -X POST "http://localhost:${VLLM_PORT}/v1/chat/completions" \
    -d "{\"model\": \"${lora_name}\", \"messages\": [{\"role\": \"user\", \"content\": \"Hi\"}], \"max_tokens\": 1}" \
    > /dev/null
done

# 4. Sobe proxy — NLB só roteia agora
python /healthcheck_proxy.py --proxy-port 8080 &

# 5. Mantém container vivo
wait $VLLM_PID