Puesta en Producción de Modelos ML

# ═══════════════════════════════════════════════════
# Dockerfile · FastAPI + PyTorch inference server
# ═══════════════════════════════════════════════════

# ── Stage 1: base con CUDA (si necesitas GPU) ──
FROM nvidia/cuda:12.1.1-cudnn8-runtime-ubuntu22.04 AS base

ENV DEBIAN_FRONTEND=noninteractive \
    PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1

RUN apt-get update && apt-get install -y --no-install-recommends \
    python3.11 python3-pip python3.11-venv && \
    rm -rf /var/lib/apt/lists/*

# ── Stage 2: dependencias Python (cacheadas) ──
FROM base AS deps
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# ── Stage 3: código y modelo ──
FROM deps AS final
COPY src/ ./src/
COPY models/model.pt ./models/

# Puerto de la API
EXPOSE 8000

# Health check
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
    CMD curl -f http://localhost:8000/health || exit 1

# Arrancar con uvicorn (workers = nº CPUs)
CMD ["uvicorn", "src.app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "2"]

# ═══════════════════════════════════════════════════
# Dockerfile · TensorFlow Serving con modelo propio
# ═══════════════════════════════════════════════════

FROM tensorflow/serving:2.14.0-gpu

# Copiar modelo exportado en formato SavedModel
# Estructura: models/my_model/1/saved_model.pb
COPY models/my_model /models/my_model

# Variables de entorno
ENV MODEL_NAME=my_model \
    MODEL_BASE_PATH=/models

# TFServing escucha en 8501 (REST) y 8500 (gRPC)
EXPOSE 8500 8501

# El entrypoint de la imagen base ya arranca tf_model_server

# ═══════════════════════════════════════════════════
# Instalar NVIDIA Container Toolkit (Ubuntu)
# ═══════════════════════════════════════════════════

# 1. Añadir repositorio
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
    sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \
    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
    sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

# 2. Instalar
sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

# 3. Verificar
docker run --rm --gpus all nvidia/cuda:12.1.1-base-ubuntu22.04 nvidia-smi

# ═══════════════════════════════════════════════════
# Ejecutar tu contenedor con GPU
# ═══════════════════════════════════════════════════

# Todas las GPUs
docker run --gpus all -p 8000:8000 my-model:latest

# GPU específica
docker run --gpus '"device=0"' -p 8000:8000 my-model:latest

# Limitar memoria GPU
docker run --gpus all --shm-size=2g -p 8000:8000 my-model:latest

# ═══════════════════════════════════════════════════
# docker-compose.yml · Stack de inferencia ML
# ═══════════════════════════════════════════════════
version: "3.8"

services:
  # ── Modelo de inferencia ──
  inference:
    build:
      context: .
      dockerfile: Dockerfile
    ports:
      - "8000:8000"
    volumes:
      - ./models:/app/models:ro    # Pesos del modelo (read-only)
    environment:
      - MODEL_PATH=/app/models/model.pt
      - WORKERS=2
      - DEVICE=cuda
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 5s
      retries: 3
    restart: unless-stopped

  # ── Redis para caching de predicciones ──
  redis:
    image: redis:7-alpine
    ports:
      - "6379:6379"
    volumes:
      - redis-data:/data

  # ── Prometheus para métricas ──
  prometheus:
    image: prom/prometheus:v2.48.0
    ports:
      - "9090:9090"
    volumes:
      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro

  # ── Grafana para dashboards ──
  grafana:
    image: grafana/grafana:10.2.0
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    depends_on:
      - prometheus

volumes:
  redis-data:

# ═══════════════════════════════════════════════════
# Push a Amazon ECR
# ═══════════════════════════════════════════════════

# 1. Login
aws ecr get-login-password --region eu-west-1 | \
    docker login --username AWS --password-stdin 123456789.dkr.ecr.eu-west-1.amazonaws.com

# 2. Tag
docker tag my-model:latest 123456789.dkr.ecr.eu-west-1.amazonaws.com/my-model:v1.2.0

# 3. Push
docker push 123456789.dkr.ecr.eu-west-1.amazonaws.com/my-model:v1.2.0

# ═══════════════════════════════════════════════════
# Push a Google Artifact Registry
# ═══════════════════════════════════════════════════

# 1. Auth
gcloud auth configure-docker europe-west1-docker.pkg.dev

# 2. Tag
docker tag my-model:latest europe-west1-docker.pkg.dev/my-project/ml-models/my-model:v1.2.0

# 3. Push
docker push europe-west1-docker.pkg.dev/my-project/ml-models/my-model:v1.2.0

# ═══════════════════════════════════════════════════
# deployment.yaml · Inference server con GPU
# ═══════════════════════════════════════════════════
apiVersion: apps/v1
kind: Deployment
metadata:
  name: inference-server
  labels:
    app: inference
    version: v2.1.0
spec:
  replicas: 3                        # Réplicas iniciales
  selector:
    matchLabels:
      app: inference
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1                    # +1 pod durante update
      maxUnavailable: 0              # 0 downtime
  template:
    metadata:
      labels:
        app: inference
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8000"
    spec:
      containers:
        - name: model
          image: 123456789.dkr.ecr.eu-west-1.amazonaws.com/my-model:v2.1.0
          ports:
            - containerPort: 8000
          resources:
            requests:
              cpu: "2"
              memory: "4Gi"
              nvidia.com/gpu: "1"     # ← Solicita 1 GPU
            limits:
              cpu: "4"
              memory: "8Gi"
              nvidia.com/gpu: "1"
          readinessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 30    # Tiempo para cargar modelo
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 60
            periodSeconds: 30
          env:
            - name: MODEL_PATH
              value: /models/model.pt
            - name: MAX_BATCH_SIZE
              value: "16"
          volumeMounts:
            - name: model-weights
              mountPath: /models
              readOnly: true
      volumes:
        - name: model-weights
          persistentVolumeClaim:
            claimName: model-pvc
      tolerations:                     # Permitir scheduling en nodos GPU
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
---
# ═══════════════════════════════════════════════════
# Service · Load balancer interno
# ═══════════════════════════════════════════════════
apiVersion: v1
kind: Service
metadata:
  name: inference-service
spec:
  selector:
    app: inference
  ports:
    - port: 80
      targetPort: 8000
  type: ClusterIP
---
# ═══════════════════════════════════════════════════
# HPA · Autoscaling basado en CPU
# ═══════════════════════════════════════════════════
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: inference-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: inference-server
  minReplicas: 2
  maxReplicas: 20
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300   # Esperar 5 min antes de reducir

# ═══════════════════════════════════════════════════
# 1. Exportar modelo como SavedModel
# ═══════════════════════════════════════════════════
import tensorflow as tf

model = tf.keras.models.load_model("my_model.keras")

# Exportar con versión (directorio numérico)
export_path = "models/my_model/1"    # ← versión 1
model.export(export_path)
print(f"✅ Modelo exportado a {export_path}")

# ═══════════════════════════════════════════════════
# 2. Levantar TF Serving con Docker
# ═══════════════════════════════════════════════════
# docker run -p 8501:8501 \
#     -v $(pwd)/models/my_model:/models/my_model \
#     -e MODEL_NAME=my_model \
#     tensorflow/serving:2.14.0-gpu

# ═══════════════════════════════════════════════════
# 3. Hacer predicciones via REST
# ═══════════════════════════════════════════════════
import requests
import numpy as np

data = {"instances": np.random.randn(1, 224, 224, 3).tolist()}
url = "http://localhost:8501/v1/models/my_model:predict"

response = requests.post(url, json=data)
predictions = response.json()["predictions"]
print(f"🎯 Predicción: {np.argmax(predictions[0])}")

# ═══════════════════════════════════════════════════
# 1. Crear Model Archive (.mar)
# ═══════════════════════════════════════════════════
# torch-model-archiver \
#     --model-name resnet50 \
#     --version 1.0 \
#     --serialized-file model.pt \
#     --handler image_classifier \
#     --export-path model_store/

# ═══════════════════════════════════════════════════
# 2. Arrancar TorchServe
# ═══════════════════════════════════════════════════
# torchserve --start --model-store model_store \
#     --models resnet50=resnet50.mar \
#     --ts-config config.properties

# ═══════════════════════════════════════════════════
# 3. Handler personalizado (opcional)
# ═══════════════════════════════════════════════════
from ts.torch_handler.base_handler import BaseHandler
import torch
from torchvision import transforms
from PIL import Image
import io

class MyHandler(BaseHandler):
    def preprocess(self, data):
        """Pre-proceso: decode imagen + transformaciones."""
        transform = transforms.Compose([
            transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406],
                                 [0.229, 0.224, 0.225]),
        ])
        images = []
        for row in data:
            image = Image.open(io.BytesIO(row["body"]))
            images.append(transform(image))
        return torch.stack(images)

    def inference(self, data):
        """Inferencia sobre batch."""
        with torch.no_grad():
            return self.model(data)

    def postprocess(self, output):
        """Post-proceso: softmax + top-5."""
        probs = torch.nn.functional.softmax(output, dim=1)
        top5 = torch.topk(probs, 5, dim=1)
        return [{"top5": top5.indices[i].tolist(),
                 "probs": top5.values[i].tolist()}
                for i in range(len(output))]

# ═══════════════════════════════════════════════════
# 4. Hacer predicciones
# ═══════════════════════════════════════════════════
import requests

with open("cat.jpg", "rb") as f:
    response = requests.post(
        "http://localhost:8080/predictions/resnet50",
        files={"data": f}
    )
print(response.json())

# ═══════════════════════════════════════════════════
# Estructura del model repository
# ═══════════════════════════════════════════════════
# model_repository/
# ├── resnet50/
# │   ├── config.pbtxt
# │   └── 1/
# │       └── model.onnx
# └── bert_qa/
#     ├── config.pbtxt
#     └── 1/
#         └── model.plan          ← TensorRT engine

# ═══════════════════════════════════════════════════
# config.pbtxt (ejemplo para ONNX)
# ═══════════════════════════════════════════════════
# name: "resnet50"
# platform: "onnxruntime_onnx"
# max_batch_size: 32
# input [{
#   name: "input"
#   data_type: TYPE_FP32
#   dims: [3, 224, 224]
# }]
# output [{
#   name: "output"
#   data_type: TYPE_FP32
#   dims: [1000]
# }]
# dynamic_batching {
#   preferred_batch_size: [8, 16]
#   max_queue_delay_microseconds: 5000
# }
# instance_group [{
#   count: 2
#   kind: KIND_GPU
# }]

# ═══════════════════════════════════════════════════
# Docker run
# ═══════════════════════════════════════════════════
# docker run --gpus all -p 8000:8000 -p 8001:8001 -p 8002:8002 \
#     -v $(pwd)/model_repository:/models \
#     nvcr.io/nvidia/tritonserver:24.01-py3 \
#     tritonserver --model-repository=/models

# ═══════════════════════════════════════════════════
# Cliente Python (tritonclient)
# ═══════════════════════════════════════════════════
import tritonclient.http as httpclient
import numpy as np

client = httpclient.InferenceServerClient(url="localhost:8000")

# Verificar que el modelo está cargado
assert client.is_model_ready("resnet50"), "Modelo no disponible"

# Preparar input
input_data = np.random.randn(1, 3, 224, 224).astype(np.float32)
inputs = [httpclient.InferInput("input", input_data.shape, "FP32")]
inputs[0].set_data_from_numpy(input_data)

# Inferir
result = client.infer("resnet50", inputs)
output = result.as_numpy("output")
print(f"🎯 Clase predicha: {np.argmax(output)}")

# ═══════════════════════════════════════════════════
# 1. Guardar modelo en BentoML model store
# ═══════════════════════════════════════════════════
import bentoml
import torch

model = torch.load("model.pt")
bentoml.pytorch.save_model("my_classifier", model)

# ═══════════════════════════════════════════════════
# 2. Definir Service (service.py)
# ═══════════════════════════════════════════════════
import bentoml
import numpy as np
from PIL import Image

@bentoml.service(
    resources={"gpu": 1, "memory": "4Gi"},
    traffic={"timeout": 30},
)
class ImageClassifier:
    def __init__(self):
        self.model = bentoml.pytorch.load_model("my_classifier")
        self.model.eval()

    @bentoml.api
    async def predict(self, image: Image.Image) -> dict:
        # Preprocesar
        tensor = self.preprocess(image)
        # Inferir
        with torch.no_grad():
            output = self.model(tensor.unsqueeze(0))
        # Devolver resultado
        prob = torch.softmax(output, dim=1)
        return {"class": int(prob.argmax()), "confidence": float(prob.max())}

# ═══════════════════════════════════════════════════
# 3. Servir localmente
# ═══════════════════════════════════════════════════
# bentoml serve service:ImageClassifier

# ═══════════════════════════════════════════════════
# 4. Containerizar y desplegar
# ═══════════════════════════════════════════════════
# bentoml build                    # Crear Bento
# bentoml containerize my_classifier:latest  # Docker image
# docker push my-registry/my_classifier:v1

# ═══════════════════════════════════════════════════
# KServe InferenceService con canary deployment
# ═══════════════════════════════════════════════════
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: image-classifier
spec:
  predictor:
    # ── Versión actual (90% tráfico) ──
    canaryTrafficPercent: 10
    model:
      modelFormat:
        name: pytorch
      storageUri: "gs://my-bucket/models/resnet50/v1"
      resources:
        limits:
          nvidia.com/gpu: "1"
          memory: "8Gi"
    # ── Versión canary (10% tráfico) ──
    canary:
      model:
        modelFormat:
          name: pytorch
        storageUri: "gs://my-bucket/models/resnet50/v2"
        resources:
          limits:
            nvidia.com/gpu: "1"
            memory: "8Gi"
  transformer:
    containers:
      - name: preprocess
        image: my-registry/preprocess:v1
        env:
          - name: STORAGE_URI
            value: "gs://my-bucket/config"

# ═══════════════════════════════════════════════════
# Demo con Gradio: clasificación de imágenes
# ═══════════════════════════════════════════════════
import gradio as gr
from transformers import pipeline

classifier = pipeline("image-classification", model="google/vit-base-patch16-224")

demo = gr.Interface(
    fn=lambda img: {p["label"]: p["score"] for p in classifier(img)},
    inputs=gr.Image(type="pil"),
    outputs=gr.Label(num_top_classes=5),
    title="🖼️ Clasificador de imágenes (ViT)",
    description="Sube una imagen y el modelo la clasificará."
)

demo.launch()   # → localhost:7860
# Para desplegar en HF Spaces: git push al repo del Space

# ═══════════════════════════════════════════════════
# Despliegue en Vertex AI con Python SDK
# ═══════════════════════════════════════════════════
from google.cloud import aiplatform

aiplatform.init(project="my-project", location="europe-west1")

# 1. Subir modelo al Model Registry
model = aiplatform.Model.upload(
    display_name="image-classifier-v2",
    artifact_uri="gs://my-bucket/models/resnet50/",
    serving_container_image_uri=(
        "europe-docker.pkg.dev/vertex-ai/prediction/"
        "tf2-gpu.2-14:latest"
    ),
)

# 2. Crear endpoint
endpoint = aiplatform.Endpoint.create(
    display_name="classifier-endpoint",
)

# 3. Desplegar modelo en endpoint
model.deploy(
    endpoint=endpoint,
    machine_type="n1-standard-4",
    accelerator_type="NVIDIA_TESLA_T4",
    accelerator_count=1,
    min_replica_count=1,
    max_replica_count=5,         # Autoscaling
    traffic_percentage=100,
)

# 4. Hacer predicciones
import numpy as np
instances = [np.random.randn(224, 224, 3).tolist()]
prediction = endpoint.predict(instances=instances)
print(prediction.predictions)

# ═══════════════════════════════════════════════════
# Despliegue en SageMaker con Python SDK
# ═══════════════════════════════════════════════════
import sagemaker
from sagemaker.pytorch import PyTorchModel

session = sagemaker.Session()
role = "arn:aws:iam::123456789:role/SageMakerRole"

# 1. Crear modelo
pytorch_model = PyTorchModel(
    model_data="s3://my-bucket/models/model.tar.gz",
    role=role,
    framework_version="2.1",
    py_version="py310",
    entry_point="inference.py",     # Script de pre/post proceso
)

# 2. Desplegar endpoint
predictor = pytorch_model.deploy(
    instance_type="ml.g4dn.xlarge",   # GPU T4
    initial_instance_count=1,
    endpoint_name="my-classifier-v2",
)

# 3. Predicción
import numpy as np
response = predictor.predict(np.random.randn(1, 3, 224, 224).tolist())
print(response)

# 4. Limpiar (¡no olvidar para no seguir pagando!)
predictor.delete_endpoint()

# ═══════════════════════════════════════════════════
# .github/workflows/ml-cicd.yml
# ═══════════════════════════════════════════════════
name: ML CI/CD Pipeline

on:
  push:
    branches: [main]
    paths: ["src/**", "models/**", "requirements.txt"]

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.11" }
      - run: pip install -r requirements.txt
      - run: pytest tests/ -v --tb=short

  evaluate:
    needs: test
    runs-on: ubuntu-latest     # O self-hosted con GPU
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.11" }
      - run: pip install -r requirements.txt

      # Evaluar modelo nuevo vs producción
      - name: Evaluate model
        run: |
          python evaluate.py \
            --model models/model_new.pt \
            --data data/test_set.csv \
            --output metrics.json

      # Quality gate: fail si accuracy < 0.90
      - name: Quality gate
        run: |
          python -c "
          import json
          m = json.load(open('metrics.json'))
          assert m['accuracy'] >= 0.90, f'Accuracy {m[\"accuracy\"]} < 0.90'
          assert m['latency_p99'] <= 200, f'Latency {m[\"latency_p99\"]}ms > 200ms'
          print('✅ Quality gate passed')
          "

  deploy:
    needs: evaluate
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build & push Docker image
        run: |
          docker build -t ghcr.io/${{ github.repository }}/model:${{ github.sha }} .
          docker push ghcr.io/${{ github.repository }}/model:${{ github.sha }}

      - name: Deploy canary (10%)
        run: |
          kubectl set image deployment/inference \
            model=ghcr.io/${{ github.repository }}/model:${{ github.sha }}
          kubectl patch hpa inference-hpa \
            -p '{"metadata":{"annotations":{"canary":"true"}}}'

# ═══════════════════════════════════════════════════
# MLflow: track experiment + register model
# ═══════════════════════════════════════════════════
import mlflow
import mlflow.pytorch

mlflow.set_tracking_uri("http://mlflow-server:5000")
mlflow.set_experiment("image-classification")

with mlflow.start_run(run_name="resnet50-v2"):
    # Log hiperparámetros
    mlflow.log_params({
        "model": "resnet50",
        "lr": 0.001,
        "batch_size": 64,
        "epochs": 20,
        "optimizer": "AdamW",
    })

    # Entrenar modelo...
    model = train(...)

    # Log métricas
    mlflow.log_metrics({
        "accuracy": 0.923,
        "f1_score": 0.918,
        "latency_p99_ms": 45,
    })

    # Log modelo → Model Registry
    mlflow.pytorch.log_model(
        model,
        artifact_path="model",
        registered_model_name="image-classifier",
    )

# ═══════════════════════════════════════════════════
# Promover modelo a producción
# ═══════════════════════════════════════════════════
from mlflow import MlflowClient

client = MlflowClient()
client.transition_model_version_stage(
    name="image-classifier",
    version=2,
    stage="Production",
    archive_existing_versions=True,   # Archiva la v1
)

# ═══════════════════════════════════════════════════
# Evidently: test de data drift
# ═══════════════════════════════════════════════════
from evidently.test_suite import TestSuite
from evidently.tests import (
    TestShareOfDriftedColumns,
    TestColumnDrift,
)
import pandas as pd

# Datos de referencia (entrenamiento) y actuales (producción)
reference = pd.read_csv("data/train_features.csv")
current = pd.read_csv("data/production_features_today.csv")

# Crear suite de tests
suite = TestSuite(tests=[
    TestShareOfDriftedColumns(lt=0.3),     # < 30% columnas con drift
    TestColumnDrift("age"),                 # Test específico por columna
    TestColumnDrift("income"),
])

suite.run(reference_data=reference, current_data=current)

# Resultado
result = suite.as_dict()
if not result["summary"]["all_passed"]:
    print("⚠️ DRIFT DETECTADO — considerar re-entrenamiento")
    # Enviar alerta a Slack / PagerDuty
else:
    print("✅ Sin drift significativo")

# Generar dashboard HTML
suite.save_html("drift_report.html")