init stacks

2026-03-21 14:12:55 +01:00
commit d3e8da2e25
1919 changed files with 200787 additions and 0 deletions
@@ -0,0 +1,21 @@
+FROM python:3.12-slim
+
+ENV PYTHONPATH=/app
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+EXPOSE 8000
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
@@ -0,0 +1,3 @@
+def send(incident, summary):
+    print("🚨 INCIDENT :", incident["id"])
+    print(summary)
@@ -0,0 +1,2 @@
+def should_alert(incident):
+    return incident["severity"] in ("high", "critical")
@@ -0,0 +1,57 @@
+from fastapi import APIRouter
+from api.connectors.prometheus import query_scalar
+
+router = APIRouter()
+
+CPU_AVG_QUERY = """
+avg_over_time(
+  100 - (
+    avg by (instance) (
+      rate(node_cpu_seconds_total{mode="idle"}[1m])
+    ) * 100
+  )
+[20m])
+"""
+
+CPU_SLOPE_QUERY = """
+deriv(
+  100 - (
+    avg by (instance) (
+      rate(node_cpu_seconds_total{mode="idle"}[1m])
+    ) * 100
+  )
+[20m])
+"""
+
+def analyze_cpu(cpu_avg: float, cpu_slope: float) -> dict:
+    if cpu_avg > 90:
+        status = "RISK_CPU"
+        reason = "CPU très élevé"
+    elif cpu_avg > 80:
+        status = "WARN_CPU"
+        reason = "CPU élevé"
+    else:
+        status = "OK"
+        reason = "CPU normal"
+
+    trend = "increasing" if cpu_slope > 0.5 else "stable"
+
+    return {
+        "status": status,
+        "avg": round(cpu_avg, 2),
+        "trend": trend,
+        "reason": reason
+    }
+
+@router.get("/cpu")
+def cpu_analysis():
+    avg = query_scalar(CPU_AVG_QUERY)
+    slope = query_scalar(CPU_SLOPE_QUERY)
+
+    if avg is None or slope is None:
+        return {
+            "status": "UNKNOWN",
+            "reason": "Prometheus data unavailable"
+        }
+
+    return analyze_cpu(avg, slope)
@@ -0,0 +1,6 @@
+def fetch_alerts():
+    """
+    Placeholder Grafana connector.
+    Will later query Grafana API.
+    """
+    return []
@@ -0,0 +1,65 @@
+import os
+import httpx
+import requests
+
+PROMETHEUS_URL = os.getenv("PROMETHEUS_URL")
+
+def query_scalar(query: str) -> float | None:
+    try:
+        r = httpx.get(
+            f"{PROMETHEUS_URL}/api/v1/query",
+            params={"query": query},
+            timeout=5.0,
+        )
+        r.raise_for_status()
+        data = r.json()
+
+        if data["status"] != "success":
+            return None
+
+        result = data["data"]["result"]
+        if not result:
+            return None
+
+        return float(result[0]["value"][1])
+
+    except Exception:
+        return None
+
+def query(promql: str):
+    r = requests.get(
+        f"{PROMETHEUS_URL}/api/v1/query",
+        params={"query": promql},
+        timeout=3,
+    )
+    r.raise_for_status()
+    return r.json()["data"]["result"]
+
+
+def get_cpu_usage():
+    result = query(
+        '100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[1m])) * 100)'
+    )
+    return float(result[0]["value"][1]) if result else 0.0
+
+
+def get_memory_usage():
+    result = query(
+        '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100'
+    )
+    return float(result[0]["value"][1]) if result else 0.0
+
+
+def get_disk_usage():
+    result = query(
+        '(1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / '
+        'node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100'
+    )
+    return float(result[0]["value"][1]) if result else 0.0
+
+
+def get_container_down():
+    result = query(
+        'count(container_last_seen{job="cadvisor"} < time() - 60)'
+    )
+    return int(result[0]["value"][1]) if result else 0
@@ -0,0 +1,31 @@
+def analyze(metrics: dict):
+    alerts = []
+
+    if metrics.get("cpu"):
+        cpu = float(metrics["cpu"][0]["value"][1])
+        if cpu > 85:
+            alerts.append({
+                "title": "CPU élevée",
+                "cause": f"{cpu:.1f} %",
+                "severity": "warning",
+            })
+
+    if metrics.get("memory"):
+        mem = float(metrics["memory"][0]["value"][1])
+        if mem > 90:
+            alerts.append({
+                "title": "Mémoire saturée",
+                "cause": f"{mem:.1f} %",
+                "severity": "critical",
+            })
+
+    if metrics.get("containers_down"):
+        down = int(float(metrics["containers_down"][0]["value"][1]))
+        if down > 0:
+            alerts.append({
+                "title": "Containers arrêtés",
+                "cause": f"{down} détecté(s)",
+                "severity": "critical",
+            })
+
+    return alerts
@@ -0,0 +1,14 @@
+from api.connectors.prometheus import (
+    get_cpu_usage,
+    get_memory_usage,
+    get_disk_usage,
+    get_container_down,
+)
+
+def collect_metrics():
+    return {
+        "cpu": get_cpu_usage(),
+        "memory": get_memory_usage(),
+        "disk": get_disk_usage(),
+        "containers_down": get_container_down(),
+    }
@@ -0,0 +1,44 @@
+import yaml
+
+with open("correlation/rules.yaml") as f:
+    RULES = yaml.safe_load(f)
+
+def correlate(alert):
+    service = alert["service"]
+    rules = RULES["services"].get(service, {})
+    return {
+        "service": service,
+        "severity": alert["severity"],
+        "metrics": alert["metrics"],
+        "depends_on": rules.get("depends_on", []),
+        "impact": rules.get("impact", [])
+    }
+
+def evaluate(rules: list, metrics: dict):
+    incidents = []
+
+    for rule in rules:
+        matched = True
+
+        for metric, condition in rule["when"].items():
+            value = metrics.get(metric)
+
+            if value is None:
+                matched = False
+                break
+
+            if "gt" in condition and value <= condition["gt"]:
+                matched = False
+
+            if "lt" in condition and value >= condition["lt"]:
+                matched = False
+
+        if matched:
+            incidents.append({
+                "id": rule["id"],
+                "severity": rule["severity"],
+                "impact": rule["impact"],
+                "metrics": metrics,
+            })
+
+    return incidents
@@ -0,0 +1,14 @@
+import yaml
+from pathlib import Path
+
+RULES_FILE = Path(__file__).parent / "rules.yaml"
+
+
+def load_rules():
+    if not RULES_FILE.exists():
+        return []
+
+    with RULES_FILE.open("r") as f:
+        data = yaml.safe_load(f) or {}
+
+    return data.get("rules", [])
@@ -0,0 +1,34 @@
+services:
+  jellyfin:
+    depends_on:
+      - tdarr
+      - cpu
+    impact:
+      - media
+      - nextcloud
+
+  nextcloud:
+    depends_on:
+      - database
+      - redis
+
+rules:
+  - id: cpu_ram_pressure
+    description: CPU élevé + mémoire sous pression
+    when:
+      cpu:
+        gt: 80
+        for: 5m
+      memory_available:
+        lt: 15
+    impact: "Risque fort de ralentissement général"
+    severity: high
+
+  - id: io_wait_saturation
+    description: IO wait élevé
+    when:
+      io_wait:
+        gt: 20
+        for: 3m
+    impact: "Disque saturé, latences applicatives"
+    severity: high
@@ -0,0 +1,8 @@
+from api.llm.ollama import summarize
+
+
+def explain_incident(incident, metrics):
+    try:
+        return summarize(incident, metrics)
+    except Exception:
+        return "Résumé indisponible (LLM non joignable)."
@@ -0,0 +1,14 @@
+import httpx
+
+def summarize(incident, metrics):
+    payload = {
+        "model": "mistral",
+        "prompt": INCIDENT_SUMMARY_PROMPT.format(
+            incident=incident,
+            metrics=metrics,
+        ),
+        "stream": False,
+    }
+
+    r = httpx.post("http://ollama:11434/api/generate", json=payload, timeout=30)
+    return r.json()["response"]
@@ -0,0 +1,28 @@
+SUMMARY_PROMPT = """
+Tu résumes une alerte d'infrastructure.
+
+Règles :
+- Utilise uniquement les données fournies
+- Indique l'heure
+- Cause probable
+- Impact
+- Si doute : dis "cause indéterminée"
+
+Données :
+{context}
+"""
+
+INCIDENT_SUMMARY_PROMPT = """
+Tu es un SRE senior.
+
+Incident détecté :
+{incident}
+
+Métriques :
+{metrics}
+
+Explique en français clair :
+- la cause probable
+- l’impact utilisateur
+- une action recommandée
+"""
@@ -0,0 +1,15 @@
+from fastapi import FastAPI
+
+from api.routers import health, alerts
+from api.analysis import cpu
+from api.scheduler import start_scheduler
+
+app = FastAPI(title="Infra Assistant")
+
+app.include_router(health.router)
+app.include_router(alerts.router)
+app.include_router(cpu.router, prefix="/analysis")
+
+@app.on_event("startup")
+async def startup():
+    start_scheduler()
@@ -0,0 +1,5 @@
+from datetime import datetime
+
+def log_alert(message: str):
+    ts = datetime.now().strftime("%Y-%m-%d %H:%M")
+    print(f"[ALERTE][{ts}]\n{message}\n")
@@ -0,0 +1,15 @@
+import smtplib
+from email.message import EmailMessage
+import os
+
+def send_mail(body: str):
+    msg = EmailMessage()
+    msg["Subject"] = "Alerte Infrastructure"
+    msg["From"] = os.getenv("SMTP_USER")
+    msg["To"] = os.getenv("MAIL_TO")
+    msg.set_content(body)
+
+    with smtplib.SMTP(os.getenv("SMTP_HOST"), int(os.getenv("SMTP_PORT"))) as s:
+        s.starttls()
+        s.login(os.getenv("SMTP_USER"), os.getenv("SMTP_PASS"))
+        s.send_message(msg)
@@ -0,0 +1,6 @@
+fastapi==0.128.0
+uvicorn==0.40.0
+httpx==0.28.1
+apscheduler==3.11.2
+pyyaml==6.0.3
+requests>=2.31
@@ -0,0 +1,32 @@
+from fastapi import APIRouter
+from api.correlation.collectors import collect_metrics
+from api.correlation.engine import evaluate, RULES
+from api.llm.client import explain_incident
+
+router = APIRouter()
+
+
+@router.get("/alerts")
+def alerts():
+    metrics = collect_metrics()
+
+    # Récupère la liste de règles depuis RULES ; fallback sur [] si introuvable
+    rules_list = []
+    if isinstance(RULES, dict):
+        rules_list = RULES.get("rules", []) or []
+    elif isinstance(RULES, list):
+        rules_list = RULES
+
+    incidents = evaluate(rules_list, metrics)
+
+    if not incidents:
+        return {"status": "ok"}
+
+    for incident in incidents:
+        # explain_incident doit être résilient — voir api/llm/client.py pour fallback
+        incident["summary"] = explain_incident(incident, metrics)
+
+    return {
+        "status": "incident",
+        "incidents": incidents,
+    }
@@ -0,0 +1,7 @@
+from fastapi import APIRouter
+
+router = APIRouter()
+
+@router.get("/health")
+def health():
+    return {"status": "ok"}
@@ -0,0 +1,52 @@
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+import logging
+
+scheduler = AsyncIOScheduler()
+logger = logging.getLogger("infra-assistant")
+
+
+def start_scheduler():
+    scheduler.add_job(run, "interval", minutes=5)
+    scheduler.start()
+    logger.info("Scheduler started (interval=5min)")
+
+
+async def run():
+    # Imports SAFE (aucun crash au boot)
+    try:
+        from connectors.grafana import fetch_alerts
+    except ImportError:
+        logger.warning("Grafana connector not available")
+        return
+
+    try:
+        from correlation.engine import correlate
+    except ImportError:
+        logger.warning("Correlation engine not available")
+        return
+
+    try:
+        from llm.client import summarize
+    except ImportError:
+        summarize = None
+
+    try:
+        from notifications.logger import log_alert
+        from notifications.mailer import send_mail
+    except ImportError:
+        logger.warning("Notification system not available")
+        return
+
+    # --- Pipeline ---
+    alerts = await fetch_alerts()
+
+    for alert in alerts:
+        context = correlate(alert)
+
+        if summarize:
+            message = summarize(context)
+        else:
+            message = "Alert received but LLM unavailable"
+
+        log_alert(message)
+        send_mail(message)