Files
scrap/analytics-ui/app.py
T
Gilles Soulier 740c3d7516 before claude
2026-01-18 06:26:17 +01:00

706 lines
26 KiB
Python

import os
from typing import Any, Dict, List, Optional, Tuple
from decimal import Decimal
from psycopg2.extras import RealDictCursor
import psycopg2
import redis
from flask import Flask, jsonify, render_template_string
app = Flask(__name__)
def _env_int(name: str, default: int) -> int:
try:
return int(os.getenv(name, "") or default)
except ValueError:
return default
def get_db_connection():
return psycopg2.connect(
host=os.getenv("PW_DB_HOST", "postgres"),
port=_env_int("PW_DB_PORT", 5432),
dbname=os.getenv("PW_DB_NAME", "pricewatch"),
user=os.getenv("PW_DB_USER", "pricewatch"),
password=os.getenv("PW_DB_PASSWORD", "pricewatch"),
)
def fetch_db_metrics() -> Tuple[Dict[str, Any], Optional[str]]:
data: Dict[str, Any] = {"counts": {}, "latest_products": []}
try:
with get_db_connection() as conn:
with conn.cursor() as cur:
cur.execute("SELECT COUNT(*) FROM products")
data["counts"]["products"] = cur.fetchone()[0]
cur.execute("SELECT COUNT(*) FROM price_history")
data["counts"]["price_history"] = cur.fetchone()[0]
cur.execute(
"SELECT COUNT(*) FROM scraping_logs"
)
data["counts"]["scraping_logs"] = cur.fetchone()[0]
cur.execute(
"""
SELECT id, source, reference, title, last_updated_at
FROM products
ORDER BY last_updated_at DESC
LIMIT 5
"""
)
rows = cur.fetchall()
data["latest_products"] = [
{
"id": row[0],
"source": row[1],
"reference": row[2],
"title": row[3] or "Sans titre",
"updated": row[4].strftime("%Y-%m-%d %H:%M:%S")
if row[4]
else "n/a",
}
for row in rows
]
return data, None
except Exception as exc: # pragma: no cover (simple explorer)
return data, str(exc)
def _serialize_decimal(value):
if isinstance(value, Decimal):
return float(value)
return value
def fetch_product_history(product_id: int) -> Tuple[List[Dict[str, Any]], Optional[str]]:
"""Récupère l'historique complet des scraps pour un produit."""
rows: List[Dict[str, Any]] = []
try:
with get_db_connection() as conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute(
"""
SELECT
ph.id,
ph.price,
ph.shipping_cost,
ph.stock_status,
ph.fetch_method,
ph.fetch_status,
ph.fetched_at
FROM price_history ph
WHERE ph.product_id = %s
ORDER BY ph.fetched_at DESC
""",
(product_id,),
)
fetched = cur.fetchall()
for item in fetched:
serialized = {key: _serialize_decimal(value) for key, value in item.items()}
if serialized.get("fetched_at"):
serialized["fetched_at"] = serialized["fetched_at"].strftime(
"%Y-%m-%d %H:%M:%S"
)
rows.append(serialized)
return rows, None
except Exception as exc:
return rows, str(exc)
def fetch_all_price_history(limit: int = 500) -> Tuple[List[Dict[str, Any]], Optional[str]]:
"""Récupère toutes les entrées de price_history avec infos produit."""
rows: List[Dict[str, Any]] = []
try:
with get_db_connection() as conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute(
"""
SELECT
ph.id,
ph.product_id,
p.source,
p.reference,
p.title,
ph.price,
ph.shipping_cost,
ph.stock_status,
ph.fetch_method,
ph.fetch_status,
ph.fetched_at
FROM price_history ph
LEFT JOIN products p ON p.id = ph.product_id
ORDER BY ph.fetched_at DESC
LIMIT %s
""",
(limit,),
)
fetched = cur.fetchall()
for item in fetched:
serialized = {key: _serialize_decimal(value) for key, value in item.items()}
if serialized.get("fetched_at"):
serialized["fetched_at"] = serialized["fetched_at"].strftime(
"%Y-%m-%d %H:%M:%S"
)
rows.append(serialized)
return rows, None
except Exception as exc:
return rows, str(exc)
def fetch_products_list(limit: int = 200) -> Tuple[List[Dict[str, Any]], Optional[str]]:
rows: List[Dict[str, Any]] = []
try:
with get_db_connection() as conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute(
"""
SELECT
p.id,
p.source,
p.reference,
p.title,
p.url,
p.category,
p.description,
p.currency,
p.msrp,
p.last_updated_at,
ph.price,
ph.stock_status,
ph.fetch_status,
ph.fetch_method,
ph.fetched_at
FROM products p
LEFT JOIN LATERAL (
SELECT price, stock_status, fetch_status, fetch_method, fetched_at
FROM price_history
WHERE product_id = p.id
ORDER BY fetched_at DESC
LIMIT 1
) ph ON true
ORDER BY p.last_updated_at DESC
LIMIT %s
""",
(limit,),
)
fetched = cur.fetchall()
for item in fetched:
serialized = {key: _serialize_decimal(value) for key, value in item.items()}
if serialized.get("last_updated_at"):
serialized["last_updated_at"] = serialized["last_updated_at"].strftime(
"%Y-%m-%d %H:%M:%S"
)
if serialized.get("fetched_at"):
serialized["fetched_at"] = serialized["fetched_at"].strftime(
"%Y-%m-%d %H:%M:%S"
)
rows.append(serialized)
return rows, None
except Exception as exc:
return rows, str(exc)
def get_redis_client() -> redis.Redis:
return redis.Redis(
host=os.getenv("PW_REDIS_HOST", "redis"),
port=_env_int("PW_REDIS_PORT", 6379),
db=_env_int("PW_REDIS_DB", 0),
socket_connect_timeout=2,
socket_timeout=2,
)
def check_redis() -> Tuple[str, Optional[str]]:
client = get_redis_client()
try:
client.ping()
return "OK", None
except Exception as exc:
return "KO", str(exc)
TEMPLATE = """
<!doctype html>
<html lang="fr">
<head>
<meta charset="utf-8" />
<title>PriceWatch Analytics UI</title>
<style>
body { font-family: "JetBrains Mono", system-ui, monospace; background:#1f1f1b; color:#ebe0c8; margin:0; padding:32px; }
main { max-width: 960px; margin: 0 auto; }
h1 { margin-bottom: 0; }
section { margin-top: 24px; background:#282828; border:1px solid rgba(255,255,255,0.08); padding:16px; border-radius:14px; box-shadow:0 14px 30px rgba(0,0,0,0.35); }
table { width:100%; border-collapse:collapse; margin-top:12px; }
th, td { text-align:left; padding:6px 8px; border-bottom:1px solid rgba(255,255,255,0.08); }
.status { display:inline-flex; align-items:center; gap:6px; font-size:14px; padding:4px 10px; border-radius:999px; background:rgba(255,255,255,0.05); }
.status.ok { background:rgba(184,187,38,0.15); }
.status.ko { background:rgba(251,73,52,0.2); }
.muted { color:rgba(255,255,255,0.5); font-size:13px; }
.browser-panel { margin-top: 16px; display: flex; flex-direction: column; gap: 12px; }
.browser-controls { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }
.browser-controls button { border-radius: 8px; border: 1px solid rgba(255,255,255,0.12); background: rgba(255,255,255,0.04); color: inherit; padding: 6px 12px; cursor: pointer; transition: transform 0.15s ease; }
.browser-controls button:hover { transform: translateY(-1px); }
.browser-display { padding: 12px; border-radius: 12px; background: rgba(255,255,255,0.02); border: 1px solid rgba(255,255,255,0.08); min-height: 150px; font-size: 0.85rem; }
.browser-display dt { font-weight: 700; }
.browser-display dd { margin: 0 0 8px 0; }
.browser-indicator { font-size: 0.9rem; }
</style>
</head>
<body>
<main>
<header>
<h1>PriceWatch Analytics UI</h1>
<p class="muted">PostgreSQL : {{ db_status }} · Redis : {{ redis_status }}</p>
</header>
<section>
<h2>Vue rapide</h2>
<div class="status {{ 'ok' if db_error is none else 'ko' }}">
Base : {{ db_status }}
</div>
<div class="status {{ 'ok' if redis_status == 'OK' else 'ko' }}">
Redis : {{ redis_status }}
</div>
{% if db_error or redis_error %}
<p class="muted">Erreurs : {{ db_error or '' }} {{ redis_error or '' }}</p>
{% endif %}
</section>
<section>
<h2>Stats métier</h2>
<table>
<tr><th>Produits</th><td>{{ metrics.counts.products }}</td></tr>
<tr><th>Historique prix</th><td>{{ metrics.counts.price_history }}</td></tr>
<tr><th>Logs de scraping</th><td>{{ metrics.counts.scraping_logs }}</td></tr>
</table>
</section>
<section>
<h2>Produits récemment mis à jour</h2>
{% if metrics.latest_products %}
<table>
<thead>
<tr><th>ID</th><th>Store</th><th>Référence</th><th>Révision</th><th>Mis à jour</th></tr>
</thead>
<tbody>
{% for item in metrics.latest_products %}
<tr>
<td>{{ item.id }}</td>
<td>{{ item.source }}</td>
<td>{{ item.reference }}</td>
<td>{{ item.title[:40] }}{% if item.title|length > 40 %}…{% endif %}</td>
<td>{{ item.updated }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p class="muted">Aucun produit enregistré.</p>
{% endif %}
</section>
<section>
<h2>Parcourir la base (produits)</h2>
<div class="browser-panel">
<div class="browser-controls">
<button id="load-products">Charger les produits</button>
<button id="product-prev" disabled>Précédent</button>
<button id="product-next" disabled>Suivant</button>
<strong class="browser-indicator" id="product-indicator">0 / 0</strong>
<span class="muted" id="product-message"></span>
</div>
<dl class="browser-display" id="product-details">
<dt data-field="title">Titre</dt>
<dd id="product-title">-</dd>
<dt data-field="store">Store</dt>
<dd data-field="store">-</dd>
<dt data-field="reference">Référence</dt>
<dd data-field="reference">-</dd>
<dt data-field="price">Dernier prix</dt>
<dd data-field="price">-</dd>
<dt data-field="currency">Devise</dt>
<dd data-field="currency">-</dd>
<dt data-field="msrp">Prix conseillé</dt>
<dd data-field="msrp">-</dd>
<dt data-field="stock_status">Stock</dt>
<dd data-field="stock_status">-</dd>
<dt data-field="category">Catégorie</dt>
<dd data-field="category">-</dd>
<dt data-field="description">Description</dt>
<dd data-field="description">-</dd>
<dt data-field="last_updated_at">Dernière mise à jour</dt>
<dd data-field="last_updated_at">-</dd>
<dt data-field="fetched_at">Historique dernier scrap</dt>
<dd data-field="fetched_at">-</dd>
</dl>
</div>
</section>
<section>
<h2>Historique complet des scraps</h2>
<div class="browser-panel">
<div class="browser-controls">
<button id="load-history">Charger l'historique du produit sélectionné</button>
<span class="muted" id="history-message"></span>
</div>
<div class="history-table-container" style="max-height: 400px; overflow-y: auto; margin-top: 12px;">
<table id="history-table">
<thead>
<tr>
<th>Date</th>
<th>Prix</th>
<th>Frais port</th>
<th>Stock</th>
<th>Méthode</th>
<th>Statut</th>
</tr>
</thead>
<tbody id="history-body">
<tr><td colspan="6" class="muted">Sélectionnez un produit puis cliquez sur "Charger l'historique"</td></tr>
</tbody>
</table>
</div>
</div>
</section>
<section>
<h2>Parcourir la table price_history</h2>
<div class="browser-panel">
<div class="browser-controls">
<button id="load-price-history">Charger price_history</button>
<button id="ph-prev" disabled>Précédent</button>
<button id="ph-next" disabled>Suivant</button>
<strong class="browser-indicator" id="ph-indicator">0 / 0</strong>
<span class="muted" id="ph-message"></span>
</div>
<dl class="browser-display" id="ph-details">
<dt>ID</dt>
<dd id="ph-id">-</dd>
<dt>Product ID</dt>
<dd id="ph-product-id">-</dd>
<dt>Store</dt>
<dd id="ph-source">-</dd>
<dt>Référence</dt>
<dd id="ph-reference">-</dd>
<dt>Titre produit</dt>
<dd id="ph-title">-</dd>
<dt>Prix</dt>
<dd id="ph-price">-</dd>
<dt>Frais de port</dt>
<dd id="ph-shipping">-</dd>
<dt>Stock</dt>
<dd id="ph-stock">-</dd>
<dt>Méthode</dt>
<dd id="ph-method">-</dd>
<dt>Statut</dt>
<dd id="ph-status">-</dd>
<dt>Date scraping</dt>
<dd id="ph-fetched-at">-</dd>
</dl>
</div>
</section>
</main>
<script>
document.addEventListener("DOMContentLoaded", () => {
const loadBtn = document.getElementById("load-products");
const prevBtn = document.getElementById("product-prev");
const nextBtn = document.getElementById("product-next");
const indicator = document.getElementById("product-indicator");
const message = document.getElementById("product-message");
const titleEl = document.getElementById("product-title");
const fields = Array.from(document.querySelectorAll("[data-field]")).reduce((acc, el) => {
acc[el.getAttribute("data-field")] = el;
return acc;
}, {});
let products = [];
let cursor = 0;
const setStatus = (text) => {
message.textContent = text || "";
};
const renderProduct = () => {
if (!products.length) {
indicator.textContent = "0 / 0";
titleEl.textContent = "-";
Object.values(fields).forEach((el) => (el.textContent = "-"));
prevBtn.disabled = true;
nextBtn.disabled = true;
return;
}
const current = products[cursor];
indicator.textContent = `${cursor + 1} / ${products.length}`;
titleEl.textContent = current.title || "Sans titre";
const mapField = {
store: current.source,
reference: current.reference,
price: current.price !== null && current.price !== undefined ? current.price : "n/a",
currency: current.currency || "EUR",
msrp: current.msrp || "-",
stock_status: current.stock_status || "n/a",
category: current.category || "n/a",
description: (current.description || "n/a").slice(0, 200),
last_updated_at: current.last_updated_at || "n/a",
fetched_at: current.fetched_at || "n/a",
};
Object.entries(mapField).forEach(([key, value]) => {
if (fields[key]) {
fields[key].textContent = value;
}
});
prevBtn.disabled = cursor === 0;
nextBtn.disabled = cursor >= products.length - 1;
};
const fetchProducts = async () => {
setStatus("Chargement…");
try {
const response = await fetch("/products.json");
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
const data = await response.json();
if (!Array.isArray(data)) {
throw new Error("Réponse invalide");
}
products = data;
cursor = 0;
setStatus(`Chargé ${products.length} produit(s)`);
renderProduct();
} catch (err) {
setStatus(`Erreur: ${err.message}`);
products = [];
renderProduct();
}
};
loadBtn.addEventListener("click", fetchProducts);
prevBtn.addEventListener("click", () => {
if (cursor > 0) {
cursor -= 1;
renderProduct();
}
});
nextBtn.addEventListener("click", () => {
if (cursor + 1 < products.length) {
cursor += 1;
renderProduct();
}
});
// Historique des scraps
const loadHistoryBtn = document.getElementById("load-history");
const historyMessage = document.getElementById("history-message");
const historyBody = document.getElementById("history-body");
const setHistoryStatus = (text) => {
historyMessage.textContent = text || "";
};
const formatStock = (status) => {
const stockMap = {
"in_stock": "✓ En stock",
"out_of_stock": "✗ Rupture",
"limited": "⚠ Limité",
"preorder": "⏳ Précommande",
"unknown": "? Inconnu"
};
return stockMap[status] || status || "-";
};
const formatMethod = (method) => {
return method === "playwright" ? "🎭 Playwright" : "📡 HTTP";
};
const formatStatus = (status) => {
const statusMap = {
"success": "✓ Succès",
"partial": "⚠ Partiel",
"failed": "✗ Échec"
};
return statusMap[status] || status || "-";
};
const renderHistory = (history) => {
if (!history.length) {
historyBody.innerHTML = '<tr><td colspan="6" class="muted">Aucun historique disponible pour ce produit.</td></tr>';
return;
}
historyBody.innerHTML = history.map(entry => `
<tr>
<td>${entry.fetched_at || "-"}</td>
<td>${entry.price !== null ? entry.price + "" : "-"}</td>
<td>${entry.shipping_cost !== null ? entry.shipping_cost + "" : "-"}</td>
<td>${formatStock(entry.stock_status)}</td>
<td>${formatMethod(entry.fetch_method)}</td>
<td>${formatStatus(entry.fetch_status)}</td>
</tr>
`).join("");
};
const fetchHistory = async () => {
if (!products.length) {
setHistoryStatus("Chargez d'abord les produits.");
return;
}
const current = products[cursor];
if (!current || !current.id) {
setHistoryStatus("Aucun produit sélectionné.");
return;
}
setHistoryStatus(`Chargement de l'historique pour le produit #${current.id}…`);
try {
const response = await fetch(`/product/${current.id}/history.json`);
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
const data = await response.json();
if (!Array.isArray(data)) {
throw new Error("Réponse invalide");
}
setHistoryStatus(`${data.length} entrée(s) pour "${(current.title || "Sans titre").slice(0, 30)}…"`);
renderHistory(data);
} catch (err) {
setHistoryStatus(`Erreur: ${err.message}`);
historyBody.innerHTML = '<tr><td colspan="6" class="muted">Erreur lors du chargement.</td></tr>';
}
};
loadHistoryBtn.addEventListener("click", fetchHistory);
// Parcourir price_history
const loadPhBtn = document.getElementById("load-price-history");
const phPrevBtn = document.getElementById("ph-prev");
const phNextBtn = document.getElementById("ph-next");
const phIndicator = document.getElementById("ph-indicator");
const phMessage = document.getElementById("ph-message");
let priceHistoryData = [];
let phCursor = 0;
const setPhStatus = (text) => {
phMessage.textContent = text || "";
};
const renderPriceHistory = () => {
const els = {
id: document.getElementById("ph-id"),
productId: document.getElementById("ph-product-id"),
source: document.getElementById("ph-source"),
reference: document.getElementById("ph-reference"),
title: document.getElementById("ph-title"),
price: document.getElementById("ph-price"),
shipping: document.getElementById("ph-shipping"),
stock: document.getElementById("ph-stock"),
method: document.getElementById("ph-method"),
status: document.getElementById("ph-status"),
fetchedAt: document.getElementById("ph-fetched-at"),
};
if (!priceHistoryData.length) {
phIndicator.textContent = "0 / 0";
Object.values(els).forEach((el) => (el.textContent = "-"));
phPrevBtn.disabled = true;
phNextBtn.disabled = true;
return;
}
const current = priceHistoryData[phCursor];
phIndicator.textContent = `${phCursor + 1} / ${priceHistoryData.length}`;
els.id.textContent = current.id || "-";
els.productId.textContent = current.product_id || "-";
els.source.textContent = current.source || "-";
els.reference.textContent = current.reference || "-";
els.title.textContent = current.title ? (current.title.length > 60 ? current.title.slice(0, 60) + "" : current.title) : "-";
els.price.textContent = current.price !== null ? current.price + "" : "-";
els.shipping.textContent = current.shipping_cost !== null ? current.shipping_cost + "" : "-";
els.stock.textContent = formatStock(current.stock_status);
els.method.textContent = formatMethod(current.fetch_method);
els.status.textContent = formatStatus(current.fetch_status);
els.fetchedAt.textContent = current.fetched_at || "-";
phPrevBtn.disabled = phCursor === 0;
phNextBtn.disabled = phCursor >= priceHistoryData.length - 1;
};
const fetchPriceHistory = async () => {
setPhStatus("Chargement…");
try {
const response = await fetch("/price_history.json");
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
const data = await response.json();
if (!Array.isArray(data)) {
throw new Error("Réponse invalide");
}
priceHistoryData = data;
phCursor = 0;
setPhStatus(`Chargé ${priceHistoryData.length} entrée(s)`);
renderPriceHistory();
} catch (err) {
setPhStatus(`Erreur: ${err.message}`);
priceHistoryData = [];
renderPriceHistory();
}
};
loadPhBtn.addEventListener("click", fetchPriceHistory);
phPrevBtn.addEventListener("click", () => {
if (phCursor > 0) {
phCursor -= 1;
renderPriceHistory();
}
});
phNextBtn.addEventListener("click", () => {
if (phCursor + 1 < priceHistoryData.length) {
phCursor += 1;
renderPriceHistory();
}
});
});
</script>
</body>
</html>
"""
@app.route("/")
def root():
metrics, db_error = fetch_db_metrics()
redis_status, redis_error = check_redis()
return render_template_string(
TEMPLATE,
metrics=metrics,
db_status="connecté" if db_error is None else "erreur",
db_error=db_error,
redis_status=redis_status,
redis_error=redis_error,
)
@app.route("/products.json")
def products_json():
products, error = fetch_products_list()
if error:
return jsonify({"error": error}), 500
return jsonify(products)
@app.route("/product/<int:product_id>/history.json")
def product_history_json(product_id: int):
history, error = fetch_product_history(product_id)
if error:
return jsonify({"error": error}), 500
return jsonify(history)
@app.route("/price_history.json")
def all_price_history_json():
history, error = fetch_all_price_history()
if error:
return jsonify({"error": error}), 500
return jsonify(history)
if __name__ == "__main__":
app.run(host="0.0.0.0", port=80)