From 82f3464565e246313d146f1e266cbf0fde0113ef Mon Sep 17 00:00:00 2001 From: alexandrump Date: Mon, 9 Feb 2026 01:02:53 +0100 Subject: [PATCH] first commit --- .dockerignore | 6 + .gitignore | 11 + PLAN.md | 251 ++++++++ README.md | 56 ++ apps/api/.env.example | 4 + apps/api/Dockerfile | 15 + apps/api/README.md | 5 + apps/api/package.json | 37 ++ apps/api/src/admin/admin-alerts.controller.ts | 20 + apps/api/src/admin/admin-backup.controller.ts | 19 + .../api/src/admin/admin-monitor.controller.ts | 65 +++ apps/api/src/admin/admin-plans.controller.ts | 24 + apps/api/src/admin/admin-users.controller.ts | 42 ++ apps/api/src/admin/admin.controller.ts | 38 ++ apps/api/src/admin/admin.module.ts | 46 ++ apps/api/src/alerts/alerts.controller.ts | 44 ++ apps/api/src/alerts/alerts.module.ts | 23 + apps/api/src/alerts/alerts.scheduler.ts | 25 + apps/api/src/alerts/alerts.service.ts | 210 +++++++ apps/api/src/alerts/mailer.service.ts | 28 + apps/api/src/alerts/telegram.service.ts | 25 + apps/api/src/app.controller.ts | 12 + apps/api/src/app.module.ts | 40 ++ apps/api/src/app.service.ts | 8 + apps/api/src/auth/admin.guard.ts | 44 ++ apps/api/src/auth/auth.controller.ts | 23 + apps/api/src/auth/auth.module.ts | 31 + apps/api/src/auth/auth.service.ts | 46 ++ apps/api/src/auth/jwt.strategy.ts | 19 + apps/api/src/auth/user.entity.ts | 30 + apps/api/src/backup/backup.module.ts | 32 + apps/api/src/backup/backup.scheduler.ts | 24 + apps/api/src/backup/backup.service.ts | 140 +++++ apps/api/src/catalog/catalog.controller.ts | 34 ++ apps/api/src/catalog/catalog.module.ts | 14 + apps/api/src/catalog/catalog.service.ts | 143 +++++ apps/api/src/catalog/dto/catalog-item.dto.ts | 9 + .../catalog/dto/create-catalog-item.dto.ts | 8 + apps/api/src/classifier/classifier.module.ts | 11 + apps/api/src/classifier/classifier.service.ts | 77 +++ .../api/src/dashboard/dashboard.controller.ts | 104 ++++ apps/api/src/dashboard/dashboard.module.ts | 12 + .../api/src/discovery/discovery.controller.ts | 63 ++ apps/api/src/discovery/discovery.module.ts | 11 + .../api/src/entities/alert-delivery.entity.ts | 25 + apps/api/src/entities/alert-profile.entity.ts | 45 ++ apps/api/src/entities/alert-run.entity.ts | 28 + .../entities/catalog-item-version.entity.ts | 19 + apps/api/src/entities/catalog-item.entity.ts | 31 + .../src/entities/classification-tag.entity.ts | 19 + apps/api/src/entities/ingest-run.entity.ts | 31 + apps/api/src/entities/plan.entity.ts | 34 ++ apps/api/src/ingest/ingest.controller.ts | 25 + apps/api/src/ingest/ingest.module.ts | 30 + apps/api/src/ingest/ingest.processor.ts | 18 + apps/api/src/ingest/ingest.scheduler.ts | 25 + apps/api/src/ingest/ingest.service.ts | 105 ++++ apps/api/src/main.ts | 14 + apps/api/src/normalizer/normalizer.module.ts | 8 + apps/api/src/normalizer/normalizer.service.ts | 46 ++ apps/api/src/normalizer/utils.ts | 51 ++ apps/api/src/plans/plans.controller.ts | 12 + apps/api/src/plans/plans.module.ts | 13 + apps/api/src/plans/plans.service.ts | 80 +++ apps/api/tsconfig.json | 14 + apps/web/Dockerfile | 13 + apps/web/README.md | 5 + apps/web/components/Header.vue | 90 +++ apps/web/layouts/default.vue | 12 + apps/web/nuxt.config.ts | 21 + apps/web/package.json | 13 + apps/web/pages/admin.vue | 549 ++++++++++++++++++ apps/web/pages/alerts.vue | 290 +++++++++ apps/web/pages/catalog.vue | 129 ++++ apps/web/pages/discover.vue | 150 +++++ apps/web/pages/index.vue | 271 +++++++++ apps/web/pages/ingest.vue | 110 ++++ apps/web/pages/landing.vue | 163 ++++++ apps/web/pages/plans.vue | 95 +++ coolify.yml | 35 ++ docker-compose.override.yml | 31 + docker-compose.yml | 54 ++ docs/ARCHITECTURE.md | 29 + docs/DEPLOYMENT.md | 46 ++ docs/KPIS.md | 17 + docs/OPERATIONS.md | 19 + package.json | 18 + packages/shared/package.json | 7 + pnpm-workspace.yaml | 3 + turbo.json | 16 + 90 files changed, 4788 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 PLAN.md create mode 100644 README.md create mode 100644 apps/api/.env.example create mode 100644 apps/api/Dockerfile create mode 100644 apps/api/README.md create mode 100644 apps/api/package.json create mode 100644 apps/api/src/admin/admin-alerts.controller.ts create mode 100644 apps/api/src/admin/admin-backup.controller.ts create mode 100644 apps/api/src/admin/admin-monitor.controller.ts create mode 100644 apps/api/src/admin/admin-plans.controller.ts create mode 100644 apps/api/src/admin/admin-users.controller.ts create mode 100644 apps/api/src/admin/admin.controller.ts create mode 100644 apps/api/src/admin/admin.module.ts create mode 100644 apps/api/src/alerts/alerts.controller.ts create mode 100644 apps/api/src/alerts/alerts.module.ts create mode 100644 apps/api/src/alerts/alerts.scheduler.ts create mode 100644 apps/api/src/alerts/alerts.service.ts create mode 100644 apps/api/src/alerts/mailer.service.ts create mode 100644 apps/api/src/alerts/telegram.service.ts create mode 100644 apps/api/src/app.controller.ts create mode 100644 apps/api/src/app.module.ts create mode 100644 apps/api/src/app.service.ts create mode 100644 apps/api/src/auth/admin.guard.ts create mode 100644 apps/api/src/auth/auth.controller.ts create mode 100644 apps/api/src/auth/auth.module.ts create mode 100644 apps/api/src/auth/auth.service.ts create mode 100644 apps/api/src/auth/jwt.strategy.ts create mode 100644 apps/api/src/auth/user.entity.ts create mode 100644 apps/api/src/backup/backup.module.ts create mode 100644 apps/api/src/backup/backup.scheduler.ts create mode 100644 apps/api/src/backup/backup.service.ts create mode 100644 apps/api/src/catalog/catalog.controller.ts create mode 100644 apps/api/src/catalog/catalog.module.ts create mode 100644 apps/api/src/catalog/catalog.service.ts create mode 100644 apps/api/src/catalog/dto/catalog-item.dto.ts create mode 100644 apps/api/src/catalog/dto/create-catalog-item.dto.ts create mode 100644 apps/api/src/classifier/classifier.module.ts create mode 100644 apps/api/src/classifier/classifier.service.ts create mode 100644 apps/api/src/dashboard/dashboard.controller.ts create mode 100644 apps/api/src/dashboard/dashboard.module.ts create mode 100644 apps/api/src/discovery/discovery.controller.ts create mode 100644 apps/api/src/discovery/discovery.module.ts create mode 100644 apps/api/src/entities/alert-delivery.entity.ts create mode 100644 apps/api/src/entities/alert-profile.entity.ts create mode 100644 apps/api/src/entities/alert-run.entity.ts create mode 100644 apps/api/src/entities/catalog-item-version.entity.ts create mode 100644 apps/api/src/entities/catalog-item.entity.ts create mode 100644 apps/api/src/entities/classification-tag.entity.ts create mode 100644 apps/api/src/entities/ingest-run.entity.ts create mode 100644 apps/api/src/entities/plan.entity.ts create mode 100644 apps/api/src/ingest/ingest.controller.ts create mode 100644 apps/api/src/ingest/ingest.module.ts create mode 100644 apps/api/src/ingest/ingest.processor.ts create mode 100644 apps/api/src/ingest/ingest.scheduler.ts create mode 100644 apps/api/src/ingest/ingest.service.ts create mode 100644 apps/api/src/main.ts create mode 100644 apps/api/src/normalizer/normalizer.module.ts create mode 100644 apps/api/src/normalizer/normalizer.service.ts create mode 100644 apps/api/src/normalizer/utils.ts create mode 100644 apps/api/src/plans/plans.controller.ts create mode 100644 apps/api/src/plans/plans.module.ts create mode 100644 apps/api/src/plans/plans.service.ts create mode 100644 apps/api/tsconfig.json create mode 100644 apps/web/Dockerfile create mode 100644 apps/web/README.md create mode 100644 apps/web/components/Header.vue create mode 100644 apps/web/layouts/default.vue create mode 100644 apps/web/nuxt.config.ts create mode 100644 apps/web/package.json create mode 100644 apps/web/pages/admin.vue create mode 100644 apps/web/pages/alerts.vue create mode 100644 apps/web/pages/catalog.vue create mode 100644 apps/web/pages/discover.vue create mode 100644 apps/web/pages/index.vue create mode 100644 apps/web/pages/ingest.vue create mode 100644 apps/web/pages/landing.vue create mode 100644 apps/web/pages/plans.vue create mode 100644 coolify.yml create mode 100644 docker-compose.override.yml create mode 100644 docker-compose.yml create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/DEPLOYMENT.md create mode 100644 docs/KPIS.md create mode 100644 docs/OPERATIONS.md create mode 100644 package.json create mode 100644 packages/shared/package.json create mode 100644 pnpm-workspace.yaml create mode 100644 turbo.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b6769ed --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +node_modules +dist +.turbo +.env +.env.* +**/node_modules diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4ad9999 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +node_modules/ +dist/ +.turbo/ +.env +/.DS_Store +node_modules +dist +.env +.DS_Store +/.turbo +/.nuxt diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..66a3f3e --- /dev/null +++ b/PLAN.md @@ -0,0 +1,251 @@ +# PLAN - Radar automático de datos públicos (gob-alert) + +## Resumen +- **Idea**: Radar automático sobre datos.gob.es — detección, curación y normalización. +- **Stack elegido**: Nuxt (frontend), Nest (API), con turborepo monorepo. +- **Clientes objetivo**: consultoras/gestorías, pymes, universidades. + +## Objetivo del MVP +- Indexar catálogo y guardar historial. +- Detectar nuevos/actualizados datasets. +- Normalización básica y perfiles de alerta. +- Dashboard simple y alertas por Telegram/email. + +## MVP 30 días + +### Semana 1 — Catálogo e historización +- Indexar `datos.gob.es` (API + SPARQL). +- Guardar metadatos y versiones en Postgres. + +### Semana 2 — Clasificación y normalización +- Clasificadores (organismo, territorio, tema). +- Normalizadores CSV/JSON → esquema común. + +### Semana 3 — Alertas y perfiles +- Perfiles no técnicos, motor de reglas, entrega (Telegram/email). + +### Semana 4 — Dashboard y pilotos +- Dashboard: novedades, cambios, tendencias. +- Onboard 3 pilotos y recoger feedback. + +## Arquitectura (2 mini-PCs) + +- **Mini-PC 1 — Ingesta & procesamiento**: workers Python (cron/queues), ingesta API/SPARQL, normalización, Postgres. +- **Mini-PC 2 — Producto**: `Nest` API, `Nuxt` dashboard, alertas (Telegram/SMTP), backups. +- **Comunicación**: REST + colas ligeras (Redis opcional). + +## Componentes clave +- Ingestor, Normalizador, Clasificador, Sistema de Alertas, Dashboard, Admin (planes/usuarios). + +## Monetización +- Plan Básico 15€/mes, Pro 39€/mes, Empresa 99€/mes. + +## KPIs iniciales +- Tiempo medio de detección, cobertura del catálogo, tasa de alertas útiles, conversión piloto→cliente. + +## Roadmap 6 meses +- Integrar BOE y portales autonómicos, ML para clasificación, exportes e integraciones B2B. + +--- + +Este documento complementa la `TODO` principal y sirve como referencia rápida para el MVP. + +# PLAN — Radar automático de datos públicos + +## Resumen +- Idea: Radar automático de datos públicos — alertas y normalización sobre datos.gob.es. +- Valor: Detección temprana + curación + normalización; entregas accionables, no datos crudos. +- Clientes objetivo: consultoras/gestorías, pymes, universidades/proyectos. + +## Objetivo del MVP +- Indexar catálogo y guardar histórico. +- Detectar nuevos/actualizados datasets. +- Normalización básica y perfiles de alertas. +- Dashboard simple + alertas Telegram/email. +- Piloto con 3 clientes en 30 días. + +## MVP 30 Días (entregables por semana) +- Semana 1 — Catálogo e historización: indexar datos.gob.es (API + SPARQL), almacenar metadatos y versiones en Postgres, endpoint interno para consultar catálogo. +- Semana 2 — Clasificación y normalización básica: clasificadores por organismo/territorio/tema, normalizadores CSV→esquema común (fechas, importes, provincias). +- Semana 3 — Alertas y perfiles: UI mínima para definir perfiles, motor de reglas, entrega por Telegram/email. +- Semana 4 — Dashboard y pilotos: dashboard con novedades/cambios/tendencias, onboarding 3 clientes piloto, ajustes según feedback. + +## Arquitectura (2 mini-PCs) +- Mini-PC 1 — Ingesta y procesamiento: Python workers (cron/queues), consumidores API+SPARQL, normalizadores, Postgres (principal), SQLite auxiliar para caches. +- Mini-PC 2 — Producto y entrega: Nest (API pública), frontend (Nuxt) dashboard, servicios de alertas (Telegram bot, SMTP), backups y monitorización. +- Comunicación: API HTTP/REST + colas ligeras (Redis opcional). +- Stack recomendado: Nuxt (frontend), Nest (backend), Python para ingestion, Playwright solo si PDF scraping necesario. + +## Componentes clave +- Ingestor: jobs programados, detección de cambios, versionado. +- Normalizador: mapeos configurables, transformaciones reutilizables. +- Clasificador: reglas + etiquetas (organismo, tema, territorio). +- Alerta: perfiles de usuario, reglas de entrega, deduplicación. +- Dashboard: novedades, cambios, tendencias, histórico. +- Admin: usuarios, planes, facturación, logs. + +## Monetización +- Plan Básico – 15€/mes +- Plan Pro – 39€/mes +- Plan Empresa – 99€/mes + +## KPIs iniciales +- Tiempo medio hasta detectar un nuevo dataset. +- Cobertura del catálogo (% datasets indexados). +- Tasa de alertas útiles (feedback piloto). +- Conversiones piloto→pagos. + +## Riesgos y mitigaciones +- Datos heterogéneos: empezar con CSV/JSON. +- Dependencia API: historizar metadatos y tener plan B. +- Escalabilidad en mini-PCs: diseñar componentes desacoplados. + +## Roadmap 6 meses +- Integrar BOE y portales autonómicos. +- ML para clasificación automática de temas. +- Exportes y reportes automáticos. +- Integraciones B2B (SFTP, webhooks). + +--- + +Archivo creado automáticamente: estructura inicial de monorepo sugerida en `README.md`. + +## Estado de tareas (resumen) + +- [x] Definir alcance y métricas +- [x] Diseñar arquitectura 2 mini-PCs +- [x] Indexar catálogo datos.gob.es +- [x] Normalización y enriquecimiento (en progreso) +- [x] Implementar motor de descubrimiento +- [x] Sistema de alertas profesionales +- [x] Panel MVP (Dashboard) +- [x] Auth, usuarios y planes (en progreso) +- [ ] Piloto con clientes +- [x] Monetización y política de precios +- [x] Monitorización, logs y backups +- [x] Documentación y landing +- [ ] Pruebas y despliegue en mini-PCs +- [ ] Iteración y roadmap 6 meses + +### Tareas ya realizadas + +- [x] Esqueleto Nest en `apps/api` +- [x] Dev `docker-compose.override.yml` (desarrollo) +- [x] Ingestor básico (parcial) +- [x] Normalización y enriquecimiento (parcial) +- [x] Versionado / Histórico +- [x] Programar ingesta periódica (scheduler) +- [x] Admin endpoints for scheduler (pause/resume) +- [x] Queue-based ingestion (Bull + Redis) +- [x] Secure admin endpoints and frontend integration (API key + JWT fallback) + +``` # PLAN - Radar automático de datos públicos (gob-alert) + +## Resumen +- **Idea**: Radar automático sobre datos.gob.es — detección, curación y normalización. +- **Stack elegido**: Nuxt (frontend), Nest (API), con turborepo monorepo. +- **Clientes objetivo**: consultoras/gestorías, pymes, universidades. + +## Objetivo del MVP +- Indexar catálogo y guardar historial. +- Detectar nuevos/actualizados datasets. +- Normalización básica y perfiles de alerta. +- Dashboard simple y alertas por Telegram/email. + +## MVP 30 días + +### Semana 1 — Catálogo e historización +- Indexar `datos.gob.es` (API + SPARQL). +- Guardar metadatos y versiones en Postgres. + +### Semana 2 — Clasificación y normalización +- Clasificadores (organismo, territorio, tema). +- Normalizadores CSV/JSON → esquema común. + +### Semana 3 — Alertas y perfiles +- Perfiles no técnicos, motor de reglas, entrega (Telegram/email). + +### Semana 4 — Dashboard y pilotos +- Dashboard: novedades, cambios, tendencias. +- Onboard 3 pilotos y recoger feedback. + +## Arquitectura (2 mini-PCs) + +- **Mini-PC 1 — Ingesta & procesamiento**: workers Python (cron/queues), ingesta API/SPARQL, normalización, Postgres. +- **Mini-PC 2 — Producto**: `Nest` API, `Nuxt` dashboard, alertas (Telegram/SMTP), backups. +- **Comunicación**: REST + colas ligeras (Redis opcional). + +## Componentes clave +- Ingestor, Normalizador, Clasificador, Sistema de Alertas, Dashboard, Admin (planes/usuarios). + +## Monetización +- Plan Básico 15€/mes, Pro 39€/mes, Empresa 99€/mes. + +## KPIs iniciales +- Tiempo medio de detección, cobertura del catálogo, tasa de alertas útiles, conversión piloto→cliente. + +## Roadmap 6 meses +- Integrar BOE y portales autonómicos, ML para clasificación, exportes e integraciones B2B. + +--- + +Este documento complementa la `TODO` principal y sirve como referencia rápida para el MVP. +# PLAN — Radar automático de datos públicos + +## Resumen +- Idea: Radar automático de datos públicos — alertas y normalización sobre datos.gob.es. +- Valor: Detección temprana + curación + normalización; entregas accionables, no datos crudos. +- Clientes objetivo: consultoras/gestorías, pymes, universidades/proyectos. + +## Objetivo del MVP +- Indexar catálogo y guardar histórico. +- Detectar nuevos/actualizados datasets. +- Normalización básica y perfiles de alertas. +- Dashboard simple + alertas Telegram/email. +- Piloto con 3 clientes en 30 días. + +## MVP 30 Días (entregables por semana) +- Semana 1 — Catálogo e historización: indexar datos.gob.es (API + SPARQL), almacenar metadatos y versiones en Postgres, endpoint interno para consultar catálogo. +- Semana 2 — Clasificación y normalización básica: clasificadores por organismo/territorio/tema, normalizadores CSV→esquema común (fechas, importes, provincias). +- Semana 3 — Alertas y perfiles: UI mínima para definir perfiles, motor de reglas, entrega por Telegram/email. +- Semana 4 — Dashboard y pilotos: dashboard con novedades/cambios/tendencias, onboarding 3 clientes piloto, ajustes según feedback. + +## Arquitectura (2 mini-PCs) +- Mini-PC 1 — Ingesta y procesamiento: Python workers (cron/queues), consumidores API+SPARQL, normalizadores, Postgres (principal), SQLite auxiliar para caches. +- Mini-PC 2 — Producto y entrega: Nest (API pública), frontend (Nuxt) dashboard, servicios de alertas (Telegram bot, SMTP), backups y monitorización. +- Comunicación: API HTTP/REST + colas ligeras (Redis opcional). +- Stack recomendado: Nuxt (frontend), Nest (backend), Python para ingestion, Playwright solo si PDF scraping necesario. + +## Componentes clave +- Ingestor: jobs programados, detección de cambios, versionado. +- Normalizador: mapeos configurables, transformaciones reutilizables. +- Clasificador: reglas + etiquetas (organismo, tema, territorio). +- Alerta: perfiles de usuario, reglas de entrega, deduplicación. +- Dashboard: novedades, cambios, tendencias, histórico. +- Admin: usuarios, planes, facturación, logs. + +## Monetización +- Plan Básico – 15€/mes +- Plan Pro – 39€/mes +- Plan Empresa – 99€/mes + +## KPIs iniciales +- Tiempo medio hasta detectar un nuevo dataset. +- Cobertura del catálogo (% datasets indexados). +- Tasa de alertas útiles (feedback piloto). +- Conversiones piloto→pagos. + +## Riesgos y mitigaciones +- Datos heterogéneos: empezar con CSV/JSON. +- Dependencia API: historizar metadatos y tener plan B. +- Escalabilidad en mini-PCs: diseñar componentes desacoplados. + +## Roadmap 6 meses +- Integrar BOE y portales autonómicos. +- ML para clasificación automática de temas. +- Exportes y reportes automáticos. +- Integraciones B2B (SFTP, webhooks). + +--- + +Archivo creado automáticamente: estructura inicial de monorepo sugerida en `README.md`. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2bd4a91 --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +# gob-alert + +Monorepo inicial para el proyecto "gob-alert". + +Stack: `Nuxt` (frontend), `Nest` (API), `turborepo`. + +Instrucciones rápidas: + +1. Instalar `pnpm` (recomendado) y `turbo` si no están instalados. + +```bash +npm install -g pnpm +pnpm install +pnpm run dev +``` + +2. Desarrollo: ejecuta `pnpm run dev` (lanza `turbo run dev`). + +3. Docker / despliegue rápido (útil para Coolify o despliegue local): + +```bash +# Desde la raíz del repo +docker compose build +docker compose up -d +``` + +Coolify: este `docker-compose.yml` incluye `api`, `web`, `postgres`, `redis` y `adminer` — Coolify puede usar las imágenes construidas aquí o construir directamente desde el contexto del repo. + +Notas: +- Las apps `apps/web` y `apps/api` contienen paquetes iniciales (placeholders). +- Si quieres que genere el esqueleto completo de `Nuxt` y `Nest`, lo hago en el siguiente paso. +# gob-alert + +Monorepo skeleton for the "Radar automático de datos públicos" project. + +Structure: +- `apps/api` — minimal Nest backend +- `apps/web` — minimal Nuxt frontend +- `packages/shared` — shared types/utilities + +Quick dev (root): + +```bash +npm install +npm run dev +``` + +This will run `turbo run dev` and start both `api` and `web` in dev mode (if dependencies installed). + +See `PLAN.md` for project plan and MVP milestones. + +Documentation: +- `docs/ARCHITECTURE.md` +- `docs/DEPLOYMENT.md` +- `docs/OPERATIONS.md` +- `docs/KPIS.md` diff --git a/apps/api/.env.example b/apps/api/.env.example new file mode 100644 index 0000000..a50b01f --- /dev/null +++ b/apps/api/.env.example @@ -0,0 +1,4 @@ +PORT=3000 +DATABASE_URL=postgres://user:pass@localhost:5432/gob_alert +PORT=3000 +NODE_ENV=development diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile new file mode 100644 index 0000000..aaf64da --- /dev/null +++ b/apps/api/Dockerfile @@ -0,0 +1,15 @@ +FROM node:18-alpine + +WORKDIR /usr/src/app + +# install deps +COPY package*.json ./ +RUN npm install + +# copy sources and build +COPY . . +RUN npm run build + +ENV NODE_ENV=production +EXPOSE 3000 +CMD ["node", "dist/main.js"] diff --git a/apps/api/README.md b/apps/api/README.md new file mode 100644 index 0000000..846a90c --- /dev/null +++ b/apps/api/README.md @@ -0,0 +1,5 @@ +# apps/api + +Directorio para la API `Nest`. + +Contiene los scripts básicos; puedo generar el esqueleto de `Nest` (controladores, módulos, servicios) si confirmas. diff --git a/apps/api/package.json b/apps/api/package.json new file mode 100644 index 0000000..107d8ec --- /dev/null +++ b/apps/api/package.json @@ -0,0 +1,37 @@ +{ + "name": "api", + "version": "0.0.1", + "private": true, + "main": "dist/main.js", + "scripts": { + "dev": "ts-node-dev --respawn --transpile-only src/main.ts", + "build": "tsc -p tsconfig.json", + "start": "node dist/main.js" + }, + "dependencies": { + "@nestjs/bull": "^0.5.0", + "@nestjs/common": "^10.0.0", + "@nestjs/config": "^3.1.1", + "@nestjs/core": "^10.0.0", + "@nestjs/jwt": "^10.0.0", + "@nestjs/passport": "^10.0.0", + "@nestjs/platform-express": "^10.0.0", + "@nestjs/schedule": "^2.1.0", + "@nestjs/typeorm": "^10.0.0", + "axios": "^1.4.0", + "bcrypt": "^5.1.0", + "bull": "^3.29.4", + "nodemailer": "^6.9.13", + "passport": "^0.6.0", + "passport-jwt": "^4.0.1", + "pg": "^8.10.0", + "reflect-metadata": "^0.1.13", + "rxjs": "^7.8.0", + "typeorm": "^0.3.17" + }, + "devDependencies": { + "@types/nodemailer": "^6.4.14", + "ts-node-dev": "^2.0.0", + "typescript": "^4.9.5" + } +} diff --git a/apps/api/src/admin/admin-alerts.controller.ts b/apps/api/src/admin/admin-alerts.controller.ts new file mode 100644 index 0000000..a90dc88 --- /dev/null +++ b/apps/api/src/admin/admin-alerts.controller.ts @@ -0,0 +1,20 @@ +import { Controller, Get, Query, UseGuards } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { AdminAuthGuard } from '../auth/admin.guard'; +import { AlertRunEntity } from '../entities/alert-run.entity'; + +@Controller('admin/alerts') +@UseGuards(AdminAuthGuard) +export class AdminAlertsController { + constructor( + @InjectRepository(AlertRunEntity) + private readonly runs: Repository, + ) {} + + @Get('runs') + async listRuns(@Query('limit') limit?: string) { + const take = limit ? Math.min(Number(limit) || 20, 200) : 20; + return this.runs.find({ order: { startedAt: 'DESC' }, take }); + } +} diff --git a/apps/api/src/admin/admin-backup.controller.ts b/apps/api/src/admin/admin-backup.controller.ts new file mode 100644 index 0000000..2bf1500 --- /dev/null +++ b/apps/api/src/admin/admin-backup.controller.ts @@ -0,0 +1,19 @@ +import { Controller, Get, Post, UseGuards } from '@nestjs/common'; +import { AdminAuthGuard } from '../auth/admin.guard'; +import { BackupService } from '../backup/backup.service'; + +@Controller('admin/backup') +@UseGuards(AdminAuthGuard) +export class AdminBackupController { + constructor(private readonly backup: BackupService) {} + + @Post('run') + async run() { + return this.backup.runBackup(); + } + + @Get('list') + async list() { + return this.backup.listBackups(); + } +} diff --git a/apps/api/src/admin/admin-monitor.controller.ts b/apps/api/src/admin/admin-monitor.controller.ts new file mode 100644 index 0000000..d026794 --- /dev/null +++ b/apps/api/src/admin/admin-monitor.controller.ts @@ -0,0 +1,65 @@ +import { Controller, Get, UseGuards } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { AdminAuthGuard } from '../auth/admin.guard'; +import { CatalogItemEntity } from '../entities/catalog-item.entity'; +import { CatalogItemVersionEntity } from '../entities/catalog-item-version.entity'; +import { AlertProfileEntity } from '../entities/alert-profile.entity'; +import { AlertDeliveryEntity } from '../entities/alert-delivery.entity'; +import { IngestRunEntity } from '../entities/ingest-run.entity'; +import { AlertRunEntity } from '../entities/alert-run.entity'; +import { UserEntity } from '../auth/user.entity'; + +@Controller('admin/monitor') +@UseGuards(AdminAuthGuard) +export class AdminMonitorController { + constructor( + @InjectRepository(CatalogItemEntity) + private readonly items: Repository, + @InjectRepository(CatalogItemVersionEntity) + private readonly versions: Repository, + @InjectRepository(AlertProfileEntity) + private readonly profiles: Repository, + @InjectRepository(AlertDeliveryEntity) + private readonly deliveries: Repository, + @InjectRepository(IngestRunEntity) + private readonly ingestRuns: Repository, + @InjectRepository(AlertRunEntity) + private readonly alertRuns: Repository, + @InjectRepository(UserEntity) + private readonly users: Repository, + ) {} + + @Get('status') + async status() { + const [ + items, + versions, + profiles, + deliveries, + users, + lastIngest, + lastAlerts, + ] = await Promise.all([ + this.items.count(), + this.versions.count(), + this.profiles.count(), + this.deliveries.count(), + this.users.count(), + this.ingestRuns.find({ order: { startedAt: 'DESC' }, take: 1 }), + this.alertRuns.find({ order: { startedAt: 'DESC' }, take: 1 }), + ]); + + return { + counts: { + catalogItems: items, + catalogVersions: versions, + alertProfiles: profiles, + alertDeliveries: deliveries, + users, + }, + lastIngest: lastIngest[0] || null, + lastAlerts: lastAlerts[0] || null, + }; + } +} diff --git a/apps/api/src/admin/admin-plans.controller.ts b/apps/api/src/admin/admin-plans.controller.ts new file mode 100644 index 0000000..e643ba0 --- /dev/null +++ b/apps/api/src/admin/admin-plans.controller.ts @@ -0,0 +1,24 @@ +import { Controller, Get, Post, Body, Param, UseGuards } from '@nestjs/common'; +import { AdminAuthGuard } from '../auth/admin.guard'; +import { PlansService } from '../plans/plans.service'; + +@Controller('admin/plans') +@UseGuards(AdminAuthGuard) +export class AdminPlansController { + constructor(private readonly plans: PlansService) {} + + @Get() + async list() { + return this.plans.listAll(); + } + + @Post() + async create(@Body() body: any) { + return this.plans.create(body); + } + + @Post(':id') + async update(@Param('id') id: string, @Body() body: any) { + return this.plans.update(id, body); + } +} diff --git a/apps/api/src/admin/admin-users.controller.ts b/apps/api/src/admin/admin-users.controller.ts new file mode 100644 index 0000000..801457e --- /dev/null +++ b/apps/api/src/admin/admin-users.controller.ts @@ -0,0 +1,42 @@ +import { Controller, Get, Post, Body, Param, Query, UseGuards } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { AdminAuthGuard } from '../auth/admin.guard'; +import { UserEntity } from '../auth/user.entity'; + +@Controller('admin/users') +@UseGuards(AdminAuthGuard) +export class AdminUsersController { + constructor( + @InjectRepository(UserEntity) + private readonly users: Repository, + ) {} + + @Get() + async list(@Query('limit') limit?: string) { + const take = limit ? Math.min(Number(limit) || 50, 200) : 50; + return this.users.find({ + order: { createdAt: 'DESC' }, + take, + relations: ['plan'], + }); + } + + @Post(':id/plan') + async assignPlan(@Param('id') id: string, @Body() body: any) { + const user = await this.users.findOne({ where: { id } }); + if (!user) return { ok: false, error: 'User not found' }; + user.planId = body.planId || null; + await this.users.save(user); + return { ok: true, userId: user.id, planId: user.planId }; + } + + @Post(':id/role') + async updateRole(@Param('id') id: string, @Body() body: any) { + const user = await this.users.findOne({ where: { id } }); + if (!user) return { ok: false, error: 'User not found' }; + if (body.role) user.role = body.role; + await this.users.save(user); + return { ok: true, userId: user.id, role: user.role }; + } +} diff --git a/apps/api/src/admin/admin.controller.ts b/apps/api/src/admin/admin.controller.ts new file mode 100644 index 0000000..2a63ff3 --- /dev/null +++ b/apps/api/src/admin/admin.controller.ts @@ -0,0 +1,38 @@ +import { Controller, Post, HttpCode, UseGuards, Get, Query } from '@nestjs/common'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { AdminAuthGuard } from '../auth/admin.guard'; +import { IngestRunEntity } from '../entities/ingest-run.entity'; + +@Controller('admin/ingest') +@UseGuards(AdminAuthGuard) +export class AdminIngestController { + constructor( + private readonly schedulerRegistry: SchedulerRegistry, + @InjectRepository(IngestRunEntity) + private readonly runs: Repository, + ) {} + + @Post('pause') + @HttpCode(200) + pause() { + const job = this.schedulerRegistry.getCronJob('ingest_job'); + job.stop(); + return { ok: true, paused: true }; + } + + @Post('resume') + @HttpCode(200) + resume() { + const job = this.schedulerRegistry.getCronJob('ingest_job'); + job.start(); + return { ok: true, resumed: true }; + } + + @Get('runs') + async listRuns(@Query('limit') limit?: string) { + const take = limit ? Math.min(Number(limit) || 20, 200) : 20; + return this.runs.find({ order: { startedAt: 'DESC' }, take }); + } +} diff --git a/apps/api/src/admin/admin.module.ts b/apps/api/src/admin/admin.module.ts new file mode 100644 index 0000000..2aa7159 --- /dev/null +++ b/apps/api/src/admin/admin.module.ts @@ -0,0 +1,46 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { AuthModule } from '../auth/auth.module'; +import { AdminIngestController } from './admin.controller'; +import { IngestRunEntity } from '../entities/ingest-run.entity'; +import { AlertRunEntity } from '../entities/alert-run.entity'; +import { AdminAlertsController } from './admin-alerts.controller'; +import { AdminPlansController } from './admin-plans.controller'; +import { AdminUsersController } from './admin-users.controller'; +import { UserEntity } from '../auth/user.entity'; +import { PlansModule } from '../plans/plans.module'; +import { BackupModule } from '../backup/backup.module'; +import { AdminBackupController } from './admin-backup.controller'; +import { AdminMonitorController } from './admin-monitor.controller'; +import { CatalogItemEntity } from '../entities/catalog-item.entity'; +import { CatalogItemVersionEntity } from '../entities/catalog-item-version.entity'; +import { AlertProfileEntity } from '../entities/alert-profile.entity'; +import { AlertDeliveryEntity } from '../entities/alert-delivery.entity'; +import { AdminAuthGuard } from '../auth/admin.guard'; + +@Module({ + imports: [ + AuthModule, + PlansModule, + BackupModule, + TypeOrmModule.forFeature([ + IngestRunEntity, + AlertRunEntity, + UserEntity, + CatalogItemEntity, + CatalogItemVersionEntity, + AlertProfileEntity, + AlertDeliveryEntity, + ]), + ], + controllers: [ + AdminIngestController, + AdminAlertsController, + AdminPlansController, + AdminUsersController, + AdminBackupController, + AdminMonitorController, + ], + providers: [AdminAuthGuard], +}) +export class AdminModule {} diff --git a/apps/api/src/alerts/alerts.controller.ts b/apps/api/src/alerts/alerts.controller.ts new file mode 100644 index 0000000..aa7d5a1 --- /dev/null +++ b/apps/api/src/alerts/alerts.controller.ts @@ -0,0 +1,44 @@ +import { Controller, Get, Post, Body, Param, Delete } from '@nestjs/common'; +import { AlertsService } from './alerts.service'; + +@Controller('alerts') +export class AlertsController { + constructor(private readonly alerts: AlertsService) {} + + private getUserId() { + return process.env.DEFAULT_ALERT_USER || '00000000-0000-0000-0000-000000000001'; + } + + @Get('profiles') + async listProfiles() { + return this.alerts.listProfiles(this.getUserId()); + } + + @Post('profiles') + async createProfile(@Body() body: any) { + return this.alerts.createProfile(this.getUserId(), body); + } + + @Post('profiles/:id') + async updateProfile(@Param('id') id: string, @Body() body: any) { + return this.alerts.updateProfile(this.getUserId(), id, body); + } + + @Delete('profiles/:id') + async deleteProfile(@Param('id') id: string) { + return this.alerts.deleteProfile(this.getUserId(), id); + } + + @Post('run') + async runAll() { + return this.alerts.runAllActive(); + } + + @Post('run/:id') + async runProfile(@Param('id') id: string) { + const profiles = await this.alerts.listProfiles(this.getUserId()); + const profile = profiles.find((p) => p.id === id); + if (!profile) return { ok: false, error: 'Profile not found' }; + return this.alerts.runProfile(profile); + } +} diff --git a/apps/api/src/alerts/alerts.module.ts b/apps/api/src/alerts/alerts.module.ts new file mode 100644 index 0000000..5bc9abe --- /dev/null +++ b/apps/api/src/alerts/alerts.module.ts @@ -0,0 +1,23 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { AlertProfileEntity } from '../entities/alert-profile.entity'; +import { AlertDeliveryEntity } from '../entities/alert-delivery.entity'; +import { AlertRunEntity } from '../entities/alert-run.entity'; +import { AlertsService } from './alerts.service'; +import { AlertsController } from './alerts.controller'; +import { CatalogModule } from '../catalog/catalog.module'; +import { MailerService } from './mailer.service'; +import { TelegramService } from './telegram.service'; +import { AlertsScheduler } from './alerts.scheduler'; +import { ClassificationTagEntity } from '../entities/classification-tag.entity'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([AlertProfileEntity, AlertDeliveryEntity, AlertRunEntity, ClassificationTagEntity]), + CatalogModule, + ], + providers: [AlertsService, MailerService, TelegramService, AlertsScheduler], + controllers: [AlertsController], + exports: [AlertsService], +}) +export class AlertsModule {} diff --git a/apps/api/src/alerts/alerts.scheduler.ts b/apps/api/src/alerts/alerts.scheduler.ts new file mode 100644 index 0000000..9402695 --- /dev/null +++ b/apps/api/src/alerts/alerts.scheduler.ts @@ -0,0 +1,25 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { AlertsService } from './alerts.service'; + +@Injectable() +export class AlertsScheduler { + private readonly logger = new Logger(AlertsScheduler.name); + + constructor(private readonly alerts: AlertsService) {} + + @Cron(process.env.ALERTS_CRON || CronExpression.EVERY_DAY_AT_8AM, { name: 'alerts_job' }) + async handleCron() { + if (String(process.env.ALERTS_ENABLED || 'true').toLowerCase() === 'false') { + return; + } + this.logger.log('Scheduled alerts run'); + try { + const res = await this.alerts.runAllActive(); + const total = res?.results?.reduce((acc: number, r: any) => acc + (r.sent || 0), 0) || 0; + this.logger.log(`Alerts sent: ${total}`); + } catch (err: any) { + this.logger.error('Alerts scheduler failed', err?.message || err); + } + } +} diff --git a/apps/api/src/alerts/alerts.service.ts b/apps/api/src/alerts/alerts.service.ts new file mode 100644 index 0000000..e6416cd --- /dev/null +++ b/apps/api/src/alerts/alerts.service.ts @@ -0,0 +1,210 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { In, Repository } from 'typeorm'; +import { AlertProfileEntity } from '../entities/alert-profile.entity'; +import { AlertDeliveryEntity } from '../entities/alert-delivery.entity'; +import { AlertRunEntity } from '../entities/alert-run.entity'; +import { ClassificationTagEntity } from '../entities/classification-tag.entity'; +import { CatalogService } from '../catalog/catalog.service'; +import { MailerService } from './mailer.service'; +import { TelegramService } from './telegram.service'; + +@Injectable() +export class AlertsService { + private readonly logger = new Logger(AlertsService.name); + + constructor( + @InjectRepository(AlertProfileEntity) + private readonly profiles: Repository, + @InjectRepository(AlertDeliveryEntity) + private readonly deliveries: Repository, + @InjectRepository(AlertRunEntity) + private readonly runs: Repository, + @InjectRepository(ClassificationTagEntity) + private readonly tags: Repository, + private readonly catalog: CatalogService, + private readonly mailer: MailerService, + private readonly telegram: TelegramService, + ) {} + + async listProfiles(userId: string) { + return this.profiles.find({ where: { userId }, order: { createdAt: 'DESC' } }); + } + + async createProfile(userId: string, payload: Partial) { + const profile = this.profiles.create({ + userId, + name: payload.name || 'Perfil', + isActive: payload.isActive ?? true, + frequency: payload.frequency || 'daily', + channel: payload.channel || 'email', + channelTarget: payload.channelTarget, + queryText: payload.queryText, + rules: payload.rules, + }); + return this.profiles.save(profile); + } + + async updateProfile(userId: string, id: string, payload: Partial) { + const profile = await this.profiles.findOne({ where: { id, userId } }); + if (!profile) return null; + Object.assign(profile, payload); + return this.profiles.save(profile); + } + + async deleteProfile(userId: string, id: string) { + const profile = await this.profiles.findOne({ where: { id, userId } }); + if (!profile) return null; + await this.profiles.delete({ id }); + return profile; + } + + async runProfile(profile: AlertProfileEntity) { + const days = profile.rules?.updatedWithinDays; + const changes = days ? await this.catalog.listChangesSince(this.startOfDayOffset(days)) : await this.catalog.listChanges(50); + const itemIds = Array.from(new Set(changes.map((c) => c.itemId).filter(Boolean))); + const tags = itemIds.length + ? await this.tags.find({ where: { itemId: In(itemIds) } }) + : []; + const tagsByItem = new Map(); + for (const tag of tags) { + const list = tagsByItem.get(tag.itemId) || []; + list.push(tag); + tagsByItem.set(tag.itemId, list); + } + + const matches = changes.filter((c) => this.matchProfile(profile, c, tagsByItem.get(c.itemId) || [])); + + if (!matches.length) return { ok: true, sent: 0 }; + + let sent = 0; + for (const change of matches) { + const already = await this.deliveries.findOne({ where: { profileId: profile.id, itemId: change.itemId } }); + if (already) continue; + + const message = this.formatMessage(profile, change, tagsByItem.get(change.itemId) || []); + let status: 'sent' | 'failed' = 'sent'; + let details = ''; + try { + if (profile.channel === 'telegram') { + await this.telegram.send(profile.channelTarget, message); + } else { + await this.mailer.send(profile.channelTarget, `gob-alert: ${profile.name}`, message); + } + } catch (err: any) { + status = 'failed'; + details = err?.message || String(err); + this.logger.error('Alert delivery failed', details); + } + + await this.deliveries.save(this.deliveries.create({ + profileId: profile.id, + itemId: change.itemId, + channel: profile.channel, + status, + details, + })); + if (status === 'sent') sent++; + } + + return { ok: true, sent }; + } + + async runAllActive() { + const runStart = Date.now(); + const run = await this.runs.save(this.runs.create({ status: 'running' })); + const profiles = await this.profiles.find({ where: { isActive: true } }); + const results = [] as any[]; + let sentTotal = 0; + let failed = 0; + for (const profile of profiles) { + try { + const res = await this.runProfile(profile); + results.push({ profileId: profile.id, ...res }); + sentTotal += res.sent || 0; + } catch (err: any) { + failed += 1; + results.push({ profileId: profile.id, ok: false, error: err?.message || String(err) }); + } + } + run.status = failed ? 'failed' : 'success'; + run.profiles = profiles.length; + run.sent = sentTotal; + run.failed = failed; + run.finishedAt = new Date(); + run.durationMs = Date.now() - runStart; + await this.runs.save(run); + return { ok: true, runId: run.id, results }; + } + + private matchProfile(profile: AlertProfileEntity, change: any, tags: ClassificationTagEntity[]) { + if (!change || !change.snapshot) return false; + const snapshot = change.snapshot; + + if (profile.queryText) { + const text = `${snapshot.title || ''} ${snapshot.description || ''}`.toLowerCase(); + if (!text.includes(profile.queryText.toLowerCase())) return false; + } + + const rules = profile.rules || {}; + if (rules.publishers && rules.publishers.length) { + if (!snapshot.publisher) return false; + if (!rules.publishers.some((p) => snapshot.publisher.toLowerCase().includes(p.toLowerCase()))) return false; + } + if (rules.formats && rules.formats.length) { + if (!snapshot.format) return false; + const formatValue = String(snapshot.format).toLowerCase(); + if (!rules.formats.some((f) => formatValue.includes(f.toLowerCase()))) return false; + } + if (rules.updatedWithinDays) { + if (!snapshot.updatedAt) return false; + const updated = new Date(snapshot.updatedAt); + const limit = new Date(); + limit.setDate(limit.getDate() - rules.updatedWithinDays); + if (updated < limit) return false; + } + + if (rules.topics && rules.topics.length) { + if (!this.matchTags(tags, 'topic', rules.topics)) return false; + } + + if (rules.territories && rules.territories.length) { + if (!this.matchTags(tags, 'territory', rules.territories)) return false; + } + return true; + } + + private formatMessage(profile: AlertProfileEntity, change: any, tags: ClassificationTagEntity[]) { + const snap = change.snapshot || {}; + const topics = this.collectTags(tags, 'topic'); + const territories = this.collectTags(tags, 'territory'); + return [ + `Perfil: ${profile.name}`, + `Evento: ${change.eventType}`, + `Título: ${snap.title || 'sin título'}`, + snap.publisher ? `Publicador: ${snap.publisher}` : null, + topics.length ? `Temas: ${topics.join(', ')}` : null, + territories.length ? `Territorios: ${territories.join(', ')}` : null, + snap.format ? `Formato: ${snap.format}` : null, + snap.updatedAt ? `Actualizado: ${snap.updatedAt}` : null, + snap.sourceUrl ? `Fuente: ${snap.sourceUrl}` : null, + ].filter(Boolean).join('\n'); + } + + private matchTags(tags: ClassificationTagEntity[], type: ClassificationTagEntity['type'], rules: string[]) { + const values = tags.filter((t) => t.type === type).map((t) => t.value.toLowerCase()); + if (!values.length) return false; + return rules.some((rule) => values.some((value) => value.includes(rule.toLowerCase()))); + } + + private collectTags(tags: ClassificationTagEntity[], type: ClassificationTagEntity['type']) { + return tags.filter((t) => t.type === type).map((t) => t.value); + } + + private startOfDayOffset(offsetDays: number) { + const date = new Date(); + date.setHours(0, 0, 0, 0); + date.setDate(date.getDate() - offsetDays); + return date; + } +} diff --git a/apps/api/src/alerts/mailer.service.ts b/apps/api/src/alerts/mailer.service.ts new file mode 100644 index 0000000..f210b27 --- /dev/null +++ b/apps/api/src/alerts/mailer.service.ts @@ -0,0 +1,28 @@ +import { Injectable, Logger } from '@nestjs/common'; +import nodemailer from 'nodemailer'; + +@Injectable() +export class MailerService { + private readonly logger = new Logger(MailerService.name); + private readonly transport = nodemailer.createTransport({ + host: process.env.SMTP_HOST || 'localhost', + port: process.env.SMTP_PORT ? Number(process.env.SMTP_PORT) : 25, + secure: false, + auth: process.env.SMTP_USER + ? { user: process.env.SMTP_USER, pass: process.env.SMTP_PASS || '' } + : undefined, + }); + + async send(to: string | undefined, subject: string, text: string) { + if (!to) { + this.logger.warn('Mailer target missing'); + return; + } + await this.transport.sendMail({ + from: process.env.SMTP_FROM || 'gob-alert@localhost', + to, + subject, + text, + }); + } +} diff --git a/apps/api/src/alerts/telegram.service.ts b/apps/api/src/alerts/telegram.service.ts new file mode 100644 index 0000000..dfaf31e --- /dev/null +++ b/apps/api/src/alerts/telegram.service.ts @@ -0,0 +1,25 @@ +import { Injectable, Logger } from '@nestjs/common'; +import axios from 'axios'; + +@Injectable() +export class TelegramService { + private readonly logger = new Logger(TelegramService.name); + + async send(target: string | undefined, text: string) { + const token = process.env.TELEGRAM_BOT_TOKEN; + const chatId = target || process.env.TELEGRAM_DEFAULT_CHAT; + if (!token || !chatId) { + this.logger.warn('Telegram not configured'); + return; + } + try { + await axios.post(`https://api.telegram.org/bot${token}/sendMessage`, { + chat_id: chatId, + text, + }); + } catch (err: any) { + this.logger.error('Telegram send failed', err?.message || err); + throw err; + } + } +} diff --git a/apps/api/src/app.controller.ts b/apps/api/src/app.controller.ts new file mode 100644 index 0000000..834616d --- /dev/null +++ b/apps/api/src/app.controller.ts @@ -0,0 +1,12 @@ +import { Controller, Get } from '@nestjs/common'; +import { AppService } from './app.service'; + +@Controller() +export class AppController { + constructor(private readonly appService: AppService) {} + + @Get() + getHealth() { + return { ok: true, service: 'gob-alert-api' }; + } +} diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts new file mode 100644 index 0000000..f73fab0 --- /dev/null +++ b/apps/api/src/app.module.ts @@ -0,0 +1,40 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { ScheduleModule } from '@nestjs/schedule'; +import { CatalogModule } from './catalog/catalog.module'; +import { AdminModule } from './admin/admin.module'; +import { IngestModule } from './ingest/ingest.module'; +import { NormalizerModule } from './normalizer/normalizer.module'; +import { AuthModule } from './auth/auth.module'; +import { AlertsModule } from './alerts/alerts.module'; +import { ClassifierModule } from './classifier/classifier.module'; +import { DashboardModule } from './dashboard/dashboard.module'; +import { PlansModule } from './plans/plans.module'; +import { BackupModule } from './backup/backup.module'; +import { DiscoveryModule } from './discovery/discovery.module'; + +@Module({ + imports: [ + ConfigModule.forRoot({ isGlobal: true }), + TypeOrmModule.forRoot({ + type: 'postgres', + url: process.env.DATABASE_URL || 'postgres://user:pass@localhost:5432/gob_alert', + synchronize: true, + autoLoadEntities: true, + }), + ScheduleModule.forRoot(), + CatalogModule, + NormalizerModule, + AdminModule, + IngestModule, + AuthModule, + AlertsModule, + ClassifierModule, + DashboardModule, + PlansModule, + BackupModule, + DiscoveryModule, + ], +}) +export class AppModule {} diff --git a/apps/api/src/app.service.ts b/apps/api/src/app.service.ts new file mode 100644 index 0000000..19f08e9 --- /dev/null +++ b/apps/api/src/app.service.ts @@ -0,0 +1,8 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class AppService { + getHello(): string { + return 'Hello from gob-alert API'; + } +} diff --git a/apps/api/src/auth/admin.guard.ts b/apps/api/src/auth/admin.guard.ts new file mode 100644 index 0000000..0358a87 --- /dev/null +++ b/apps/api/src/auth/admin.guard.ts @@ -0,0 +1,44 @@ +import { Injectable, CanActivate, ExecutionContext, UnauthorizedException } from '@nestjs/common'; +import { JwtService } from '@nestjs/jwt'; +import { ConfigService } from '@nestjs/config'; + +@Injectable() +export class AdminAuthGuard implements CanActivate { + constructor(private readonly jwt: JwtService, private readonly config: ConfigService) {} + + async canActivate(context: ExecutionContext): Promise { + const req = context.switchToHttp().getRequest(); + const authHeader = req.headers['authorization']; + const apiKeyHeader = req.headers['x-api-key']; + const envKey = this.config.get('API_ADMIN_KEY') || process.env.API_ADMIN_KEY || 'dev-admin-key'; + + let token: string | undefined; + if (authHeader) { + const raw = Array.isArray(authHeader) ? authHeader[0] : authHeader; + if (typeof raw === 'string' && raw.startsWith('Bearer ')) token = raw.slice(7).trim(); + else if (typeof raw === 'string' && !raw.startsWith('Bearer ')) token = raw.trim(); + } + + if (token) { + try { + const secret = this.config.get('JWT_SECRET') || process.env.JWT_SECRET || 'dev-secret'; + const payload: any = this.jwt.verify(token, { secret }); + if (payload && payload.role === 'admin') { + req.user = { id: payload.sub, email: payload.email, role: payload.role }; + return true; + } + } catch (e) { + // fall through to api key check + } + } + + // fallback to API key header (x-api-key) or plain authorization header value + let provided: string | undefined; + if (apiKeyHeader) provided = Array.isArray(apiKeyHeader) ? apiKeyHeader[0] : apiKeyHeader; + if (!provided && authHeader && typeof authHeader === 'string' && !authHeader.startsWith('Bearer ')) provided = authHeader; + + if (provided && provided === envKey) return true; + + throw new UnauthorizedException('Invalid admin credentials'); + } +} diff --git a/apps/api/src/auth/auth.controller.ts b/apps/api/src/auth/auth.controller.ts new file mode 100644 index 0000000..ade4265 --- /dev/null +++ b/apps/api/src/auth/auth.controller.ts @@ -0,0 +1,23 @@ +import { Controller, Post, Body, BadRequestException } from '@nestjs/common'; +import { AuthService } from './auth.service'; + +class RegisterDto { email: string; password: string } +class LoginDto { email: string; password: string } + +@Controller('auth') +export class AuthController { + constructor(private readonly auth: AuthService) {} + + @Post('register') + async register(@Body() body: RegisterDto) { + if (!body.email || !body.password) throw new BadRequestException('email and password required'); + return this.auth.register(body.email, body.password); + } + + @Post('login') + async login(@Body() body: LoginDto) { + const user = await this.auth.validateUser(body.email, body.password); + if (!user) throw new BadRequestException('invalid credentials'); + return this.auth.login(user); + } +} diff --git a/apps/api/src/auth/auth.module.ts b/apps/api/src/auth/auth.module.ts new file mode 100644 index 0000000..dacb4ac --- /dev/null +++ b/apps/api/src/auth/auth.module.ts @@ -0,0 +1,31 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { JwtModule } from '@nestjs/jwt'; +import { PassportModule } from '@nestjs/passport'; +import { ConfigModule, ConfigService } from '@nestjs/config'; +import { AuthService } from './auth.service'; +import { AuthController } from './auth.controller'; +import { UserEntity } from './user.entity'; +import { JwtStrategy } from './jwt.strategy'; +import { PlansModule } from '../plans/plans.module'; + +@Module({ + imports: [ + ConfigModule, + TypeOrmModule.forFeature([UserEntity]), + PlansModule, + PassportModule, + JwtModule.registerAsync({ + imports: [ConfigModule], + inject: [ConfigService], + useFactory: (cfg: ConfigService) => ({ + secret: cfg.get('JWT_SECRET') || 'dev-secret', + signOptions: { expiresIn: '7d' }, + }), + }), + ], + providers: [AuthService, JwtStrategy], + controllers: [AuthController], + exports: [AuthService, JwtModule], +}) +export class AuthModule {} diff --git a/apps/api/src/auth/auth.service.ts b/apps/api/src/auth/auth.service.ts new file mode 100644 index 0000000..6fa32e8 --- /dev/null +++ b/apps/api/src/auth/auth.service.ts @@ -0,0 +1,46 @@ +import { Injectable, ConflictException, UnauthorizedException } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { UserEntity } from './user.entity'; +import * as bcrypt from 'bcrypt'; +import { JwtService } from '@nestjs/jwt'; +import { PlansService } from '../plans/plans.service'; + +@Injectable() +export class AuthService { + constructor( + @InjectRepository(UserEntity) private readonly users: Repository, + private readonly jwtService: JwtService, + private readonly plans: PlansService, + ) {} + + async register(email: string, password: string, role = 'user') { + const existing = await this.users.findOne({ where: { email } }); + if (existing) throw new ConflictException('Email already registered'); + const hash = await bcrypt.hash(password, 10); + const defaultPlan = await this.plans.getDefaultPlan(); + const user = this.users.create({ email, passwordHash: hash, role, planId: defaultPlan?.id }); + await this.users.save(user); + return { id: user.id, email: user.email, role: user.role }; + } + + async validateUser(email: string, password: string) { + const user = await this.users.findOne({ + where: { email }, + select: ['id', 'email', 'passwordHash', 'role', 'planId'], + }); + if (!user) return null; + const ok = await bcrypt.compare(password, user.passwordHash); + if (!ok) return null; + return user; + } + + async login(user: UserEntity) { + const payload = { sub: user.id, email: user.email, role: user.role }; + return { access_token: this.jwtService.sign(payload) }; + } + + async findById(id: string) { + return this.users.findOne({ where: { id } }); + } +} diff --git a/apps/api/src/auth/jwt.strategy.ts b/apps/api/src/auth/jwt.strategy.ts new file mode 100644 index 0000000..d873e87 --- /dev/null +++ b/apps/api/src/auth/jwt.strategy.ts @@ -0,0 +1,19 @@ +import { Injectable } from '@nestjs/common'; +import { PassportStrategy } from '@nestjs/passport'; +import { ExtractJwt, Strategy } from 'passport-jwt'; +import { ConfigService } from '@nestjs/config'; + +@Injectable() +export class JwtStrategy extends PassportStrategy(Strategy) { + constructor(private readonly config: ConfigService) { + super({ + jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(), + ignoreExpiration: false, + secretOrKey: config.get('JWT_SECRET') || 'dev-secret', + }); + } + + async validate(payload: any) { + return { userId: payload.sub, email: payload.email, role: payload.role }; + } +} diff --git a/apps/api/src/auth/user.entity.ts b/apps/api/src/auth/user.entity.ts new file mode 100644 index 0000000..c39ae8d --- /dev/null +++ b/apps/api/src/auth/user.entity.ts @@ -0,0 +1,30 @@ +import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, UpdateDateColumn, ManyToOne, JoinColumn } from 'typeorm'; +import { PlanEntity } from '../entities/plan.entity'; + +@Entity('users') +export class UserEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ unique: true }) + email: string; + + @Column({ nullable: false, select: false }) + passwordHash: string; + + @Column({ default: 'user' }) + role: string; // 'user' | 'admin' + + @Column({ type: 'uuid', nullable: true }) + planId?: string | null; + + @ManyToOne(() => PlanEntity, { nullable: true }) + @JoinColumn({ name: 'planId' }) + plan?: PlanEntity | null; + + @CreateDateColumn() + createdAt: Date; + + @UpdateDateColumn() + updatedAt: Date; +} diff --git a/apps/api/src/backup/backup.module.ts b/apps/api/src/backup/backup.module.ts new file mode 100644 index 0000000..b1ca045 --- /dev/null +++ b/apps/api/src/backup/backup.module.ts @@ -0,0 +1,32 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { BackupService } from './backup.service'; +import { BackupScheduler } from './backup.scheduler'; +import { CatalogItemEntity } from '../entities/catalog-item.entity'; +import { CatalogItemVersionEntity } from '../entities/catalog-item-version.entity'; +import { AlertProfileEntity } from '../entities/alert-profile.entity'; +import { AlertDeliveryEntity } from '../entities/alert-delivery.entity'; +import { AlertRunEntity } from '../entities/alert-run.entity'; +import { IngestRunEntity } from '../entities/ingest-run.entity'; +import { UserEntity } from '../auth/user.entity'; +import { PlanEntity } from '../entities/plan.entity'; +import { ClassificationTagEntity } from '../entities/classification-tag.entity'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ + CatalogItemEntity, + CatalogItemVersionEntity, + AlertProfileEntity, + AlertDeliveryEntity, + AlertRunEntity, + IngestRunEntity, + UserEntity, + PlanEntity, + ClassificationTagEntity, + ]), + ], + providers: [BackupService, BackupScheduler], + exports: [BackupService], +}) +export class BackupModule {} diff --git a/apps/api/src/backup/backup.scheduler.ts b/apps/api/src/backup/backup.scheduler.ts new file mode 100644 index 0000000..c0e0aa0 --- /dev/null +++ b/apps/api/src/backup/backup.scheduler.ts @@ -0,0 +1,24 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { BackupService } from './backup.service'; + +@Injectable() +export class BackupScheduler { + private readonly logger = new Logger(BackupScheduler.name); + + constructor(private readonly backup: BackupService) {} + + @Cron(process.env.BACKUP_CRON || CronExpression.EVERY_DAY_AT_2AM, { name: 'backup_job' }) + async handleCron() { + if (String(process.env.BACKUP_ENABLED || 'true').toLowerCase() === 'false') { + return; + } + this.logger.log('Scheduled backup triggered'); + try { + await this.backup.runBackup(); + this.logger.log('Backup completed'); + } catch (err: any) { + this.logger.error('Backup failed', err?.message || err); + } + } +} diff --git a/apps/api/src/backup/backup.service.ts b/apps/api/src/backup/backup.service.ts new file mode 100644 index 0000000..8f7d0f7 --- /dev/null +++ b/apps/api/src/backup/backup.service.ts @@ -0,0 +1,140 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { promises as fs } from 'fs'; +import path from 'path'; +import { CatalogItemEntity } from '../entities/catalog-item.entity'; +import { CatalogItemVersionEntity } from '../entities/catalog-item-version.entity'; +import { AlertProfileEntity } from '../entities/alert-profile.entity'; +import { AlertDeliveryEntity } from '../entities/alert-delivery.entity'; +import { AlertRunEntity } from '../entities/alert-run.entity'; +import { IngestRunEntity } from '../entities/ingest-run.entity'; +import { UserEntity } from '../auth/user.entity'; +import { PlanEntity } from '../entities/plan.entity'; +import { ClassificationTagEntity } from '../entities/classification-tag.entity'; + +@Injectable() +export class BackupService { + private readonly logger = new Logger(BackupService.name); + + constructor( + @InjectRepository(CatalogItemEntity) + private readonly catalogItems: Repository, + @InjectRepository(CatalogItemVersionEntity) + private readonly catalogVersions: Repository, + @InjectRepository(AlertProfileEntity) + private readonly alertProfiles: Repository, + @InjectRepository(AlertDeliveryEntity) + private readonly alertDeliveries: Repository, + @InjectRepository(AlertRunEntity) + private readonly alertRuns: Repository, + @InjectRepository(IngestRunEntity) + private readonly ingestRuns: Repository, + @InjectRepository(UserEntity) + private readonly users: Repository, + @InjectRepository(PlanEntity) + private readonly plans: Repository, + @InjectRepository(ClassificationTagEntity) + private readonly tags: Repository, + ) {} + + async runBackup() { + const dir = await this.ensureDir(); + const timestamp = this.formatTimestamp(new Date()); + const filename = `gob-alert-backup-${timestamp}.json`; + const filepath = path.join(dir, filename); + + const limit = this.getLimit(); + const [ + catalogItems, + catalogVersions, + alertProfiles, + alertDeliveries, + alertRuns, + ingestRuns, + users, + plans, + tags, + ] = await Promise.all([ + this.catalogItems.find({ take: limit }), + this.catalogVersions.find({ take: limit }), + this.alertProfiles.find({ take: limit }), + this.alertDeliveries.find({ take: limit }), + this.alertRuns.find({ take: limit }), + this.ingestRuns.find({ take: limit }), + this.users.find({ take: limit, relations: ['plan'] }), + this.plans.find({ take: limit }), + this.tags.find({ take: limit }), + ]); + + const payload = { + meta: { + createdAt: new Date().toISOString(), + limit, + }, + catalogItems, + catalogVersions, + alertProfiles, + alertDeliveries, + alertRuns, + ingestRuns, + users, + plans, + tags, + }; + + await fs.writeFile(filepath, JSON.stringify(payload, null, 2), 'utf8'); + this.logger.log(`Backup written: ${filepath}`); + + return { + ok: true, + file: filepath, + counts: { + catalogItems: catalogItems.length, + catalogVersions: catalogVersions.length, + alertProfiles: alertProfiles.length, + alertDeliveries: alertDeliveries.length, + alertRuns: alertRuns.length, + ingestRuns: ingestRuns.length, + users: users.length, + plans: plans.length, + tags: tags.length, + }, + }; + } + + async listBackups() { + const dir = await this.ensureDir(); + const entries = await fs.readdir(dir); + return entries + .filter((name) => name.startsWith('gob-alert-backup-') && name.endsWith('.json')) + .sort() + .reverse(); + } + + private async ensureDir() { + const dir = path.resolve(process.env.BACKUP_DIR || './backups'); + await fs.mkdir(dir, { recursive: true }); + return dir; + } + + private formatTimestamp(date: Date) { + const pad = (v: number) => String(v).padStart(2, '0'); + return [ + date.getFullYear(), + pad(date.getMonth() + 1), + pad(date.getDate()), + pad(date.getHours()), + pad(date.getMinutes()), + pad(date.getSeconds()), + ].join(''); + } + + private getLimit() { + const raw = process.env.BACKUP_LIMIT; + if (!raw) return undefined; + const value = Number(raw); + if (!Number.isFinite(value) || value <= 0) return undefined; + return Math.min(value, 50000); + } +} diff --git a/apps/api/src/catalog/catalog.controller.ts b/apps/api/src/catalog/catalog.controller.ts new file mode 100644 index 0000000..435a5fd --- /dev/null +++ b/apps/api/src/catalog/catalog.controller.ts @@ -0,0 +1,34 @@ +import { Controller, Get, Post, Body, Param, Query } from '@nestjs/common'; +import { CatalogService } from './catalog.service'; +import { CreateCatalogItemDto } from './dto/create-catalog-item.dto'; + +@Controller('catalog') +export class CatalogController { + constructor(private readonly catalogService: CatalogService) {} + + @Get('health') + health() { + return { ok: true, service: 'gob-alert-catalog' }; + } + + @Get() + async list() { + return this.catalogService.list(); + } + + @Get('changes') + async changes(@Query('limit') limit?: string) { + const take = limit ? Math.min(Number(limit) || 20, 200) : 20; + return this.catalogService.listChanges(take); + } + + @Post('ingest') + async ingest(@Body() dto: CreateCatalogItemDto) { + return this.catalogService.upsertWithVersion(dto); + } + + @Get(':id/versions') + async versions(@Param('id') id: string) { + return this.catalogService.getVersions(id); + } +} diff --git a/apps/api/src/catalog/catalog.module.ts b/apps/api/src/catalog/catalog.module.ts new file mode 100644 index 0000000..9fd48d6 --- /dev/null +++ b/apps/api/src/catalog/catalog.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { CatalogService } from './catalog.service'; +import { CatalogController } from './catalog.controller'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { CatalogItemEntity } from '../entities/catalog-item.entity'; +import { CatalogItemVersionEntity } from '../entities/catalog-item-version.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([CatalogItemEntity, CatalogItemVersionEntity])], + controllers: [CatalogController], + providers: [CatalogService], + exports: [CatalogService], +}) +export class CatalogModule {} diff --git a/apps/api/src/catalog/catalog.service.ts b/apps/api/src/catalog/catalog.service.ts new file mode 100644 index 0000000..60dbbf6 --- /dev/null +++ b/apps/api/src/catalog/catalog.service.ts @@ -0,0 +1,143 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { MoreThanOrEqual, Repository } from 'typeorm'; +import { CatalogItemEntity } from '../entities/catalog-item.entity'; +import { CatalogItemVersionEntity } from '../entities/catalog-item-version.entity'; +import { CreateCatalogItemDto } from './dto/create-catalog-item.dto'; + +export type CatalogUpsertResult = { + item: CatalogItemEntity; + created: boolean; + updated: boolean; +}; + +@Injectable() +export class CatalogService { + constructor( + @InjectRepository(CatalogItemEntity) + private readonly repo: Repository, + @InjectRepository(CatalogItemVersionEntity) + private readonly versions: Repository, + ) {} + + async list(): Promise { + return this.repo.find({ order: { createdAt: 'DESC' } }); + } + + async listChanges(limit = 20) { + const rows = await this.versions.find({ order: { createdAt: 'DESC' }, take: limit }); + return rows.map((r) => ({ + id: r.id, + itemId: r.itemId, + eventType: r.eventType, + snapshot: this.parseSnapshot(r.snapshot), + createdAt: r.createdAt, + })); + } + + async listChangesSince(since: Date) { + const rows = await this.versions.find({ + where: { createdAt: MoreThanOrEqual(since) }, + order: { createdAt: 'ASC' }, + }); + return rows.map((r) => ({ + id: r.id, + itemId: r.itemId, + eventType: r.eventType, + snapshot: this.parseSnapshot(r.snapshot), + createdAt: r.createdAt, + })); + } + + async create(dto: CreateCatalogItemDto): Promise { + const item = this.repo.create({ + title: dto.title, + description: dto.description, + sourceUrl: dto.sourceUrl, + format: dto.format, + publisher: dto.publisher, + updatedAt: dto.updatedAt ? new Date(dto.updatedAt) : undefined, + }); + const saved = await this.repo.save(item); + await this.saveVersion(saved, 'created', this.toSnapshot(saved)); + return saved; + } + + // Upsert with versioning: if an existing item is found and differs, save a snapshot and update + async upsertWithVersion(dto: CreateCatalogItemDto): Promise { + const where: Array> = [{ title: dto.title }]; + if (dto.sourceUrl) where.unshift({ sourceUrl: dto.sourceUrl }); + const existing = await this.repo.findOne({ where }); + + if (!existing) { + const created = await this.create(dto); + return { item: created, created: true, updated: false }; + } + + const changed = ( + existing.title !== dto.title || + (existing.description || '') !== (dto.description || '') || + (existing.format || '') !== (dto.format || '') || + (existing.publisher || '') !== (dto.publisher || '') || + ((existing.updatedAt && dto.updatedAt) + ? existing.updatedAt.toISOString() !== new Date(dto.updatedAt).toISOString() + : Boolean(existing.updatedAt) !== Boolean(dto.updatedAt)) + ); + + if (!changed) return { item: existing, created: false, updated: false }; + + await this.saveVersion(existing, 'updated', this.toSnapshot(existing)); + + existing.title = dto.title; + existing.description = dto.description; + existing.sourceUrl = dto.sourceUrl; + existing.format = dto.format; + existing.publisher = dto.publisher; + existing.updatedAt = dto.updatedAt ? new Date(dto.updatedAt) : undefined; + + const saved = await this.repo.save(existing); + return { item: saved, created: false, updated: true }; + } + + async getVersions(itemId: string) { + const rows = await this.versions.find({ where: { itemId }, order: { createdAt: 'DESC' } }); + return rows.map((r) => ({ + id: r.id, + itemId: r.itemId, + eventType: r.eventType, + snapshot: this.parseSnapshot(r.snapshot), + createdAt: r.createdAt, + })); + } + + private async saveVersion(item: CatalogItemEntity, eventType: 'created' | 'updated', snapshot: Record) { + const row = this.versions.create({ + itemId: item.id, + eventType, + snapshot: JSON.stringify(snapshot), + }); + await this.versions.save(row); + } + + private toSnapshot(item: CatalogItemEntity) { + return { + id: item.id, + title: item.title, + description: item.description, + sourceUrl: item.sourceUrl, + format: item.format, + publisher: item.publisher, + updatedAt: item.updatedAt ? item.updatedAt.toISOString() : null, + createdAt: item.createdAt ? item.createdAt.toISOString() : null, + modifiedAt: item.modifiedAt ? item.modifiedAt.toISOString() : null, + }; + } + + private parseSnapshot(snapshot: string) { + try { + return JSON.parse(snapshot); + } catch { + return { raw: snapshot }; + } + } +} diff --git a/apps/api/src/catalog/dto/catalog-item.dto.ts b/apps/api/src/catalog/dto/catalog-item.dto.ts new file mode 100644 index 0000000..69a9ddc --- /dev/null +++ b/apps/api/src/catalog/dto/catalog-item.dto.ts @@ -0,0 +1,9 @@ +export class CatalogItemDto { + id: string; + title: string; + description?: string; + sourceUrl?: string; + updatedAt?: string; + format?: string; + publisher?: string; +} diff --git a/apps/api/src/catalog/dto/create-catalog-item.dto.ts b/apps/api/src/catalog/dto/create-catalog-item.dto.ts new file mode 100644 index 0000000..4e42273 --- /dev/null +++ b/apps/api/src/catalog/dto/create-catalog-item.dto.ts @@ -0,0 +1,8 @@ +export class CreateCatalogItemDto { + title: string; + description?: string; + sourceUrl?: string; + updatedAt?: string; // ISO + format?: string; + publisher?: string; +} diff --git a/apps/api/src/classifier/classifier.module.ts b/apps/api/src/classifier/classifier.module.ts new file mode 100644 index 0000000..b25678d --- /dev/null +++ b/apps/api/src/classifier/classifier.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { ClassificationTagEntity } from '../entities/classification-tag.entity'; +import { ClassifierService } from './classifier.service'; + +@Module({ + imports: [TypeOrmModule.forFeature([ClassificationTagEntity])], + providers: [ClassifierService], + exports: [ClassifierService], +}) +export class ClassifierModule {} diff --git a/apps/api/src/classifier/classifier.service.ts b/apps/api/src/classifier/classifier.service.ts new file mode 100644 index 0000000..67e7a47 --- /dev/null +++ b/apps/api/src/classifier/classifier.service.ts @@ -0,0 +1,77 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { CatalogItemEntity } from '../entities/catalog-item.entity'; +import { ClassificationTagEntity } from '../entities/classification-tag.entity'; + +export type ClassificationResult = { + publisher?: string; + format?: string[]; + territory?: string[]; + topics?: string[]; +}; + +@Injectable() +export class ClassifierService { + constructor( + @InjectRepository(ClassificationTagEntity) + private readonly tags: Repository, + ) {} + + async classify(item: CatalogItemEntity): Promise { + const res: ClassificationResult = { + publisher: item.publisher || undefined, + format: this.splitFormats(item.format), + }; + + const text = `${item.title || ''} ${item.description || ''}`.toLowerCase(); + + const territory: string[] = []; + const topics: string[] = []; + + if (text.includes('madrid')) territory.push('Madrid'); + if (text.includes('andaluc')) territory.push('Andalucía'); + if (text.includes('catalu')) territory.push('Cataluña'); + + if (text.includes('subvenc')) topics.push('Subvenciones'); + if (text.includes('licit')) topics.push('Licitaciones'); + if (text.includes('contrat')) topics.push('Contratación'); + if (text.includes('presupuesto')) topics.push('Presupuestos'); + + res.territory = territory.length ? territory : undefined; + res.topics = topics.length ? topics : undefined; + + return res; + } + + async applyTags(item: CatalogItemEntity, classification: ClassificationResult) { + await this.tags.delete({ itemId: item.id }); + const next: ClassificationTagEntity[] = []; + + if (classification.publisher) { + next.push(this.tags.create({ itemId: item.id, type: 'publisher', value: classification.publisher })); + } + + for (const format of classification.format || []) { + next.push(this.tags.create({ itemId: item.id, type: 'format', value: format })); + } + + for (const topic of classification.topics || []) { + next.push(this.tags.create({ itemId: item.id, type: 'topic', value: topic })); + } + + for (const territory of classification.territory || []) { + next.push(this.tags.create({ itemId: item.id, type: 'territory', value: territory })); + } + + if (next.length) await this.tags.save(next); + } + + private splitFormats(value?: string): string[] { + if (!value) return []; + return value + .split(',') + .map((v) => v.trim()) + .filter(Boolean); + } +} diff --git a/apps/api/src/dashboard/dashboard.controller.ts b/apps/api/src/dashboard/dashboard.controller.ts new file mode 100644 index 0000000..02b322a --- /dev/null +++ b/apps/api/src/dashboard/dashboard.controller.ts @@ -0,0 +1,104 @@ +import { Controller, Get, Query } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { CatalogService } from '../catalog/catalog.service'; +import { AlertsService } from '../alerts/alerts.service'; +import { ClassificationTagEntity } from '../entities/classification-tag.entity'; + +@Controller('dashboard') +export class DashboardController { + constructor( + private readonly catalog: CatalogService, + private readonly alerts: AlertsService, + @InjectRepository(ClassificationTagEntity) + private readonly tags: Repository, + ) {} + + @Get('summary') + async summary(@Query('days') days?: string) { + const daysCount = this.clampDays(days ? Number(days) : 14); + const items = await this.catalog.list(); + const changes = await this.catalog.listChanges(20); + const since = this.startOfDayOffset(daysCount - 1); + const trendChanges = await this.catalog.listChangesSince(since); + const profiles = await this.alerts.listProfiles(process.env.DEFAULT_ALERT_USER || '00000000-0000-0000-0000-000000000001'); + const tags = await this.tags.find(); + + const perPublisher: Record = {}; + const perFormat: Record = {}; + const perTopic: Record = {}; + const perTerritory: Record = {}; + const trend = this.buildTrend(daysCount); + + for (const item of items) { + if (item.publisher) perPublisher[item.publisher] = (perPublisher[item.publisher] || 0) + 1; + if (item.format) { + for (const format of String(item.format).split(',')) { + const value = format.trim(); + if (!value) continue; + perFormat[value] = (perFormat[value] || 0) + 1; + } + } + } + + for (const tag of tags) { + if (tag.type === 'topic') perTopic[tag.value] = (perTopic[tag.value] || 0) + 1; + if (tag.type === 'territory') perTerritory[tag.value] = (perTerritory[tag.value] || 0) + 1; + } + + for (const change of trendChanges) { + const key = this.formatDateKey(change.createdAt); + const bucket = trend[key]; + if (!bucket) continue; + if (change.eventType === 'created') bucket.created += 1; + else bucket.updated += 1; + bucket.total += 1; + } + + return { + totals: { + items: items.length, + changes: changes.length, + alertProfiles: profiles.length, + }, + latestChanges: changes.slice(0, 8), + trend: Object.values(trend), + topPublishers: this.toSorted(perPublisher).slice(0, 6), + topFormats: this.toSorted(perFormat).slice(0, 6), + topTopics: this.toSorted(perTopic).slice(0, 6), + topTerritories: this.toSorted(perTerritory).slice(0, 6), + }; + } + + private toSorted(input: Record) { + return Object.entries(input) + .map(([label, value]) => ({ label, value })) + .sort((a, b) => b.value - a.value); + } + + private clampDays(value: number) { + if (!value || Number.isNaN(value)) return 14; + return Math.min(Math.max(value, 7), 60); + } + + private startOfDayOffset(offsetDays: number) { + const date = new Date(); + date.setHours(0, 0, 0, 0); + date.setDate(date.getDate() - offsetDays); + return date; + } + + private formatDateKey(value: Date) { + return new Date(value).toISOString().slice(0, 10); + } + + private buildTrend(days: number) { + const out: Record = {}; + for (let i = days - 1; i >= 0; i -= 1) { + const d = this.startOfDayOffset(i); + const key = this.formatDateKey(d); + out[key] = { date: key, created: 0, updated: 0, total: 0 }; + } + return out; + } +} diff --git a/apps/api/src/dashboard/dashboard.module.ts b/apps/api/src/dashboard/dashboard.module.ts new file mode 100644 index 0000000..7b3afce --- /dev/null +++ b/apps/api/src/dashboard/dashboard.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { DashboardController } from './dashboard.controller'; +import { CatalogModule } from '../catalog/catalog.module'; +import { AlertsModule } from '../alerts/alerts.module'; +import { ClassificationTagEntity } from '../entities/classification-tag.entity'; + +@Module({ + imports: [CatalogModule, AlertsModule, TypeOrmModule.forFeature([ClassificationTagEntity])], + controllers: [DashboardController], +}) +export class DashboardModule {} diff --git a/apps/api/src/discovery/discovery.controller.ts b/apps/api/src/discovery/discovery.controller.ts new file mode 100644 index 0000000..e009fa7 --- /dev/null +++ b/apps/api/src/discovery/discovery.controller.ts @@ -0,0 +1,63 @@ +import { Controller, Get, Query } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { In, Repository } from 'typeorm'; +import { CatalogService } from '../catalog/catalog.service'; +import { ClassificationTagEntity } from '../entities/classification-tag.entity'; + +@Controller('discover') +export class DiscoveryController { + constructor( + private readonly catalog: CatalogService, + @InjectRepository(ClassificationTagEntity) + private readonly tags: Repository, + ) {} + + @Get('changes') + async changes(@Query('days') days?: string, @Query('limit') limit?: string) { + const daysCount = this.clampDays(days ? Number(days) : 7); + const take = limit ? Math.min(Number(limit) || 20, 200) : 20; + const since = this.startOfDayOffset(daysCount - 1); + + let changes = await this.catalog.listChangesSince(since); + changes = changes.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()); + changes = changes.slice(0, take); + + const itemIds = Array.from(new Set(changes.map((c) => c.itemId).filter(Boolean))); + const tags = itemIds.length + ? await this.tags.find({ where: { itemId: In(itemIds) } }) + : []; + + const tagsByItem = new Map(); + for (const tag of tags) { + const list = tagsByItem.get(tag.itemId) || []; + list.push(tag); + tagsByItem.set(tag.itemId, list); + } + + return changes.map((change) => ({ + ...change, + tags: this.groupTags(tagsByItem.get(change.itemId) || []), + })); + } + + private groupTags(tags: ClassificationTagEntity[]) { + return { + topics: tags.filter((t) => t.type === 'topic').map((t) => t.value), + territories: tags.filter((t) => t.type === 'territory').map((t) => t.value), + formats: tags.filter((t) => t.type === 'format').map((t) => t.value), + publishers: tags.filter((t) => t.type === 'publisher').map((t) => t.value), + }; + } + + private clampDays(value: number) { + if (!value || Number.isNaN(value)) return 7; + return Math.min(Math.max(value, 1), 90); + } + + private startOfDayOffset(offsetDays: number) { + const date = new Date(); + date.setHours(0, 0, 0, 0); + date.setDate(date.getDate() - offsetDays); + return date; + } +} diff --git a/apps/api/src/discovery/discovery.module.ts b/apps/api/src/discovery/discovery.module.ts new file mode 100644 index 0000000..a7f147a --- /dev/null +++ b/apps/api/src/discovery/discovery.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { CatalogModule } from '../catalog/catalog.module'; +import { ClassificationTagEntity } from '../entities/classification-tag.entity'; +import { DiscoveryController } from './discovery.controller'; + +@Module({ + imports: [CatalogModule, TypeOrmModule.forFeature([ClassificationTagEntity])], + controllers: [DiscoveryController], +}) +export class DiscoveryModule {} diff --git a/apps/api/src/entities/alert-delivery.entity.ts b/apps/api/src/entities/alert-delivery.entity.ts new file mode 100644 index 0000000..27ccce9 --- /dev/null +++ b/apps/api/src/entities/alert-delivery.entity.ts @@ -0,0 +1,25 @@ +import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn } from 'typeorm'; + +@Entity('alert_deliveries') +export class AlertDeliveryEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ type: 'uuid' }) + profileId: string; + + @Column({ type: 'uuid' }) + itemId: string; + + @Column({ type: 'varchar', length: 20 }) + channel: string; + + @Column({ type: 'varchar', length: 20, default: 'sent' }) + status: 'sent' | 'failed' | 'skipped'; + + @Column({ type: 'text', nullable: true }) + details?: string; + + @CreateDateColumn() + createdAt: Date; +} diff --git a/apps/api/src/entities/alert-profile.entity.ts b/apps/api/src/entities/alert-profile.entity.ts new file mode 100644 index 0000000..7827c5e --- /dev/null +++ b/apps/api/src/entities/alert-profile.entity.ts @@ -0,0 +1,45 @@ +import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, UpdateDateColumn } from 'typeorm'; + +export type AlertChannel = 'email' | 'telegram'; + +@Entity('alert_profiles') +export class AlertProfileEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ type: 'uuid' }) + userId: string; + + @Column({ type: 'varchar', length: 120 }) + name: string; + + @Column({ type: 'boolean', default: true }) + isActive: boolean; + + @Column({ type: 'varchar', length: 20, default: 'daily' }) + frequency: 'instant' | 'daily'; + + @Column({ type: 'varchar', length: 20, default: 'email' }) + channel: AlertChannel; + + @Column({ type: 'varchar', length: 200, nullable: true }) + channelTarget?: string; + + @Column({ type: 'text', nullable: true }) + queryText?: string; + + @Column({ type: 'simple-json', nullable: true }) + rules?: { + publishers?: string[]; + formats?: string[]; + territories?: string[]; + topics?: string[]; + updatedWithinDays?: number; + }; + + @CreateDateColumn() + createdAt: Date; + + @UpdateDateColumn() + updatedAt: Date; +} diff --git a/apps/api/src/entities/alert-run.entity.ts b/apps/api/src/entities/alert-run.entity.ts new file mode 100644 index 0000000..2c4d63b --- /dev/null +++ b/apps/api/src/entities/alert-run.entity.ts @@ -0,0 +1,28 @@ +import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn } from 'typeorm'; + +@Entity('alert_runs') +export class AlertRunEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ type: 'varchar', length: 20, default: 'running' }) + status: 'running' | 'success' | 'failed'; + + @Column({ type: 'int', default: 0 }) + profiles: number; + + @Column({ type: 'int', default: 0 }) + sent: number; + + @Column({ type: 'int', default: 0 }) + failed: number; + + @Column({ type: 'int', nullable: true }) + durationMs?: number; + + @Column({ type: 'timestamp', nullable: true }) + finishedAt?: Date; + + @CreateDateColumn() + startedAt: Date; +} diff --git a/apps/api/src/entities/catalog-item-version.entity.ts b/apps/api/src/entities/catalog-item-version.entity.ts new file mode 100644 index 0000000..3d00bde --- /dev/null +++ b/apps/api/src/entities/catalog-item-version.entity.ts @@ -0,0 +1,19 @@ +import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn } from 'typeorm'; + +@Entity('catalog_item_versions') +export class CatalogItemVersionEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ type: 'uuid' }) + itemId: string; + + @Column({ type: 'varchar', length: 20, default: 'updated' }) + eventType: 'created' | 'updated'; + + @Column({ type: 'text' }) + snapshot: string; // JSON snapshot of previous state + + @CreateDateColumn() + createdAt: Date; +} diff --git a/apps/api/src/entities/catalog-item.entity.ts b/apps/api/src/entities/catalog-item.entity.ts new file mode 100644 index 0000000..84c06cc --- /dev/null +++ b/apps/api/src/entities/catalog-item.entity.ts @@ -0,0 +1,31 @@ +import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, UpdateDateColumn } from 'typeorm'; + +@Entity('catalog_items') +export class CatalogItemEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ type: 'varchar', length: 1000 }) + title: string; + + @Column({ type: 'text', nullable: true }) + description?: string; + + @Column({ type: 'varchar', length: 2000, nullable: true }) + sourceUrl?: string; + + @Column({ type: 'varchar', length: 50, nullable: true }) + format?: string; + + @Column({ type: 'varchar', length: 200, nullable: true }) + publisher?: string; + + @Column({ type: 'timestamp', nullable: true }) + updatedAt?: Date; + + @CreateDateColumn() + createdAt: Date; + + @UpdateDateColumn() + modifiedAt: Date; +} diff --git a/apps/api/src/entities/classification-tag.entity.ts b/apps/api/src/entities/classification-tag.entity.ts new file mode 100644 index 0000000..77c5fbd --- /dev/null +++ b/apps/api/src/entities/classification-tag.entity.ts @@ -0,0 +1,19 @@ +import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn } from 'typeorm'; + +@Entity('classification_tags') +export class ClassificationTagEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ type: 'uuid' }) + itemId: string; + + @Column({ type: 'varchar', length: 30 }) + type: 'publisher' | 'format' | 'topic' | 'territory'; + + @Column({ type: 'varchar', length: 120 }) + value: string; + + @CreateDateColumn() + createdAt: Date; +} diff --git a/apps/api/src/entities/ingest-run.entity.ts b/apps/api/src/entities/ingest-run.entity.ts new file mode 100644 index 0000000..16b94e0 --- /dev/null +++ b/apps/api/src/entities/ingest-run.entity.ts @@ -0,0 +1,31 @@ +import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn } from 'typeorm'; + +@Entity('ingest_runs') +export class IngestRunEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ type: 'varchar', length: 20, default: 'running' }) + status: 'running' | 'success' | 'partial' | 'failed'; + + @Column({ type: 'int', default: 0 }) + imported: number; + + @Column({ type: 'int', default: 0 }) + updated: number; + + @Column({ type: 'int', default: 0 }) + errorCount: number; + + @Column({ type: 'simple-json', nullable: true }) + errors?: string[]; + + @Column({ type: 'int', nullable: true }) + durationMs?: number; + + @Column({ type: 'timestamp', nullable: true }) + finishedAt?: Date; + + @CreateDateColumn() + startedAt: Date; +} diff --git a/apps/api/src/entities/plan.entity.ts b/apps/api/src/entities/plan.entity.ts new file mode 100644 index 0000000..5f9db99 --- /dev/null +++ b/apps/api/src/entities/plan.entity.ts @@ -0,0 +1,34 @@ +import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, UpdateDateColumn } from 'typeorm'; + +@Entity('plans') +export class PlanEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ type: 'varchar', length: 30, unique: true }) + code: string; + + @Column({ type: 'varchar', length: 80 }) + name: string; + + @Column({ type: 'int', default: 0 }) + priceMonthly: number; + + @Column({ type: 'varchar', length: 3, default: 'EUR' }) + currency: string; + + @Column({ type: 'text', nullable: true }) + description?: string; + + @Column({ type: 'simple-json', nullable: true }) + features?: Record; + + @Column({ type: 'boolean', default: true }) + isActive: boolean; + + @CreateDateColumn() + createdAt: Date; + + @UpdateDateColumn() + updatedAt: Date; +} diff --git a/apps/api/src/ingest/ingest.controller.ts b/apps/api/src/ingest/ingest.controller.ts new file mode 100644 index 0000000..e496cca --- /dev/null +++ b/apps/api/src/ingest/ingest.controller.ts @@ -0,0 +1,25 @@ +import { Controller, Get, Post, UseGuards } from '@nestjs/common'; +import { IngestService } from './ingest.service'; +import { InjectQueue } from '@nestjs/bull'; +import { Queue } from 'bull'; +import { AdminAuthGuard } from '../auth/admin.guard'; + +@Controller('ingest') +export class IngestController { + constructor( + private readonly ingestService: IngestService, + @InjectQueue('ingest') private readonly ingestQueue: Queue, + ) {} + + @Get('run') + async run() { + return this.ingestService.runOnce(); + } + + @Post('queue') + @UseGuards(AdminAuthGuard) + async queue() { + const job = await this.ingestQueue.add({}, { removeOnComplete: true, removeOnFail: false }); + return { ok: true, jobId: job.id }; + } +} diff --git a/apps/api/src/ingest/ingest.module.ts b/apps/api/src/ingest/ingest.module.ts new file mode 100644 index 0000000..c7d83ad --- /dev/null +++ b/apps/api/src/ingest/ingest.module.ts @@ -0,0 +1,30 @@ +import { Module } from '@nestjs/common'; +import { IngestService } from './ingest.service'; +import { IngestController } from './ingest.controller'; +import { CatalogModule } from '../catalog/catalog.module'; +import { NormalizerModule } from '../normalizer/normalizer.module'; +import { IngestScheduler } from './ingest.scheduler'; +import { BullModule } from '@nestjs/bull'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { IngestProcessor } from './ingest.processor'; +import { ClassifierModule } from '../classifier/classifier.module'; +import { IngestRunEntity } from '../entities/ingest-run.entity'; + +@Module({ + imports: [ + CatalogModule, + NormalizerModule, + ClassifierModule, + TypeOrmModule.forFeature([IngestRunEntity]), + BullModule.registerQueue({ + name: 'ingest', + redis: { + host: process.env.REDIS_HOST || 'redis', + port: Number(process.env.REDIS_PORT || 6379), + }, + }), + ], + providers: [IngestService, IngestScheduler, IngestProcessor], + controllers: [IngestController], +}) +export class IngestModule {} diff --git a/apps/api/src/ingest/ingest.processor.ts b/apps/api/src/ingest/ingest.processor.ts new file mode 100644 index 0000000..3a48847 --- /dev/null +++ b/apps/api/src/ingest/ingest.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { Injectable, Logger } from '@nestjs/common'; +import { IngestService } from './ingest.service'; + +@Processor('ingest') +@Injectable() +export class IngestProcessor { + private readonly logger = new Logger(IngestProcessor.name); + + constructor(private readonly ingestService: IngestService) {} + + @Process() + async handle(job: Job) { + this.logger.log(`Processing ingest job id=${job.id}`); + return this.ingestService.runOnce(); + } +} diff --git a/apps/api/src/ingest/ingest.scheduler.ts b/apps/api/src/ingest/ingest.scheduler.ts new file mode 100644 index 0000000..1029dac --- /dev/null +++ b/apps/api/src/ingest/ingest.scheduler.ts @@ -0,0 +1,25 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { IngestService } from './ingest.service'; + +@Injectable() +export class IngestScheduler { + private readonly logger = new Logger(IngestScheduler.name); + + constructor(private readonly ingestService: IngestService) {} + + // Cron expression configurable via env `INGEST_CRON` (default: every hour) + @Cron(process.env.INGEST_CRON || CronExpression.EVERY_HOUR, { name: 'ingest_job' }) + async handleCron() { + if (String(process.env.INGEST_ENABLED || 'true').toLowerCase() === 'false') { + return; + } + this.logger.log('Scheduled ingest triggered'); + try { + const res = await this.ingestService.runOnce(); + this.logger.log(`Ingest finished: imported=${res.imported} errors=${res.errors.length}`); + } catch (err: any) { + this.logger.error('Scheduled ingest failed', err?.message || err); + } + } +} diff --git a/apps/api/src/ingest/ingest.service.ts b/apps/api/src/ingest/ingest.service.ts new file mode 100644 index 0000000..0a88579 --- /dev/null +++ b/apps/api/src/ingest/ingest.service.ts @@ -0,0 +1,105 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import axios from 'axios'; +import { CatalogService } from '../catalog/catalog.service'; +import { CreateCatalogItemDto } from '../catalog/dto/create-catalog-item.dto'; +import { NormalizerService } from '../normalizer/normalizer.service'; +import { ClassifierService } from '../classifier/classifier.service'; +import { IngestRunEntity } from '../entities/ingest-run.entity'; + +@Injectable() +export class IngestService { + private readonly logger = new Logger(IngestService.name); + + constructor( + private readonly catalogService: CatalogService, + private readonly normalizer: NormalizerService, + private readonly classifier: ClassifierService, + @InjectRepository(IngestRunEntity) + private readonly runs: Repository, + ) {} + + async runOnce(): Promise<{ runId: string; imported: number; updated: number; errors: string[] }> { + const apiBase = process.env.DATOS_API_BASE || 'https://datos.gob.es/apidata/3/action/package_search'; + this.logger.log(`Running ingest against ${apiBase}`); + const runStart = Date.now(); + const run = await this.runs.save(this.runs.create({ status: 'running' })); + try { + const res = await axios.get(apiBase, { params: { q: '', rows: 20 } }); + const data = res.data; + + let rows: any[] = []; + if (Array.isArray(data)) rows = data; + else if (data?.result?.results) rows = data.result.results; + else if (data?.result && Array.isArray(data.result)) rows = data.result; + else if (data?.items) rows = data.items; + + let imported = 0; + let updated = 0; + const errors: string[] = []; + + for (const r of rows) { + try { + const dto: CreateCatalogItemDto = { + title: r.title || r.name || r.id || 'sin-titulo', + description: r.notes || r.description || r.title || '', + sourceUrl: undefined, + format: undefined, + publisher: (r.organization && (r.organization.title || r.organization.name)) || r.owner_org || r.author || undefined, + updatedAt: r.metadata_modified || r.last_modified || r.updated || undefined, + }; + + if (Array.isArray(r.resources) && r.resources.length > 0) { + dto.sourceUrl = r.resources[0].url || r.resources[0].access_url || dto.sourceUrl; + const formats = Array.from(new Set( + r.resources + .map((x: any) => (x.format || x.format_description || '').toString().toUpperCase()) + .filter(Boolean) + )); + if (formats.length) dto.format = formats.join(','); + } + + if (!dto.sourceUrl && (r.url || r.link)) dto.sourceUrl = r.url || r.link; + if (!dto.sourceUrl && r.id) dto.sourceUrl = `urn:dataset:${r.id}`; + + const normalized = await this.normalizer.normalize(dto); + const result = await this.catalogService.upsertWithVersion(normalized); + if (result.created) imported++; + if (result.updated) updated++; + + if (result.item) { + const classification = await this.classifier.classify(result.item); + await this.classifier.applyTags(result.item, classification); + } + } catch (err: any) { + this.logger.error('Error ingesting row', err?.message || err); + errors.push(err?.message || String(err)); + } + } + + run.status = errors.length ? 'partial' : 'success'; + run.imported = imported; + run.updated = updated; + run.errorCount = errors.length; + run.errors = errors.slice(0, 50); + run.finishedAt = new Date(); + run.durationMs = Date.now() - runStart; + await this.runs.save(run); + + return { runId: run.id, imported, updated, errors }; + } catch (err: any) { + this.logger.error('Ingest request failed', err?.message || err); + const errors = [err?.message || String(err)]; + run.status = 'failed'; + run.imported = 0; + run.updated = 0; + run.errorCount = errors.length; + run.errors = errors; + run.finishedAt = new Date(); + run.durationMs = Date.now() - runStart; + await this.runs.save(run); + return { runId: run.id, imported: 0, updated: 0, errors }; + } + } +} diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts new file mode 100644 index 0000000..869d434 --- /dev/null +++ b/apps/api/src/main.ts @@ -0,0 +1,14 @@ +import 'reflect-metadata'; +import { NestFactory } from '@nestjs/core'; +import { AppModule } from './app.module'; + +async function bootstrap() { + const app = await NestFactory.create(AppModule); + app.enableCors(); + const port = process.env.PORT ? Number(process.env.PORT) : 3000; + await app.listen(port); + // eslint-disable-next-line no-console + console.log(`API listening on http://localhost:${port}`); +} + +bootstrap(); diff --git a/apps/api/src/normalizer/normalizer.module.ts b/apps/api/src/normalizer/normalizer.module.ts new file mode 100644 index 0000000..90ba9e4 --- /dev/null +++ b/apps/api/src/normalizer/normalizer.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common' +import { NormalizerService } from './normalizer.service' + +@Module({ + providers: [NormalizerService], + exports: [NormalizerService], +}) +export class NormalizerModule {} diff --git a/apps/api/src/normalizer/normalizer.service.ts b/apps/api/src/normalizer/normalizer.service.ts new file mode 100644 index 0000000..d48f2c2 --- /dev/null +++ b/apps/api/src/normalizer/normalizer.service.ts @@ -0,0 +1,46 @@ +import { Injectable, Logger } from '@nestjs/common' +import { CreateCatalogItemDto } from '../catalog/dto/create-catalog-item.dto' +import { parseDate, parseAmount, normalizeProvince, verifyUrl } from './utils' + +@Injectable() +export class NormalizerService { + private readonly logger = new Logger(NormalizerService.name) + + async normalize(dto: CreateCatalogItemDto): Promise { + const out: CreateCatalogItemDto = { ...dto } + + // Normalize dates + try { + if (dto.updatedAt) { + const d = parseDate(dto.updatedAt) + if (d) out.updatedAt = d + } + } catch (e) { + this.logger.debug('Date normalization failed', e) + } + + // Example: normalise publisher/province if present + if (dto.publisher) { + out.publisher = normalizeProvince(dto.publisher) + } + + // Verify sourceUrl + if (dto.sourceUrl) { + try { + const ok = await verifyUrl(dto.sourceUrl) + if (!ok) { + this.logger.warn(`Source URL not reachable: ${dto.sourceUrl}`) + // keep it but flag by appending note + out.sourceUrl = dto.sourceUrl + } + } catch (e) { + this.logger.debug('verifyUrl error', e) + } + } + + // Amounts are not part of the DTO now, but kept for extension + // if ((dto as any).amount) (dto as any).amount = parseAmount((dto as any).amount) + + return out + } +} diff --git a/apps/api/src/normalizer/utils.ts b/apps/api/src/normalizer/utils.ts new file mode 100644 index 0000000..8fdc3c7 --- /dev/null +++ b/apps/api/src/normalizer/utils.ts @@ -0,0 +1,51 @@ +import axios from 'axios' + +export function parseDate(value?: string): string | undefined { + if (!value) return undefined + const d = new Date(value) + if (!isNaN(d.getTime())) return d.toISOString() + + // try common formats dd/mm/yyyy or dd-mm-yyyy + const m = value.match(/(\d{1,2})[\/\-](\d{1,2})[\/\-](\d{4})/) + if (m) { + const day = parseInt(m[1], 10) + const month = parseInt(m[2], 10) - 1 + const year = parseInt(m[3], 10) + const d2 = new Date(year, month, day) + if (!isNaN(d2.getTime())) return d2.toISOString() + } + return undefined +} + +export function parseAmount(value?: string | number): number | undefined { + if (value === undefined || value === null) return undefined + if (typeof value === 'number') return value + // remove currency symbols and thousands separators + const cleaned = String(value).replace(/[^0-9,\.\-]/g, '').replace(/\./g, '').replace(/,/g, '.') + const n = parseFloat(cleaned) + return isNaN(n) ? undefined : n +} + +export function normalizeProvince(value?: string): string | undefined { + if (!value) return undefined + // simple normalizations + const v = value.trim().toLowerCase() + const map: Record = { + 'andalucía': 'Andalucía', 'andalucia': 'Andalucía', 'andaluzia': 'Andalucía', + 'madrid': 'Madrid', 'comunidad de madrid': 'Madrid', + 'cataluña': 'Cataluña', 'cataluna': 'Cataluña', 'barcelona': 'Barcelona' + } + if (map[v]) return map[v] + // Title case fallback + return value.replace(/\b\w/g, (l) => l.toUpperCase()) +} + +export async function verifyUrl(url?: string): Promise { + if (!url) return false + try { + const res = await axios.head(url, { timeout: 5000, maxRedirects: 3 }) + return res.status >= 200 && res.status < 400 + } catch (err) { + return false + } +} diff --git a/apps/api/src/plans/plans.controller.ts b/apps/api/src/plans/plans.controller.ts new file mode 100644 index 0000000..7b099a0 --- /dev/null +++ b/apps/api/src/plans/plans.controller.ts @@ -0,0 +1,12 @@ +import { Controller, Get } from '@nestjs/common'; +import { PlansService } from './plans.service'; + +@Controller('plans') +export class PlansController { + constructor(private readonly plans: PlansService) {} + + @Get() + async listActive() { + return this.plans.listActive(); + } +} diff --git a/apps/api/src/plans/plans.module.ts b/apps/api/src/plans/plans.module.ts new file mode 100644 index 0000000..4e8c899 --- /dev/null +++ b/apps/api/src/plans/plans.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { PlanEntity } from '../entities/plan.entity'; +import { PlansService } from './plans.service'; +import { PlansController } from './plans.controller'; + +@Module({ + imports: [TypeOrmModule.forFeature([PlanEntity])], + providers: [PlansService], + controllers: [PlansController], + exports: [PlansService], +}) +export class PlansModule {} diff --git a/apps/api/src/plans/plans.service.ts b/apps/api/src/plans/plans.service.ts new file mode 100644 index 0000000..bc41362 --- /dev/null +++ b/apps/api/src/plans/plans.service.ts @@ -0,0 +1,80 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { PlanEntity } from '../entities/plan.entity'; + +@Injectable() +export class PlansService implements OnModuleInit { + private readonly logger = new Logger(PlansService.name); + private initialized = false; + + constructor( + @InjectRepository(PlanEntity) + private readonly plans: Repository, + ) {} + + async onModuleInit() { + await this.ensureDefaults(); + } + + async ensureDefaults() { + if (this.initialized) return; + this.initialized = true; + const defaults = [ + { + code: 'basic', + name: 'Plan Básico', + priceMonthly: 15, + description: 'Alertas esenciales y dashboard básico', + features: { alertProfiles: 3, refresh: 'daily' }, + }, + { + code: 'pro', + name: 'Plan Pro', + priceMonthly: 39, + description: 'Más alertas, filtros avanzados y tendencias', + features: { alertProfiles: 10, refresh: 'hourly' }, + }, + { + code: 'empresa', + name: 'Plan Empresa', + priceMonthly: 99, + description: 'Personalización y soporte dedicado', + features: { alertProfiles: 50, refresh: 'custom' }, + }, + ]; + + for (const entry of defaults) { + const existing = await this.plans.findOne({ where: { code: entry.code } }); + if (!existing) { + await this.plans.save(this.plans.create(entry)); + this.logger.log(`Seeded plan ${entry.code}`); + } + } + } + + async listActive() { + return this.plans.find({ where: { isActive: true }, order: { priceMonthly: 'ASC' } }); + } + + async listAll() { + return this.plans.find({ order: { priceMonthly: 'ASC' } }); + } + + async getDefaultPlan() { + await this.ensureDefaults(); + return this.plans.findOne({ where: { code: 'basic' } }); + } + + async create(payload: Partial) { + const plan = this.plans.create(payload); + return this.plans.save(plan); + } + + async update(id: string, payload: Partial) { + const plan = await this.plans.findOne({ where: { id } }); + if (!plan) return null; + Object.assign(plan, payload); + return this.plans.save(plan); + } +} diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json new file mode 100644 index 0000000..abc9b7e --- /dev/null +++ b/apps/api/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2020", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true + }, + "include": ["src/**/*.ts"] +} diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile new file mode 100644 index 0000000..9f89d96 --- /dev/null +++ b/apps/web/Dockerfile @@ -0,0 +1,13 @@ +FROM node:18-alpine + +WORKDIR /usr/src/app + +COPY package*.json ./ +RUN npm install + +COPY . . +RUN npm run build + +ENV NODE_ENV=production +EXPOSE 3000 +CMD ["npm", "run", "start"] diff --git a/apps/web/README.md b/apps/web/README.md new file mode 100644 index 0000000..6ec5d2c --- /dev/null +++ b/apps/web/README.md @@ -0,0 +1,5 @@ +# apps/web + +Directorio para la aplicación Nuxt (frontend dashboard). + +Para crear el esqueleto de Nuxt (si deseas que lo genere automáticamente), puedo ejecutar los comandos del creador de Nuxt y añadir la configuración inicial. diff --git a/apps/web/components/Header.vue b/apps/web/components/Header.vue new file mode 100644 index 0000000..885dc7e --- /dev/null +++ b/apps/web/components/Header.vue @@ -0,0 +1,90 @@ + + + + + diff --git a/apps/web/layouts/default.vue b/apps/web/layouts/default.vue new file mode 100644 index 0000000..e4348ec --- /dev/null +++ b/apps/web/layouts/default.vue @@ -0,0 +1,12 @@ + + + + + diff --git a/apps/web/nuxt.config.ts b/apps/web/nuxt.config.ts new file mode 100644 index 0000000..d5956b3 --- /dev/null +++ b/apps/web/nuxt.config.ts @@ -0,0 +1,21 @@ +import { defineNuxtConfig } from 'nuxt' + +export default defineNuxtConfig({ + ssr: false, + app: { + head: { + title: 'gob-alert', + link: [ + { + rel: 'stylesheet', + href: 'https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&display=swap', + }, + ], + }, + }, + runtimeConfig: { + public: { + apiBase: process.env.API_URL || 'http://localhost:3000' + } + } +}) diff --git a/apps/web/package.json b/apps/web/package.json new file mode 100644 index 0000000..d129137 --- /dev/null +++ b/apps/web/package.json @@ -0,0 +1,13 @@ +{ + "name": "web", + "version": "0.0.1", + "private": true, + "scripts": { + "dev": "nuxt dev", + "build": "nuxt build", + "start": "nuxt start" + }, + "dependencies": { + "nuxt": "^3.8.0" + } +} diff --git a/apps/web/pages/admin.vue b/apps/web/pages/admin.vue new file mode 100644 index 0000000..3e44b80 --- /dev/null +++ b/apps/web/pages/admin.vue @@ -0,0 +1,549 @@ + + + + + diff --git a/apps/web/pages/alerts.vue b/apps/web/pages/alerts.vue new file mode 100644 index 0000000..eb42eef --- /dev/null +++ b/apps/web/pages/alerts.vue @@ -0,0 +1,290 @@ + + + + + diff --git a/apps/web/pages/catalog.vue b/apps/web/pages/catalog.vue new file mode 100644 index 0000000..55c132a --- /dev/null +++ b/apps/web/pages/catalog.vue @@ -0,0 +1,129 @@ + + + + + diff --git a/apps/web/pages/discover.vue b/apps/web/pages/discover.vue new file mode 100644 index 0000000..6eae09a --- /dev/null +++ b/apps/web/pages/discover.vue @@ -0,0 +1,150 @@ + + + + + diff --git a/apps/web/pages/index.vue b/apps/web/pages/index.vue new file mode 100644 index 0000000..f6fd2f0 --- /dev/null +++ b/apps/web/pages/index.vue @@ -0,0 +1,271 @@ + + + + + diff --git a/apps/web/pages/ingest.vue b/apps/web/pages/ingest.vue new file mode 100644 index 0000000..6374ae6 --- /dev/null +++ b/apps/web/pages/ingest.vue @@ -0,0 +1,110 @@ + + + + + diff --git a/apps/web/pages/landing.vue b/apps/web/pages/landing.vue new file mode 100644 index 0000000..b5cb654 --- /dev/null +++ b/apps/web/pages/landing.vue @@ -0,0 +1,163 @@ + + + + + diff --git a/apps/web/pages/plans.vue b/apps/web/pages/plans.vue new file mode 100644 index 0000000..28f608c --- /dev/null +++ b/apps/web/pages/plans.vue @@ -0,0 +1,95 @@ + + + + + diff --git a/coolify.yml b/coolify.yml new file mode 100644 index 0000000..cc7b385 --- /dev/null +++ b/coolify.yml @@ -0,0 +1,35 @@ +apps: + - name: gob-alert-postgres + image: postgres:15-alpine + env: + POSTGRES_USER: gob + POSTGRES_PASSWORD: gobpass + POSTGRES_DB: gob_alert + + - name: gob-alert-redis + image: redis:7-alpine + + - name: gob-alert-api + context: ./apps/api + dockerfile: Dockerfile + env: + - key: DATABASE_URL + value: postgres://gob:gobpass@localhost:5432/gob_alert + - key: API_ADMIN_KEY + value: changeme + ports: + - 3000 + + - name: gob-alert-web + context: ./apps/web + dockerfile: Dockerfile + env: + - key: API_URL + value: http://gob-alert-api:3000 + ports: + - 3001 + +notes: | + This file provides a simple mapping for Coolify to build and deploy services from this repository. + Adjust `DATABASE_URL` host when Coolify provides internal network hostnames. Use the Coolify UI + to set secrets/variables in the target environment. diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..c8f35b2 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,31 @@ +version: '3.8' +services: + api: + build: + context: ./apps/api + volumes: + - ./apps/api:/usr/src/app + - /usr/src/app/node_modules + environment: + DATABASE_URL: postgres://gob:gobpass@postgres:5432/gob_alert + NODE_ENV: development + command: npm run dev + ports: + - "3000:3000" + depends_on: + - postgres + + web: + build: + context: ./apps/web + volumes: + - ./apps/web:/usr/src/app + - /usr/src/app/node_modules + environment: + API_URL: http://api:3000 + NODE_ENV: development + command: npm run dev + ports: + - "3001:3000" + depends_on: + - api diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8d3912a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,54 @@ +version: '3.8' +services: + postgres: + image: postgres:15-alpine + environment: + POSTGRES_USER: gob + POSTGRES_PASSWORD: gobpass + POSTGRES_DB: gob_alert + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + + redis: + image: redis:7-alpine + ports: + - "6379:6379" + volumes: + - redis_data:/data + + adminer: + image: adminer + restart: unless-stopped + ports: + - "8080:8080" + + api: + build: + context: ./apps/api + environment: + DATABASE_URL: postgres://gob:gobpass@postgres:5432/gob_alert + API_ADMIN_KEY: "changeme" + NODE_ENV: production + depends_on: + - postgres + ports: + - "3000:3000" + restart: unless-stopped + + web: + build: + context: ./apps/web + environment: + API_URL: http://api:3000 + NODE_ENV: production + depends_on: + - api + ports: + - "3001:3000" + restart: unless-stopped + +volumes: + postgres_data: + redis_data: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..b04f681 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,29 @@ +# Arquitectura (2 mini-PCs) + +## Objetivo +Separar la ingesta/normalización del producto (API + dashboard) para reducir carga y permitir escalado gradual. + +## Mini-PC 1 — Ingesta y procesamiento +- **Servicios**: workers de ingesta, normalizadores, colas ligeras. +- **Responsabilidad**: consultar datos.gob.es, normalizar y guardar en Postgres. +- **Componentes clave**: + - Scheduler de ingesta (`INGEST_CRON`) + - Bull + Redis (opcional) + - Normalizador y clasificador + +## Mini-PC 2 — Producto y entrega +- **Servicios**: API Nest, Nuxt dashboard, alertas Telegram/SMTP, backups. +- **Responsabilidad**: servir datos, dashboards y notificaciones. +- **Componentes clave**: + - API `apps/api` + - Dashboard `apps/web` + - Scheduler alertas (`ALERTS_CRON`) y backups (`BACKUP_CRON`) + +## Comunicación +- API HTTP/REST entre servicios. +- Redis opcional si se utiliza colas de ingestión. + +## Datos +- Postgres centralizado (en Mini‑PC 1 o 2 según recursos). +- Backups periódicos en `BACKUP_DIR`. + diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..b7784fb --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,46 @@ +# Despliegue en mini‑PCs + +## Requisitos +- Docker + Docker Compose +- Postgres y Redis (opcional) + +## Variables de entorno recomendadas +- `DATABASE_URL` +- `API_ADMIN_KEY` +- `JWT_SECRET` +- `INGEST_CRON` +- `ALERTS_CRON` +- `BACKUP_CRON` +- `BACKUP_DIR` +- `INGEST_ENABLED` +- `ALERTS_ENABLED` +- `BACKUP_ENABLED` +- `SMTP_HOST`, `SMTP_PORT`, `SMTP_USER`, `SMTP_PASS`, `SMTP_FROM` +- `TELEGRAM_BOT_TOKEN`, `TELEGRAM_DEFAULT_CHAT` + +## Arranque rápido (todo‑en‑uno) +```bash +# Desde la raíz del repo +pnpm install +pnpm build +pnpm start +``` + +## Arranque con Docker Compose +```bash +docker compose build +docker compose up -d +``` + +## Estrategia recomendada (2 mini‑PCs) +1. **Mini‑PC 1**: ejecutar Postgres + Redis + jobs de ingesta. +2. **Mini‑PC 2**: ejecutar API + web + backups + alertas. +3. Definir `DATABASE_URL` apuntando al Postgres del Mini‑PC 1. + +Sugerencia de flags: +- Mini‑PC 1: `INGEST_ENABLED=true`, `ALERTS_ENABLED=false`, `BACKUP_ENABLED=false`. +- Mini‑PC 2: `INGEST_ENABLED=false`, `ALERTS_ENABLED=true`, `BACKUP_ENABLED=true`. + +## Backups +- Programados por `BACKUP_CRON`. +- Manual: `POST /admin/backup/run`. diff --git a/docs/KPIS.md b/docs/KPIS.md new file mode 100644 index 0000000..b1dcd5e --- /dev/null +++ b/docs/KPIS.md @@ -0,0 +1,17 @@ +# KPIs y métricas + +## KPIs iniciales +- **Tiempo medio de detección**: minutos desde la publicación hasta la ingesta. +- **Cobertura del catálogo**: % de datasets detectados vs. catálogo total. +- **Tasa de alertas útiles**: % de alertas calificadas como relevantes en pilotos. +- **Conversión piloto → cliente**: ratio de pilotos que pagan al mes 2. + +## Métricas operativas (hoy) +- `ingest_runs`: duración, errores, datasets nuevos/actualizados. +- `alert_runs`: perfiles procesados, alertas enviadas/fallidas. +- `/admin/monitor/status`: conteos globales. + +## Recolección recomendada +- Log de timestamps de ingesta por dataset (para medir detección). +- Feedback por alerta (útil/no útil) en UI de piloto. + diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 0000000..3afbc07 --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,19 @@ +# Operaciones y monitorización + +## Endpoints admin +- `POST /admin/ingest/pause` / `resume` +- `POST /admin/ingest/queue` +- `GET /admin/ingest/runs` +- `POST /admin/backup/run` +- `GET /admin/backup/list` +- `GET /admin/alerts/runs` +- `GET /admin/monitor/status` + +## Checks rápidos +- `/dashboard/summary` (salud del dashboard) +- `/catalog/health` (API catálogo) + +## Logs +- Logs de ingesta y alertas quedan en stdout. +- `ingest_runs` y `alert_runs` guardan historial resumido. + diff --git a/package.json b/package.json new file mode 100644 index 0000000..dc5b774 --- /dev/null +++ b/package.json @@ -0,0 +1,18 @@ +{ + "name": "gob-alert", + "private": true, + "version": "0.1.0", + "workspaces": [ + "apps/*", + "packages/*" + ], + "scripts": { + "bootstrap": "pnpm install", + "dev": "turbo run dev", + "build": "turbo run build", + "start": "turbo run start" + }, + "devDependencies": { + "turbo": "^1.10.11" + } +} diff --git a/packages/shared/package.json b/packages/shared/package.json new file mode 100644 index 0000000..177dcd0 --- /dev/null +++ b/packages/shared/package.json @@ -0,0 +1,7 @@ +{ + "name": "@gob-alert/shared", + "version": "0.0.1", + "private": true, + "main": "index.js", + "files": [] +} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..e9b0dad --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,3 @@ +packages: + - 'apps/*' + - 'packages/*' diff --git a/turbo.json b/turbo.json new file mode 100644 index 0000000..5c837a4 --- /dev/null +++ b/turbo.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://turbo.build/schema.json", + "pipeline": { + "dev": { + "dependsOn": [], + "cache": false + }, + "build": { + "dependsOn": ["^build"], + "outputs": [] + }, + "start": { + "dependsOn": ["build"] + } + } +}