From f1a1f093e6d4c98f6ef5b2558ac3432a11829dd6 Mon Sep 17 00:00:00 2001 From: wisecolt Date: Sun, 15 Feb 2026 23:12:24 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20altyaz=C4=B1=20otomasyon=20sistemi=20MV?= =?UTF-8?q?P'sini=20ekle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker tabanlı mikro servis mimarisi ile altyazı otomasyon sistemi altyapısı kuruldu. - Core (Node.js): Chokidar dosya izleyici, BullMQ iş kuyrukları, ffprobe medya analizi, MongoDB entegrasyonu ve dosya yazma işlemleri. - API (Fastify): Mock sağlayıcılar, arşiv güvenliği (zip-slip), altyazı doğrulama, puanlama ve aday seçim motoru. - UI (React/Vite): İş yönetimi paneli, canlı SSE log akışı, manuel inceleme arayüzü ve sistem ayarları. - Altyapı: Docker Compose (dev/prod), Redis, Mongo ve çevresel değişken yapılandırmaları. --- .env.example | 14 + .gitignore | 8 + README.md | 160 ++++++++++ _media/movie/.gitkeep | 0 _media/tv/.gitkeep | 0 compose.dev.yml | 94 ++++++ compose.yml | 83 ++++++ services/api/Dockerfile | 24 ++ services/api/package.json | 28 ++ services/api/src/app.ts | 20 ++ services/api/src/config/env.ts | 11 + services/api/src/index.ts | 26 ++ services/api/src/lib/deterministic.ts | 18 ++ services/api/src/lib/mockArtifact.ts | 51 ++++ services/api/src/lib/scoring.ts | 89 ++++++ services/api/src/lib/security.ts | 30 ++ services/api/src/lib/subtitleEngine.ts | 200 +++++++++++++ services/api/src/lib/validators.ts | 25 ++ .../src/providers/OpenSubtitlesProvider.ts | 45 +++ .../src/providers/TurkceAltyaziProvider.ts | 44 +++ services/api/src/routes/subtitles.ts | 65 +++++ services/api/src/types/index.ts | 49 ++++ services/api/test/scoring.test.ts | 48 +++ services/api/test/security.test.ts | 14 + services/api/test/validators.test.ts | 21 ++ services/api/tsconfig.json | 14 + services/core/Dockerfile | 24 ++ services/core/package.json | 31 ++ services/core/src/app.ts | 34 +++ services/core/src/config/env.ts | 18 ++ services/core/src/db/mongo.ts | 7 + services/core/src/db/redis.ts | 8 + services/core/src/index.ts | 49 ++++ services/core/src/models/Job.ts | 35 +++ services/core/src/models/JobLog.ts | 15 + services/core/src/models/MediaFile.ts | 24 ++ services/core/src/models/Setting.ts | 27 ++ services/core/src/models/WatchedPath.ts | 14 + services/core/src/queues/queues.ts | 11 + services/core/src/routes/debug.ts | 27 ++ services/core/src/routes/health.ts | 5 + services/core/src/routes/jobs.ts | 69 +++++ services/core/src/routes/review.ts | 103 +++++++ services/core/src/routes/settings.ts | 37 +++ services/core/src/routes/watchedPaths.ts | 49 ++++ services/core/src/utils/apiClient.ts | 14 + services/core/src/utils/ffprobe.ts | 46 +++ services/core/src/utils/file.ts | 79 +++++ services/core/src/utils/logger.ts | 45 +++ services/core/src/utils/parser.ts | 50 ++++ services/core/src/watcher/index.ts | 56 ++++ services/core/src/workers/pipeline.ts | 274 ++++++++++++++++++ services/core/test/parser.test.ts | 26 ++ services/core/tsconfig.json | 15 + services/ui/Dockerfile | 21 ++ services/ui/index.html | 12 + services/ui/package.json | 22 ++ services/ui/src/App.tsx | 34 +++ services/ui/src/api/client.ts | 9 + services/ui/src/components/JobTable.tsx | 26 ++ services/ui/src/components/Layout.tsx | 20 ++ services/ui/src/hooks/usePoll.ts | 16 + services/ui/src/main.tsx | 9 + services/ui/src/pages/DashboardPage.tsx | 44 +++ services/ui/src/pages/JobDetailPage.tsx | 95 ++++++ services/ui/src/pages/JobsPage.tsx | 32 ++ services/ui/src/pages/ReviewPage.tsx | 29 ++ services/ui/src/pages/SettingsPage.tsx | 32 ++ services/ui/src/pages/WatchedPathsPage.tsx | 60 ++++ services/ui/src/types.ts | 20 ++ services/ui/tsconfig.json | 12 + services/ui/vite.config.ts | 16 + 72 files changed, 2882 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 _media/movie/.gitkeep create mode 100644 _media/tv/.gitkeep create mode 100644 compose.dev.yml create mode 100644 compose.yml create mode 100644 services/api/Dockerfile create mode 100644 services/api/package.json create mode 100644 services/api/src/app.ts create mode 100644 services/api/src/config/env.ts create mode 100644 services/api/src/index.ts create mode 100644 services/api/src/lib/deterministic.ts create mode 100644 services/api/src/lib/mockArtifact.ts create mode 100644 services/api/src/lib/scoring.ts create mode 100644 services/api/src/lib/security.ts create mode 100644 services/api/src/lib/subtitleEngine.ts create mode 100644 services/api/src/lib/validators.ts create mode 100644 services/api/src/providers/OpenSubtitlesProvider.ts create mode 100644 services/api/src/providers/TurkceAltyaziProvider.ts create mode 100644 services/api/src/routes/subtitles.ts create mode 100644 services/api/src/types/index.ts create mode 100644 services/api/test/scoring.test.ts create mode 100644 services/api/test/security.test.ts create mode 100644 services/api/test/validators.test.ts create mode 100644 services/api/tsconfig.json create mode 100644 services/core/Dockerfile create mode 100644 services/core/package.json create mode 100644 services/core/src/app.ts create mode 100644 services/core/src/config/env.ts create mode 100644 services/core/src/db/mongo.ts create mode 100644 services/core/src/db/redis.ts create mode 100644 services/core/src/index.ts create mode 100644 services/core/src/models/Job.ts create mode 100644 services/core/src/models/JobLog.ts create mode 100644 services/core/src/models/MediaFile.ts create mode 100644 services/core/src/models/Setting.ts create mode 100644 services/core/src/models/WatchedPath.ts create mode 100644 services/core/src/queues/queues.ts create mode 100644 services/core/src/routes/debug.ts create mode 100644 services/core/src/routes/health.ts create mode 100644 services/core/src/routes/jobs.ts create mode 100644 services/core/src/routes/review.ts create mode 100644 services/core/src/routes/settings.ts create mode 100644 services/core/src/routes/watchedPaths.ts create mode 100644 services/core/src/utils/apiClient.ts create mode 100644 services/core/src/utils/ffprobe.ts create mode 100644 services/core/src/utils/file.ts create mode 100644 services/core/src/utils/logger.ts create mode 100644 services/core/src/utils/parser.ts create mode 100644 services/core/src/watcher/index.ts create mode 100644 services/core/src/workers/pipeline.ts create mode 100644 services/core/test/parser.test.ts create mode 100644 services/core/tsconfig.json create mode 100644 services/ui/Dockerfile create mode 100644 services/ui/index.html create mode 100644 services/ui/package.json create mode 100644 services/ui/src/App.tsx create mode 100644 services/ui/src/api/client.ts create mode 100644 services/ui/src/components/JobTable.tsx create mode 100644 services/ui/src/components/Layout.tsx create mode 100644 services/ui/src/hooks/usePoll.ts create mode 100644 services/ui/src/main.tsx create mode 100644 services/ui/src/pages/DashboardPage.tsx create mode 100644 services/ui/src/pages/JobDetailPage.tsx create mode 100644 services/ui/src/pages/JobsPage.tsx create mode 100644 services/ui/src/pages/ReviewPage.tsx create mode 100644 services/ui/src/pages/SettingsPage.tsx create mode 100644 services/ui/src/pages/WatchedPathsPage.tsx create mode 100644 services/ui/src/types.ts create mode 100644 services/ui/tsconfig.json create mode 100644 services/ui/vite.config.ts diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b0dbced --- /dev/null +++ b/.env.example @@ -0,0 +1,14 @@ +NODE_ENV=development +MONGO_URI=mongodb://mongo:27017/subwatcher +REDIS_HOST=redis +REDIS_PORT=6379 +CORE_PORT=3001 +API_PORT=3002 +UI_PORT=5173 +API_BASE_URL=http://api:3002 +CORE_BASE_URL=http://core:3001 +TEMP_ROOT=/temp +MEDIA_TV_PATH=/media/tv +MEDIA_MOVIE_PATH=/media/movie +ENABLE_API_KEY=false +API_KEY= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..daa04d6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +node_modules +dist +.env +.DS_Store +coverage +*.log +services/*/node_modules +services/*/dist diff --git a/README.md b/README.md new file mode 100644 index 0000000..ce04695 --- /dev/null +++ b/README.md @@ -0,0 +1,160 @@ +# subwatcher + +Docker tabanli altyazi otomasyon sistemi. + +- `core`: watcher + ffprobe + BullMQ + Mongo job/log API + review akisi +- `api`: mock provider subtitle engine (TurkceAltyazi/OpenSubtitles stub) + archive extraction + security + scoring +- `ui`: React/Vite panel (dashboard, jobs, detail live logs, review, settings, watched paths) + +## Mimari + +- Mongo koleksiyonlari: `watched_paths`, `settings`, `media_files`, `jobs`, `job_logs` +- Redis/BullMQ kuyruklari: + - `fileEvents` + - `mediaAnalysis` + - `subtitleFetch` + - `finalizeWrite` +- Core -> API servis cagrisi (docker network): `http://api:3002` +- UI -> Core API: `http://localhost:3001/api` (CORS acik) +- Temp alan: `/temp/{jobToken}` + +## Mock Provider Notu + +Gercek scraping/API cagrilari bu MVP'de yoktur. + +- `TurkceAltyaziProvider`: mock + TODO +- `OpenSubtitlesProvider`: mock + TODO + +Deterministik candidate uretimi vardir (aynı input = ayni aday davranisi). + +## Gelistirme (Dev) + +1. Ortam dosyasi: + +```bash +cp .env.example .env +``` + +2. Servisleri kaldir: + +```bash +docker compose -f compose.dev.yml up --build +``` + +3. Portlar: + +- UI: `http://localhost:5173` +- Core: `http://localhost:3001` +- API: `http://localhost:3002` +- Mongo: `localhost:27017` +- Redis: `localhost:6379` + +4. Media dosyasi yerlestirme: + +- TV: `./_media/tv` +- Movie: `./_media/movie` + +Gercek `.mkv` dosyasi ekleyince watcher pipeline'i tetikler. + +5. Debug enqueue (dev-only): + +```bash +curl -X POST http://localhost:3001/api/debug/enqueue \ + -H 'content-type: application/json' \ + -d '{"path":"/media/movie/example.mkv","kind":"movie"}' +``` + +## Production + +```bash +docker compose -f compose.yml up --build -d +``` + +Portlar: + +- UI: `http://localhost:3000` +- Core: `http://localhost:3001` +- API: `http://localhost:3002` + +## UI Ozellikleri + +- Dashboard: son 24h ozet + son isler +- Jobs: filtreleme + job detayi +- Job Detail: metadata, mediaInfo, sonuc dosyalari, canli SSE log paneli +- Review List: `NEEDS_REVIEW` isler +- Manual override: metadata ile ara + candidate sec + finalize write +- Settings: language/overwrite/stability/security ayarlari +- Watched Paths: ekle/sil/enable/disable + +## API Endpointleri + +### Core + +- `GET /api/health` +- `GET /api/settings` +- `POST /api/settings` +- `GET /api/watched-paths` +- `POST /api/watched-paths` +- `GET /api/jobs` +- `GET /api/jobs/:id` +- `GET /api/jobs/:id/logs` +- `GET /api/jobs/:id/stream` (SSE) +- `GET /api/review` +- `POST /api/review/:jobId/search` +- `POST /api/review/:jobId/choose` +- `POST /api/debug/enqueue` (dev) + +### Subtitle API + +- `GET /v1/health` +- `POST /v1/subtitles/search` +- `POST /v1/subtitles/choose` +- `POST /v1/subtitles/cleanup` + +## Guvenlik ve Dogrulama + +- Archive extraction: `7z` +- Zip slip kontrolu: realpath root disina cikis reddi +- Limit kontrolleri: + - max file count + - max total size + - max single file size +- SRT/ASS extension'a gore degil, icerige gore validate edilir +- Gecersiz altyazilar aninda silinir (`INVALID_SUBTITLE_DELETED`) + +## Encoding + +Core finalize adiminda: + +- BOM kontrol +- UTF-8 / windows-1254 / latin1 fallback +- LF newline normalizasyonu +- hedef adlandirma: `{base}.{lang}.{ext}` +- overwrite false ise `.2`, `.3`... + +## Testler + +Core: + +```bash +cd services/core && npm test +``` + +API: + +```bash +cd services/api && npm test +``` + +Kapsam: + +- filename parser +- SRT/ASS validator +- scoring + ambiguous karari +- zip slip helper + +## Gelecek (v2) + +- Gercek TurkceAltyazi scraping +- Gercek OpenSubtitles API entegrasyonu +- ClamAV tarama (feature flag hazir) diff --git a/_media/movie/.gitkeep b/_media/movie/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/_media/tv/.gitkeep b/_media/tv/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/compose.dev.yml b/compose.dev.yml new file mode 100644 index 0000000..1861f53 --- /dev/null +++ b/compose.dev.yml @@ -0,0 +1,94 @@ +services: + mongo: + image: mongo:7 + container_name: subwatcher-mongo-dev + ports: + - "27017:27017" + volumes: + - mongo_data:/data/db + + redis: + image: redis:7-alpine + container_name: subwatcher-redis-dev + ports: + - "6379:6379" + volumes: + - redis_data:/data + + api: + build: + context: ./services/api + target: dev + container_name: subwatcher-api-dev + env_file: + - .env + environment: + - NODE_ENV=development + - API_PORT=3002 + - TEMP_ROOT=/temp + - ENABLE_API_KEY=false + ports: + - "3002:3002" + volumes: + - ./services/api:/app + - api_node_modules:/app/node_modules + - temp_data:/temp + depends_on: + - mongo + - redis + + core: + build: + context: ./services/core + target: dev + container_name: subwatcher-core-dev + env_file: + - .env + environment: + - NODE_ENV=development + - CORE_PORT=3001 + - MONGO_URI=mongodb://mongo:27017/subwatcher + - REDIS_HOST=redis + - REDIS_PORT=6379 + - API_BASE_URL=http://api:3002 + - TEMP_ROOT=/temp + - MEDIA_TV_PATH=/media/tv + - MEDIA_MOVIE_PATH=/media/movie + - ENABLE_API_KEY=false + ports: + - "3001:3001" + volumes: + - ./services/core:/app + - core_node_modules:/app/node_modules + - temp_data:/temp:ro + - ./_media/tv:/media/tv + - ./_media/movie:/media/movie + depends_on: + - mongo + - redis + - api + + ui: + build: + context: ./services/ui + target: dev + container_name: subwatcher-ui-dev + environment: + - NODE_ENV=development + - VITE_CORE_URL=http://core:3001 + - VITE_PUBLIC_CORE_URL=http://localhost:3001 + ports: + - "5173:5173" + volumes: + - ./services/ui:/app + - ui_node_modules:/app/node_modules + depends_on: + - core + +volumes: + mongo_data: + redis_data: + temp_data: + core_node_modules: + api_node_modules: + ui_node_modules: diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..6ca3e71 --- /dev/null +++ b/compose.yml @@ -0,0 +1,83 @@ +services: + mongo: + image: mongo:7 + container_name: subwatcher-mongo + restart: unless-stopped + volumes: + - mongo_data:/data/db + + redis: + image: redis:7-alpine + container_name: subwatcher-redis + restart: unless-stopped + volumes: + - redis_data:/data + + api: + build: + context: ./services/api + target: prod + container_name: subwatcher-api + restart: unless-stopped + env_file: + - .env + environment: + - NODE_ENV=production + - API_PORT=3002 + - TEMP_ROOT=/temp + ports: + - "3002:3002" + volumes: + - temp_data:/temp + depends_on: + - mongo + - redis + + core: + build: + context: ./services/core + target: prod + container_name: subwatcher-core + restart: unless-stopped + env_file: + - .env + environment: + - NODE_ENV=production + - CORE_PORT=3001 + - MONGO_URI=mongodb://mongo:27017/subwatcher + - REDIS_HOST=redis + - REDIS_PORT=6379 + - API_BASE_URL=http://api:3002 + - TEMP_ROOT=/temp + - MEDIA_TV_PATH=/media/tv + - MEDIA_MOVIE_PATH=/media/movie + ports: + - "3001:3001" + volumes: + - temp_data:/temp:ro + - ./_media/tv:/media/tv + - ./_media/movie:/media/movie + depends_on: + - mongo + - redis + - api + + ui: + build: + context: ./services/ui + target: prod + container_name: subwatcher-ui + restart: unless-stopped + environment: + - NODE_ENV=production + - PORT=3000 + - CORE_PROXY_URL=http://core:3001 + ports: + - "3000:3000" + depends_on: + - core + +volumes: + mongo_data: + redis_data: + temp_data: diff --git a/services/api/Dockerfile b/services/api/Dockerfile new file mode 100644 index 0000000..0c3d54e --- /dev/null +++ b/services/api/Dockerfile @@ -0,0 +1,24 @@ +FROM node:20-bookworm AS base +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends p7zip-full unrar-free && rm -rf /var/lib/apt/lists/* +COPY package*.json ./ + +FROM base AS dev +RUN npm install +COPY . . +EXPOSE 3002 +CMD ["npm", "run", "dev"] + +FROM base AS build +RUN npm ci +COPY . . +RUN npm run build + +FROM node:20-bookworm AS prod +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends p7zip-full unrar-free && rm -rf /var/lib/apt/lists/* +COPY --from=build /app/package*.json ./ +COPY --from=build /app/node_modules ./node_modules +COPY --from=build /app/dist ./dist +EXPOSE 3002 +CMD ["npm", "run", "start"] diff --git a/services/api/package.json b/services/api/package.json new file mode 100644 index 0000000..ab0f8ef --- /dev/null +++ b/services/api/package.json @@ -0,0 +1,28 @@ +{ + "name": "subwatcher-api", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "tsx watch src/index.ts", + "build": "tsc -p tsconfig.json", + "start": "node dist/index.js", + "test": "vitest run" + }, + "dependencies": { + "@fastify/cors": "^11.0.0", + "adm-zip": "^0.5.16", + "dotenv": "^16.4.7", + "fastify": "^5.2.1", + "fs-extra": "^11.3.0", + "zod": "^3.24.1" + }, + "devDependencies": { + "@types/adm-zip": "^0.5.7", + "@types/fs-extra": "^11.0.4", + "@types/node": "^22.13.1", + "tsx": "^4.19.2", + "typescript": "^5.7.3", + "vitest": "^3.0.5" + } +} diff --git a/services/api/src/app.ts b/services/api/src/app.ts new file mode 100644 index 0000000..e40e09a --- /dev/null +++ b/services/api/src/app.ts @@ -0,0 +1,20 @@ +import Fastify from 'fastify'; +import cors from '@fastify/cors'; +import { env } from './config/env.js'; +import { subtitleRoutes } from './routes/subtitles.js'; + +export async function buildApp() { + const app = Fastify({ logger: true }); + await app.register(cors, { origin: true }); + + if (env.enableApiKey && env.apiKey) { + app.addHook('preHandler', async (req, reply) => { + if (req.headers['x-api-key'] !== env.apiKey) { + return reply.status(401).send({ error: 'invalid api key' }); + } + }); + } + + await subtitleRoutes(app); + return app; +} diff --git a/services/api/src/config/env.ts b/services/api/src/config/env.ts new file mode 100644 index 0000000..dec6948 --- /dev/null +++ b/services/api/src/config/env.ts @@ -0,0 +1,11 @@ +import dotenv from 'dotenv'; + +dotenv.config(); + +export const env = { + nodeEnv: process.env.NODE_ENV ?? 'development', + port: Number(process.env.API_PORT ?? 3002), + tempRoot: process.env.TEMP_ROOT ?? '/temp', + enableApiKey: process.env.ENABLE_API_KEY === 'true', + apiKey: process.env.API_KEY ?? '' +}; diff --git a/services/api/src/index.ts b/services/api/src/index.ts new file mode 100644 index 0000000..2ee29b0 --- /dev/null +++ b/services/api/src/index.ts @@ -0,0 +1,26 @@ +import fs from 'node:fs/promises'; +import { buildApp } from './app.js'; +import { cleanupOldTemp } from './lib/subtitleEngine.js'; +import { env } from './config/env.js'; + +async function bootstrap() { + await fs.mkdir(env.tempRoot, { recursive: true }); + const app = await buildApp(); + + setInterval(async () => { + try { + const deleted = await cleanupOldTemp(24); + if (deleted > 0) console.log(`[api] cleanup removed ${deleted} temp folders`); + } catch (e) { + console.error('[api] cleanup failed', e); + } + }, 60 * 60 * 1000); + + await app.listen({ port: env.port, host: '0.0.0.0' }); + console.log(`[api] running on :${env.port}`); +} + +bootstrap().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/services/api/src/lib/deterministic.ts b/services/api/src/lib/deterministic.ts new file mode 100644 index 0000000..814c229 --- /dev/null +++ b/services/api/src/lib/deterministic.ts @@ -0,0 +1,18 @@ +export function hashString(input: string): number { + let h = 2166136261; + for (let i = 0; i < input.length; i++) { + h ^= input.charCodeAt(i); + h = Math.imul(h, 16777619); + } + return h >>> 0; +} + +export function seeded(seed: number): () => number { + let t = seed; + return () => { + t += 0x6d2b79f5; + let x = Math.imul(t ^ (t >>> 15), t | 1); + x ^= x + Math.imul(x ^ (x >>> 7), x | 61); + return ((x ^ (x >>> 14)) >>> 0) / 4294967296; + }; +} diff --git a/services/api/src/lib/mockArtifact.ts b/services/api/src/lib/mockArtifact.ts new file mode 100644 index 0000000..d767cae --- /dev/null +++ b/services/api/src/lib/mockArtifact.ts @@ -0,0 +1,51 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import AdmZip from 'adm-zip'; +import type { Candidate, SearchParams } from '../types/index.js'; +import { hashString, seeded } from './deterministic.js'; + +function buildSrt(title: string, season?: number, episode?: number): string { + const ep = season && episode ? ` S${String(season).padStart(2, '0')}E${String(episode).padStart(2, '0')}` : ''; + return `1\n00:00:01,000 --> 00:00:04,000\n${title}${ep} satir 1\n\n2\n00:00:05,000 --> 00:00:08,000\n${title}${ep} satir 2\n\n3\n00:00:09,000 --> 00:00:12,000\n${title}${ep} satir 3\n`; +} + +function buildAss(title: string): string { + return `[Script Info]\nTitle: ${title}\n[Events]\nDialogue: 0,0:00:01.00,0:00:04.00,Default,,0,0,0,,Ass satiri\n`; +} + +export async function generateMockArtifact(candidate: Candidate, params: SearchParams, jobToken: string, downloadDir: string): Promise<{ type: 'archive' | 'direct'; filePath: string }> { + await fs.mkdir(downloadDir, { recursive: true }); + const seed = hashString(`${jobToken}|${params.title}|${candidate.id}`); + const rnd = seeded(seed); + + if (candidate.downloadType === 'direct') { + const filePath = path.join(downloadDir, `${candidate.id}.srt`); + await fs.writeFile(filePath, buildSrt(params.title, params.season, params.episode), 'utf8'); + return { type: 'direct', filePath }; + } + + const zip = new AdmZip(); + if (params.type === 'tv') { + const s = params.season ?? 1; + const e = params.episode ?? 1; + const base = params.title.replace(/\s+/g, '.'); + zip.addFile(`${base}.S${String(s).padStart(2, '0')}E${String(e).padStart(2, '0')}.1080p.srt`, Buffer.from(buildSrt(params.title, s, e))); + zip.addFile(`${base}.S${String(s).padStart(2, '0')}E${String(e + 1).padStart(2, '0')}.srt`, Buffer.from(buildSrt(params.title, s, e + 1))); + zip.addFile(`${base}.S${String(s).padStart(2, '0')}E${String(Math.max(1, e - 1)).padStart(2, '0')}.srt`, Buffer.from(buildSrt(params.title, s, Math.max(1, e - 1)))); + if (rnd() > 0.5) { + zip.addFile(`${base}.S${String(s).padStart(2, '0')}E${String(e).padStart(2, '0')}.ass`, Buffer.from(buildAss(params.title))); + } + } else { + const name = `${params.title.replace(/\s+/g, '.')}.${params.year ?? 2020}`; + zip.addFile(`${name}.tr.srt`, Buffer.from(buildSrt(params.title))); + if (rnd() > 0.3) { + zip.addFile(`${name}.txt`, Buffer.from('this is not subtitle')); + } + } + + zip.addFile('invalid.bin', Buffer.from([0, 159, 255, 0, 18])); + + const archivePath = path.join(downloadDir, `${candidate.id}.zip`); + zip.writeZip(archivePath); + return { type: 'archive', filePath: archivePath }; +} diff --git a/services/api/src/lib/scoring.ts b/services/api/src/lib/scoring.ts new file mode 100644 index 0000000..354e0a0 --- /dev/null +++ b/services/api/src/lib/scoring.ts @@ -0,0 +1,89 @@ +import path from 'node:path'; +import type { Candidate, SearchParams } from '../types/index.js'; + +export interface ScoredSubtitle { + id: string; + candidateId: string; + provider: string; + filePath: string; + ext: 'srt' | 'ass'; + lang: string; + score: number; + reasons: string[]; +} + +function tokenize(s?: string): string[] { + if (!s) return []; + return s + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean); +} + +export function scoreCandidateFile(filePath: string, ext: 'srt' | 'ass', candidate: Candidate, params: SearchParams): ScoredSubtitle | null { + const fn = path.basename(filePath).toLowerCase(); + let score = 0; + const reasons: string[] = []; + + if (params.type === 'tv') { + const sePattern = /s(\d{1,2})e(\d{1,2})/i; + const match = fn.match(sePattern); + if (!match || Number(match[1]) !== params.season || Number(match[2]) !== params.episode) { + return null; + } + score += 100; + reasons.push('season_episode_match'); + } + + const releaseTokens = tokenize(params.release); + const fileTokens = tokenize(fn).concat(candidate.releaseHints.map((x) => x.toLowerCase())); + const releaseMatches = releaseTokens.filter((t) => fileTokens.includes(t)).length; + score += Math.min(25, releaseMatches * 6); + if (releaseMatches > 0) reasons.push('release_match'); + + if (candidate.lang === (params.languages[0] || 'tr')) { + score += 10; + reasons.push('lang_match'); + } + + const height = params.mediaInfo?.video?.height; + if (height && (fn.includes(String(height)) || candidate.releaseHints.includes(`${height}p`))) { + score += 8; + reasons.push('resolution_match'); + } + + if (params.preferHI && candidate.isHI) { + score += 4; + reasons.push('prefer_hi'); + } + + if (params.preferForced && candidate.isForced) { + score += 4; + reasons.push('prefer_forced'); + } + + return { + id: `${candidate.id}:${path.basename(filePath)}`, + candidateId: candidate.id, + provider: candidate.provider, + filePath, + ext, + lang: candidate.lang, + score, + reasons + }; +} + +export function chooseBest(scored: ScoredSubtitle[]): { status: 'FOUND' | 'AMBIGUOUS' | 'NOT_FOUND'; best?: ScoredSubtitle; confidence?: number; candidates: ScoredSubtitle[] } { + if (scored.length === 0) return { status: 'NOT_FOUND', candidates: [] }; + const sorted = [...scored].sort((a, b) => b.score - a.score); + const best = sorted[0]; + const second = sorted[1]; + + if (second && Math.abs(best.score - second.score) <= 3) { + return { status: 'AMBIGUOUS', candidates: sorted.slice(0, 10) }; + } + + const confidence = Math.max(0.5, Math.min(0.99, best.score / 130)); + return { status: 'FOUND', best, confidence, candidates: sorted.slice(0, 10) }; +} diff --git a/services/api/src/lib/security.ts b/services/api/src/lib/security.ts new file mode 100644 index 0000000..cb44019 --- /dev/null +++ b/services/api/src/lib/security.ts @@ -0,0 +1,30 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; + +export async function ensureInsideRoot(root: string, candidatePath: string): Promise { + const [rootReal, candReal] = await Promise.all([fs.realpath(root), fs.realpath(candidatePath)]); + return candReal === rootReal || candReal.startsWith(rootReal + path.sep); +} + +export async function collectFilesRecursive(dir: string): Promise { + const entries = await fs.readdir(dir, { withFileTypes: true }); + const files: string[] = []; + for (const e of entries) { + const p = path.join(dir, e.name); + if (e.isDirectory()) files.push(...(await collectFilesRecursive(p))); + else if (e.isFile()) files.push(p); + } + return files; +} + +export async function validateExtractionLimits(files: string[], limits: { maxFiles: number; maxTotalBytes: number; maxSingleBytes: number }): Promise<{ ok: boolean; reason?: string; totalBytes: number }> { + if (files.length > limits.maxFiles) return { ok: false, reason: 'max files exceeded', totalBytes: 0 }; + let total = 0; + for (const file of files) { + const st = await fs.stat(file); + if (st.size > limits.maxSingleBytes) return { ok: false, reason: `single file exceeded: ${file}`, totalBytes: total }; + total += st.size; + if (total > limits.maxTotalBytes) return { ok: false, reason: 'total bytes exceeded', totalBytes: total }; + } + return { ok: true, totalBytes: total }; +} diff --git a/services/api/src/lib/subtitleEngine.ts b/services/api/src/lib/subtitleEngine.ts new file mode 100644 index 0000000..d844e14 --- /dev/null +++ b/services/api/src/lib/subtitleEngine.ts @@ -0,0 +1,200 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import fse from 'fs-extra'; +import { env } from '../config/env.js'; +import type { SearchParams, TraceLog } from '../types/index.js'; +import { SubtitleProvider, Candidate } from '../types/index.js'; +import { TurkceAltyaziProvider } from '../providers/TurkceAltyaziProvider.js'; +import { OpenSubtitlesProvider } from '../providers/OpenSubtitlesProvider.js'; +import { collectFilesRecursive, ensureInsideRoot, validateExtractionLimits } from './security.js'; +import { detectSubtitleType, isProbablyText } from './validators.js'; +import { chooseBest, scoreCandidateFile } from './scoring.js'; + +const execFileAsync = promisify(execFile); + +const providers: SubtitleProvider[] = [new TurkceAltyaziProvider(), new OpenSubtitlesProvider()]; + +function defaultLimits() { + return { maxFiles: 300, maxTotalBytes: 250 * 1024 * 1024, maxSingleBytes: 10 * 1024 * 1024 }; +} + +async function ensureJobDirs(jobToken: string) { + const base = path.join(env.tempRoot, jobToken); + const download = path.join(base, 'download'); + const extracted = path.join(base, 'extracted'); + await fs.mkdir(download, { recursive: true }); + await fs.mkdir(extracted, { recursive: true }); + return { base, download, extracted }; +} + +async function extractArchive(archivePath: string, extractedDir: string, trace: TraceLog[]): Promise { + trace.push({ level: 'info', step: 'EXTRACT_STARTED', message: archivePath }); + await execFileAsync('7z', ['x', '-y', archivePath, `-o${extractedDir}`]); + const files = await collectFilesRecursive(extractedDir); + trace.push({ level: 'info', step: 'EXTRACT_DONE', message: `Extracted ${files.length} files` }); + return files; +} + +export async function searchSubtitles(input: SearchParams) { + const jobToken = input.jobToken ?? `job-${Date.now()}`; + const trace: TraceLog[] = []; + const limits = input.securityLimits ?? defaultLimits(); + const dirs = await ensureJobDirs(jobToken); + + const allCandidates: Candidate[] = []; + for (const p of providers) { + const c = await p.search(input); + allCandidates.push(...c); + } + + const scored: any[] = []; + + for (const candidate of allCandidates) { + const provider = providers.find((p: any) => p.constructor.name.toLowerCase().includes(candidate.provider === 'turkcealtyazi' ? 'turkce' : 'open')); + if (!provider) continue; + + const dl = await provider.download(candidate, input, jobToken); + trace.push({ level: 'info', step: 'ARCHIVE_DOWNLOADED', message: `${candidate.provider}:${candidate.id}`, meta: { path: dl.filePath, type: dl.type } }); + + let files: string[] = []; + if (dl.type === 'archive') { + const perCandidateExtractDir = path.join(dirs.extracted, candidate.id); + await fs.mkdir(perCandidateExtractDir, { recursive: true }); + files = await extractArchive(dl.filePath, perCandidateExtractDir, trace); + + for (const file of files) { + const inside = await ensureInsideRoot(perCandidateExtractDir, file); + if (!inside) { + trace.push({ level: 'warn', step: 'ZIPSLIP_REJECTED', message: `Rejected path traversal candidate: ${file}` }); + await fse.remove(file); + } + } + + files = await collectFilesRecursive(perCandidateExtractDir); + const lim = await validateExtractionLimits(files, limits); + if (!lim.ok) { + trace.push({ level: 'warn', step: 'LIMIT_REJECTED', message: lim.reason ?? 'limit rejected' }); + continue; + } + } else { + files = [dl.filePath]; + } + + for (const file of files) { + const buf = await fs.readFile(file); + if (!isProbablyText(buf)) { + await fse.remove(file); + trace.push({ level: 'warn', step: 'INVALID_SUBTITLE_DELETED', message: `Deleted binary/invalid: ${file}` }); + continue; + } + + const text = buf.toString('utf8'); + const ext = detectSubtitleType(text); + if (!ext) { + await fse.remove(file); + trace.push({ level: 'warn', step: 'INVALID_SUBTITLE_DELETED', message: `Deleted unknown subtitle content: ${file}` }); + continue; + } + + const s = scoreCandidateFile(file, ext, candidate, input); + if (s) scored.push(s); + } + } + + trace.push({ level: 'info', step: 'CANDIDATES_SCANNED', message: `Scored ${scored.length} subtitle files` }); + + const decision = chooseBest(scored); + const manifestPath = path.join(dirs.base, 'manifest.json'); + await fs.writeFile(manifestPath, JSON.stringify({ jobToken, input, scored: decision.candidates }, null, 2), 'utf8'); + + if (decision.status === 'FOUND' && decision.best) { + const bestPath = path.join(dirs.base, `best.${decision.best.ext}`); + await fs.copyFile(decision.best.filePath, bestPath); + trace.push({ level: 'info', step: 'BEST_SELECTED', message: `Selected ${decision.best.filePath}`, meta: { score: decision.best.score } }); + + return { + status: 'FOUND', + jobToken, + bestPath, + confidence: decision.confidence, + source: decision.best.provider, + candidates: decision.candidates, + trace + }; + } + + if (decision.status === 'AMBIGUOUS') { + trace.push({ level: 'warn', step: 'AMBIGUOUS_NEEDS_REVIEW', message: 'Top candidates too close' }); + return { + status: 'AMBIGUOUS', + jobToken, + confidence: 0.5, + source: 'multi', + candidates: decision.candidates, + trace + }; + } + + trace.push({ level: 'warn', step: 'NOT_FOUND_NEEDS_REVIEW', message: 'No valid subtitle file found' }); + return { + status: 'NOT_FOUND', + jobToken, + confidence: 0, + source: 'none', + candidates: [], + trace + }; +} + +export async function chooseSubtitle(jobToken: string, chosenCandidateId?: string, chosenPath?: string) { + const base = path.join(env.tempRoot, jobToken); + const manifestPath = path.join(base, 'manifest.json'); + const raw = await fs.readFile(manifestPath, 'utf8'); + const manifest = JSON.parse(raw); + const list = manifest.scored ?? []; + + const found = chosenPath + ? list.find((x: any) => x.filePath === chosenPath || x.id === chosenPath) + : list.find((x: any) => x.id === chosenCandidateId || x.candidateId === chosenCandidateId); + + if (!found) { + return { status: 'NOT_FOUND', message: 'Chosen candidate not found' }; + } + + const bestPath = path.join(base, `best.${found.ext}`); + await fs.copyFile(found.filePath, bestPath); + + return { + status: 'FOUND', + bestPath, + confidence: Math.max(0.5, Math.min(0.98, found.score / 130)), + source: found.provider + }; +} + +export async function cleanupJobToken(jobToken: string) { + const dir = path.join(env.tempRoot, jobToken); + await fse.remove(dir); +} + +export async function cleanupOldTemp(hours = 24): Promise { + await fs.mkdir(env.tempRoot, { recursive: true }); + const entries = await fs.readdir(env.tempRoot, { withFileTypes: true }); + const now = Date.now(); + let count = 0; + + for (const e of entries) { + if (!e.isDirectory()) continue; + const p = path.join(env.tempRoot, e.name); + const st = await fs.stat(p); + const ageHours = (now - st.mtimeMs) / 1000 / 3600; + if (ageHours > hours) { + await fse.remove(p); + count += 1; + } + } + + return count; +} diff --git a/services/api/src/lib/validators.ts b/services/api/src/lib/validators.ts new file mode 100644 index 0000000..467866d --- /dev/null +++ b/services/api/src/lib/validators.ts @@ -0,0 +1,25 @@ +export function isProbablyText(buffer: Buffer): boolean { + if (buffer.includes(0x00)) return false; + let nonPrintable = 0; + for (const b of buffer) { + const printable = b === 9 || b === 10 || b === 13 || (b >= 32 && b <= 126) || b >= 160; + if (!printable) nonPrintable += 1; + } + return buffer.length === 0 ? false : nonPrintable / buffer.length < 0.2; +} + +export function validateSrt(text: string): boolean { + const lines = text.split(/\r?\n/); + const tc = lines.filter((l) => /^\d{2}:\d{2}:\d{2},\d{3}\s-->\s\d{2}:\d{2}:\d{2},\d{3}$/.test(l.trim())); + return tc.length >= 3; +} + +export function validateAss(text: string): boolean { + return text.includes('[Script Info]') && text.includes('[Events]') && /Dialogue:/.test(text); +} + +export function detectSubtitleType(text: string): 'srt' | 'ass' | null { + if (validateSrt(text)) return 'srt'; + if (validateAss(text)) return 'ass'; + return null; +} diff --git a/services/api/src/providers/OpenSubtitlesProvider.ts b/services/api/src/providers/OpenSubtitlesProvider.ts new file mode 100644 index 0000000..3d9b56a --- /dev/null +++ b/services/api/src/providers/OpenSubtitlesProvider.ts @@ -0,0 +1,45 @@ +import type { Candidate, SearchParams, SubtitleProvider } from '../types/index.js'; +import { generateMockArtifact } from '../lib/mockArtifact.js'; +import { hashString, seeded } from '../lib/deterministic.js'; +import { env } from '../config/env.js'; + +export class OpenSubtitlesProvider implements SubtitleProvider { + async search(params: SearchParams): Promise { + // TODO(v2): real OpenSubtitles API integration. + const key = `${params.title}|${params.year}|${params.season}|${params.episode}|os`; + const rnd = seeded(hashString(key)); + const base = params.title.replace(/\s+/g, '.'); + const directForMovie = params.type === 'movie' && rnd() > 0.4; + return [ + { + id: `os-${hashString(`${key}-a`)}`, + provider: 'opensubtitles', + displayName: `OS ${base} Official`, + downloadType: directForMovie ? 'direct' : 'archiveZip', + downloadUrl: directForMovie ? `mock://os/${base}/direct.srt` : `mock://os/${base}/archive.zip`, + lang: 'tr', + releaseHints: ['1080p', rnd() > 0.5 ? 'x265' : 'x264', 'flux'], + scoreHints: ['api_match'], + isHI: rnd() > 0.8, + isForced: rnd() > 0.92 + }, + { + id: `os-${hashString(`${key}-b`)}`, + provider: 'opensubtitles', + displayName: `OS ${base} Backup`, + downloadType: 'archiveZip', + downloadUrl: `mock://os/${base}/backup.zip`, + lang: 'tr', + releaseHints: ['720p', 'x264'], + scoreHints: ['backup'], + isHI: false, + isForced: false + } + ]; + } + + async download(candidate: Candidate, params: SearchParams, jobToken: string) { + const artifact = await generateMockArtifact(candidate, params, jobToken, `${env.tempRoot}/${jobToken}/download`); + return { type: artifact.type, filePath: artifact.filePath, candidateId: candidate.id }; + } +} diff --git a/services/api/src/providers/TurkceAltyaziProvider.ts b/services/api/src/providers/TurkceAltyaziProvider.ts new file mode 100644 index 0000000..f718fac --- /dev/null +++ b/services/api/src/providers/TurkceAltyaziProvider.ts @@ -0,0 +1,44 @@ +import type { Candidate, SearchParams, SubtitleProvider } from '../types/index.js'; +import { generateMockArtifact } from '../lib/mockArtifact.js'; +import { hashString, seeded } from '../lib/deterministic.js'; +import { env } from '../config/env.js'; + +export class TurkceAltyaziProvider implements SubtitleProvider { + async search(params: SearchParams): Promise { + // TODO(v2): real TurkceAltyazi scraping implementation. + const key = `${params.title}|${params.year}|${params.season}|${params.episode}|ta`; + const rnd = seeded(hashString(key)); + const base = params.title.replace(/\s+/g, '.'); + return [ + { + id: `ta-${hashString(`${key}-a`)}`, + provider: 'turkcealtyazi', + displayName: `TA ${base} Ana Surum`, + downloadType: 'archiveZip', + downloadUrl: `mock://ta/${base}/a.zip`, + lang: 'tr', + releaseHints: [rnd() > 0.4 ? '1080p' : '720p', 'x265', 'flux'], + scoreHints: ['trusted', 'crowd'], + isHI: rnd() > 0.7, + isForced: false + }, + { + id: `ta-${hashString(`${key}-b`)}`, + provider: 'turkcealtyazi', + displayName: `TA ${base} Alternatif`, + downloadType: 'archiveZip', + downloadUrl: `mock://ta/${base}/b.zip`, + lang: 'tr', + releaseHints: ['webrip', 'x264'], + scoreHints: ['alt'], + isHI: false, + isForced: false + } + ]; + } + + async download(candidate: Candidate, params: SearchParams, jobToken: string) { + const artifact = await generateMockArtifact(candidate, params, jobToken, `${env.tempRoot}/${jobToken}/download`); + return { type: artifact.type, filePath: artifact.filePath, candidateId: candidate.id }; + } +} diff --git a/services/api/src/routes/subtitles.ts b/services/api/src/routes/subtitles.ts new file mode 100644 index 0000000..28ac06f --- /dev/null +++ b/services/api/src/routes/subtitles.ts @@ -0,0 +1,65 @@ +import { FastifyInstance } from 'fastify'; +import { z } from 'zod'; +import { chooseSubtitle, cleanupJobToken, searchSubtitles } from '../lib/subtitleEngine.js'; + +const SearchSchema = z.object({ + jobToken: z.string().optional(), + type: z.enum(['movie', 'tv']), + title: z.string().min(1), + year: z.number().optional(), + release: z.string().optional(), + languages: z.array(z.string()).min(1), + season: z.number().optional(), + episode: z.number().optional(), + mediaInfo: z.any().optional(), + preferHI: z.boolean().optional(), + preferForced: z.boolean().optional(), + securityLimits: z + .object({ + maxFiles: z.number().min(1), + maxTotalBytes: z.number().min(1024), + maxSingleBytes: z.number().min(1024) + }) + .optional() +}); + +const ChooseSchema = z.object({ + jobToken: z.string().min(1), + chosenCandidateId: z.string().optional(), + chosenPath: z.string().optional() +}); + +export async function subtitleRoutes(app: FastifyInstance): Promise { + app.get('/v1/health', async () => ({ ok: true, service: 'api' })); + + app.post('/v1/subtitles/search', async (req, reply) => { + const parsed = SearchSchema.safeParse(req.body); + if (!parsed.success) return reply.status(400).send({ error: parsed.error.flatten() }); + + try { + const result = await searchSubtitles(parsed.data); + return result; + } catch (err: any) { + return reply.status(500).send({ status: 'ERROR', message: err.message, trace: [{ level: 'error', step: 'JOB_ERROR', message: err.message }] }); + } + }); + + app.post('/v1/subtitles/choose', async (req, reply) => { + const parsed = ChooseSchema.safeParse(req.body); + if (!parsed.success) return reply.status(400).send({ error: parsed.error.flatten() }); + + try { + const result = await chooseSubtitle(parsed.data.jobToken, parsed.data.chosenCandidateId, parsed.data.chosenPath); + return result; + } catch (err: any) { + return reply.status(500).send({ status: 'ERROR', message: err.message }); + } + }); + + app.post('/v1/subtitles/cleanup', async (req, reply) => { + const body = z.object({ jobToken: z.string().min(1) }).safeParse(req.body); + if (!body.success) return reply.status(400).send({ error: body.error.flatten() }); + await cleanupJobToken(body.data.jobToken); + return { ok: true }; + }); +} diff --git a/services/api/src/types/index.ts b/services/api/src/types/index.ts new file mode 100644 index 0000000..9bc1ab4 --- /dev/null +++ b/services/api/src/types/index.ts @@ -0,0 +1,49 @@ +export interface SearchParams { + jobToken?: string; + type: 'movie' | 'tv'; + title: string; + year?: number; + release?: string; + languages: string[]; + season?: number; + episode?: number; + mediaInfo?: any; + preferHI?: boolean; + preferForced?: boolean; + securityLimits?: { + maxFiles: number; + maxTotalBytes: number; + maxSingleBytes: number; + }; +} + +export interface Candidate { + id: string; + provider: 'turkcealtyazi' | 'opensubtitles'; + displayName: string; + downloadType: 'archiveZip' | 'direct'; + downloadUrl: string; + lang: string; + releaseHints: string[]; + scoreHints: string[]; + isHI: boolean; + isForced: boolean; +} + +export interface DownloadedArtifact { + type: 'archive' | 'direct'; + filePath: string; + candidateId: string; +} + +export interface SubtitleProvider { + search(params: SearchParams): Promise; + download(candidate: Candidate, params: SearchParams, jobToken: string): Promise; +} + +export interface TraceLog { + level: 'info' | 'warn' | 'error'; + step: string; + message: string; + meta?: any; +} diff --git a/services/api/test/scoring.test.ts b/services/api/test/scoring.test.ts new file mode 100644 index 0000000..6ca43a0 --- /dev/null +++ b/services/api/test/scoring.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import { chooseBest, scoreCandidateFile } from '../src/lib/scoring.js'; + +const candidate: any = { + id: 'c1', + provider: 'opensubtitles', + displayName: 'x', + downloadType: 'archiveZip', + downloadUrl: 'mock://x', + lang: 'tr', + releaseHints: ['1080p', 'x265', 'flux'], + scoreHints: [], + isHI: false, + isForced: false +}; + +describe('scoring', () => { + it('scores tv season/episode match strongly', () => { + const s = scoreCandidateFile('/tmp/show.S01E02.1080p.srt', 'srt', candidate, { + type: 'tv', + title: 'show', + season: 1, + episode: 2, + release: 'FLUX', + languages: ['tr'] + }); + expect(s?.score).toBeGreaterThan(100); + }); + + it('disqualifies wrong episode for tv', () => { + const s = scoreCandidateFile('/tmp/show.S01E03.srt', 'srt', candidate, { + type: 'tv', + title: 'show', + season: 1, + episode: 2, + languages: ['tr'] + }); + expect(s).toBeNull(); + }); + + it('returns ambiguous when top scores are close', () => { + const d = chooseBest([ + { id: 'a', candidateId: 'a', provider: 'x', filePath: '/a', ext: 'srt', lang: 'tr', score: 90, reasons: [] }, + { id: 'b', candidateId: 'b', provider: 'x', filePath: '/b', ext: 'srt', lang: 'tr', score: 88, reasons: [] } + ] as any); + expect(d.status).toBe('AMBIGUOUS'); + }); +}); diff --git a/services/api/test/security.test.ts b/services/api/test/security.test.ts new file mode 100644 index 0000000..72593de --- /dev/null +++ b/services/api/test/security.test.ts @@ -0,0 +1,14 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ensureInsideRoot } from '../src/lib/security.js'; + +describe('zip slip helper', () => { + it('accepts path inside root', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'sw-root-')); + const file = path.join(root, 'a.txt'); + await fs.writeFile(file, 'x'); + expect(await ensureInsideRoot(root, file)).toBe(true); + }); +}); diff --git a/services/api/test/validators.test.ts b/services/api/test/validators.test.ts new file mode 100644 index 0000000..dfcee05 --- /dev/null +++ b/services/api/test/validators.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest'; +import { detectSubtitleType, isProbablyText, validateAss, validateSrt } from '../src/lib/validators.js'; + +describe('subtitle validators', () => { + it('validates srt content', () => { + const srt = `1\n00:00:01,000 --> 00:00:02,000\na\n\n2\n00:00:03,000 --> 00:00:04,000\nb\n\n3\n00:00:05,000 --> 00:00:06,000\nc\n`; + expect(validateSrt(srt)).toBe(true); + expect(detectSubtitleType(srt)).toBe('srt'); + }); + + it('validates ass content', () => { + const ass = `[Script Info]\n[Events]\nDialogue: 0,0:00:01.00,0:00:02.00,Default,,0,0,0,,x`; + expect(validateAss(ass)).toBe(true); + expect(detectSubtitleType(ass)).toBe('ass'); + }); + + it('rejects binary for text detector', () => { + const b = Buffer.from([0, 255, 3, 0, 9]); + expect(isProbablyText(b)).toBe(false); + }); +}); diff --git a/services/api/tsconfig.json b/services/api/tsconfig.json new file mode 100644 index 0000000..bdd12fc --- /dev/null +++ b/services/api/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src/**/*"] +} diff --git a/services/core/Dockerfile b/services/core/Dockerfile new file mode 100644 index 0000000..a04a306 --- /dev/null +++ b/services/core/Dockerfile @@ -0,0 +1,24 @@ +FROM node:20-bookworm AS base +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* +COPY package*.json ./ + +FROM base AS dev +RUN npm install +COPY . . +EXPOSE 3001 +CMD ["npm", "run", "dev"] + +FROM base AS build +RUN npm ci +COPY . . +RUN npm run build + +FROM node:20-bookworm AS prod +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* +COPY --from=build /app/package*.json ./ +COPY --from=build /app/node_modules ./node_modules +COPY --from=build /app/dist ./dist +EXPOSE 3001 +CMD ["npm", "run", "start"] diff --git a/services/core/package.json b/services/core/package.json new file mode 100644 index 0000000..304798d --- /dev/null +++ b/services/core/package.json @@ -0,0 +1,31 @@ +{ + "name": "subwatcher-core", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "tsx watch src/index.ts", + "build": "tsc -p tsconfig.json", + "start": "node dist/index.js", + "test": "vitest run" + }, + "dependencies": { + "axios": "^1.8.2", + "bullmq": "^5.40.3", + "chokidar": "^4.0.3", + "dotenv": "^16.4.7", + "fastify": "^5.2.1", + "@fastify/cors": "^11.0.0", + "ioredis": "^5.4.2", + "iconv-lite": "^0.6.3", + "jschardet": "^3.1.4", + "mongoose": "^8.10.0", + "zod": "^3.24.1" + }, + "devDependencies": { + "@types/node": "^22.13.1", + "tsx": "^4.19.2", + "typescript": "^5.7.3", + "vitest": "^3.0.5" + } +} diff --git a/services/core/src/app.ts b/services/core/src/app.ts new file mode 100644 index 0000000..ef909cc --- /dev/null +++ b/services/core/src/app.ts @@ -0,0 +1,34 @@ +import Fastify from 'fastify'; +import cors from '@fastify/cors'; +import { env } from './config/env.js'; +import { healthRoutes } from './routes/health.js'; +import { settingsRoutes } from './routes/settings.js'; +import { watchedPathRoutes } from './routes/watchedPaths.js'; +import { jobRoutes } from './routes/jobs.js'; +import { reviewRoutes } from './routes/review.js'; +import { debugRoutes } from './routes/debug.js'; + +export async function buildApp() { + const app = Fastify({ logger: true }); + + await app.register(cors, { origin: true }); + + if (env.enableApiKey && env.apiKey) { + app.addHook('preHandler', async (req, reply) => { + if (!req.url.startsWith('/api')) return; + const key = req.headers['x-api-key']; + if (key !== env.apiKey) { + return reply.status(401).send({ error: 'invalid api key' }); + } + }); + } + + await healthRoutes(app); + await settingsRoutes(app); + await watchedPathRoutes(app); + await jobRoutes(app); + await reviewRoutes(app); + await debugRoutes(app); + + return app; +} diff --git a/services/core/src/config/env.ts b/services/core/src/config/env.ts new file mode 100644 index 0000000..fe254c4 --- /dev/null +++ b/services/core/src/config/env.ts @@ -0,0 +1,18 @@ +import dotenv from 'dotenv'; + +dotenv.config(); + +export const env = { + nodeEnv: process.env.NODE_ENV ?? 'development', + port: Number(process.env.CORE_PORT ?? 3001), + mongoUri: process.env.MONGO_URI ?? 'mongodb://mongo:27017/subwatcher', + redisHost: process.env.REDIS_HOST ?? 'redis', + redisPort: Number(process.env.REDIS_PORT ?? 6379), + apiBaseUrl: process.env.API_BASE_URL ?? 'http://api:3002', + tempRoot: process.env.TEMP_ROOT ?? '/temp', + mediaTvPath: process.env.MEDIA_TV_PATH ?? '/media/tv', + mediaMoviePath: process.env.MEDIA_MOVIE_PATH ?? '/media/movie', + enableApiKey: process.env.ENABLE_API_KEY === 'true', + apiKey: process.env.API_KEY ?? '', + isDev: (process.env.NODE_ENV ?? 'development') !== 'production' +}; diff --git a/services/core/src/db/mongo.ts b/services/core/src/db/mongo.ts new file mode 100644 index 0000000..cbfb216 --- /dev/null +++ b/services/core/src/db/mongo.ts @@ -0,0 +1,7 @@ +import mongoose from 'mongoose'; +import { env } from '../config/env.js'; + +export async function connectMongo(): Promise { + await mongoose.connect(env.mongoUri); + console.log(`[core] Mongo connected: ${env.mongoUri}`); +} diff --git a/services/core/src/db/redis.ts b/services/core/src/db/redis.ts new file mode 100644 index 0000000..04a35d0 --- /dev/null +++ b/services/core/src/db/redis.ts @@ -0,0 +1,8 @@ +import IORedis from 'ioredis'; +import { env } from '../config/env.js'; + +export const redis = new IORedis({ + host: env.redisHost, + port: env.redisPort, + maxRetriesPerRequest: null +}); diff --git a/services/core/src/index.ts b/services/core/src/index.ts new file mode 100644 index 0000000..b471cca --- /dev/null +++ b/services/core/src/index.ts @@ -0,0 +1,49 @@ +import { connectMongo } from './db/mongo.js'; +import { env } from './config/env.js'; +import { buildApp } from './app.js'; +import { SettingModel } from './models/Setting.js'; +import { ensureDefaultWatchedPaths, startWatcher } from './watcher/index.js'; +import { startWorkers } from './workers/pipeline.js'; + +async function bootstrap() { + await connectMongo(); + + await SettingModel.findByIdAndUpdate( + 'global', + { + $setOnInsert: { + _id: 'global', + languages: ['tr'], + multiSubtitleEnabled: false, + overwriteExisting: false, + fileStableSeconds: 20, + stableChecks: 3, + stableIntervalSeconds: 10, + autoWriteThreshold: 0.75, + securityLimits: { + maxFiles: 300, + maxTotalBytes: 250 * 1024 * 1024, + maxSingleBytes: 10 * 1024 * 1024 + }, + preferHI: false, + preferForced: false, + features: { clamavEnabled: false } + } + }, + { upsert: true, new: true } + ); + + await ensureDefaultWatchedPaths(env.mediaTvPath, env.mediaMoviePath); + + startWorkers(); + await startWatcher(); + + const app = await buildApp(); + await app.listen({ port: env.port, host: '0.0.0.0' }); + console.log(`[core] running on :${env.port}`); +} + +bootstrap().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/services/core/src/models/Job.ts b/services/core/src/models/Job.ts new file mode 100644 index 0000000..e4c62f7 --- /dev/null +++ b/services/core/src/models/Job.ts @@ -0,0 +1,35 @@ +import { Schema, model } from 'mongoose'; + +export const JobStatuses = [ + 'PENDING', + 'WAITING_FILE_STABLE', + 'PARSED', + 'ANALYZED', + 'REQUESTING_API', + 'FOUND_TEMP', + 'NORMALIZING_ENCODING', + 'WRITING_SUBTITLE', + 'DONE', + 'NEEDS_REVIEW', + 'NOT_FOUND', + 'AMBIGUOUS', + 'SECURITY_REJECTED', + 'ERROR' +] as const; + +export type JobStatus = (typeof JobStatuses)[number]; + +const jobSchema = new Schema( + { + mediaFileId: { type: Schema.Types.ObjectId, ref: 'media_files', required: true }, + status: { type: String, enum: JobStatuses, default: 'PENDING' }, + attempts: { type: Number, default: 0 }, + requestSnapshot: Schema.Types.Mixed, + apiSnapshot: Schema.Types.Mixed, + result: Schema.Types.Mixed, + error: Schema.Types.Mixed + }, + { timestamps: true } +); + +export const JobModel = model('jobs', jobSchema); diff --git a/services/core/src/models/JobLog.ts b/services/core/src/models/JobLog.ts new file mode 100644 index 0000000..d367651 --- /dev/null +++ b/services/core/src/models/JobLog.ts @@ -0,0 +1,15 @@ +import { Schema, model } from 'mongoose'; + +const jobLogSchema = new Schema( + { + jobId: { type: Schema.Types.ObjectId, ref: 'jobs', required: true, index: true }, + ts: { type: Date, default: Date.now, index: true }, + level: { type: String, enum: ['info', 'warn', 'error'], default: 'info' }, + step: { type: String, required: true }, + message: { type: String, required: true }, + meta: Schema.Types.Mixed + }, + { versionKey: false } +); + +export const JobLogModel = model('job_logs', jobLogSchema); diff --git a/services/core/src/models/MediaFile.ts b/services/core/src/models/MediaFile.ts new file mode 100644 index 0000000..b2c5760 --- /dev/null +++ b/services/core/src/models/MediaFile.ts @@ -0,0 +1,24 @@ +import { Schema, model } from 'mongoose'; + +export type MediaType = 'movie' | 'tv'; + +const mediaFileSchema = new Schema( + { + path: { type: String, required: true, unique: true }, + type: { type: String, enum: ['movie', 'tv'], required: true }, + title: String, + year: Number, + season: Number, + episode: Number, + release: String, + size: Number, + mtime: Date, + status: { type: String, enum: ['ACTIVE', 'MISSING'], default: 'ACTIVE' }, + lastSeenAt: Date, + mediaInfo: Schema.Types.Mixed, + analyzedAt: Date + }, + { timestamps: true } +); + +export const MediaFileModel = model('media_files', mediaFileSchema); diff --git a/services/core/src/models/Setting.ts b/services/core/src/models/Setting.ts new file mode 100644 index 0000000..cbde7d4 --- /dev/null +++ b/services/core/src/models/Setting.ts @@ -0,0 +1,27 @@ +import { Schema, model } from 'mongoose'; + +const settingSchema = new Schema( + { + _id: { type: String, default: 'global' }, + languages: { type: [String], default: ['tr'] }, + multiSubtitleEnabled: { type: Boolean, default: false }, + overwriteExisting: { type: Boolean, default: false }, + fileStableSeconds: { type: Number, default: 20 }, + stableChecks: { type: Number, default: 3 }, + stableIntervalSeconds: { type: Number, default: 10 }, + autoWriteThreshold: { type: Number, default: 0.75 }, + securityLimits: { + maxFiles: { type: Number, default: 300 }, + maxTotalBytes: { type: Number, default: 250 * 1024 * 1024 }, + maxSingleBytes: { type: Number, default: 10 * 1024 * 1024 } + }, + preferHI: { type: Boolean, default: false }, + preferForced: { type: Boolean, default: false }, + features: { + clamavEnabled: { type: Boolean, default: false } + } + }, + { timestamps: true } +); + +export const SettingModel = model('settings', settingSchema); diff --git a/services/core/src/models/WatchedPath.ts b/services/core/src/models/WatchedPath.ts new file mode 100644 index 0000000..538720d --- /dev/null +++ b/services/core/src/models/WatchedPath.ts @@ -0,0 +1,14 @@ +import { Schema, model } from 'mongoose'; + +export type WatchedKind = 'tv' | 'movie' | 'mixed'; + +const watchedPathSchema = new Schema( + { + path: { type: String, required: true, unique: true }, + kind: { type: String, enum: ['tv', 'movie', 'mixed'], required: true }, + enabled: { type: Boolean, default: true } + }, + { timestamps: { createdAt: true, updatedAt: false } } +); + +export const WatchedPathModel = model('watched_paths', watchedPathSchema); diff --git a/services/core/src/queues/queues.ts b/services/core/src/queues/queues.ts new file mode 100644 index 0000000..3d5f4e4 --- /dev/null +++ b/services/core/src/queues/queues.ts @@ -0,0 +1,11 @@ +import { Queue, Worker } from 'bullmq'; +import { redis } from '../db/redis.js'; + +export const fileEventsQueue = new Queue('fileEvents', { connection: redis }); +export const mediaAnalysisQueue = new Queue('mediaAnalysis', { connection: redis }); +export const subtitleFetchQueue = new Queue('subtitleFetch', { connection: redis }); +export const finalizeWriteQueue = new Queue('finalizeWrite', { connection: redis }); + +export function createWorker(name: string, processor: any): Worker { + return new Worker(name, processor, { connection: redis, concurrency: 2 }); +} diff --git a/services/core/src/routes/debug.ts b/services/core/src/routes/debug.ts new file mode 100644 index 0000000..da74743 --- /dev/null +++ b/services/core/src/routes/debug.ts @@ -0,0 +1,27 @@ +import fs from 'node:fs/promises'; +import { FastifyInstance } from 'fastify'; +import { z } from 'zod'; +import { env } from '../config/env.js'; +import { createJobForPath } from '../workers/pipeline.js'; +import { MediaFileModel } from '../models/MediaFile.js'; +import { fileEventsQueue } from '../queues/queues.js'; + +export async function debugRoutes(app: FastifyInstance): Promise { + if (!env.isDev) return; + + app.post('/api/debug/enqueue', async (req, reply) => { + const body = z.object({ path: z.string(), kind: z.enum(['tv', 'movie']).default('movie') }).parse(req.body); + try { + await fs.access(body.path); + } catch { + return reply.status(400).send({ error: 'Path does not exist in container' }); + } + + const jobId = await createJobForPath(body.path, body.kind); + const media = await MediaFileModel.findOne({ path: body.path }).lean(); + if (!media) return reply.status(500).send({ error: 'media not persisted' }); + + await fileEventsQueue.add('debug', { jobId, mediaFileId: String(media._id), path: body.path }); + return { ok: true, jobId }; + }); +} diff --git a/services/core/src/routes/health.ts b/services/core/src/routes/health.ts new file mode 100644 index 0000000..bd17868 --- /dev/null +++ b/services/core/src/routes/health.ts @@ -0,0 +1,5 @@ +import { FastifyInstance } from 'fastify'; + +export async function healthRoutes(app: FastifyInstance): Promise { + app.get('/api/health', async () => ({ ok: true, service: 'core' })); +} diff --git a/services/core/src/routes/jobs.ts b/services/core/src/routes/jobs.ts new file mode 100644 index 0000000..202f562 --- /dev/null +++ b/services/core/src/routes/jobs.ts @@ -0,0 +1,69 @@ +import { FastifyInstance } from 'fastify'; +import { z } from 'zod'; +import { JobModel } from '../models/Job.js'; +import { JobLogModel } from '../models/JobLog.js'; +import { subscribeLogs } from '../utils/logger.js'; + +export async function jobRoutes(app: FastifyInstance): Promise { + app.get('/api/jobs', async (req) => { + const q = z + .object({ + page: z.coerce.number().default(1), + limit: z.coerce.number().default(20), + status: z.string().optional(), + search: z.string().optional() + }) + .parse(req.query); + + const filter: any = {}; + if (q.status) filter.status = q.status; + if (q.search) filter.$or = [{ 'requestSnapshot.title': { $regex: q.search, $options: 'i' } }]; + + const skip = (q.page - 1) * q.limit; + const [items, total] = await Promise.all([ + JobModel.find(filter).sort({ createdAt: -1 }).skip(skip).limit(q.limit).lean(), + JobModel.countDocuments(filter) + ]); + + return { items, total, page: q.page, limit: q.limit }; + }); + + app.get('/api/jobs/:id', async (req, reply) => { + const p = z.object({ id: z.string() }).parse(req.params); + const item = await JobModel.findById(p.id).populate('mediaFileId').lean(); + if (!item) return reply.status(404).send({ error: 'Not found' }); + return item; + }); + + app.get('/api/jobs/:id/logs', async (req) => { + const p = z.object({ id: z.string() }).parse(req.params); + const q = z.object({ page: z.coerce.number().default(1), limit: z.coerce.number().default(100) }).parse(req.query); + const skip = (q.page - 1) * q.limit; + const [items, total] = await Promise.all([ + JobLogModel.find({ jobId: p.id }).sort({ ts: 1 }).skip(skip).limit(q.limit).lean(), + JobLogModel.countDocuments({ jobId: p.id }) + ]); + return { items, total, page: q.page, limit: q.limit }; + }); + + app.get('/api/jobs/:id/stream', async (req, reply) => { + const p = z.object({ id: z.string() }).parse(req.params); + + reply.raw.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive' + }); + + const unsubscribe = subscribeLogs(p.id, (log) => { + reply.raw.write(`data: ${JSON.stringify(log)}\n\n`); + }); + + const ping = setInterval(() => reply.raw.write(': ping\n\n'), 15000); + req.raw.on('close', () => { + clearInterval(ping); + unsubscribe(); + reply.raw.end(); + }); + }); +} diff --git a/services/core/src/routes/review.ts b/services/core/src/routes/review.ts new file mode 100644 index 0000000..6444864 --- /dev/null +++ b/services/core/src/routes/review.ts @@ -0,0 +1,103 @@ +import { FastifyInstance } from 'fastify'; +import { z } from 'zod'; +import { JobModel } from '../models/Job.js'; +import { MediaFileModel } from '../models/MediaFile.js'; +import { SettingModel } from '../models/Setting.js'; +import { apiClient } from '../utils/apiClient.js'; +import { finalizeWriteQueue } from '../queues/queues.js'; +import { writeJobLog } from '../utils/logger.js'; + +export async function reviewRoutes(app: FastifyInstance): Promise { + app.get('/api/review', async () => { + return JobModel.find({ status: 'NEEDS_REVIEW' }).sort({ updatedAt: -1 }).limit(200).lean(); + }); + + app.post('/api/review/:jobId/search', async (req, reply) => { + const p = z.object({ jobId: z.string() }).parse(req.params); + const body = z + .object({ + title: z.string().optional(), + year: z.number().optional(), + release: z.string().optional(), + season: z.number().optional(), + episode: z.number().optional(), + languages: z.array(z.string()).optional() + }) + .parse(req.body); + + const j = await JobModel.findById(p.jobId).lean(); + if (!j) return reply.status(404).send({ error: 'Job not found' }); + const media = await MediaFileModel.findById(j.mediaFileId).lean(); + const settings = await SettingModel.findById('global').lean(); + if (!media || !settings) return reply.status(404).send({ error: 'Related data not found' }); + + const payload = { + jobToken: p.jobId, + type: media.type, + title: body.title ?? media.title, + year: body.year ?? media.year, + release: body.release ?? media.release, + season: body.season ?? media.season, + episode: body.episode ?? media.episode, + languages: body.languages ?? settings.languages, + mediaInfo: media.mediaInfo, + preferHI: settings.preferHI, + preferForced: settings.preferForced, + securityLimits: settings.securityLimits + }; + + const res = await apiClient.post('/v1/subtitles/search', payload); + await JobModel.findByIdAndUpdate(p.jobId, { + apiSnapshot: { + ...(j.apiSnapshot ?? {}), + manualSearch: payload, + candidates: res.data.candidates, + status: res.data.status + } + }); + + if (Array.isArray(res.data.trace)) { + for (const t of res.data.trace) { + await writeJobLog({ + jobId: p.jobId, + step: t.step, + message: t.message, + level: t.level || 'info', + meta: t.meta + }); + } + } + + return res.data; + }); + + app.post('/api/review/:jobId/choose', async (req, reply) => { + const p = z.object({ jobId: z.string() }).parse(req.params); + const body = z.object({ chosenCandidateId: z.string().optional(), chosenPath: z.string().optional(), lang: z.string().default('tr') }).parse(req.body); + + const j = await JobModel.findById(p.jobId).lean(); + if (!j) return reply.status(404).send({ error: 'Job not found' }); + + const chooseRes = await apiClient.post('/v1/subtitles/choose', { + jobToken: p.jobId, + chosenCandidateId: body.chosenCandidateId, + chosenPath: body.chosenPath + }); + + if (chooseRes.data.status !== 'FOUND' || !chooseRes.data.bestPath) { + return reply.status(400).send({ error: 'Could not produce best subtitle', details: chooseRes.data }); + } + + await finalizeWriteQueue.add('finalize', { + jobId: p.jobId, + mediaFileId: String(j.mediaFileId), + bestPath: chooseRes.data.bestPath, + lang: body.lang, + source: chooseRes.data.source ?? 'manual', + confidence: chooseRes.data.confidence ?? 0.7 + }); + + await JobModel.findByIdAndUpdate(p.jobId, { status: 'FOUND_TEMP' }); + return { ok: true, queued: true }; + }); +} diff --git a/services/core/src/routes/settings.ts b/services/core/src/routes/settings.ts new file mode 100644 index 0000000..5cd7b42 --- /dev/null +++ b/services/core/src/routes/settings.ts @@ -0,0 +1,37 @@ +import { FastifyInstance } from 'fastify'; +import { z } from 'zod'; +import { SettingModel } from '../models/Setting.js'; + +const SettingsSchema = z.object({ + languages: z.array(z.string()).min(1), + multiSubtitleEnabled: z.boolean(), + overwriteExisting: z.boolean(), + fileStableSeconds: z.number(), + stableChecks: z.number().min(1), + stableIntervalSeconds: z.number().min(1), + autoWriteThreshold: z.number(), + securityLimits: z.object({ + maxFiles: z.number().min(1), + maxTotalBytes: z.number().min(1024), + maxSingleBytes: z.number().min(1024) + }), + preferHI: z.boolean(), + preferForced: z.boolean(), + features: z.object({ + clamavEnabled: z.boolean() + }).optional() +}); + +export async function settingsRoutes(app: FastifyInstance): Promise { + app.get('/api/settings', async () => { + const settings = await SettingModel.findById('global').lean(); + return settings; + }); + + app.post('/api/settings', async (req, reply) => { + const parsed = SettingsSchema.safeParse(req.body); + if (!parsed.success) return reply.status(400).send({ error: parsed.error.flatten() }); + const doc = await SettingModel.findByIdAndUpdate('global', parsed.data, { upsert: true, new: true }); + return doc; + }); +} diff --git a/services/core/src/routes/watchedPaths.ts b/services/core/src/routes/watchedPaths.ts new file mode 100644 index 0000000..57454fa --- /dev/null +++ b/services/core/src/routes/watchedPaths.ts @@ -0,0 +1,49 @@ +import { FastifyInstance } from 'fastify'; +import { z } from 'zod'; +import { WatchedPathModel } from '../models/WatchedPath.js'; + +const BodySchema = z.object({ + action: z.enum(['add', 'remove', 'toggle', 'update']), + path: z.string(), + kind: z.enum(['tv', 'movie', 'mixed']).optional(), + enabled: z.boolean().optional() +}); + +export async function watchedPathRoutes(app: FastifyInstance): Promise { + app.get('/api/watched-paths', async () => { + return WatchedPathModel.find().sort({ createdAt: -1 }).lean(); + }); + + app.post('/api/watched-paths', async (req, reply) => { + const parsed = BodySchema.safeParse(req.body); + if (!parsed.success) return reply.status(400).send({ error: parsed.error.flatten() }); + + const body = parsed.data; + if (body.action === 'add') { + return WatchedPathModel.findOneAndUpdate( + { path: body.path }, + { $setOnInsert: { path: body.path, kind: body.kind ?? 'mixed', enabled: true } }, + { upsert: true, new: true } + ); + } + + if (body.action === 'remove') { + await WatchedPathModel.deleteOne({ path: body.path }); + return { ok: true }; + } + + if (body.action === 'toggle') { + const current = await WatchedPathModel.findOne({ path: body.path }); + if (!current) return reply.status(404).send({ error: 'Not found' }); + current.enabled = body.enabled ?? !current.enabled; + await current.save(); + return current; + } + + return WatchedPathModel.findOneAndUpdate( + { path: body.path }, + { kind: body.kind ?? 'mixed', enabled: body.enabled ?? true }, + { new: true } + ); + }); +} diff --git a/services/core/src/utils/apiClient.ts b/services/core/src/utils/apiClient.ts new file mode 100644 index 0000000..f5e798b --- /dev/null +++ b/services/core/src/utils/apiClient.ts @@ -0,0 +1,14 @@ +import axios from 'axios'; +import { env } from '../config/env.js'; + +export const apiClient = axios.create({ + baseURL: env.apiBaseUrl, + timeout: 30000 +}); + +if (env.enableApiKey && env.apiKey) { + apiClient.interceptors.request.use((config) => { + config.headers['x-api-key'] = env.apiKey; + return config; + }); +} diff --git a/services/core/src/utils/ffprobe.ts b/services/core/src/utils/ffprobe.ts new file mode 100644 index 0000000..921f1b3 --- /dev/null +++ b/services/core/src/utils/ffprobe.ts @@ -0,0 +1,46 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export async function analyzeWithFfprobe(path: string): Promise { + const { stdout } = await execFileAsync('ffprobe', [ + '-v', + 'error', + '-print_format', + 'json', + '-show_streams', + '-show_format', + path + ]); + const json = JSON.parse(stdout); + const video = (json.streams || []).find((s: any) => s.codec_type === 'video'); + const audio = (json.streams || []) + .filter((s: any) => s.codec_type === 'audio') + .map((s: any) => ({ codec_name: s.codec_name, channels: s.channels, language: s.tags?.language })); + + return { + video: video + ? { + codec_name: video.codec_name, + width: video.width, + height: video.height, + r_frame_rate: video.r_frame_rate + } + : null, + audio, + format: { + duration: json.format?.duration, + bit_rate: json.format?.bit_rate, + format_name: json.format?.format_name + } + }; +} + +export function fallbackMediaInfo(): any { + return { + video: { codec_name: 'unknown', width: 1920, height: 1080, r_frame_rate: '24/1' }, + audio: [{ codec_name: 'unknown', channels: 2, language: 'und' }], + format: { duration: '0', bit_rate: '0', format_name: 'matroska' } + }; +} diff --git a/services/core/src/utils/file.ts b/services/core/src/utils/file.ts new file mode 100644 index 0000000..d3b7d53 --- /dev/null +++ b/services/core/src/utils/file.ts @@ -0,0 +1,79 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import iconv from 'iconv-lite'; +import jschardet from 'jschardet'; + +export function isVideoFile(filePath: string): boolean { + const p = filePath.toLowerCase(); + if (!p.endsWith('.mkv')) return false; + return !/(\.part|\.tmp|\.partial)$/.test(p); +} + +export async function waitForStable(filePath: string, checks: number, intervalSeconds: number): Promise { + let prev: { size: number; mtimeMs: number } | null = null; + + for (let i = 0; i < checks; i++) { + const st = await fs.stat(filePath); + const current = { size: st.size, mtimeMs: st.mtimeMs }; + if (prev && prev.size === current.size && prev.mtimeMs === current.mtimeMs) { + // continue + } else if (prev) { + prev = current; + await new Promise((r) => setTimeout(r, intervalSeconds * 1000)); + continue; + } + prev = current; + await new Promise((r) => setTimeout(r, intervalSeconds * 1000)); + } + + const a = await fs.stat(filePath); + await new Promise((r) => setTimeout(r, intervalSeconds * 1000)); + const b = await fs.stat(filePath); + return a.size === b.size && a.mtimeMs === b.mtimeMs; +} + +export function normalizeSubtitleBuffer(input: Buffer): string { + if (input.length >= 3 && input[0] === 0xef && input[1] === 0xbb && input[2] === 0xbf) { + return input.toString('utf8').replace(/^\uFEFF/, '').replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + } + + const detected = jschardet.detect(input); + const enc = (detected.encoding || '').toLowerCase(); + + let decoded: string; + if (enc.includes('utf')) { + decoded = input.toString('utf8'); + } else if (enc.includes('windows-1254') || enc.includes('iso-8859-9')) { + decoded = iconv.decode(input, 'windows-1254'); + } else { + decoded = iconv.decode(input, 'latin1'); + } + + return decoded.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n').replace(/\r/g, '\n'); +} + +export async function nextSubtitlePath(basePathWithoutExt: string, lang: string, ext: 'srt' | 'ass', overwrite: boolean): Promise { + const preferred = `${basePathWithoutExt}.${lang}.${ext}`; + if (overwrite) return preferred; + + try { + await fs.access(preferred); + } catch { + return preferred; + } + + let i = 2; + while (true) { + const p = `${basePathWithoutExt}.${lang}.${i}.${ext}`; + try { + await fs.access(p); + i += 1; + } catch { + return p; + } + } +} + +export function extensionFromPath(p: string): 'srt' | 'ass' { + return path.extname(p).toLowerCase() === '.ass' ? 'ass' : 'srt'; +} diff --git a/services/core/src/utils/logger.ts b/services/core/src/utils/logger.ts new file mode 100644 index 0000000..4debcda --- /dev/null +++ b/services/core/src/utils/logger.ts @@ -0,0 +1,45 @@ +import { Types } from 'mongoose'; +import { JobLogModel } from '../models/JobLog.js'; + +interface LogInput { + jobId: string | Types.ObjectId; + level?: 'info' | 'warn' | 'error'; + step: string; + message: string; + meta?: any; +} + +type Subscriber = (log: any) => void; +const subscribers = new Map>(); + +export function subscribeLogs(jobId: string, cb: Subscriber): () => void { + if (!subscribers.has(jobId)) subscribers.set(jobId, new Set()); + subscribers.get(jobId)!.add(cb); + return () => subscribers.get(jobId)?.delete(cb); +} + +export async function writeJobLog(input: LogInput): Promise { + const doc = await JobLogModel.create({ + jobId: input.jobId, + level: input.level ?? 'info', + step: input.step, + message: input.message, + meta: input.meta, + ts: new Date() + }); + + const key = String(input.jobId); + const set = subscribers.get(key); + if (set) { + const payload = { + _id: String(doc._id), + jobId: key, + level: doc.level, + step: doc.step, + message: doc.message, + meta: doc.meta, + ts: doc.ts + }; + set.forEach((fn) => fn(payload)); + } +} diff --git a/services/core/src/utils/parser.ts b/services/core/src/utils/parser.ts new file mode 100644 index 0000000..7e3c171 --- /dev/null +++ b/services/core/src/utils/parser.ts @@ -0,0 +1,50 @@ +export interface ParsedMedia { + type: 'tv' | 'movie'; + title: string; + year?: number; + season?: number; + episode?: number; + release?: string; +} + +function cleanTitle(raw: string): string { + return raw + .replace(/[._]+/g, ' ') + .replace(/\s+/g, ' ') + .replace(/\b(1080p|720p|2160p|x264|x265|h264|bluray|webrip|web-dl|dvdrip|hdrip)\b/gi, '') + .trim(); +} + +export function parseMediaFilename(filename: string): ParsedMedia { + const noExt = filename.replace(/\.[^.]+$/, ''); + const tvPatterns = [ + /(.+?)[ ._-]+S(\d{1,2})E(\d{1,2})(?:[ ._-]+(.*))?/i, + /(.+?)[ ._-]+(\d{1,2})x(\d{1,2})(?:[ ._-]+(.*))?/i, + /(.+?)[ ._-]+S(\d{1,2})[ ._-]*x[ ._-]*E(\d{1,2})(?:[ ._-]+(.*))?/i + ]; + + for (const p of tvPatterns) { + const m = noExt.match(p); + if (m) { + const yearMatch = noExt.match(/\b(19\d{2}|20\d{2})\b/); + return { + type: 'tv', + title: cleanTitle(m[1]), + season: Number(m[2]), + episode: Number(m[3]), + year: yearMatch ? Number(yearMatch[1]) : undefined, + release: m[4]?.replace(/[._]/g, ' ').trim() + }; + } + } + + const yearMatch = noExt.match(/\b(19\d{2}|20\d{2})\b/); + const splitByYear = yearMatch ? noExt.split(yearMatch[1])[0] : noExt; + const releaseMatch = noExt.match(/-(\w[\w.-]*)$/); + return { + type: 'movie', + title: cleanTitle(splitByYear), + year: yearMatch ? Number(yearMatch[1]) : undefined, + release: releaseMatch?.[1] + }; +} diff --git a/services/core/src/watcher/index.ts b/services/core/src/watcher/index.ts new file mode 100644 index 0000000..bf6f81c --- /dev/null +++ b/services/core/src/watcher/index.ts @@ -0,0 +1,56 @@ +import chokidar from 'chokidar'; +import { WatchedPathModel } from '../models/WatchedPath.js'; +import { createJobForPath } from '../workers/pipeline.js'; +import { fileEventsQueue } from '../queues/queues.js'; +import { isVideoFile } from '../utils/file.js'; + +export async function ensureDefaultWatchedPaths(tvPath: string, moviePath: string): Promise { + await WatchedPathModel.updateOne({ path: tvPath }, { $setOnInsert: { path: tvPath, kind: 'tv', enabled: true } }, { upsert: true }); + await WatchedPathModel.updateOne({ path: moviePath }, { $setOnInsert: { path: moviePath, kind: 'movie', enabled: true } }, { upsert: true }); +} + +export async function startWatcher(): Promise { + const watched = await WatchedPathModel.find({ enabled: true }).lean(); + const paths = watched.map((w) => w.path); + if (paths.length === 0) { + console.log('[core] no watched paths enabled'); + return; + } + + const byPath = new Map(watched.map((w) => [w.path, w.kind])); + const watcher = chokidar.watch(paths, { ignoreInitial: false, awaitWriteFinish: false, persistent: true }); + + watcher.on('add', async (p) => { + if (!isVideoFile(p)) return; + const kind = resolveKind(p, byPath); + const jobId = await createJobForPath(p, kind === 'movie' ? 'movie' : 'tv'); + const media = await import('../models/MediaFile.js'); + const mediaDoc = await media.MediaFileModel.findOne({ path: p }).lean(); + if (!mediaDoc) return; + await fileEventsQueue.add('add', { jobId, mediaFileId: String(mediaDoc._id), path: p }); + }); + + watcher.on('change', async (p) => { + if (!isVideoFile(p)) return; + const kind = resolveKind(p, byPath); + const jobId = await createJobForPath(p, kind === 'movie' ? 'movie' : 'tv'); + const media = await import('../models/MediaFile.js'); + const mediaDoc = await media.MediaFileModel.findOne({ path: p }).lean(); + if (!mediaDoc) return; + await fileEventsQueue.add('change', { jobId, mediaFileId: String(mediaDoc._id), path: p }); + }); + + watcher.on('unlink', async (p) => { + const media = await import('../models/MediaFile.js'); + await media.MediaFileModel.updateOne({ path: p }, { status: 'MISSING', lastSeenAt: new Date() }); + }); + + console.log(`[core] watcher started for: ${paths.join(', ')}`); +} + +function resolveKind(filePath: string, byPath: Map): 'tv' | 'movie' { + const matched = [...byPath.entries()].find(([root]) => filePath.startsWith(root)); + if (!matched) return 'movie'; + if (matched[1] === 'mixed') return 'movie'; + return matched[1]; +} diff --git a/services/core/src/workers/pipeline.ts b/services/core/src/workers/pipeline.ts new file mode 100644 index 0000000..446251f --- /dev/null +++ b/services/core/src/workers/pipeline.ts @@ -0,0 +1,274 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { Types } from 'mongoose'; +import { JobModel } from '../models/Job.js'; +import { MediaFileModel } from '../models/MediaFile.js'; +import { SettingModel } from '../models/Setting.js'; +import { parseMediaFilename } from '../utils/parser.js'; +import { analyzeWithFfprobe, fallbackMediaInfo } from '../utils/ffprobe.js'; +import { extensionFromPath, isVideoFile, nextSubtitlePath, normalizeSubtitleBuffer, waitForStable } from '../utils/file.js'; +import { writeJobLog } from '../utils/logger.js'; +import { apiClient } from '../utils/apiClient.js'; +import { createWorker, finalizeWriteQueue, mediaAnalysisQueue, subtitleFetchQueue } from '../queues/queues.js'; + +interface FileEventData { + jobId: string; + mediaFileId: string; + path: string; +} + +interface SubtitleFetchData { + jobId: string; + mediaFileId: string; +} + +interface FinalizeData { + jobId: string; + mediaFileId: string; + bestPath: string; + lang: string; + source: string; + confidence: number; +} + +const activeWorkers: any[] = []; + +export function startWorkers(): void { + activeWorkers.push(createWorker('fileEvents', async (job) => { + const data = job.data as FileEventData; + const settings = await SettingModel.findById('global').lean(); + if (!settings) throw new Error('settings missing'); + + await JobModel.findByIdAndUpdate(data.jobId, { status: 'WAITING_FILE_STABLE' }); + await writeJobLog({ jobId: data.jobId, step: 'WAIT_FILE_STABLE_CHECK', message: `Stability check started for ${data.path}` }); + + if (!isVideoFile(data.path)) { + await writeJobLog({ jobId: data.jobId, step: 'WATCH_EVENT_RECEIVED', level: 'warn', message: `Ignored non-video file: ${data.path}` }); + return; + } + + const stable = await waitForStable(data.path, settings.stableChecks, settings.stableIntervalSeconds); + if (!stable) { + await JobModel.findByIdAndUpdate(data.jobId, { status: 'ERROR', error: { code: 'FILE_NOT_STABLE', message: 'file not stable' } }); + await writeJobLog({ jobId: data.jobId, step: 'JOB_ERROR', level: 'error', message: 'File did not become stable' }); + return; + } + + const parsed = parseMediaFilename(path.basename(data.path)); + await MediaFileModel.findByIdAndUpdate(data.mediaFileId, { + type: parsed.type, + title: parsed.title, + year: parsed.year, + season: parsed.season, + episode: parsed.episode, + release: parsed.release, + lastSeenAt: new Date(), + status: 'ACTIVE' + }); + + await JobModel.findByIdAndUpdate(data.jobId, { + status: 'PARSED', + requestSnapshot: parsed + }); + await writeJobLog({ + jobId: data.jobId, + step: 'PARSE_DONE', + message: `Parsed as ${parsed.type}: ${parsed.title}`, + meta: { + title: parsed.title, + year: parsed.year, + season: parsed.season, + episode: parsed.episode, + release: parsed.release + } + }); + await writeJobLog({ jobId: data.jobId, step: 'FILE_STABLE_CONFIRMED', message: data.path }); + + await mediaAnalysisQueue.add('analyze', { jobId: data.jobId, mediaFileId: data.mediaFileId }); + })); + + activeWorkers.push(createWorker('mediaAnalysis', async (job) => { + const { jobId, mediaFileId } = job.data as SubtitleFetchData; + const media = await MediaFileModel.findById(mediaFileId).lean(); + if (!media) return; + + await writeJobLog({ jobId, step: 'FFPROBE_STARTED', message: media.path }); + let mediaInfo: any; + try { + mediaInfo = await analyzeWithFfprobe(media.path); + } catch (err: any) { + mediaInfo = fallbackMediaInfo(); + await writeJobLog({ + jobId, + step: 'FFPROBE_DONE', + level: 'warn', + message: 'ffprobe failed, fallback metadata used', + meta: { error: err?.message } + }); + } + + await MediaFileModel.findByIdAndUpdate(mediaFileId, { mediaInfo, analyzedAt: new Date() }); + await JobModel.findByIdAndUpdate(jobId, { status: 'ANALYZED' }); + await writeJobLog({ jobId, step: 'FFPROBE_DONE', message: 'Media analysis done', meta: mediaInfo }); + + await subtitleFetchQueue.add('search', { jobId, mediaFileId }); + })); + + activeWorkers.push(createWorker('subtitleFetch', async (job) => { + const { jobId, mediaFileId } = job.data as SubtitleFetchData; + const media = await MediaFileModel.findById(mediaFileId).lean(); + const settings = await SettingModel.findById('global').lean(); + if (!media || !settings) return; + + await JobModel.findByIdAndUpdate(jobId, { status: 'REQUESTING_API' }); + await writeJobLog({ jobId, step: 'SUBTITLE_SEARCH_STARTED', message: 'Searching subtitle API' }); + + const payload = { + jobToken: jobId, + type: media.type, + title: media.title, + year: media.year, + release: media.release, + languages: settings.multiSubtitleEnabled ? settings.languages : [settings.languages[0] ?? 'tr'], + season: media.season, + episode: media.episode, + mediaInfo: media.mediaInfo, + preferHI: settings.preferHI, + preferForced: settings.preferForced, + securityLimits: settings.securityLimits + }; + + const res = await apiClient.post('/v1/subtitles/search', payload); + const data = res.data; + + if (Array.isArray(data.trace)) { + for (const t of data.trace) { + await writeJobLog({ + jobId, + step: t.step, + message: t.message, + level: t.level || 'info', + meta: t.meta + }); + } + } + + await writeJobLog({ jobId, step: 'SUBTITLE_SEARCH_DONE', message: `API status ${data.status}` }); + + if (data.status === 'FOUND' && data.bestPath) { + await JobModel.findByIdAndUpdate(jobId, { + status: 'FOUND_TEMP', + apiSnapshot: { status: data.status, confidence: data.confidence, source: data.source, candidates: data.candidates } + }); + await finalizeWriteQueue.add('finalize', { + jobId, + mediaFileId, + bestPath: data.bestPath, + lang: (payload.languages[0] ?? 'tr') as string, + source: data.source ?? 'mock', + confidence: data.confidence ?? 0.8 + } satisfies FinalizeData); + return; + } + + const status = data.status === 'NOT_FOUND' ? 'NOT_FOUND' : 'AMBIGUOUS'; + await JobModel.findByIdAndUpdate(jobId, { + status: 'NEEDS_REVIEW', + apiSnapshot: { + status, + confidence: data.confidence, + source: data.source, + candidates: data.candidates + } + }); + + await writeJobLog({ + jobId, + step: data.status === 'NOT_FOUND' ? 'NOT_FOUND_NEEDS_REVIEW' : 'AMBIGUOUS_NEEDS_REVIEW', + message: 'Manual review required' + }); + })); + + activeWorkers.push(createWorker('finalizeWrite', async (job) => { + const data = job.data as FinalizeData; + const media = await MediaFileModel.findById(data.mediaFileId).lean(); + const settings = await SettingModel.findById('global').lean(); + if (!media || !settings) return; + + await JobModel.findByIdAndUpdate(data.jobId, { status: 'NORMALIZING_ENCODING' }); + + const raw = await fs.readFile(data.bestPath); + const normalized = normalizeSubtitleBuffer(raw); + await writeJobLog({ jobId: data.jobId, step: 'ENCODING_NORMALIZED_UTF8', message: 'Subtitle normalized to UTF-8/LF' }); + + await JobModel.findByIdAndUpdate(data.jobId, { status: 'WRITING_SUBTITLE' }); + + const parsed = path.parse(media.path); + const target = await nextSubtitlePath(path.join(parsed.dir, parsed.name), data.lang, extensionFromPath(data.bestPath), settings.overwriteExisting); + + const targetExists = target !== `${path.join(parsed.dir, parsed.name)}.${data.lang}.${extensionFromPath(data.bestPath)}`; + if (targetExists) { + await writeJobLog({ jobId: data.jobId, step: 'WRITE_TARGET_SKIPPED_EXISTS', message: 'Target exists, using incremented filename', meta: { target } }); + } + + await fs.writeFile(target, normalized, 'utf8'); + await writeJobLog({ jobId: data.jobId, step: 'WRITE_TARGET_DONE', message: target }); + + await JobModel.findByIdAndUpdate(data.jobId, { + status: 'DONE', + result: { + subtitles: [ + { + lang: data.lang, + source: data.source, + confidence: data.confidence, + writtenPath: target, + ext: extensionFromPath(data.bestPath) + } + ] + } + }); + + await writeJobLog({ jobId: data.jobId, step: 'JOB_DONE', message: 'Pipeline completed' }); + + try { + await apiClient.post('/v1/subtitles/cleanup', { jobToken: data.jobId }); + await writeJobLog({ jobId: data.jobId, step: 'CLEANUP_TEMP_DONE', message: 'Temporary files cleanup requested' }); + } catch { + await writeJobLog({ jobId: data.jobId, step: 'CLEANUP_TEMP_DONE', level: 'warn', message: 'Cleanup endpoint unavailable, periodic cleanup will handle it' }); + } + })); +} + +export async function createJobForPath(filePath: string, mediaTypeHint: 'tv' | 'movie'): Promise { + const st = await fs.stat(filePath); + + const media = await MediaFileModel.findOneAndUpdate( + { path: filePath }, + { + $set: { + path: filePath, + type: mediaTypeHint, + size: st.size, + mtime: st.mtime, + status: 'ACTIVE', + lastSeenAt: new Date() + } + }, + { upsert: true, new: true } + ); + + const created = await JobModel.create({ + mediaFileId: media._id, + status: 'PENDING', + attempts: 0 + }); + + await writeJobLog({ + jobId: created._id as Types.ObjectId, + step: 'WATCH_EVENT_RECEIVED', + message: `Queued watch event for ${filePath}` + }); + + return String(created._id); +} diff --git a/services/core/test/parser.test.ts b/services/core/test/parser.test.ts new file mode 100644 index 0000000..bcaf9b7 --- /dev/null +++ b/services/core/test/parser.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest'; +import { parseMediaFilename } from '../src/utils/parser.js'; + +describe('parseMediaFilename', () => { + it('parses SxxEyy pattern', () => { + const p = parseMediaFilename('The.Last.of.Us.S01E02.1080p-FLUX.mkv'); + expect(p.type).toBe('tv'); + expect(p.title).toBe('The Last of Us'); + expect(p.season).toBe(1); + expect(p.episode).toBe(2); + }); + + it('parses 1x02 pattern', () => { + const p = parseMediaFilename('Dark.1x02.WEBRip.mkv'); + expect(p.type).toBe('tv'); + expect(p.season).toBe(1); + expect(p.episode).toBe(2); + }); + + it('parses movie pattern', () => { + const p = parseMediaFilename('John.Wick.2014.1080p.BluRay-FLUX.mkv'); + expect(p.type).toBe('movie'); + expect(p.title).toBe('John Wick'); + expect(p.year).toBe(2014); + }); +}); diff --git a/services/core/tsconfig.json b/services/core/tsconfig.json new file mode 100644 index 0000000..4396df9 --- /dev/null +++ b/services/core/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "types": ["node"] + }, + "include": ["src/**/*"] +} diff --git a/services/ui/Dockerfile b/services/ui/Dockerfile new file mode 100644 index 0000000..b93c2f5 --- /dev/null +++ b/services/ui/Dockerfile @@ -0,0 +1,21 @@ +FROM node:20-bookworm AS base +WORKDIR /app +COPY package*.json ./ + +FROM base AS dev +RUN npm install +COPY . . +EXPOSE 5173 +CMD ["npm", "run", "dev"] + +FROM base AS build +RUN npm ci +COPY . . +RUN npm run build + +FROM node:20-bookworm AS prod +WORKDIR /app +RUN npm install -g serve +COPY --from=build /app/dist ./dist +EXPOSE 3000 +CMD ["serve", "-s", "dist", "-l", "3000"] diff --git a/services/ui/index.html b/services/ui/index.html new file mode 100644 index 0000000..0743eff --- /dev/null +++ b/services/ui/index.html @@ -0,0 +1,12 @@ + + + + + + subwatcher + + +
+ + + diff --git a/services/ui/package.json b/services/ui/package.json new file mode 100644 index 0000000..68e85b1 --- /dev/null +++ b/services/ui/package.json @@ -0,0 +1,22 @@ +{ + "name": "subwatcher-ui", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite --host 0.0.0.0 --port 5173", + "build": "tsc -b && vite build", + "start": "serve -s dist -l 3000" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@types/react": "^18.3.18", + "@types/react-dom": "^18.3.5", + "@vitejs/plugin-react": "^4.3.4", + "typescript": "^5.7.3", + "vite": "^6.0.11" + } +} \ No newline at end of file diff --git a/services/ui/src/App.tsx b/services/ui/src/App.tsx new file mode 100644 index 0000000..5f203de --- /dev/null +++ b/services/ui/src/App.tsx @@ -0,0 +1,34 @@ +import { useState } from 'react'; +import { Layout, Tab } from './components/Layout'; +import { DashboardPage } from './pages/DashboardPage'; +import { JobsPage } from './pages/JobsPage'; +import { JobDetailPage } from './pages/JobDetailPage'; +import { ReviewPage } from './pages/ReviewPage'; +import { SettingsPage } from './pages/SettingsPage'; +import { WatchedPathsPage } from './pages/WatchedPathsPage'; + +export default function App() { + const [tab, setTab] = useState('dashboard'); + const [selectedJob, setSelectedJob] = useState(null); + + return ( + + {selectedJob && ( +
+ +
+ )} + {selectedJob ? ( + + ) : ( + <> + {tab === 'dashboard' && } + {tab === 'jobs' && } + {tab === 'review' && } + {tab === 'settings' && } + {tab === 'paths' && } + + )} + + ); +} diff --git a/services/ui/src/api/client.ts b/services/ui/src/api/client.ts new file mode 100644 index 0000000..7e2805f --- /dev/null +++ b/services/ui/src/api/client.ts @@ -0,0 +1,9 @@ +export async function api(path: string, options?: RequestInit): Promise { + const base = import.meta.env.VITE_PUBLIC_CORE_URL || 'http://localhost:3001'; + const res = await fetch(`${base}${path}`, { + headers: { 'content-type': 'application/json', ...(options?.headers || {}) }, + ...options + }); + if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); + return res.json(); +} diff --git a/services/ui/src/components/JobTable.tsx b/services/ui/src/components/JobTable.tsx new file mode 100644 index 0000000..e724edd --- /dev/null +++ b/services/ui/src/components/JobTable.tsx @@ -0,0 +1,26 @@ +import { Job } from '../types'; + +export function JobTable({ jobs, onSelect }: { jobs: Job[]; onSelect: (id: string) => void }) { + return ( + + + + + + + + + + + {jobs.map((j) => ( + onSelect(j._id)} style={{ cursor: 'pointer', borderTop: '1px solid #e2e8f0' }}> + + + + + + ))} + +
IDDurumBaslikGuncelleme
{j._id.slice(-8)}{j.status}{j.requestSnapshot?.title || '-'}{new Date(j.updatedAt).toLocaleString()}
+ ); +} diff --git a/services/ui/src/components/Layout.tsx b/services/ui/src/components/Layout.tsx new file mode 100644 index 0000000..eba60bf --- /dev/null +++ b/services/ui/src/components/Layout.tsx @@ -0,0 +1,20 @@ +import React from 'react'; + +const tabs = ['dashboard', 'jobs', 'review', 'settings', 'paths'] as const; +export type Tab = (typeof tabs)[number]; + +export function Layout({ tab, setTab, children }: { tab: Tab; setTab: (t: Tab) => void; children: React.ReactNode }) { + return ( +
+
+ subwatcher + {tabs.map((t) => ( + + ))} +
+
{children}
+
+ ); +} diff --git a/services/ui/src/hooks/usePoll.ts b/services/ui/src/hooks/usePoll.ts new file mode 100644 index 0000000..cee1cd3 --- /dev/null +++ b/services/ui/src/hooks/usePoll.ts @@ -0,0 +1,16 @@ +import { useEffect } from 'react'; + +export function usePoll(fn: () => void | Promise, ms: number) { + useEffect(() => { + let active = true; + const tick = async () => { + if (!active) return; + await fn(); + setTimeout(tick, ms); + }; + tick(); + return () => { + active = false; + }; + }, [fn, ms]); +} diff --git a/services/ui/src/main.tsx b/services/ui/src/main.tsx new file mode 100644 index 0000000..9707d82 --- /dev/null +++ b/services/ui/src/main.tsx @@ -0,0 +1,9 @@ +import React from 'react'; +import ReactDOM from 'react-dom/client'; +import App from './App'; + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + +); diff --git a/services/ui/src/pages/DashboardPage.tsx b/services/ui/src/pages/DashboardPage.tsx new file mode 100644 index 0000000..6326037 --- /dev/null +++ b/services/ui/src/pages/DashboardPage.tsx @@ -0,0 +1,44 @@ +import { useCallback, useState } from 'react'; +import { api } from '../api/client'; +import { Job } from '../types'; +import { usePoll } from '../hooks/usePoll'; +import { JobTable } from '../components/JobTable'; + +export function DashboardPage({ onSelectJob }: { onSelectJob: (id: string) => void }) { + const [jobs, setJobs] = useState([]); + + const load = useCallback(async () => { + const data = await api<{ items: Job[] }>('/api/jobs?limit=20'); + setJobs(data.items); + }, []); + + usePoll(load, 5000); + + const since = Date.now() - 24 * 3600 * 1000; + const recent = jobs.filter((x) => new Date(x.createdAt).getTime() >= since); + const done = recent.filter((x) => x.status === 'DONE').length; + const review = recent.filter((x) => x.status === 'NEEDS_REVIEW').length; + const errors = recent.filter((x) => x.status === 'ERROR').length; + + return ( +
+
+ + + + +
+

Son Isler

+ +
+ ); +} + +function Stat({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ); +} diff --git a/services/ui/src/pages/JobDetailPage.tsx b/services/ui/src/pages/JobDetailPage.tsx new file mode 100644 index 0000000..007408a --- /dev/null +++ b/services/ui/src/pages/JobDetailPage.tsx @@ -0,0 +1,95 @@ +import { useEffect, useState } from 'react'; +import { api } from '../api/client'; +import { Job, JobLog } from '../types'; + +export function JobDetailPage({ jobId }: { jobId: string }) { + const [job, setJob] = useState(null); + const [logs, setLogs] = useState([]); + const [override, setOverride] = useState({}); + const [candidates, setCandidates] = useState([]); + + useEffect(() => { + let es: EventSource | null = null; + (async () => { + const j = await api(`/api/jobs/${jobId}`); + setJob(j); + const l = await api<{ items: JobLog[] }>(`/api/jobs/${jobId}/logs?limit=200`); + setLogs(l.items); + if (j.apiSnapshot?.candidates) setCandidates(j.apiSnapshot.candidates); + + es = new EventSource(`/api/jobs/${jobId}/stream`); + es.onmessage = (ev) => { + const item = JSON.parse(ev.data); + setLogs((prev) => [...prev, item]); + }; + })(); + + return () => { + if (es) es.close(); + }; + }, [jobId]); + + async function manualSearch() { + const res = await api(`/api/review/${jobId}/search`, { method: 'POST', body: JSON.stringify(override) }); + setCandidates(res.candidates || []); + } + + async function choose(c: any) { + await api(`/api/review/${jobId}/choose`, { method: 'POST', body: JSON.stringify({ chosenCandidateId: c.id, lang: c.lang || 'tr' }) }); + const j = await api(`/api/jobs/${jobId}`); + setJob(j); + } + + if (!job) return
Yukleniyor...
; + + const media = job.mediaFileId as any; + return ( +
+

Job #{job._id.slice(-8)} - {job.status}

+
+
Baslik: {job.requestSnapshot?.title || '-'}
+
Tip: {job.requestSnapshot?.type || '-'}
+
Yil: {job.requestSnapshot?.year || '-'}
+
Release: {job.requestSnapshot?.release || '-'}
+
Season/Episode: {job.requestSnapshot?.season ?? '-'} / {job.requestSnapshot?.episode ?? '-'}
+
Media: {media?.path || '-'}
+
Video: {media?.mediaInfo?.video?.codec_name || '-'} {media?.mediaInfo?.video?.width || '-'}x{media?.mediaInfo?.video?.height || '-'}
+
Sonuc: {job.result?.subtitles?.map((s: any) => s.writtenPath).join(', ') || '-'}
+
+ + {job.status === 'NEEDS_REVIEW' && ( +
+

Manual Override

+
+ setOverride((x: any) => ({ ...x, title: e.target.value }))} /> + setOverride((x: any) => ({ ...x, year: Number(e.target.value) }))} /> + setOverride((x: any) => ({ ...x, release: e.target.value }))} /> + setOverride((x: any) => ({ ...x, season: Number(e.target.value) }))} /> + setOverride((x: any) => ({ ...x, episode: Number(e.target.value) }))} /> + +
+
    + {candidates.map((c) => ( +
  • + {c.provider} | {c.id} | score={c.score} + +
    • + ))} +
+
+ )} + +
+

Canli Loglar

+
+ {logs.map((l) => ( +
+ [{new Date(l.ts).toLocaleTimeString()}] {l.step} - {l.message} + {l.meta ? ` | meta=${JSON.stringify(l.meta)}` : ''} +
+ ))} +
+
+
+ ); +} diff --git a/services/ui/src/pages/JobsPage.tsx b/services/ui/src/pages/JobsPage.tsx new file mode 100644 index 0000000..d1bb384 --- /dev/null +++ b/services/ui/src/pages/JobsPage.tsx @@ -0,0 +1,32 @@ +import { useCallback, useState } from 'react'; +import { api } from '../api/client'; +import { Job } from '../types'; +import { usePoll } from '../hooks/usePoll'; +import { JobTable } from '../components/JobTable'; + +export function JobsPage({ onSelectJob }: { onSelectJob: (id: string) => void }) { + const [jobs, setJobs] = useState([]); + const [status, setStatus] = useState(''); + const [search, setSearch] = useState(''); + + const load = useCallback(async () => { + const q = new URLSearchParams({ limit: '100' }); + if (status) q.set('status', status); + if (search) q.set('search', search); + const data = await api<{ items: Job[] }>(`/api/jobs?${q.toString()}`); + setJobs(data.items); + }, [status, search]); + + usePoll(load, 4000); + + return ( +
+
+ setStatus(e.target.value)} /> + setSearch(e.target.value)} /> + +
+ +
+ ); +} diff --git a/services/ui/src/pages/ReviewPage.tsx b/services/ui/src/pages/ReviewPage.tsx new file mode 100644 index 0000000..5993867 --- /dev/null +++ b/services/ui/src/pages/ReviewPage.tsx @@ -0,0 +1,29 @@ +import { useCallback, useState } from 'react'; +import { api } from '../api/client'; +import { Job } from '../types'; +import { usePoll } from '../hooks/usePoll'; + +export function ReviewPage({ onSelectJob }: { onSelectJob: (id: string) => void }) { + const [jobs, setJobs] = useState([]); + + const load = useCallback(async () => { + const data = await api('/api/review'); + setJobs(data); + }, []); + + usePoll(load, 5000); + + return ( +
+

Needs Review

+
    + {jobs.map((j) => ( +
  • + {j.requestSnapshot?.title || '-'} ({j.status}) + +
    • + ))} +
+
+ ); +} diff --git a/services/ui/src/pages/SettingsPage.tsx b/services/ui/src/pages/SettingsPage.tsx new file mode 100644 index 0000000..ea5017a --- /dev/null +++ b/services/ui/src/pages/SettingsPage.tsx @@ -0,0 +1,32 @@ +import { useEffect, useState } from 'react'; +import { api } from '../api/client'; + +export function SettingsPage() { + const [settings, setSettings] = useState(null); + + useEffect(() => { + api('/api/settings').then(setSettings); + }, []); + + async function save() { + await api('/api/settings', { method: 'POST', body: JSON.stringify(settings) }); + } + + if (!settings) return
Yukleniyor...
; + + return ( +
+ + + + + + + + + +
+ ); +} diff --git a/services/ui/src/pages/WatchedPathsPage.tsx b/services/ui/src/pages/WatchedPathsPage.tsx new file mode 100644 index 0000000..e24a92b --- /dev/null +++ b/services/ui/src/pages/WatchedPathsPage.tsx @@ -0,0 +1,60 @@ +import { useCallback, useState } from 'react'; +import { api } from '../api/client'; +import { usePoll } from '../hooks/usePoll'; + +export function WatchedPathsPage() { + const [items, setItems] = useState([]); + const [path, setPath] = useState(''); + const [kind, setKind] = useState('mixed'); + + const load = useCallback(async () => { + const data = await api('/api/watched-paths'); + setItems(data); + }, []); + + usePoll(load, 5000); + + async function add() { + await api('/api/watched-paths', { method: 'POST', body: JSON.stringify({ action: 'add', path, kind }) }); + setPath(''); + await load(); + } + + async function toggle(item: any) { + await api('/api/watched-paths', { method: 'POST', body: JSON.stringify({ action: 'toggle', path: item.path, enabled: !item.enabled }) }); + await load(); + } + + async function remove(item: any) { + await api('/api/watched-paths', { method: 'POST', body: JSON.stringify({ action: 'remove', path: item.path }) }); + await load(); + } + + return ( +
+
+ setPath(e.target.value)} /> + + +
+ + + + {items.map((i) => ( + + + + + ))} + +
PathKindEnabled
{i.path}{i.kind}{String(i.enabled)} + + +
+
+ ); +} diff --git a/services/ui/src/types.ts b/services/ui/src/types.ts new file mode 100644 index 0000000..393819a --- /dev/null +++ b/services/ui/src/types.ts @@ -0,0 +1,20 @@ +export interface Job { + _id: string; + status: string; + requestSnapshot?: any; + apiSnapshot?: any; + result?: any; + mediaFileId?: any; + createdAt: string; + updatedAt: string; +} + +export interface JobLog { + _id: string; + jobId: string; + step: string; + message: string; + level: 'info' | 'warn' | 'error'; + ts: string; + meta?: any; +} diff --git a/services/ui/tsconfig.json b/services/ui/tsconfig.json new file mode 100644 index 0000000..d28907b --- /dev/null +++ b/services/ui/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "jsx": "react-jsx", + "strict": true, + "skipLibCheck": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/services/ui/vite.config.ts b/services/ui/vite.config.ts new file mode 100644 index 0000000..fae6c55 --- /dev/null +++ b/services/ui/vite.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; + +export default defineConfig({ + plugins: [react()], + server: { + port: 5173, + host: '0.0.0.0', + proxy: { + '/api': { + target: process.env.VITE_CORE_URL || 'http://core:3001', + changeOrigin: true + } + } + } +});