diff --git a/.env.example b/.env.example index b9fb6f0..b848ac9 100644 --- a/.env.example +++ b/.env.example @@ -12,7 +12,11 @@ MEDIA_TV_PATH=/media/tv MEDIA_MOVIE_PATH=/media/movie ENABLE_API_KEY=false API_KEY= +CORE_WATCHER_DEDUP_WINDOW_MS=15000 ENABLE_TA_STEP_LOGS=false +CLAMAV_AUTO_UPDATE=true +CLAMAV_FAIL_ON_UPDATE_ERROR=false +CLAMAV_DB_DIR=/var/lib/clamav ENABLE_TURKCEALTYAZI_REAL=false TURKCEALTYAZI_BASE_URL=https://turkcealtyazi.org TURKCEALTYAZI_TIMEOUT_MS=12000 diff --git a/compose.dev.yml b/compose.dev.yml index 1540d23..36a2ef9 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -27,6 +27,7 @@ services: - API_PORT=3002 - TEMP_ROOT=/temp - ENABLE_API_KEY=false + - CLAMAV_DB_DIR=/var/lib/clamav ports: - "3002:3002" command: sh -c "npm install && npm run dev" @@ -34,6 +35,7 @@ services: - ./services/api:/app - api_node_modules:/app/node_modules - temp_data:/temp + - clamav_data:/var/lib/clamav depends_on: - mongo - redis @@ -95,3 +97,4 @@ volumes: core_node_modules: api_node_modules: ui_node_modules: + clamav_data: diff --git a/compose.yml b/compose.yml index 6ca3e71..0b1a4d7 100644 --- a/compose.yml +++ b/compose.yml @@ -25,10 +25,12 @@ services: - NODE_ENV=production - API_PORT=3002 - TEMP_ROOT=/temp + - CLAMAV_DB_DIR=/var/lib/clamav ports: - "3002:3002" volumes: - temp_data:/temp + - clamav_data:/var/lib/clamav depends_on: - mongo - redis @@ -81,3 +83,4 @@ volumes: mongo_data: redis_data: temp_data: + clamav_data: diff --git a/services/api/Dockerfile b/services/api/Dockerfile index 0c3d54e..4419753 100644 --- a/services/api/Dockerfile +++ b/services/api/Dockerfile @@ -1,6 +1,6 @@ FROM node:20-bookworm AS base WORKDIR /app -RUN apt-get update && apt-get install -y --no-install-recommends p7zip-full unrar-free && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends p7zip-full unrar-free clamav && rm -rf /var/lib/apt/lists/* COPY package*.json ./ FROM base AS dev @@ -16,7 +16,7 @@ RUN npm run build FROM node:20-bookworm AS prod WORKDIR /app -RUN apt-get update && apt-get install -y --no-install-recommends p7zip-full unrar-free && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends p7zip-full unrar-free clamav && rm -rf /var/lib/apt/lists/* COPY --from=build /app/package*.json ./ COPY --from=build /app/node_modules ./node_modules COPY --from=build /app/dist ./dist diff --git a/services/api/package-lock.json b/services/api/package-lock.json index 40abb9c..1fb0cfa 100644 --- a/services/api/package-lock.json +++ b/services/api/package-lock.json @@ -26,6 +26,278 @@ "vitest": "^3.0.5" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", + "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", + "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", + "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", + "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", + "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", + "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", + "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", + "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", + "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", + "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", + "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", + "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", + "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", + "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", + "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", + "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/linux-x64": { "version": "0.27.3", "cpu": [ @@ -41,6 +313,159 @@ "node": ">=18" } }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", + "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", + "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", + "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", + "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", + "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", + "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", + "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", + "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", + "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@fastify/ajv-compiler": { "version": "4.0.5", "funding": [ @@ -167,6 +592,244 @@ "version": "0.4.0", "license": "MIT" }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.57.1.tgz", + "integrity": "sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.57.1.tgz", + "integrity": "sha512-dQaAddCY9YgkFHZcFNS/606Exo8vcLHwArFZ7vxXq4rigo2bb494/xKMMwRRQW6ug7Js6yXmBZhSBRuBvCCQ3w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.57.1.tgz", + "integrity": "sha512-crNPrwJOrRxagUYeMn/DZwqN88SDmwaJ8Cvi/TN1HnWBU7GwknckyosC2gd0IqYRsHDEnXf328o9/HC6OkPgOg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.57.1.tgz", + "integrity": "sha512-Ji8g8ChVbKrhFtig5QBV7iMaJrGtpHelkB3lsaKzadFBe58gmjfGXAOfI5FV0lYMH8wiqsxKQ1C9B0YTRXVy4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.57.1.tgz", + "integrity": "sha512-R+/WwhsjmwodAcz65guCGFRkMb4gKWTcIeLy60JJQbXrJ97BOXHxnkPFrP+YwFlaS0m+uWJTstrUA9o+UchFug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.57.1.tgz", + "integrity": "sha512-IEQTCHeiTOnAUC3IDQdzRAGj3jOAYNr9kBguI7MQAAZK3caezRrg0GxAb6Hchg4lxdZEI5Oq3iov/w/hnFWY9Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.57.1.tgz", + "integrity": "sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.57.1.tgz", + "integrity": "sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.57.1.tgz", + "integrity": "sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.57.1.tgz", + "integrity": "sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.57.1.tgz", + "integrity": "sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.57.1.tgz", + "integrity": "sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.57.1.tgz", + "integrity": "sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.57.1.tgz", + "integrity": "sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.57.1.tgz", + "integrity": "sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.57.1.tgz", + "integrity": "sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.57.1.tgz", + "integrity": "sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, "node_modules/@rollup/rollup-linux-x64-gnu": { "version": "4.57.1", "cpu": [ @@ -179,6 +842,104 @@ "linux" ] }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.57.1.tgz", + "integrity": "sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.57.1.tgz", + "integrity": "sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.57.1.tgz", + "integrity": "sha512-4wYoDpNg6o/oPximyc/NG+mYUejZrCU2q+2w6YZqrAs2UcNUChIZXjtafAiiZSUc7On8v5NyNj34Kzj/Ltk6dQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.57.1.tgz", + "integrity": "sha512-O54mtsV/6LW3P8qdTcamQmuC990HDfR71lo44oZMZlXU4tzLrbvTii87Ni9opq60ds0YzuAlEr/GNwuNluZyMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.57.1.tgz", + "integrity": "sha512-P3dLS+IerxCT/7D2q2FYcRdWRl22dNbrbBEtxdWhXrfIMPP9lQhb5h4Du04mdl5Woq05jVCDPCMF7Ub0NAjIew==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.57.1.tgz", + "integrity": "sha512-VMBH2eOOaKGtIJYleXsi2B8CPVADrh+TyNxJ4mWPnKfLB/DBUmzW+5m1xUrcwWoMfSLagIRpjUFeW5CO5hyciQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.57.1.tgz", + "integrity": "sha512-mxRFDdHIWRxg3UfIIAwCm6NzvxG0jDX/wBN6KsQFTvKFqqg9vTrWUE68qEjHt19A5wwx5X5aUi2zuZT7YR0jrA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@types/adm-zip": { "version": "0.5.7", "dev": true, @@ -994,6 +1755,21 @@ "node": ">=14.14" } }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", diff --git a/services/api/src/config/env.ts b/services/api/src/config/env.ts index b15c944..393c416 100644 --- a/services/api/src/config/env.ts +++ b/services/api/src/config/env.ts @@ -9,6 +9,9 @@ export const env = { enableApiKey: process.env.ENABLE_API_KEY === 'true', apiKey: process.env.API_KEY ?? '', enableTaStepLogs: process.env.ENABLE_TA_STEP_LOGS === 'true', + clamavAutoUpdate: process.env.CLAMAV_AUTO_UPDATE !== 'false', + clamavFailOnUpdateError: process.env.CLAMAV_FAIL_ON_UPDATE_ERROR === 'true', + clamavDbDir: process.env.CLAMAV_DB_DIR ?? '/var/lib/clamav', enableTurkcealtyaziReal: process.env.ENABLE_TURKCEALTYAZI_REAL === 'true', turkcealtyaziBaseUrl: process.env.TURKCEALTYAZI_BASE_URL ?? 'https://turkcealtyazi.org', turkcealtyaziTimeoutMs: Number(process.env.TURKCEALTYAZI_TIMEOUT_MS ?? 12000), diff --git a/services/api/src/index.ts b/services/api/src/index.ts index 2ee29b0..0d9aed8 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -2,9 +2,11 @@ import fs from 'node:fs/promises'; import { buildApp } from './app.js'; import { cleanupOldTemp } from './lib/subtitleEngine.js'; import { env } from './config/env.js'; +import { ensureClamavDatabase } from './lib/clamavDb.js'; async function bootstrap() { await fs.mkdir(env.tempRoot, { recursive: true }); + await ensureClamavDatabase(); const app = await buildApp(); setInterval(async () => { diff --git a/services/api/src/lib/clamav.ts b/services/api/src/lib/clamav.ts new file mode 100644 index 0000000..4505ff2 --- /dev/null +++ b/services/api/src/lib/clamav.ts @@ -0,0 +1,30 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export interface ClamAvScanResult { + clean: boolean; + infected: boolean; + output: string; +} + +export async function scanFileWithClamav(filePath: string): Promise { + try { + const { stdout, stderr } = await execFileAsync('clamscan', ['--no-summary', filePath]); + const output = `${stdout || ''}${stderr || ''}`.trim(); + return { clean: true, infected: false, output }; + } catch (err: any) { + const output = `${err?.stdout || ''}${err?.stderr || ''}`.trim(); + const code = typeof err?.code === 'number' ? err.code : undefined; + + // clamscan exit code: + // 0 = no virus found, 1 = virus found, >1 = error + if (code === 1) { + return { clean: false, infected: true, output }; + } + + const reason = output || err?.message || 'clamav scan failed'; + throw new Error(`clamav scan error: ${reason}`); + } +} diff --git a/services/api/src/lib/clamavDb.ts b/services/api/src/lib/clamavDb.ts new file mode 100644 index 0000000..86fa559 --- /dev/null +++ b/services/api/src/lib/clamavDb.ts @@ -0,0 +1,68 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { env } from '../config/env.js'; + +const execFileAsync = promisify(execFile); + +function hasDbFile(filename: string): boolean { + return /\.(cvd|cld)$/i.test(filename); +} + +async function hasLocalDatabase(dbDir: string): Promise { + try { + const entries = await fs.readdir(dbDir, { withFileTypes: true }); + return entries.some((e) => e.isFile() && hasDbFile(e.name)); + } catch { + return false; + } +} + +function normalizeFreshclamOutput(raw: string): string { + return raw + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .filter((line) => !line.includes('NotifyClamd: Can\'t find or parse configuration file')) + .join(' '); +} + +export async function ensureClamavDatabase(): Promise { + if (!env.clamavAutoUpdate) { + console.log('[api][clamav] auto update disabled'); + return; + } + + const dbDir = env.clamavDbDir; + await fs.mkdir(dbDir, { recursive: true }); + + const before = await hasLocalDatabase(dbDir); + if (before) { + console.log(`[api][clamav] database already present in ${dbDir}`); + return; + } + + console.log(`[api][clamav] database missing, running freshclam in ${dbDir}`); + try { + const { stdout, stderr } = await execFileAsync('freshclam', ['--stdout'], { timeout: 5 * 60 * 1000 }); + const out = normalizeFreshclamOutput(`${stdout || ''}\n${stderr || ''}`); + console.log(`[api][clamav] freshclam completed: ${out}`); + } catch (err: any) { + const details = `${err?.stdout || ''}${err?.stderr || ''}${err?.message || ''}`.replace(/\s+/g, ' ').trim(); + if (env.clamavFailOnUpdateError) { + throw new Error(`clamav freshclam failed: ${details}`); + } + console.warn(`[api][clamav] freshclam failed (continuing): ${details}`); + } + + const after = await hasLocalDatabase(dbDir); + if (!after && env.clamavFailOnUpdateError) { + throw new Error(`clamav database still missing after freshclam: ${path.resolve(dbDir)}`); + } + if (after) { + console.log(`[api][clamav] database ready in ${dbDir}`); + } else { + console.warn(`[api][clamav] database not found in ${dbDir}; clamav scans may fail`); + } +} diff --git a/services/api/src/lib/errors.ts b/services/api/src/lib/errors.ts new file mode 100644 index 0000000..b5d08c4 --- /dev/null +++ b/services/api/src/lib/errors.ts @@ -0,0 +1,96 @@ +import axios from 'axios'; + +export type ErrorCategory = + | 'network' + | 'parse' + | 'blocked' + | 'rate-limit' + | 'malware' + | 'invalid-subtitle' + | 'internal'; + +export class PipelineError extends Error { + code: string; + category: ErrorCategory; + retryable: boolean; + httpStatus: number; + + constructor(opts: { + code: string; + message: string; + category: ErrorCategory; + retryable: boolean; + httpStatus?: number; + cause?: unknown; + }) { + super(opts.message); + this.name = 'PipelineError'; + this.code = opts.code; + this.category = opts.category; + this.retryable = opts.retryable; + this.httpStatus = opts.httpStatus ?? 500; + if (opts.cause !== undefined) { + (this as any).cause = opts.cause; + } + } +} + +function normalizeMsg(input: unknown): string { + return String(input ?? 'unknown error').replace(/\s+/g, ' ').trim(); +} + +export function toPipelineError(err: unknown, fallbackCode = 'INTERNAL_ERROR'): PipelineError { + if (err instanceof PipelineError) return err; + + if (axios.isAxiosError(err)) { + const status = err.response?.status; + const msg = normalizeMsg(err.message); + if (status === 429) { + return new PipelineError({ + code: 'UPSTREAM_RATE_LIMIT', + message: msg, + category: 'rate-limit', + retryable: true, + httpStatus: 503, + cause: err + }); + } + if (status === 403) { + return new PipelineError({ + code: 'UPSTREAM_BLOCKED', + message: msg, + category: 'blocked', + retryable: false, + httpStatus: 502, + cause: err + }); + } + if (status && status >= 500) { + return new PipelineError({ + code: 'UPSTREAM_5XX', + message: msg, + category: 'network', + retryable: true, + httpStatus: 502, + cause: err + }); + } + return new PipelineError({ + code: status ? `UPSTREAM_${status}` : 'NETWORK_ERROR', + message: msg, + category: 'network', + retryable: !status, + httpStatus: 502, + cause: err + }); + } + + return new PipelineError({ + code: fallbackCode, + message: normalizeMsg((err as any)?.message ?? err), + category: 'internal', + retryable: false, + httpStatus: 500, + cause: err + }); +} diff --git a/services/api/src/lib/scoring.ts b/services/api/src/lib/scoring.ts index 354e0a0..fcf1ae8 100644 --- a/services/api/src/lib/scoring.ts +++ b/services/api/src/lib/scoring.ts @@ -24,6 +24,7 @@ export function scoreCandidateFile(filePath: string, ext: 'srt' | 'ass', candida const fn = path.basename(filePath).toLowerCase(); let score = 0; const reasons: string[] = []; + const isPackageCandidate = candidate.scoreHints.includes('ta_package_candidate'); if (params.type === 'tv') { const sePattern = /s(\d{1,2})e(\d{1,2})/i; @@ -35,11 +36,12 @@ export function scoreCandidateFile(filePath: string, ext: 'srt' | 'ass', candida reasons.push('season_episode_match'); } - const releaseTokens = tokenize(params.release); + const releaseTokens = isPackageCandidate ? [] : tokenize(params.release); const fileTokens = tokenize(fn).concat(candidate.releaseHints.map((x) => x.toLowerCase())); const releaseMatches = releaseTokens.filter((t) => fileTokens.includes(t)).length; score += Math.min(25, releaseMatches * 6); if (releaseMatches > 0) reasons.push('release_match'); + if (isPackageCandidate) reasons.push('package_mode_episode_only'); if (candidate.lang === (params.languages[0] || 'tr')) { score += 10; diff --git a/services/api/src/lib/subtitleEngine.ts b/services/api/src/lib/subtitleEngine.ts index e57be5f..c289c5c 100644 --- a/services/api/src/lib/subtitleEngine.ts +++ b/services/api/src/lib/subtitleEngine.ts @@ -11,6 +11,8 @@ import { OpenSubtitlesProvider } from '../providers/OpenSubtitlesProvider.js'; import { collectFilesRecursive, ensureInsideRoot, validateExtractionLimits } from './security.js'; import { detectSubtitleType, isProbablyText } from './validators.js'; import { chooseBest, scoreCandidateFile } from './scoring.js'; +import { scanFileWithClamav } from './clamav.js'; +import { PipelineError } from './errors.js'; const execFileAsync = promisify(execFile); @@ -45,6 +47,10 @@ export async function searchSubtitles(input: SearchParams) { const trace: TraceLog[] = []; const limits = input.securityLimits ?? defaultLimits(); const dirs = await ensureJobDirs(jobToken); + const clamavEnabled = input.features?.clamavEnabled === true; + if (!clamavEnabled) { + trace.push({ level: 'info', step: 'CLAMAV_SCAN_SKIPPED', message: 'ClamAV scanning disabled for this request' }); + } const allCandidates: Candidate[] = []; for (const p of providerEntries) { @@ -56,12 +62,21 @@ export async function searchSubtitles(input: SearchParams) { trace.push({ level: 'info', step: 'SUBTITLE_SEARCH_DONE', message: `Provider search done: ${p.name}`, meta: { count: c.length } }); if (p.name === 'turkcealtyazi') { const realCount = c.filter((item) => item.scoreHints.includes('real_provider')).length; + const strategyHint = c.find((item) => item.scoreHints.some((h) => h.startsWith('ta_strategy_')))?.scoreHints.find((h) => h.startsWith('ta_strategy_')); trace.push({ level: 'info', step: 'TA_SEARCH_PARSED', message: `TurkceAltyazi candidates parsed`, - meta: { total: c.length, real: realCount } + meta: { total: c.length, real: realCount, strategy: strategyHint?.replace('ta_strategy_', '') || 'none' } }); + if (c.length === 0) { + trace.push({ + level: 'warn', + step: 'TA_SEARCH_NO_MATCH', + message: 'TurkceAltyazi returned no candidate', + meta: { title: input.title, year: input.year, release: input.release } + }); + } } allCandidates.push(...c); } @@ -71,6 +86,21 @@ export async function searchSubtitles(input: SearchParams) { for (const candidate of allCandidates) { const provider = providerEntries.find((p) => p.name === candidate.provider)?.impl; if (!provider) continue; + const isPackageCandidate = candidate.scoreHints.includes('ta_package_candidate'); + let candidateScoreCount = 0; + + if (isPackageCandidate) { + trace.push({ + level: 'info', + step: 'TA_PACKAGE_MODE_SELECTED', + message: `Package candidate selected: ${candidate.downloadUrl}`, + meta: { + candidateId: candidate.id, + season: input.season, + episode: input.episode + } + }); + } const dl = await provider.download(candidate, input, jobToken); if (Array.isArray(dl.trace)) { @@ -103,6 +133,44 @@ export async function searchSubtitles(input: SearchParams) { } for (const file of files) { + if (clamavEnabled) { + trace.push({ level: 'info', step: 'CLAMAV_SCAN_STARTED', message: file }); + let scan; + try { + scan = await scanFileWithClamav(file); + } catch (err: any) { + trace.push({ + level: 'error', + step: 'CLAMAV_SCAN_ERROR', + message: `ClamAV scan failed for ${file}`, + meta: { error: err?.message } + }); + throw new PipelineError({ + code: 'MALWARE_SCAN_ERROR', + message: err?.message || `ClamAV scan failed for ${file}`, + category: 'malware', + retryable: false, + httpStatus: 500, + cause: err + }); + } + if (scan.infected) { + await fse.remove(file); + trace.push({ + level: 'warn', + step: 'CLAMAV_SCAN_INFECTED_DELETED', + message: `Deleted infected file: ${file}`, + meta: { output: scan.output } + }); + continue; + } + trace.push({ + level: 'info', + step: 'CLAMAV_SCAN_CLEAN', + message: `Clean file: ${file}` + }); + } + const buf = await fs.readFile(file); if (!isProbablyText(buf)) { await fse.remove(file); @@ -119,7 +187,31 @@ export async function searchSubtitles(input: SearchParams) { } const s = scoreCandidateFile(file, ext, candidate, input); - if (s) scored.push(s); + if (s) { + scored.push(s); + candidateScoreCount += 1; + if (isPackageCandidate && input.type === 'tv') { + trace.push({ + level: 'info', + step: 'TA_PACKAGE_EPISODE_FILE_MATCHED', + message: path.basename(file), + meta: { + candidateId: candidate.id, + season: input.season, + episode: input.episode + } + }); + } + } + } + + if (isPackageCandidate && input.type === 'tv' && candidateScoreCount === 0) { + trace.push({ + level: 'warn', + step: 'TA_PACKAGE_EPISODE_FILE_NOT_FOUND', + message: `No subtitle file matched S${String(input.season ?? '').padStart(2, '0')}E${String(input.episode ?? '').padStart(2, '0')} in extracted package`, + meta: { candidateId: candidate.id } + }); } } @@ -133,6 +225,7 @@ export async function searchSubtitles(input: SearchParams) { const bestPath = path.join(dirs.base, `best.${decision.best.ext}`); await fs.copyFile(decision.best.filePath, bestPath); trace.push({ level: 'info', step: 'BEST_SELECTED', message: `Selected ${decision.best.filePath}`, meta: { score: decision.best.score } }); + trace.push({ level: 'info', step: 'BEST_EXPORT_DONE', message: `Exported best subtitle to ${bestPath}` }); return { status: 'FOUND', diff --git a/services/api/src/lib/turkcealtyaziReal.ts b/services/api/src/lib/turkcealtyaziReal.ts index c7ccb9d..5190522 100644 --- a/services/api/src/lib/turkcealtyaziReal.ts +++ b/services/api/src/lib/turkcealtyaziReal.ts @@ -2,9 +2,11 @@ import axios from 'axios'; import * as cheerio from 'cheerio'; import { URL } from 'node:url'; import { Buffer } from 'node:buffer'; +import { setTimeout as sleepMs } from 'node:timers/promises'; import { env } from '../config/env.js'; import type { SearchParams } from '../types/index.js'; import { taError, taInfo } from './taLog.js'; +import { PipelineError, toPipelineError } from './errors.js'; export interface RealTaCandidate { id: string; @@ -14,6 +16,8 @@ export interface RealTaCandidate { releaseHints: string[]; isHI: boolean; isForced: boolean; + strategy?: 'exact' | 'token' | 'fallback' | 'default' | 'package_fallback'; + isPackage?: boolean; } const client = axios.create({ @@ -31,6 +35,10 @@ function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } +function computeBackoffMs(attempt: number): number { + return Math.min(2000, 300 * 2 ** Math.max(0, attempt - 1)); +} + interface HttpResultText { body: string; finalUrl: string; @@ -71,7 +79,7 @@ async function getWithRetry(url: string, retries = 2, cookies?: Map 0) await sleep(250 * i); + if (i > 0) await sleepMs(computeBackoffMs(i)); taInfo('HTTP_GET_START', 'HTTP GET started', { url, attempt: i + 1, retries: retries + 1 }); const res = await client.get(url, { headers: cookies && cookies.size > 0 ? { cookie: cookieHeader(cookies) } : undefined @@ -87,8 +95,10 @@ async function getWithRetry(url: string, retries = 2, cookies?: Map !/^\d+$/.test(t)); - return toks.slice(0, 2).join(' '); + const meaningful = toks.filter((t) => !QUERY_STOPWORDS.has(t)); + const queryTokens = (meaningful.length > 0 ? meaningful : toks).slice(0, 3); + return queryTokens.join(' '); } function pickMovieLinkFromSearch(html: string, params: SearchParams, baseUrl: string): { movieUrl: string; movieTitle: string } | null { const $ = cheerio.load(html); const wantedYear = params.year; const wantedTitleTokens = tokenize(params.title); + const wantedNormalizedTitle = normalizeText(params.title); const links: Array<{ url: string; title: string; year?: number; score: number }> = []; - $('a[href^="/mov/"]').each((_, el) => { + $('a[href]').each((_, el) => { const href = ($(el).attr('href') || '').trim(); if (!href) return; + if (!/^\/(mov|tv|dizi)\//i.test(href)) return; const title = ($(el).attr('title') || $(el).text() || '').replace(/\s+/g, ' ').trim(); if (!title) return; @@ -150,8 +169,28 @@ function pickMovieLinkFromSearch(html: string, params: SearchParams, baseUrl: st const year = yearMatch ? Number(yearMatch[1]) : undefined; const titleTokens = tokenize(title); + const normalizedTitle = normalizeText(title); const overlap = wantedTitleTokens.filter((t) => titleTokens.includes(t)).length; let score = overlap; + + if (normalizedTitle === wantedNormalizedTitle) score += 30; + else if ( + normalizedTitle.includes(wantedNormalizedTitle) || + wantedNormalizedTitle.includes(normalizedTitle) + ) { + score += 18; + } + + const isFilm = /\bfilm\b/i.test(containerText); + const isTv = /\b(dizi|tv dizisi)\b/i.test(containerText); + if (params.type === 'tv') { + if (isTv) score += 8; + if (isFilm) score -= 4; + } else { + if (isFilm) score += 8; + if (isTv) score -= 4; + } + if (wantedYear && year === wantedYear) score += 10; links.push({ @@ -176,11 +215,55 @@ function pickMovieLinkFromSearch(html: string, params: SearchParams, baseUrl: st return { movieUrl: best.url, movieTitle: best.title }; } -function pickSubPageFromMovieDetail(html: string, movieUrl: string, params: SearchParams): { subUrl: string; title: string; releaseHints: string[]; isHI: boolean } | null { +function parseSeasonEpisodeFromRow($: cheerio.CheerioAPI, row: any): { season?: number; episode?: number; isPackage: boolean } { + const alcd = ($(row).find('.alcd').text() || '').replace(/\s+/g, ' ').trim(); + const m = alcd.match(/S\s*0?(\d{1,2}).*E\s*0?(\d{1,2})/i); + const isPackage = /\bpaket\b/i.test(alcd); + if (m) { + return { season: Number(m[1]), episode: Number(m[2]), isPackage }; + } + const s = alcd.match(/S\s*0?(\d{1,2})/i); + if (s) { + return { season: Number(s[1]), isPackage }; + } + return { isPackage }; +} + +function pickSubPageFromMovieDetail( + html: string, + movieUrl: string, + params: SearchParams +): { + picked?: { + subUrl: string; + title: string; + releaseHints: string[]; + isHI: boolean; + strategy: 'exact' | 'token' | 'fallback' | 'default' | 'package_fallback'; + isPackage?: boolean; + }; + noMatchReason?: 'episode_not_matched' | 'release_not_matched' | 'no_sub_rows'; +} { const $ = cheerio.load(html); const wantedRelease = normalizeText(params.release || ''); - const rows = $('.altsonsez2'); - const candidates: Array<{ subUrl: string; title: string; releaseHints: string[]; isHI: boolean; score: number }> = []; + const wantedReleaseTokens = wantedRelease.split(/\s+/).filter(Boolean); + const wantedSeason = params.type === 'tv' ? params.season : undefined; + const wantedEpisode = params.type === 'tv' ? params.episode : undefined; + const rows = $('[class*="altsonsez"]'); + const candidates: Array<{ + subUrl: string; + title: string; + releaseHints: string[]; + isHI: boolean; + score: number; + releaseExact: boolean; + releaseTokenHits: number; + trScore: number; + downloadCount: number; + season?: number; + episode?: number; + isPackage: boolean; + }> = []; rows.each((_, row) => { const linkEl = $(row).find('a[href^="/sub/"]').first(); @@ -192,35 +275,106 @@ function pickSubPageFromMovieDetail(html: string, movieUrl: string, params: Sear const relHints = normalizeReleaseHints(ripText); const normalizedRip = normalizeText(ripText); const isHI = /(sdh|hearing|isitme|hi)/i.test(ripText); + const isTr = $(row).find('.flagtr').length > 0; + const indirmeRaw = ($(row).find('.alindirme').text() || '').replace(/\./g, '').replace(/,/g, '').trim(); + const downloadCount = Number(indirmeRaw.replace(/[^\d]/g, '')) || 0; + const { season, episode, isPackage } = parseSeasonEpisodeFromRow($, row); - let score = 0; - if (wantedRelease) { - if (normalizedRip.includes(wantedRelease)) score += 20; - const releaseToken = wantedRelease.split(/\s+/).find(Boolean); - if (releaseToken && normalizedRip.includes(releaseToken)) score += 15; - } else { - score += 1; + if (params.type === 'tv') { + if (!season) return; + if (wantedSeason && season !== wantedSeason) return; + if (wantedEpisode && episode !== wantedEpisode && !isPackage) return; } - if ($(row).find('.flagtr').length > 0) score += 3; + const releaseExact = Boolean(wantedRelease && normalizedRip.includes(wantedRelease)); + const releaseTokenHits = wantedRelease + ? wantedReleaseTokens.filter((tok) => normalizedRip.includes(tok)).length + : 0; + + let score = 0; + if (!wantedRelease) { + score += 1; + } + if (releaseExact) score += 40; + if (releaseTokenHits > 0) score += Math.min(20, releaseTokenHits * 8); + if (isTr) score += 8; + score += Math.min(10, Math.floor(downloadCount / 1500)); candidates.push({ subUrl: abs(movieUrl, href), title, releaseHints: relHints, isHI, - score + score, + releaseExact, + releaseTokenHits, + trScore: isTr ? 1 : 0, + downloadCount, + season, + episode, + isPackage }); }); - if (candidates.length === 0) return null; - const picked = candidates.sort((a, b) => b.score - a.score)[0]; - if (wantedRelease && picked.score < 10) return null; - return picked; + if (candidates.length === 0) { + return { noMatchReason: params.type === 'tv' ? 'episode_not_matched' : 'no_sub_rows' }; + } + + let selectedPool = candidates; + let forcedStrategy: 'package_fallback' | undefined; + if (params.type === 'tv' && wantedEpisode) { + const episodeRows = candidates.filter((c) => c.episode === wantedEpisode); + if (episodeRows.length > 0) { + selectedPool = episodeRows; + } else { + const packageRows = candidates.filter((c) => c.isPackage); + if (packageRows.length > 0) { + selectedPool = packageRows; + forcedStrategy = 'package_fallback'; + } else { + return { noMatchReason: 'episode_not_matched' }; + } + } + } + + if (!wantedRelease || forcedStrategy === 'package_fallback') { + const picked = selectedPool.sort((a, b) => b.score - a.score || b.downloadCount - a.downloadCount)[0]; + if (forcedStrategy === 'package_fallback') { + return { picked: { ...picked, strategy: 'package_fallback', isPackage: true } }; + } + return { picked: { ...picked, strategy: 'default' } }; + } + + const exact = selectedPool + .filter((c) => c.releaseExact) + .sort((a, b) => b.score - a.score || b.downloadCount - a.downloadCount)[0]; + if (exact) { + return { picked: { ...exact, strategy: 'exact' } }; + } + + const token = selectedPool + .filter((c) => c.releaseTokenHits > 0) + .sort((a, b) => b.releaseTokenHits - a.releaseTokenHits || b.score - a.score || b.downloadCount - a.downloadCount)[0]; + if (token) { + return { picked: { ...token, strategy: 'token' } }; + } + + if (params.type === 'tv') { + // TV'de once bolum dogrulugu, sonra release gelir. Release bulunamasa da en iyi bolum satirini kullan. + const tvFallback = selectedPool + .sort((a, b) => b.trScore - a.trScore || b.downloadCount - a.downloadCount || b.score - a.score)[0]; + if (tvFallback) { + return { picked: { ...tvFallback, strategy: 'fallback' } }; + } + } + + const fallback = selectedPool + .sort((a, b) => b.trScore - a.trScore || b.downloadCount - a.downloadCount || b.score - a.score)[0]; + if (!fallback) return { noMatchReason: 'release_not_matched' }; + return { picked: { ...fallback, strategy: 'fallback' } }; } export async function searchTurkceAltyaziReal(params: SearchParams): Promise { - if (params.type !== 'movie') return []; const q = buildFindQuery(params); if (!q) return []; @@ -241,24 +395,51 @@ export async function searchTurkceAltyaziReal(params: SearchParams): Promise 0) await sleep(250 * i); + if (i > 0) await sleepMs(computeBackoffMs(i)); const form = new URLSearchParams(payload).toString(); const indUrl = `${env.turkcealtyaziBaseUrl}/ind`; taInfo('TA_IND_POST_START', 'POST /ind started', { subPageUrl, indUrl, attempt: i + 1, retries: retries + 1, altid: payload.altid }); @@ -313,8 +496,10 @@ async function postIndWithRetry(subPageUrl: string, payload: { idid: string; alt contentType: res.headers['content-type'] }; } catch (err) { - lastError = err; - taError('TA_IND_POST_FAILED', err, { subPageUrl, attempt: i + 1, retries: retries + 1 }); + const pe = toPipelineError(err, 'TA_IND_POST_FAILED'); + lastError = pe; + taError('TA_IND_POST_FAILED', pe, { subPageUrl, attempt: i + 1, retries: retries + 1, code: pe.code, retryable: pe.retryable }); + if (!pe.retryable) throw pe; } } throw lastError; @@ -330,7 +515,13 @@ export async function downloadTurkceAltyaziFile(subPageUrl: string): Promise<{ b mergeCookies(cookies, subPageRes.setCookie); const form = parseDownloadForm(subPageRes.body); if (!form) { - const err = new Error('TA sub page download form parse failed'); + const err = new PipelineError({ + code: 'TA_FORM_PARSE_FAILED', + message: 'TA sub page download form parse failed', + category: 'parse', + retryable: false, + httpStatus: 422 + }); taError('TA_FORM_PARSE_FAILED', err, { subPageUrl }); throw err; } diff --git a/services/api/src/lib/validators.ts b/services/api/src/lib/validators.ts index 467866d..671da94 100644 --- a/services/api/src/lib/validators.ts +++ b/services/api/src/lib/validators.ts @@ -10,8 +10,52 @@ export function isProbablyText(buffer: Buffer): boolean { export function validateSrt(text: string): boolean { const lines = text.split(/\r?\n/); - const tc = lines.filter((l) => /^\d{2}:\d{2}:\d{2},\d{3}\s-->\s\d{2}:\d{2}:\d{2},\d{3}$/.test(l.trim())); - return tc.length >= 3; + const tcIndexes: number[] = []; + for (let i = 0; i < lines.length; i++) { + if (/^\d{2}:\d{2}:\d{2},\d{3}\s-->\s\d{2}:\d{2}:\d{2},\d{3}$/.test(lines[i].trim())) { + tcIndexes.push(i); + } + } + if (tcIndexes.length < 3) return false; + + let prevStart = -1; + let prevIndex = -1; + let malformed = 0; + + for (const idx of tcIndexes) { + const seq = (lines[idx - 1] || '').trim(); + if (!/^\d+$/.test(seq)) malformed += 1; + + const m = lines[idx].trim().match( + /^(\d{2}):(\d{2}):(\d{2}),(\d{3})\s-->\s(\d{2}):(\d{2}):(\d{2}),(\d{3})$/ + ); + if (!m) { + malformed += 1; + continue; + } + + const start = + Number(m[1]) * 3600000 + + Number(m[2]) * 60000 + + Number(m[3]) * 1000 + + Number(m[4]); + const end = + Number(m[5]) * 3600000 + + Number(m[6]) * 60000 + + Number(m[7]) * 1000 + + Number(m[8]); + + if (end <= start) malformed += 1; + + const seqNum = Number(seq); + if (prevIndex !== -1 && seqNum <= prevIndex) malformed += 1; + if (prevStart !== -1 && start < prevStart) malformed += 1; + + prevIndex = seqNum; + prevStart = start; + } + + return malformed / tcIndexes.length < 0.35; } export function validateAss(text: string): boolean { diff --git a/services/api/src/providers/TurkceAltyaziProvider.ts b/services/api/src/providers/TurkceAltyaziProvider.ts index 473033a..01e08bf 100644 --- a/services/api/src/providers/TurkceAltyaziProvider.ts +++ b/services/api/src/providers/TurkceAltyaziProvider.ts @@ -7,15 +7,84 @@ import { searchTurkceAltyaziReal } from '../lib/turkcealtyaziReal.js'; import { taError, taInfo } from '../lib/taLog.js'; +import { detectSubtitleType, isProbablyText } from '../lib/validators.js'; +import { PipelineError } from '../lib/errors.js'; -function extensionFromDownload(url: string, contentType?: string): 'zip' | 'rar' | '7z' | 'srt' | 'ass' { - const lowerUrl = url.toLowerCase(); - if (lowerUrl.includes('.zip')) return 'zip'; - if (lowerUrl.includes('.rar')) return 'rar'; - if (lowerUrl.includes('.7z')) return '7z'; - if (lowerUrl.includes('.ass')) return 'ass'; - if (contentType?.includes('zip')) return 'zip'; - return 'srt'; +function hasPrefix(buf: Buffer, sig: number[]): boolean { + if (buf.length < sig.length) return false; + return sig.every((b, i) => buf[i] === b); +} + +function classifyDownloadedPayload( + buffer: Buffer, + finalUrl: string, + contentType?: string +): { type: 'archive' | 'direct'; ext: 'zip' | 'rar' | '7z' | 'srt' | 'ass'; reason: string } { + const ct = (contentType || '').toLowerCase(); + const url = finalUrl.toLowerCase(); + + if ( + hasPrefix(buffer, [0x50, 0x4b, 0x03, 0x04]) || + hasPrefix(buffer, [0x50, 0x4b, 0x05, 0x06]) || + hasPrefix(buffer, [0x50, 0x4b, 0x07, 0x08]) || + ct.includes('zip') || + url.includes('.zip') + ) { + return { type: 'archive', ext: 'zip', reason: 'zip signature/content-type/url' }; + } + + if ( + hasPrefix(buffer, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]) || + hasPrefix(buffer, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]) || + ct.includes('rar') || + url.includes('.rar') + ) { + return { type: 'archive', ext: 'rar', reason: 'rar signature/content-type/url' }; + } + + if ( + hasPrefix(buffer, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]) || + ct.includes('7z') || + url.includes('.7z') + ) { + return { type: 'archive', ext: '7z', reason: '7z signature/content-type/url' }; + } + + if (isProbablyText(buffer)) { + const utf8 = buffer.toString('utf8'); + const latin1 = buffer.toString('latin1'); + const ext = detectSubtitleType(utf8) || detectSubtitleType(latin1); + if (ext) { + return { type: 'direct', ext, reason: 'text + subtitle format detected' }; + } + + const probe = utf8.slice(0, 2000).toLowerCase(); + if (/ { if (!/^https?:\/\//i.test(candidate.downloadUrl)) { - throw new Error('TurkceAltyazi candidate download URL must be http(s)'); + throw new PipelineError({ + code: 'TA_INVALID_DOWNLOAD_URL', + message: 'TurkceAltyazi candidate download URL must be http(s)', + category: 'parse', + retryable: false, + httpStatus: 422 + }); } const downloadDir = `${env.tempRoot}/${jobToken}/download`; @@ -66,11 +153,18 @@ export class TurkceAltyaziProvider implements SubtitleProvider { trace.push({ level: 'info', step: 'TA_SUB_PAGE_FETCHED', message: candidate.downloadUrl }); const downloaded = await downloadTurkceAltyaziFile(candidate.downloadUrl); trace.push({ level: 'info', step: 'TA_IND_POST_DONE', message: downloaded.finalUrl }); - const ext = extensionFromDownload(downloaded.finalUrl, downloaded.contentType); + const detected = classifyDownloadedPayload(downloaded.buffer, downloaded.finalUrl, downloaded.contentType); + const ext = detected.ext; const filePath = path.join(downloadDir, `${candidate.id}.${ext}`); await fs.writeFile(filePath, downloaded.buffer); + trace.push({ + level: 'info', + step: 'TA_DOWNLOAD_PAYLOAD_CLASSIFIED', + message: `${detected.type}:${detected.ext}`, + meta: { reason: detected.reason, contentType: downloaded.contentType, finalUrl: downloaded.finalUrl } + }); - const type: 'direct' | 'archive' = ext === 'srt' || ext === 'ass' ? 'direct' : 'archive'; + const type: 'direct' | 'archive' = detected.type; taInfo('TA_PROVIDER_DOWNLOAD_RESULT', 'Provider download completed', { candidateId: candidate.id, filePath, diff --git a/services/api/src/routes/subtitles.ts b/services/api/src/routes/subtitles.ts index 28ac06f..bbc7c8e 100644 --- a/services/api/src/routes/subtitles.ts +++ b/services/api/src/routes/subtitles.ts @@ -1,6 +1,7 @@ import { FastifyInstance } from 'fastify'; import { z } from 'zod'; import { chooseSubtitle, cleanupJobToken, searchSubtitles } from '../lib/subtitleEngine.js'; +import { toPipelineError } from '../lib/errors.js'; const SearchSchema = z.object({ jobToken: z.string().optional(), @@ -20,6 +21,11 @@ const SearchSchema = z.object({ maxTotalBytes: z.number().min(1024), maxSingleBytes: z.number().min(1024) }) + .optional(), + features: z + .object({ + clamavEnabled: z.boolean().optional() + }) .optional() }); @@ -40,7 +46,22 @@ export async function subtitleRoutes(app: FastifyInstance): Promise { const result = await searchSubtitles(parsed.data); return result; } catch (err: any) { - return reply.status(500).send({ status: 'ERROR', message: err.message, trace: [{ level: 'error', step: 'JOB_ERROR', message: err.message }] }); + const pe = toPipelineError(err); + return reply.status(pe.httpStatus).send({ + status: 'ERROR', + code: pe.code, + category: pe.category, + retryable: pe.retryable, + message: pe.message, + trace: [ + { + level: 'error', + step: 'JOB_ERROR', + message: pe.message, + meta: { code: pe.code, category: pe.category, retryable: pe.retryable } + } + ] + }); } }); diff --git a/services/api/src/types/index.ts b/services/api/src/types/index.ts index e687eca..df1363a 100644 --- a/services/api/src/types/index.ts +++ b/services/api/src/types/index.ts @@ -15,6 +15,9 @@ export interface SearchParams { maxTotalBytes: number; maxSingleBytes: number; }; + features?: { + clamavEnabled?: boolean; + }; } export interface Candidate { diff --git a/services/core/package-lock.json b/services/core/package-lock.json index fe86574..9a25f49 100644 --- a/services/core/package-lock.json +++ b/services/core/package-lock.json @@ -27,6 +27,278 @@ "vitest": "^3.0.5" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", + "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", + "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", + "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", + "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", + "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", + "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", + "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", + "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", + "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", + "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", + "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", + "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", + "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", + "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", + "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", + "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/linux-x64": { "version": "0.27.3", "cpu": [ @@ -42,6 +314,159 @@ "node": ">=18" } }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", + "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", + "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", + "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", + "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", + "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", + "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", + "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", + "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", + "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@fastify/ajv-compiler": { "version": "4.0.5", "funding": [ @@ -175,6 +600,58 @@ "sparse-bitfield": "^3.0.3" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { "version": "3.0.3", "cpu": [ @@ -186,10 +663,261 @@ "linux" ] }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@pinojs/redact": { "version": "0.4.0", "license": "MIT" }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.57.1.tgz", + "integrity": "sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.57.1.tgz", + "integrity": "sha512-dQaAddCY9YgkFHZcFNS/606Exo8vcLHwArFZ7vxXq4rigo2bb494/xKMMwRRQW6ug7Js6yXmBZhSBRuBvCCQ3w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.57.1.tgz", + "integrity": "sha512-crNPrwJOrRxagUYeMn/DZwqN88SDmwaJ8Cvi/TN1HnWBU7GwknckyosC2gd0IqYRsHDEnXf328o9/HC6OkPgOg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.57.1.tgz", + "integrity": "sha512-Ji8g8ChVbKrhFtig5QBV7iMaJrGtpHelkB3lsaKzadFBe58gmjfGXAOfI5FV0lYMH8wiqsxKQ1C9B0YTRXVy4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.57.1.tgz", + "integrity": "sha512-R+/WwhsjmwodAcz65guCGFRkMb4gKWTcIeLy60JJQbXrJ97BOXHxnkPFrP+YwFlaS0m+uWJTstrUA9o+UchFug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.57.1.tgz", + "integrity": "sha512-IEQTCHeiTOnAUC3IDQdzRAGj3jOAYNr9kBguI7MQAAZK3caezRrg0GxAb6Hchg4lxdZEI5Oq3iov/w/hnFWY9Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.57.1.tgz", + "integrity": "sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.57.1.tgz", + "integrity": "sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.57.1.tgz", + "integrity": "sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.57.1.tgz", + "integrity": "sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.57.1.tgz", + "integrity": "sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.57.1.tgz", + "integrity": "sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.57.1.tgz", + "integrity": "sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.57.1.tgz", + "integrity": "sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.57.1.tgz", + "integrity": "sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.57.1.tgz", + "integrity": "sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.57.1.tgz", + "integrity": "sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, "node_modules/@rollup/rollup-linux-x64-gnu": { "version": "4.57.1", "cpu": [ @@ -202,6 +930,104 @@ "linux" ] }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.57.1.tgz", + "integrity": "sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.57.1.tgz", + "integrity": "sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.57.1.tgz", + "integrity": "sha512-4wYoDpNg6o/oPximyc/NG+mYUejZrCU2q+2w6YZqrAs2UcNUChIZXjtafAiiZSUc7On8v5NyNj34Kzj/Ltk6dQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.57.1.tgz", + "integrity": "sha512-O54mtsV/6LW3P8qdTcamQmuC990HDfR71lo44oZMZlXU4tzLrbvTii87Ni9opq60ds0YzuAlEr/GNwuNluZyMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.57.1.tgz", + "integrity": "sha512-P3dLS+IerxCT/7D2q2FYcRdWRl22dNbrbBEtxdWhXrfIMPP9lQhb5h4Du04mdl5Woq05jVCDPCMF7Ub0NAjIew==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.57.1.tgz", + "integrity": "sha512-VMBH2eOOaKGtIJYleXsi2B8CPVADrh+TyNxJ4mWPnKfLB/DBUmzW+5m1xUrcwWoMfSLagIRpjUFeW5CO5hyciQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.57.1.tgz", + "integrity": "sha512-mxRFDdHIWRxg3UfIIAwCm6NzvxG0jDX/wBN6KsQFTvKFqqg9vTrWUE68qEjHt19A5wwx5X5aUi2zuZT7YR0jrA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@types/chai": { "version": "5.2.3", "dev": true, @@ -890,6 +1716,21 @@ "node": ">= 6" } }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "license": "MIT", diff --git a/services/core/src/config/env.ts b/services/core/src/config/env.ts index fe254c4..038b6c4 100644 --- a/services/core/src/config/env.ts +++ b/services/core/src/config/env.ts @@ -14,5 +14,6 @@ export const env = { mediaMoviePath: process.env.MEDIA_MOVIE_PATH ?? '/media/movie', enableApiKey: process.env.ENABLE_API_KEY === 'true', apiKey: process.env.API_KEY ?? '', + watcherDedupWindowMs: Number(process.env.CORE_WATCHER_DEDUP_WINDOW_MS ?? 15000), isDev: (process.env.NODE_ENV ?? 'development') !== 'production' }; diff --git a/services/core/src/db/redis.ts b/services/core/src/db/redis.ts index 04a35d0..e74a119 100644 --- a/services/core/src/db/redis.ts +++ b/services/core/src/db/redis.ts @@ -1,7 +1,7 @@ -import IORedis from 'ioredis'; +import { Redis } from 'ioredis'; import { env } from '../config/env.js'; -export const redis = new IORedis({ +export const redis = new Redis({ host: env.redisHost, port: env.redisPort, maxRetriesPerRequest: null diff --git a/services/core/src/queues/queues.ts b/services/core/src/queues/queues.ts index 3d5f4e4..a5f177f 100644 --- a/services/core/src/queues/queues.ts +++ b/services/core/src/queues/queues.ts @@ -1,11 +1,17 @@ import { Queue, Worker } from 'bullmq'; -import { redis } from '../db/redis.js'; +import { env } from '../config/env.js'; -export const fileEventsQueue = new Queue('fileEvents', { connection: redis }); -export const mediaAnalysisQueue = new Queue('mediaAnalysis', { connection: redis }); -export const subtitleFetchQueue = new Queue('subtitleFetch', { connection: redis }); -export const finalizeWriteQueue = new Queue('finalizeWrite', { connection: redis }); +const connection = { + host: env.redisHost, + port: env.redisPort, + maxRetriesPerRequest: null +}; + +export const fileEventsQueue = new Queue('fileEvents', { connection }); +export const mediaAnalysisQueue = new Queue('mediaAnalysis', { connection }); +export const subtitleFetchQueue = new Queue('subtitleFetch', { connection }); +export const finalizeWriteQueue = new Queue('finalizeWrite', { connection }); export function createWorker(name: string, processor: any): Worker { - return new Worker(name, processor, { connection: redis, concurrency: 2 }); + return new Worker(name, processor, { connection, concurrency: 2 }); } diff --git a/services/core/src/routes/debug.ts b/services/core/src/routes/debug.ts index da74743..f5af94a 100644 --- a/services/core/src/routes/debug.ts +++ b/services/core/src/routes/debug.ts @@ -3,7 +3,6 @@ import { FastifyInstance } from 'fastify'; import { z } from 'zod'; import { env } from '../config/env.js'; import { createJobForPath } from '../workers/pipeline.js'; -import { MediaFileModel } from '../models/MediaFile.js'; import { fileEventsQueue } from '../queues/queues.js'; export async function debugRoutes(app: FastifyInstance): Promise { @@ -17,11 +16,10 @@ export async function debugRoutes(app: FastifyInstance): Promise { return reply.status(400).send({ error: 'Path does not exist in container' }); } - const jobId = await createJobForPath(body.path, body.kind); - const media = await MediaFileModel.findOne({ path: body.path }).lean(); - if (!media) return reply.status(500).send({ error: 'media not persisted' }); - - await fileEventsQueue.add('debug', { jobId, mediaFileId: String(media._id), path: body.path }); - return { ok: true, jobId }; + const created = await createJobForPath(body.path, body.kind); + if (created.enqueued) { + await fileEventsQueue.add('debug', { jobId: created.jobId, mediaFileId: created.mediaFileId, path: body.path }); + } + return { ok: true, jobId: created.jobId, deduped: !created.enqueued }; }); } diff --git a/services/core/src/routes/jobs.ts b/services/core/src/routes/jobs.ts index 202f562..fc5488b 100644 --- a/services/core/src/routes/jobs.ts +++ b/services/core/src/routes/jobs.ts @@ -11,19 +11,51 @@ export async function jobRoutes(app: FastifyInstance): Promise { page: z.coerce.number().default(1), limit: z.coerce.number().default(20), status: z.string().optional(), - search: z.string().optional() + search: z.string().optional(), + dedupe: z.coerce.boolean().default(true) }) .parse(req.query); - const filter: any = {}; - if (q.status) filter.status = q.status; - if (q.search) filter.$or = [{ 'requestSnapshot.title': { $regex: q.search, $options: 'i' } }]; - const skip = (q.page - 1) * q.limit; - const [items, total] = await Promise.all([ - JobModel.find(filter).sort({ createdAt: -1 }).skip(skip).limit(q.limit).lean(), - JobModel.countDocuments(filter) - ]); + let items: any[] = []; + let total = 0; + + if (!q.dedupe) { + const filter: any = {}; + if (q.status) filter.status = q.status; + if (q.search) filter.$or = [{ 'requestSnapshot.title': { $regex: q.search, $options: 'i' } }]; + + [items, total] = await Promise.all([ + JobModel.find(filter).sort({ createdAt: -1 }).skip(skip).limit(q.limit).lean(), + JobModel.countDocuments(filter) + ]); + } else { + const pipeline: any[] = [ + { $sort: { createdAt: -1 } }, + { $group: { _id: '$mediaFileId', latest: { $first: '$$ROOT' } } }, + { $replaceRoot: { newRoot: '$latest' } } + ]; + + const postMatch: any = {}; + if (q.status) postMatch.status = q.status; + if (q.search) { + postMatch.$or = [ + { 'requestSnapshot.title': { $regex: q.search, $options: 'i' } }, + { 'requestSnapshot.release': { $regex: q.search, $options: 'i' } } + ]; + } + if (Object.keys(postMatch).length > 0) { + pipeline.push({ $match: postMatch }); + } + + const [countAgg, listAgg] = await Promise.all([ + JobModel.aggregate([...pipeline, { $count: 'total' }]), + JobModel.aggregate([...pipeline, { $sort: { createdAt: -1 } }, { $skip: skip }, { $limit: q.limit }]) + ]); + + total = countAgg[0]?.total ?? 0; + items = listAgg; + } return { items, total, page: q.page, limit: q.limit }; }); diff --git a/services/core/src/utils/file.ts b/services/core/src/utils/file.ts index d3b7d53..1abbd72 100644 --- a/services/core/src/utils/file.ts +++ b/services/core/src/utils/file.ts @@ -33,23 +33,67 @@ export async function waitForStable(filePath: string, checks: number, intervalSe } export function normalizeSubtitleBuffer(input: Buffer): string { + const normalizeNewlines = (text: string): string => text.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + + const textQualityScore = (text: string): number => { + const replacements = (text.match(/\uFFFD/g) || []).length; + const turkishLetters = (text.match(/[çğıöşüÇĞİÖŞÜ]/g) || []).length; + const suspiciousMojibake = (text.match(/[þýðÞÝÐÃÂ]/g) || []).length; + return turkishLetters * 2 - replacements * 5 - suspiciousMojibake * 2; + }; + + const maybeRepairTurkishMojibake = (text: string): string => { + const suspicious = (text.match(/[þýðÞÝÐ]/g) || []).length; + if (suspicious < 2) return text; + const repaired = iconv.decode(Buffer.from(text, 'latin1'), 'windows-1254'); + return textQualityScore(repaired) > textQualityScore(text) ? repaired : text; + }; + + const finalizeText = (text: string): string => normalizeNewlines(maybeRepairTurkishMojibake(text)); + + if (input.length >= 2) { + // UTF-16 LE BOM + if (input[0] === 0xff && input[1] === 0xfe) { + return finalizeText(iconv.decode(input, 'utf16-le')); + } + // UTF-16 BE BOM + if (input[0] === 0xfe && input[1] === 0xff) { + return finalizeText(iconv.decode(input, 'utf16-be')); + } + } + if (input.length >= 3 && input[0] === 0xef && input[1] === 0xbb && input[2] === 0xbf) { - return input.toString('utf8').replace(/^\uFEFF/, '').replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + return finalizeText(input.toString('utf8')); + } + + // If bytes round-trip as UTF-8, prefer UTF-8 regardless of heuristic detection. + const utf8Decoded = input.toString('utf8'); + if (Buffer.from(utf8Decoded, 'utf8').equals(input)) { + return finalizeText(utf8Decoded); } const detected = jschardet.detect(input); const enc = (detected.encoding || '').toLowerCase(); + const confidence = typeof detected.confidence === 'number' ? detected.confidence : 0; + + const pickSingleByteDecode = (): string => { + const tr = iconv.decode(input, 'windows-1254'); + const latin = iconv.decode(input, 'latin1'); + return textQualityScore(tr) >= textQualityScore(latin) ? tr : latin; + }; let decoded: string; - if (enc.includes('utf')) { + if (enc.includes('utf') && confidence >= 0.6) { decoded = input.toString('utf8'); } else if (enc.includes('windows-1254') || enc.includes('iso-8859-9')) { decoded = iconv.decode(input, 'windows-1254'); + } else if (enc.includes('windows') || enc.includes('latin') || enc.includes('iso-8859-1') || confidence < 0.7) { + decoded = pickSingleByteDecode(); } else { decoded = iconv.decode(input, 'latin1'); } - return decoded.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + return finalizeText(decoded); } export async function nextSubtitlePath(basePathWithoutExt: string, lang: string, ext: 'srt' | 'ass', overwrite: boolean): Promise { diff --git a/services/core/src/watcher/dedup.ts b/services/core/src/watcher/dedup.ts new file mode 100644 index 0000000..b25f919 --- /dev/null +++ b/services/core/src/watcher/dedup.ts @@ -0,0 +1,20 @@ +export function createEventDeduper(windowMs: number): (filePath: string, now?: number) => boolean { + const recentByPath = new Map(); + const safeWindow = Number.isFinite(windowMs) && windowMs > 0 ? windowMs : 0; + + return (filePath: string, now = Date.now()) => { + const last = recentByPath.get(filePath); + if (typeof last === 'number' && now - last < safeWindow) { + return false; + } + + recentByPath.set(filePath, now); + + // Keep memory bounded by removing old keys opportunistically. + for (const [p, ts] of recentByPath) { + if (now - ts >= safeWindow) recentByPath.delete(p); + } + + return true; + }; +} diff --git a/services/core/src/watcher/index.ts b/services/core/src/watcher/index.ts index bf6f81c..ca94841 100644 --- a/services/core/src/watcher/index.ts +++ b/services/core/src/watcher/index.ts @@ -3,6 +3,10 @@ import { WatchedPathModel } from '../models/WatchedPath.js'; import { createJobForPath } from '../workers/pipeline.js'; import { fileEventsQueue } from '../queues/queues.js'; import { isVideoFile } from '../utils/file.js'; +import { env } from '../config/env.js'; +import { createEventDeduper } from './dedup.js'; + +let watcherStarted = false; export async function ensureDefaultWatchedPaths(tvPath: string, moviePath: string): Promise { await WatchedPathModel.updateOne({ path: tvPath }, { $setOnInsert: { path: tvPath, kind: 'tv', enabled: true } }, { upsert: true }); @@ -10,6 +14,9 @@ export async function ensureDefaultWatchedPaths(tvPath: string, moviePath: strin } export async function startWatcher(): Promise { + if (watcherStarted) return; + watcherStarted = true; + const watched = await WatchedPathModel.find({ enabled: true }).lean(); const paths = watched.map((w) => w.path); if (paths.length === 0) { @@ -18,26 +25,25 @@ export async function startWatcher(): Promise { } const byPath = new Map(watched.map((w) => [w.path, w.kind])); + const shouldProcessEvent = createEventDeduper(env.watcherDedupWindowMs); const watcher = chokidar.watch(paths, { ignoreInitial: false, awaitWriteFinish: false, persistent: true }); watcher.on('add', async (p) => { if (!isVideoFile(p)) return; + if (!shouldProcessEvent(p)) return; const kind = resolveKind(p, byPath); - const jobId = await createJobForPath(p, kind === 'movie' ? 'movie' : 'tv'); - const media = await import('../models/MediaFile.js'); - const mediaDoc = await media.MediaFileModel.findOne({ path: p }).lean(); - if (!mediaDoc) return; - await fileEventsQueue.add('add', { jobId, mediaFileId: String(mediaDoc._id), path: p }); + const created = await createJobForPath(p, kind === 'movie' ? 'movie' : 'tv'); + if (!created.enqueued) return; + await fileEventsQueue.add('add', { jobId: created.jobId, mediaFileId: created.mediaFileId, path: p }); }); watcher.on('change', async (p) => { if (!isVideoFile(p)) return; + if (!shouldProcessEvent(p)) return; const kind = resolveKind(p, byPath); - const jobId = await createJobForPath(p, kind === 'movie' ? 'movie' : 'tv'); - const media = await import('../models/MediaFile.js'); - const mediaDoc = await media.MediaFileModel.findOne({ path: p }).lean(); - if (!mediaDoc) return; - await fileEventsQueue.add('change', { jobId, mediaFileId: String(mediaDoc._id), path: p }); + const created = await createJobForPath(p, kind === 'movie' ? 'movie' : 'tv'); + if (!created.enqueued) return; + await fileEventsQueue.add('change', { jobId: created.jobId, mediaFileId: created.mediaFileId, path: p }); }); watcher.on('unlink', async (p) => { diff --git a/services/core/src/workers/pipeline.ts b/services/core/src/workers/pipeline.ts index 8c86325..55d2777 100644 --- a/services/core/src/workers/pipeline.ts +++ b/services/core/src/workers/pipeline.ts @@ -1,5 +1,6 @@ import fs from 'node:fs/promises'; import path from 'node:path'; +import type { Job } from 'bullmq'; import { Types } from 'mongoose'; import { JobModel } from '../models/Job.js'; import { MediaFileModel } from '../models/MediaFile.js'; @@ -32,9 +33,23 @@ interface FinalizeData { } const activeWorkers: any[] = []; +let workersStarted = false; +const ACTIVE_JOB_STATUSES = [ + 'PENDING', + 'WAITING_FILE_STABLE', + 'PARSED', + 'ANALYZED', + 'REQUESTING_API', + 'FOUND_TEMP', + 'NORMALIZING_ENCODING', + 'WRITING_SUBTITLE' +] as const; export function startWorkers(): void { - activeWorkers.push(createWorker('fileEvents', async (job) => { + if (workersStarted) return; + workersStarted = true; + + activeWorkers.push(createWorker('fileEvents', async (job: Job) => { const data = job.data as FileEventData; const settings = await SettingModel.findById('global').lean(); if (!settings) throw new Error('settings missing'); @@ -47,7 +62,31 @@ export function startWorkers(): void { return; } - const stable = await waitForStable(data.path, settings.stableChecks, settings.stableIntervalSeconds); + let stable = false; + try { + stable = await waitForStable(data.path, settings.stableChecks, settings.stableIntervalSeconds); + } catch (err: any) { + const missing = err?.code === 'ENOENT'; + if (missing) { + await MediaFileModel.findByIdAndUpdate(data.mediaFileId, { status: 'MISSING', lastSeenAt: new Date() }); + } + await JobModel.findByIdAndUpdate(data.jobId, { + status: 'ERROR', + error: { + code: missing ? 'FILE_NOT_FOUND' : 'FILE_STABLE_CHECK_FAILED', + message: missing ? 'file not found during stability check' : (err?.message || 'stability check failed') + } + }); + await writeJobLog({ + jobId: data.jobId, + step: 'JOB_ERROR', + level: 'error', + message: missing ? `File not found during stability check: ${data.path}` : 'Stability check failed', + meta: missing ? undefined : { error: err?.message } + }); + return; + } + if (!stable) { await JobModel.findByIdAndUpdate(data.jobId, { status: 'ERROR', error: { code: 'FILE_NOT_STABLE', message: 'file not stable' } }); await writeJobLog({ jobId: data.jobId, step: 'JOB_ERROR', level: 'error', message: 'File did not become stable' }); @@ -87,7 +126,7 @@ export function startWorkers(): void { await mediaAnalysisQueue.add('analyze', { jobId: data.jobId, mediaFileId: data.mediaFileId }); })); - activeWorkers.push(createWorker('mediaAnalysis', async (job) => { + activeWorkers.push(createWorker('mediaAnalysis', async (job: Job) => { const { jobId, mediaFileId } = job.data as SubtitleFetchData; const media = await MediaFileModel.findById(mediaFileId).lean(); if (!media) return; @@ -114,7 +153,7 @@ export function startWorkers(): void { await subtitleFetchQueue.add('search', { jobId, mediaFileId }); })); - activeWorkers.push(createWorker('subtitleFetch', async (job) => { + activeWorkers.push(createWorker('subtitleFetch', async (job: Job) => { const { jobId, mediaFileId } = job.data as SubtitleFetchData; const media = await MediaFileModel.findById(mediaFileId).lean(); const settings = await SettingModel.findById('global').lean(); @@ -135,7 +174,8 @@ export function startWorkers(): void { mediaInfo: media.mediaInfo, preferHI: settings.preferHI, preferForced: settings.preferForced, - securityLimits: settings.securityLimits + securityLimits: settings.securityLimits, + features: settings.features }; const res = await apiClient.post('/v1/subtitles/search', payload); @@ -189,7 +229,7 @@ export function startWorkers(): void { }); })); - activeWorkers.push(createWorker('finalizeWrite', async (job) => { + activeWorkers.push(createWorker('finalizeWrite', async (job: Job) => { const data = job.data as FinalizeData; const media = await MediaFileModel.findById(data.mediaFileId).lean(); const settings = await SettingModel.findById('global').lean(); @@ -205,14 +245,22 @@ export function startWorkers(): void { const parsed = path.parse(media.path); const target = await nextSubtitlePath(path.join(parsed.dir, parsed.name), data.lang, extensionFromPath(data.bestPath), settings.overwriteExisting); + await writeJobLog({ jobId: data.jobId, step: 'FINAL_TARGET_PATH_RESOLVED', message: target }); const targetExists = target !== `${path.join(parsed.dir, parsed.name)}.${data.lang}.${extensionFromPath(data.bestPath)}`; if (targetExists) { await writeJobLog({ jobId: data.jobId, step: 'WRITE_TARGET_SKIPPED_EXISTS', message: 'Target exists, using incremented filename', meta: { target } }); } - await fs.writeFile(target, normalized, 'utf8'); - await writeJobLog({ jobId: data.jobId, step: 'WRITE_TARGET_DONE', message: target }); + const tempTarget = `${target}.tmp-${Date.now()}`; + await fs.writeFile(tempTarget, normalized, 'utf8'); + await fs.rename(tempTarget, target); + await writeJobLog({ + jobId: data.jobId, + step: 'WRITE_TARGET_DONE', + message: target, + meta: { bytes: Buffer.byteLength(normalized, 'utf8') } + }); await JobModel.findByIdAndUpdate(data.jobId, { status: 'DONE', @@ -240,7 +288,10 @@ export function startWorkers(): void { })); } -export async function createJobForPath(filePath: string, mediaTypeHint: 'tv' | 'movie'): Promise { +export async function createJobForPath( + filePath: string, + mediaTypeHint: 'tv' | 'movie' +): Promise<{ jobId: string; mediaFileId: string; enqueued: boolean }> { const st = await fs.stat(filePath); const media = await MediaFileModel.findOneAndUpdate( @@ -258,6 +309,27 @@ export async function createJobForPath(filePath: string, mediaTypeHint: 'tv' | ' { upsert: true, new: true } ); + const existing = await JobModel.findOne({ + mediaFileId: media._id, + status: { $in: ACTIVE_JOB_STATUSES } + }) + .sort({ createdAt: -1 }) + .lean(); + + if (existing) { + await writeJobLog({ + jobId: existing._id as Types.ObjectId, + step: 'WATCH_EVENT_DEDUPED', + message: `Skipped duplicate watch event for ${filePath}`, + meta: { mediaFileId: String(media._id), existingStatus: existing.status } + }); + return { + jobId: String(existing._id), + mediaFileId: String(media._id), + enqueued: false + }; + } + const created = await JobModel.create({ mediaFileId: media._id, status: 'PENDING', @@ -270,5 +342,9 @@ export async function createJobForPath(filePath: string, mediaTypeHint: 'tv' | ' message: `Queued watch event for ${filePath}` }); - return String(created._id); + return { + jobId: String(created._id), + mediaFileId: String(media._id), + enqueued: true + }; } diff --git a/services/core/test/encoding.test.ts b/services/core/test/encoding.test.ts new file mode 100644 index 0000000..ba8905e --- /dev/null +++ b/services/core/test/encoding.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; +import iconv from 'iconv-lite'; +import { normalizeSubtitleBuffer } from '../src/utils/file.js'; + +describe('normalizeSubtitleBuffer', () => { + const sample = '1\r\n00:00:01,000 --> 00:00:02,000\r\nÇığ ÖşÜ ıİ\r\n'; + + it('keeps valid utf8 Turkish characters intact', () => { + const buf = Buffer.from(sample, 'utf8'); + const out = normalizeSubtitleBuffer(buf); + expect(out).toBe('1\n00:00:01,000 --> 00:00:02,000\nÇığ ÖşÜ ıİ\n'); + }); + + it('decodes windows-1254 Turkish text correctly', () => { + const buf = iconv.encode(sample, 'windows-1254'); + const out = normalizeSubtitleBuffer(buf); + expect(out).toBe('1\n00:00:01,000 --> 00:00:02,000\nÇığ ÖşÜ ıİ\n'); + }); + + it('decodes utf16-le BOM correctly', () => { + const buf = iconv.encode(sample, 'utf16-le'); + const withBom = Buffer.concat([Buffer.from([0xff, 0xfe]), buf]); + const out = normalizeSubtitleBuffer(withBom); + expect(out).toBe('1\n00:00:01,000 --> 00:00:02,000\nÇığ ÖşÜ ıİ\n'); + }); + + it('repairs mojibake form with \u00fd/\u00fe/\u00f0 Turkish corruption', () => { + const mojibake = '1\r\n00:00:01,000 --> 00:00:02,000\r\nKýz þöyle dedi: aðýr bir iþ.\r\n'; + const buf = Buffer.from(mojibake, 'utf8'); + const out = normalizeSubtitleBuffer(buf); + expect(out).toContain('Kız şöyle dedi: ağır bir iş.'); + }); +}); diff --git a/services/core/test/watcherDedup.test.ts b/services/core/test/watcherDedup.test.ts new file mode 100644 index 0000000..77f7586 --- /dev/null +++ b/services/core/test/watcherDedup.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { createEventDeduper } from '../src/watcher/dedup.js'; + +describe('createEventDeduper', () => { + it('suppresses repeated events inside window for same path', () => { + const dedupe = createEventDeduper(1000); + + expect(dedupe('/media/movie/a.mkv', 1000)).toBe(true); + expect(dedupe('/media/movie/a.mkv', 1500)).toBe(false); + expect(dedupe('/media/movie/a.mkv', 2001)).toBe(true); + }); + + it('does not suppress different paths', () => { + const dedupe = createEventDeduper(1000); + + expect(dedupe('/media/movie/a.mkv', 1000)).toBe(true); + expect(dedupe('/media/movie/b.mkv', 1200)).toBe(true); + }); +});