feat: watcher akislarini ve wscraper servis entegrasyonunu ekle
This commit is contained in:
11
.env.example
11
.env.example
@@ -9,10 +9,21 @@ SERVER_PORT=3001
|
||||
WEB_PORT=5173
|
||||
WEB_ORIGIN=http://localhost:5173
|
||||
WEB_ALLOWED_ORIGINS=http://192.168.1.205:5175,http://qbuffer.bee
|
||||
WEB_ALLOWED_HOSTS=localhost,192.168.1.205,qbuffer.bee
|
||||
POLL_INTERVAL_MS=3000
|
||||
ENFORCE_INTERVAL_MS=2000
|
||||
DEFAULT_DELAY_MS=3000
|
||||
MAX_LOOP_LIMIT=1000
|
||||
STALLED_RECOVERY_MS=300000
|
||||
TIMER_POLL_MS=60000
|
||||
WATCHER_SECRET_KEY=replace_me_watcher
|
||||
WATCHER_ENABLED=true
|
||||
WATCHER_TICK_MS=30000
|
||||
WATCHER_TIMEOUT_MS=180000
|
||||
WATCHER_RUNTIME_DIR=/tmp/qbuffer-watchers
|
||||
WSCRAPER_SERVICE_BASE_URL=http://host.docker.internal:8787
|
||||
WSCRAPER_SERVICE_TOKEN=
|
||||
WSCRAPER_SERVICE_HOST=0.0.0.0
|
||||
WSCRAPER_SERVICE_PORT=8787
|
||||
WSCRAPER_SERVICE_PYTHON_BIN=python3.12
|
||||
NODE_ENV=development
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -36,6 +36,7 @@ lcov-report/
|
||||
.vite/
|
||||
.tmp/
|
||||
.cache/
|
||||
.runtime/
|
||||
|
||||
# Docker files / volumes
|
||||
docker/*-volume/
|
||||
@@ -77,4 +78,4 @@ movie/movieData/**/backdrop.jpg
|
||||
*.log
|
||||
|
||||
# Client
|
||||
client/src/assets/avatar_.png
|
||||
client/src/assets/avatar_.png
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
FROM node:20-alpine AS base
|
||||
FROM node:20-bookworm-slim AS base
|
||||
WORKDIR /app
|
||||
RUN corepack enable
|
||||
RUN corepack enable \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
FROM base AS deps
|
||||
COPY package.json pnpm-workspace.yaml ./
|
||||
|
||||
68
README.md
68
README.md
@@ -16,7 +16,7 @@ q-buffer, qBittorrent üzerinde torrentleri kontrollü şekilde döngüye almay
|
||||
2) Geliştirme ortamını başlatın:
|
||||
|
||||
```bash
|
||||
docker-compose -f docker-compose.dev.yml up --build
|
||||
./scripts/bootstrap.sh --dev-mode
|
||||
```
|
||||
|
||||
3) Açın:
|
||||
@@ -24,6 +24,69 @@ docker-compose -f docker-compose.dev.yml up --build
|
||||
- Web: http://localhost:5173
|
||||
- API/Socket: http://localhost:3001
|
||||
|
||||
## Watcher Notu
|
||||
|
||||
`Watcher` akışı `wscraper -> scrapling -> Playwright` zincirini kullanır. Playwright DNS ve browser bağımlılıklarını Docker içine taşımak yerine `wscraper-service` host makinede çalışır; `web` ve `server` ise Docker içinde kalır. `server`, host servisle `http://host.docker.internal:8787` üzerinden konuşur.
|
||||
|
||||
`bootstrap.sh` şu işleri tek komutta yapar:
|
||||
|
||||
- Docker servislerini `up --build` ile kaldırır
|
||||
- host `wscraper-service` için Python venv hazırlar
|
||||
- eksik Python paketlerini ve Playwright bağımlılıklarını kurar
|
||||
- `wscraper-service`i başlatır
|
||||
|
||||
`wscraper-service` kurulumu her çalıştırmada sıfırdan yapılmaz. Kurulum daha önce tamamlandıysa script sadece kontrol eder ve eksik yoksa yeniden kurmaz.
|
||||
|
||||
Host makinede Python 3.10+ gerekir. Script sırasıyla `python3.12`, `python3.11`, `python3.10`, `python3` ikililerini dener ve uygun ilk sürümü seçer. Gerekirse `.env` içine `WSCRAPER_SERVICE_PYTHON_BIN=python3.12` benzeri açık bir değer verebilirsiniz.
|
||||
|
||||
Kullanılabilir bayraklar:
|
||||
|
||||
```bash
|
||||
./scripts/bootstrap.sh --dev-mode
|
||||
./scripts/bootstrap.sh --prod-mode
|
||||
./scripts/bootstrap.sh --dev-mode --skip-wscraper-install
|
||||
./scripts/bootstrap.sh --dev-mode --restart-wscraper
|
||||
```
|
||||
|
||||
Docker tarafında normal ağ erişimi hâlâ gereklidir. DNS problemi yaşarsanız Docker Desktop içinde sabit resolver (`8.8.8.8`, `1.1.1.1`) tanımlayın. Docker DNS doğru ayarlanmamışsa:
|
||||
|
||||
- `pnpm install`
|
||||
- image pull işlemleri
|
||||
- container içi paket kurulumları
|
||||
|
||||
kurulum sırasında kırılabilir.
|
||||
|
||||
Önerilen Docker Engine ayarı:
|
||||
|
||||
```json
|
||||
{
|
||||
"builder": {
|
||||
"gc": {
|
||||
"defaultKeepStorage": "20GB",
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"experimental": false,
|
||||
"dns": ["8.8.8.8", "1.1.1.1"]
|
||||
}
|
||||
```
|
||||
|
||||
Docker Desktop yeniden başladıktan sonra şu testler başarılı olmalıdır:
|
||||
|
||||
```bash
|
||||
docker run --rm alpine nslookup registry-1.docker.io
|
||||
docker run --rm alpine nslookup files.pythonhosted.org
|
||||
docker run --rm alpine nslookup cdn.playwright.dev
|
||||
```
|
||||
|
||||
Host servis için kullanılacak ortam değişkenleri:
|
||||
|
||||
- `WSCRAPER_SERVICE_BASE_URL` varsayılan: `http://host.docker.internal:8787`
|
||||
- `WSCRAPER_SERVICE_TOKEN` varsayılan: boş
|
||||
- `WSCRAPER_SERVICE_HOST` varsayılan: `0.0.0.0`
|
||||
- `WSCRAPER_SERVICE_PORT` varsayılan: `8787`
|
||||
- `WSCRAPER_SERVICE_PYTHON_BIN` örnek: `python3.12`
|
||||
|
||||
## Kullanım (Buffer)
|
||||
|
||||
1) qBittorrent’te torrentleri ekleyin (UI listeye düşer).
|
||||
@@ -41,7 +104,7 @@ docker-compose -f docker-compose.dev.yml up --build
|
||||
## Production
|
||||
|
||||
```bash
|
||||
docker-compose up --build
|
||||
./scripts/bootstrap.sh --prod-mode
|
||||
```
|
||||
|
||||
Ardından http://localhost:3001
|
||||
@@ -52,6 +115,7 @@ Ardından http://localhost:3001
|
||||
- `APP_USERNAME`, `APP_PASSWORD`, `JWT_SECRET`
|
||||
- `POLL_INTERVAL_MS`, `ENFORCE_INTERVAL_MS`, `DEFAULT_DELAY_MS`, `MAX_LOOP_LIMIT`
|
||||
- `WEB_ALLOWED_HOSTS` (ör: `localhost,qbuffer.bee,qbuffer.panda`)
|
||||
- `WSCRAPER_SERVICE_BASE_URL`, `WSCRAPER_SERVICE_TOKEN`
|
||||
|
||||
## Klasör Yapısı
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
FROM node:20-alpine
|
||||
FROM node:20-bookworm-slim
|
||||
WORKDIR /app
|
||||
RUN corepack enable
|
||||
RUN corepack enable \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
COPY package.json /app/apps/server/package.json
|
||||
COPY package.json /app/package.json
|
||||
COPY pnpm-workspace.yaml /app/pnpm-workspace.yaml
|
||||
|
||||
@@ -23,12 +23,21 @@ export const config = {
|
||||
webPort: envNumber(process.env.WEB_PORT, 5173),
|
||||
webOrigin: process.env.WEB_ORIGIN ?? "",
|
||||
webAllowedOrigins: process.env.WEB_ALLOWED_ORIGINS ?? "",
|
||||
watcherSecretKey: process.env.WATCHER_SECRET_KEY ?? "",
|
||||
watcherEnabled: (process.env.WATCHER_ENABLED ?? "true").toLowerCase() !== "false",
|
||||
watcherTickMs: envNumber(process.env.WATCHER_TICK_MS, 30_000),
|
||||
watcherTimeoutMs: envNumber(process.env.WATCHER_TIMEOUT_MS, 180_000),
|
||||
watcherRuntimeDir: process.env.WATCHER_RUNTIME_DIR ?? "/tmp/qbuffer-watchers",
|
||||
wscraperServiceBaseUrl:
|
||||
process.env.WSCRAPER_SERVICE_BASE_URL ?? "http://host.docker.internal:8787",
|
||||
wscraperServiceToken: process.env.WSCRAPER_SERVICE_TOKEN ?? "",
|
||||
dataDir: "/app/data",
|
||||
dbPath: "/app/data/db.json",
|
||||
logsPath: "/app/data/logs.json",
|
||||
loopLogsDir: "/app/data/loop-logs",
|
||||
loopLogsArchiveDir: "/app/data/loop-logs-archive",
|
||||
torrentArchiveDir: "/app/data/torrents",
|
||||
watcherDownloadsDir: "/app/data/watcher-downloads",
|
||||
webPublicDir: path.resolve("/app/apps/server/public"),
|
||||
};
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import loopRoutes from "./loop/loop.routes"
|
||||
import profilesRoutes from "./loop/profiles.routes"
|
||||
import statusRoutes from "./status/status.routes"
|
||||
import timerRoutes from "./timer/timer.routes"
|
||||
import watcherRoutes from "./watcher/watcher.routes"
|
||||
import { QbitClient } from "./qbit/qbit.client"
|
||||
import { detectCapabilities } from "./qbit/qbit.capabilities"
|
||||
import { setQbitClient, setQbitCapabilities } from "./qbit/qbit.context"
|
||||
@@ -24,6 +25,7 @@ import { initEmitter, emitQbitHealth } from "./realtime/emitter"
|
||||
import { startLoopScheduler } from "./loop/loop.scheduler"
|
||||
import { startEnforcementWorker } from "./enforcement/enforcement.worker"
|
||||
import { startTimerWorker } from "./timer/timer.worker"
|
||||
import { startWatcherWorker } from "./watcher/watcher.worker"
|
||||
import { logger } from "./utils/logger"
|
||||
|
||||
const crashLogPath = "/app/data/crash.log";
|
||||
@@ -94,6 +96,7 @@ const bootstrap = async () => {
|
||||
app.use("/api/profiles", requireAuth, profilesRoutes);
|
||||
app.use("/api/status", requireAuth, statusRoutes);
|
||||
app.use("/api/timer", requireAuth, timerRoutes);
|
||||
app.use("/api/watchers", requireAuth, watcherRoutes);
|
||||
|
||||
if (!isDev) {
|
||||
app.use(express.static(config.webPublicDir));
|
||||
@@ -126,6 +129,7 @@ const bootstrap = async () => {
|
||||
startLoopScheduler(qbit, config.pollIntervalMs);
|
||||
startEnforcementWorker(config.enforceIntervalMs);
|
||||
startTimerWorker(qbit, config.timerPollMs);
|
||||
startWatcherWorker(config.watcherTickMs);
|
||||
|
||||
server.listen(config.port, () => {
|
||||
serverStarted = true;
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { QbitClient } from "../qbit/qbit.client"
|
||||
import { tickLoopJobs } from "./loop.engine"
|
||||
import { getStatusSnapshot, refreshJobsStatus, setQbitStatus, setTorrentsStatus } from "../status/status.service"
|
||||
import { emitQbitHealth, emitStatusUpdate } from "../realtime/emitter"
|
||||
import { emitQbitHealth, emitStatusUpdate, emitWatcherItems } from "../realtime/emitter"
|
||||
import { logger } from "../utils/logger"
|
||||
import { getWatcherItems } from "../watcher/watcher.service"
|
||||
|
||||
export const startLoopScheduler = (qbit: QbitClient, intervalMs: number) => {
|
||||
setInterval(async () => {
|
||||
@@ -19,6 +20,7 @@ export const startLoopScheduler = (qbit: QbitClient, intervalMs: number) => {
|
||||
transfer,
|
||||
jobs,
|
||||
});
|
||||
emitWatcherItems(await getWatcherItems());
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
logger.error({ error }, "Loop scheduler tick failed");
|
||||
|
||||
@@ -6,6 +6,7 @@ import fs from "node:fs";
|
||||
import { config } from "../config"
|
||||
import { logger } from "../utils/logger"
|
||||
import {
|
||||
QbitCategory,
|
||||
QbitPeerList,
|
||||
QbitTorrentInfo,
|
||||
QbitTorrentProperties,
|
||||
@@ -96,6 +97,13 @@ export class QbitClient {
|
||||
return response.data;
|
||||
}
|
||||
|
||||
async getCategories(): Promise<Record<string, QbitCategory>> {
|
||||
const response = await this.request(() =>
|
||||
this.client.get<Record<string, QbitCategory>>("/api/v2/torrents/categories")
|
||||
);
|
||||
return response.data;
|
||||
}
|
||||
|
||||
async getTorrentProperties(hash: string): Promise<QbitTorrentProperties> {
|
||||
const response = await this.request(() =>
|
||||
this.client.get<QbitTorrentProperties>("/api/v2/torrents/properties", {
|
||||
|
||||
@@ -11,6 +11,8 @@ export interface QbitTorrentInfo {
|
||||
tags?: string;
|
||||
category?: string;
|
||||
tracker?: string;
|
||||
num_seeds?: number;
|
||||
num_leechs?: number;
|
||||
added_on?: number;
|
||||
seeding_time?: number;
|
||||
uploaded?: number;
|
||||
@@ -51,3 +53,8 @@ export interface QbitCapabilities {
|
||||
hasPeersEndpoint: boolean;
|
||||
hasBanEndpoint: boolean;
|
||||
}
|
||||
|
||||
export interface QbitCategory {
|
||||
name?: string;
|
||||
savePath?: string;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Server } from "socket.io";
|
||||
import { EVENTS } from "./events";
|
||||
import { StatusSnapshot } from "../status/status.service";
|
||||
import { LoopJob, TimerLog, TimerSummary } from "../types";
|
||||
import { EnrichedWatcherItem, WatcherListItem, WatcherSummaryResponse } from "../watcher/watcher.types";
|
||||
|
||||
let io: Server | null = null;
|
||||
|
||||
@@ -41,3 +42,15 @@ export const emitTimerLog = (payload: TimerLog) => {
|
||||
export const emitTimerSummary = (payload: TimerSummary) => {
|
||||
io?.emit(EVENTS.TIMER_SUMMARY, payload);
|
||||
};
|
||||
|
||||
export const emitWatchersList = (payload: WatcherListItem[]) => {
|
||||
io?.emit(EVENTS.WATCHERS_LIST, payload);
|
||||
};
|
||||
|
||||
export const emitWatcherItems = (payload: EnrichedWatcherItem[]) => {
|
||||
io?.emit(EVENTS.WATCHER_ITEMS, payload);
|
||||
};
|
||||
|
||||
export const emitWatcherSummary = (payload: WatcherSummaryResponse) => {
|
||||
io?.emit(EVENTS.WATCHER_SUMMARY, payload);
|
||||
};
|
||||
|
||||
@@ -6,4 +6,7 @@ export const EVENTS = {
|
||||
QBIT_HEALTH: "qbit:health",
|
||||
TIMER_LOG: "timer:log",
|
||||
TIMER_SUMMARY: "timer:summary",
|
||||
WATCHERS_LIST: "watchers:list",
|
||||
WATCHER_ITEMS: "watcher:items",
|
||||
WATCHER_SUMMARY: "watcher:summary",
|
||||
};
|
||||
|
||||
@@ -4,6 +4,7 @@ import { verifyToken } from "../auth/auth.service";
|
||||
import { getStatusSnapshot } from "../status/status.service";
|
||||
import { EVENTS } from "./events";
|
||||
import { config, isDev } from "../config";
|
||||
import { getWatcherItems, getWatcherSummary, listWatchers } from "../watcher/watcher.service";
|
||||
|
||||
const parseCookies = (cookieHeader?: string) => {
|
||||
const cookies: Record<string, string> = {};
|
||||
@@ -53,6 +54,9 @@ export const createSocketServer = (server: http.Server) => {
|
||||
io.on("connection", async (socket) => {
|
||||
const snapshot = await getStatusSnapshot();
|
||||
socket.emit(EVENTS.STATUS_SNAPSHOT, snapshot);
|
||||
socket.emit(EVENTS.WATCHERS_LIST, await listWatchers());
|
||||
socket.emit(EVENTS.WATCHER_SUMMARY, await getWatcherSummary());
|
||||
socket.emit(EVENTS.WATCHER_ITEMS, await getWatcherItems());
|
||||
});
|
||||
|
||||
return io;
|
||||
|
||||
@@ -26,6 +26,16 @@ const defaultDb = (): DbSchema => ({
|
||||
totalUploadedBytes: 0,
|
||||
updatedAt: new Date().toISOString(),
|
||||
},
|
||||
watchers: [],
|
||||
watcherItems: [],
|
||||
watcherRuns: [],
|
||||
watcherSummary: {
|
||||
activeWatchers: 0,
|
||||
totalImported: 0,
|
||||
totalSeen: 0,
|
||||
trackedLabels: [],
|
||||
updatedAt: new Date().toISOString(),
|
||||
},
|
||||
});
|
||||
|
||||
const rotateBackups = async (dbPath: string) => {
|
||||
@@ -71,6 +81,16 @@ export const readDb = async (): Promise<DbSchema> => {
|
||||
totalUploadedBytes: 0,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
parsed.watchers ??= [];
|
||||
parsed.watcherItems ??= [];
|
||||
parsed.watcherRuns ??= [];
|
||||
parsed.watcherSummary ??= {
|
||||
activeWatchers: 0,
|
||||
totalImported: 0,
|
||||
totalSeen: 0,
|
||||
trackedLabels: [],
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
return parsed;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
|
||||
@@ -6,4 +6,6 @@ export const ensureDataPaths = async () => {
|
||||
await fs.mkdir(config.torrentArchiveDir, { recursive: true });
|
||||
await fs.mkdir(config.loopLogsDir, { recursive: true });
|
||||
await fs.mkdir(config.loopLogsArchiveDir, { recursive: true });
|
||||
await fs.mkdir(config.watcherDownloadsDir, { recursive: true });
|
||||
await fs.mkdir(config.watcherRuntimeDir, { recursive: true });
|
||||
};
|
||||
|
||||
@@ -98,6 +98,87 @@ export interface TimerSummary {
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export type WatcherTracker = "happyfappy";
|
||||
|
||||
export type WatcherItemStatus =
|
||||
| "bookmarked"
|
||||
| "downloading_torrent"
|
||||
| "sending_to_qbit"
|
||||
| "sent_to_qbit"
|
||||
| "downloading"
|
||||
| "completed"
|
||||
| "failed";
|
||||
|
||||
export interface Watcher {
|
||||
id: string;
|
||||
tracker: WatcherTracker;
|
||||
trackerLabel: string;
|
||||
category?: string;
|
||||
cookieEncrypted: string;
|
||||
cookieHint: string;
|
||||
intervalMinutes: number;
|
||||
enabled: boolean;
|
||||
lastRunAt?: string;
|
||||
lastSuccessAt?: string;
|
||||
lastError?: string;
|
||||
nextRunAt?: string;
|
||||
totalImported: number;
|
||||
totalSeen: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface WatcherItem {
|
||||
id: string;
|
||||
watcherId: string;
|
||||
tracker: WatcherTracker;
|
||||
trackerLabel: string;
|
||||
sourceKey: string;
|
||||
pageUrl: string;
|
||||
title: string;
|
||||
imageUrl?: string;
|
||||
status: WatcherItemStatus;
|
||||
statusLabel: string;
|
||||
qbitHash?: string;
|
||||
qbitName?: string;
|
||||
qbitProgress?: number;
|
||||
qbitState?: string;
|
||||
qbitCategory?: string;
|
||||
sizeBytes?: number;
|
||||
seeders?: number;
|
||||
leechers?: number;
|
||||
trackerTorrentId?: string;
|
||||
seenAt: string;
|
||||
downloadedAt?: string;
|
||||
importedAt?: string;
|
||||
lastSyncAt: string;
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
export interface WatcherRun {
|
||||
id: string;
|
||||
watcherId: string;
|
||||
startedAt: string;
|
||||
finishedAt?: string;
|
||||
status: "RUNNING" | "SUCCESS" | "FAILED";
|
||||
newBookmarks: number;
|
||||
importedCount: number;
|
||||
failedCount: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface WatcherSummary {
|
||||
activeWatchers: number;
|
||||
totalImported: number;
|
||||
totalSeen: number;
|
||||
trackedLabels: string[];
|
||||
lastRunAt?: string;
|
||||
lastSuccessAt?: string;
|
||||
nextRunAt?: string;
|
||||
lastError?: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface AuditLog {
|
||||
id: string;
|
||||
level: "INFO" | "WARN" | "ERROR";
|
||||
@@ -133,4 +214,8 @@ export interface DbSchema {
|
||||
timerRules?: TimerRule[];
|
||||
timerLogs?: TimerLog[];
|
||||
timerSummary?: TimerSummary;
|
||||
watchers?: Watcher[];
|
||||
watcherItems?: WatcherItem[];
|
||||
watcherRuns?: WatcherRun[];
|
||||
watcherSummary?: WatcherSummary;
|
||||
}
|
||||
|
||||
42
apps/server/src/watcher/watcher.crypto.ts
Normal file
42
apps/server/src/watcher/watcher.crypto.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import crypto from "node:crypto";
|
||||
import { config } from "../config";
|
||||
|
||||
const ensureSecret = () => {
|
||||
if (!config.watcherSecretKey) {
|
||||
throw new Error("WATCHER_SECRET_KEY missing");
|
||||
}
|
||||
return crypto.createHash("sha256").update(config.watcherSecretKey).digest();
|
||||
};
|
||||
|
||||
export const encryptWatcherCookie = (value: string) => {
|
||||
const key = ensureSecret();
|
||||
const iv = crypto.randomBytes(12);
|
||||
const cipher = crypto.createCipheriv("aes-256-gcm", key, iv);
|
||||
const encrypted = Buffer.concat([cipher.update(value, "utf8"), cipher.final()]);
|
||||
const tag = cipher.getAuthTag();
|
||||
return Buffer.concat([iv, tag, encrypted]).toString("base64");
|
||||
};
|
||||
|
||||
export const decryptWatcherCookie = (payload: string) => {
|
||||
const key = ensureSecret();
|
||||
const source = Buffer.from(payload, "base64");
|
||||
const iv = source.subarray(0, 12);
|
||||
const tag = source.subarray(12, 28);
|
||||
const encrypted = source.subarray(28);
|
||||
const decipher = crypto.createDecipheriv("aes-256-gcm", key, iv);
|
||||
decipher.setAuthTag(tag);
|
||||
return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString("utf8");
|
||||
};
|
||||
|
||||
export const buildCookieHint = (value: string) => {
|
||||
const segments = value
|
||||
.split(/[;\n]/)
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean)
|
||||
.slice(0, 2)
|
||||
.map((entry) => {
|
||||
const [name] = entry.split("=", 1);
|
||||
return `${name}=...`;
|
||||
});
|
||||
return segments.length > 0 ? segments.join("; ") : "Cookie kayitli";
|
||||
};
|
||||
14
apps/server/src/watcher/watcher.registry.ts
Normal file
14
apps/server/src/watcher/watcher.registry.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { TrackerDefinition } from "./watcher.types";
|
||||
|
||||
export const trackerRegistry: TrackerDefinition[] = [
|
||||
{
|
||||
key: "happyfappy",
|
||||
label: "HappyFappy",
|
||||
cliSiteKey: "happyfappy",
|
||||
supportsRemoveBookmark: true,
|
||||
},
|
||||
];
|
||||
|
||||
export const getTrackerDefinition = (key: string) => {
|
||||
return trackerRegistry.find((tracker) => tracker.key === key) ?? null;
|
||||
};
|
||||
116
apps/server/src/watcher/watcher.routes.ts
Normal file
116
apps/server/src/watcher/watcher.routes.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
createWatcher,
|
||||
deleteWatcher,
|
||||
fetchWatcherImage,
|
||||
getQbitCategories,
|
||||
getWatcherItems,
|
||||
getWatcherSummary,
|
||||
listTrackers,
|
||||
listWatchers,
|
||||
runWatcherById,
|
||||
updateWatcher,
|
||||
} from "./watcher.service";
|
||||
|
||||
const router = Router();
|
||||
|
||||
const createWatcherSchema = z.object({
|
||||
tracker: z.string().min(1),
|
||||
cookie: z.string().min(1),
|
||||
intervalMinutes: z.number().int().min(1).max(24 * 60),
|
||||
category: z.string().optional(),
|
||||
enabled: z.boolean().optional(),
|
||||
});
|
||||
|
||||
const updateWatcherSchema = z.object({
|
||||
cookie: z.string().min(1).optional(),
|
||||
intervalMinutes: z.number().int().min(1).max(24 * 60).optional(),
|
||||
category: z.string().optional(),
|
||||
enabled: z.boolean().optional(),
|
||||
});
|
||||
|
||||
router.get("/trackers", (_req, res) => {
|
||||
return res.json(listTrackers());
|
||||
});
|
||||
|
||||
router.get("/categories", async (_req, res) => {
|
||||
try {
|
||||
return res.json(await getQbitCategories());
|
||||
} catch (error) {
|
||||
return res.status(400).json({ error: error instanceof Error ? error.message : "Failed to load qBittorrent categories" });
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/", async (_req, res) => {
|
||||
return res.json(await listWatchers());
|
||||
});
|
||||
|
||||
router.post("/", async (req, res) => {
|
||||
const parsed = createWatcherSchema.safeParse(req.body ?? {});
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({ error: "Invalid watcher payload" });
|
||||
}
|
||||
try {
|
||||
const watcher = await createWatcher(parsed.data);
|
||||
return res.status(201).json(watcher);
|
||||
} catch (error) {
|
||||
return res.status(400).json({ error: error instanceof Error ? error.message : "Failed to create watcher" });
|
||||
}
|
||||
});
|
||||
|
||||
router.patch("/:id", async (req, res) => {
|
||||
const parsed = updateWatcherSchema.safeParse(req.body ?? {});
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({ error: "Invalid watcher payload" });
|
||||
}
|
||||
try {
|
||||
const watcher = await updateWatcher(req.params.id, parsed.data);
|
||||
return res.json(watcher);
|
||||
} catch (error) {
|
||||
return res.status(400).json({ error: error instanceof Error ? error.message : "Failed to update watcher" });
|
||||
}
|
||||
});
|
||||
|
||||
router.delete("/:id", async (req, res) => {
|
||||
try {
|
||||
return res.json(await deleteWatcher(req.params.id));
|
||||
} catch (error) {
|
||||
return res.status(400).json({ error: error instanceof Error ? error.message : "Failed to delete watcher" });
|
||||
}
|
||||
});
|
||||
|
||||
router.post("/:id/run", async (req, res) => {
|
||||
try {
|
||||
await runWatcherById(req.params.id);
|
||||
return res.json({ ok: true });
|
||||
} catch (error) {
|
||||
return res.status(400).json({ error: error instanceof Error ? error.message : "Watcher run failed" });
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/summary", async (_req, res) => {
|
||||
return res.json(await getWatcherSummary());
|
||||
});
|
||||
|
||||
router.get("/items", async (_req, res) => {
|
||||
return res.json(await getWatcherItems());
|
||||
});
|
||||
|
||||
router.get("/image", async (req, res) => {
|
||||
const watcherId = String(req.query.watcherId ?? "");
|
||||
const imageUrl = String(req.query.url ?? "");
|
||||
if (!watcherId || !imageUrl) {
|
||||
return res.status(400).json({ error: "Missing watcherId or url" });
|
||||
}
|
||||
try {
|
||||
const image = await fetchWatcherImage(watcherId, imageUrl);
|
||||
res.setHeader("Content-Type", image.contentType);
|
||||
res.setHeader("Cache-Control", "private, max-age=300");
|
||||
return res.send(image.data);
|
||||
} catch (error) {
|
||||
return res.status(400).json({ error: error instanceof Error ? error.message : "Failed to fetch image" });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
102
apps/server/src/watcher/watcher.scraper.ts
Normal file
102
apps/server/src/watcher/watcher.scraper.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import axios from "axios";
|
||||
import { config } from "../config";
|
||||
import { logger } from "../utils/logger";
|
||||
import { BookmarkRecord, ScraperRunPaths } from "./watcher.types";
|
||||
|
||||
const ensureDir = async (target: string) => {
|
||||
await fs.mkdir(target, { recursive: true });
|
||||
};
|
||||
|
||||
export const createScraperRunPaths = async (watcherId: string): Promise<ScraperRunPaths> => {
|
||||
const runDir = path.join(config.watcherRuntimeDir, watcherId, randomUUID());
|
||||
const torrentDir = path.join(runDir, "torrent");
|
||||
await ensureDir(torrentDir);
|
||||
return {
|
||||
runDir,
|
||||
cookiesPath: path.join(runDir, "cookies.txt"),
|
||||
bookmarksPath: path.join(runDir, "bookmarks.json"),
|
||||
torrentDir,
|
||||
};
|
||||
};
|
||||
|
||||
const scraperClient = axios.create({
|
||||
baseURL: config.wscraperServiceBaseUrl,
|
||||
timeout: config.watcherTimeoutMs,
|
||||
});
|
||||
|
||||
const buildHeaders = () => {
|
||||
if (!config.wscraperServiceToken) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
Authorization: `Bearer ${config.wscraperServiceToken}`,
|
||||
};
|
||||
};
|
||||
|
||||
const request = async <T>(method: "GET" | "POST", url: string, body?: unknown) => {
|
||||
logger.info(
|
||||
{
|
||||
method,
|
||||
url,
|
||||
baseUrl: config.wscraperServiceBaseUrl,
|
||||
},
|
||||
"Calling wscraper service"
|
||||
);
|
||||
try {
|
||||
const response = await scraperClient.request<T>({
|
||||
method,
|
||||
url,
|
||||
data: body,
|
||||
headers: buildHeaders(),
|
||||
});
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
if (axios.isAxiosError(error)) {
|
||||
const message =
|
||||
typeof error.response?.data === "object" && error.response?.data && "error" in error.response.data
|
||||
? String((error.response.data as { error: string }).error)
|
||||
: error.message;
|
||||
throw new Error(`wscraper-service request failed: ${message}`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const runBookmarkFetch = async (
|
||||
trackerSiteKey: string,
|
||||
cookie: string
|
||||
) => {
|
||||
const response = await request<{ items: BookmarkRecord[] }>("POST", "/bookmarks", {
|
||||
tracker: trackerSiteKey,
|
||||
cookie,
|
||||
});
|
||||
return response.items;
|
||||
};
|
||||
|
||||
export const runTorrentDownload = async (
|
||||
trackerSiteKey: string,
|
||||
cookie: string,
|
||||
detailUrl: string,
|
||||
outputDir: string
|
||||
) => {
|
||||
const response = await request<{ filename: string; contentBase64: string }>("POST", "/download", {
|
||||
tracker: trackerSiteKey,
|
||||
cookie,
|
||||
url: detailUrl,
|
||||
removeBookmark: true,
|
||||
});
|
||||
const targetPath = path.join(outputDir, response.filename);
|
||||
await fs.writeFile(targetPath, Buffer.from(response.contentBase64, "base64"));
|
||||
return targetPath;
|
||||
};
|
||||
|
||||
export const cleanupRunPaths = async (paths: ScraperRunPaths) => {
|
||||
try {
|
||||
await fs.rm(paths.runDir, { recursive: true, force: true });
|
||||
} catch (error) {
|
||||
logger.warn({ error, runDir: paths.runDir }, "Failed to cleanup watcher runtime directory");
|
||||
}
|
||||
};
|
||||
742
apps/server/src/watcher/watcher.service.ts
Normal file
742
apps/server/src/watcher/watcher.service.ts
Normal file
@@ -0,0 +1,742 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import axios from "axios";
|
||||
import { getQbitClient } from "../qbit/qbit.context";
|
||||
import { QbitTorrentInfo } from "../qbit/qbit.types";
|
||||
import { readDb, writeDb } from "../storage/jsondb";
|
||||
import { Watcher, WatcherItem, WatcherRun, WatcherSummary } from "../types";
|
||||
import { nowIso } from "../utils/time";
|
||||
import { appendAuditLog, logger } from "../utils/logger";
|
||||
import { emitWatcherItems, emitWatchersList, emitWatcherSummary } from "../realtime/emitter";
|
||||
import { buildCookieHint, decryptWatcherCookie, encryptWatcherCookie } from "./watcher.crypto";
|
||||
import { getTrackerDefinition, trackerRegistry } from "./watcher.registry";
|
||||
import { cleanupRunPaths, createScraperRunPaths, runBookmarkFetch, runTorrentDownload } from "./watcher.scraper";
|
||||
import { BookmarkRecord, EnrichedWatcherItem, WatcherListItem, WatcherSummaryResponse } from "./watcher.types";
|
||||
|
||||
const MAX_WATCHER_ITEMS = 500;
|
||||
const MAX_WATCHER_RUNS = 200;
|
||||
const QBIT_VERIFY_ATTEMPTS = 5;
|
||||
const QBIT_VERIFY_DELAY_MS = 2500;
|
||||
const activeWatcherRuns = new Set<string>();
|
||||
|
||||
const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
const toListItem = (watcher: Watcher): WatcherListItem => ({
|
||||
...watcher,
|
||||
hasCookie: Boolean(watcher.cookieEncrypted),
|
||||
});
|
||||
|
||||
const statusLabel = (status: WatcherItem["status"]) => {
|
||||
switch (status) {
|
||||
case "bookmarked":
|
||||
return "Bookmark bulundu";
|
||||
case "downloading_torrent":
|
||||
return "Torrent indiriliyor";
|
||||
case "sending_to_qbit":
|
||||
return "qBittorrent'a gonderiliyor";
|
||||
case "sent_to_qbit":
|
||||
return "qBittorrent'a gonderildi";
|
||||
case "downloading":
|
||||
return "Indiriliyor";
|
||||
case "completed":
|
||||
return "Tamamlandi";
|
||||
case "failed":
|
||||
default:
|
||||
return "Hata";
|
||||
}
|
||||
};
|
||||
|
||||
const deriveTrackerTorrentId = (pageUrl: string) => {
|
||||
try {
|
||||
const url = new URL(pageUrl);
|
||||
return url.searchParams.get("id") ?? undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const normalizeImageUrl = (pageUrl: string, imageUrl?: string | null) => {
|
||||
if (!imageUrl?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return new URL(imageUrl, pageUrl).toString();
|
||||
} catch {
|
||||
return imageUrl;
|
||||
}
|
||||
};
|
||||
|
||||
const domainMatches = (targetHost: string, cookieDomain: string) => {
|
||||
const normalizedCookieDomain = cookieDomain.replace(/^\./, "").toLowerCase();
|
||||
const normalizedTargetHost = targetHost.toLowerCase();
|
||||
return (
|
||||
normalizedTargetHost === normalizedCookieDomain ||
|
||||
normalizedTargetHost.endsWith(`.${normalizedCookieDomain}`)
|
||||
);
|
||||
};
|
||||
|
||||
const toCookieHeader = (cookieValue: string, targetUrl: string) => {
|
||||
let targetHost = "";
|
||||
try {
|
||||
targetHost = new URL(targetUrl).hostname;
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
|
||||
const lines = cookieValue.split(/\r?\n/);
|
||||
const looksLikeNetscape =
|
||||
lines.length > 1 && lines.some((line) => line.includes("\t"));
|
||||
|
||||
if (looksLikeNetscape) {
|
||||
const pairs: string[] = [];
|
||||
for (const rawLine of lines) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith("#")) {
|
||||
continue;
|
||||
}
|
||||
const parts = line.split("\t");
|
||||
if (parts.length < 7) {
|
||||
continue;
|
||||
}
|
||||
const [domain, , , , , name, value] = parts;
|
||||
if (!name || !domainMatches(targetHost, domain)) {
|
||||
continue;
|
||||
}
|
||||
pairs.push(`${name}=${value}`);
|
||||
}
|
||||
return pairs.join("; ");
|
||||
}
|
||||
|
||||
return cookieValue
|
||||
.split(";")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean)
|
||||
.join("; ");
|
||||
};
|
||||
|
||||
const computeNextRunAt = (intervalMinutes: number) => {
|
||||
return new Date(Date.now() + intervalMinutes * 60_000).toISOString();
|
||||
};
|
||||
|
||||
const deriveLiveStatus = (
|
||||
itemStatus: WatcherItem["status"],
|
||||
torrent?: QbitTorrentInfo
|
||||
): WatcherItem["status"] => {
|
||||
if (!torrent) {
|
||||
return itemStatus;
|
||||
}
|
||||
if ((torrent.progress ?? 0) >= 1) {
|
||||
return "completed";
|
||||
}
|
||||
if (itemStatus === "failed") {
|
||||
return "failed";
|
||||
}
|
||||
if (
|
||||
(torrent.progress ?? 0) > 0 ||
|
||||
(torrent.dlspeed ?? 0) > 0 ||
|
||||
(torrent.state ?? "").toLowerCase().includes("download")
|
||||
) {
|
||||
return "downloading";
|
||||
}
|
||||
return "sent_to_qbit";
|
||||
};
|
||||
|
||||
const refreshWatcherSummary = async () => {
|
||||
const db = await readDb();
|
||||
const watchers = db.watchers ?? [];
|
||||
const updatedAt = nowIso();
|
||||
const summary: WatcherSummary = {
|
||||
activeWatchers: watchers.filter((watcher) => watcher.enabled).length,
|
||||
totalImported: watchers.reduce((sum, watcher) => sum + watcher.totalImported, 0),
|
||||
totalSeen: watchers.reduce((sum, watcher) => sum + watcher.totalSeen, 0),
|
||||
trackedLabels: Array.from(new Set(watchers.filter((watcher) => watcher.enabled).map((watcher) => watcher.trackerLabel))),
|
||||
lastRunAt: watchers
|
||||
.map((watcher) => watcher.lastRunAt)
|
||||
.filter(Boolean)
|
||||
.sort()
|
||||
.at(-1),
|
||||
lastSuccessAt: watchers
|
||||
.map((watcher) => watcher.lastSuccessAt)
|
||||
.filter(Boolean)
|
||||
.sort()
|
||||
.at(-1),
|
||||
nextRunAt: watchers
|
||||
.map((watcher) => watcher.nextRunAt)
|
||||
.filter(Boolean)
|
||||
.sort()[0],
|
||||
lastError: watchers.map((watcher) => watcher.lastError).filter(Boolean).at(-1),
|
||||
updatedAt,
|
||||
};
|
||||
db.watcherSummary = summary;
|
||||
await writeDb(db);
|
||||
emitWatcherSummary({
|
||||
...summary,
|
||||
watchers: watchers.map((watcher) => ({
|
||||
id: watcher.id,
|
||||
tracker: watcher.tracker,
|
||||
trackerLabel: watcher.trackerLabel,
|
||||
enabled: watcher.enabled,
|
||||
lastRunAt: watcher.lastRunAt,
|
||||
lastSuccessAt: watcher.lastSuccessAt,
|
||||
nextRunAt: watcher.nextRunAt,
|
||||
totalImported: watcher.totalImported,
|
||||
})),
|
||||
});
|
||||
return summary;
|
||||
};
|
||||
|
||||
const upsertWatcherItem = (items: WatcherItem[], nextItem: WatcherItem) => {
|
||||
const index = items.findIndex((item) => item.id === nextItem.id);
|
||||
if (index >= 0) {
|
||||
items[index] = nextItem;
|
||||
} else {
|
||||
items.unshift(nextItem);
|
||||
}
|
||||
return items.slice(0, MAX_WATCHER_ITEMS);
|
||||
};
|
||||
|
||||
const mergeQbitState = (item: WatcherItem, torrents: QbitTorrentInfo[]): EnrichedWatcherItem => {
|
||||
const matched =
|
||||
torrents.find((torrent) => item.qbitHash && torrent.hash === item.qbitHash) ??
|
||||
torrents.find((torrent) => torrent.name === (item.qbitName || item.title));
|
||||
const progress = matched?.progress ?? item.qbitProgress ?? 0;
|
||||
const status = deriveLiveStatus(item.status, matched);
|
||||
return {
|
||||
...item,
|
||||
status,
|
||||
statusLabel: statusLabel(status),
|
||||
progress,
|
||||
state: matched?.state ?? item.qbitState,
|
||||
qbitHash: matched?.hash ?? item.qbitHash,
|
||||
qbitName: matched?.name ?? item.qbitName,
|
||||
qbitProgress: progress,
|
||||
qbitState: matched?.state ?? item.qbitState,
|
||||
qbitCategory: matched?.category ?? item.qbitCategory,
|
||||
sizeBytes: matched?.size ?? item.sizeBytes,
|
||||
seeders: matched?.num_seeds ?? item.seeders,
|
||||
leechers: matched?.num_leechs ?? item.leechers,
|
||||
};
|
||||
};
|
||||
|
||||
const persistWatcher = async (watcherId: string, update: (watcher: Watcher) => Watcher) => {
|
||||
const db = await readDb();
|
||||
const watchers = db.watchers ?? [];
|
||||
const index = watchers.findIndex((watcher) => watcher.id === watcherId);
|
||||
if (index < 0) {
|
||||
throw new Error("Watcher not found");
|
||||
}
|
||||
watchers[index] = update(watchers[index]);
|
||||
db.watchers = watchers;
|
||||
await writeDb(db);
|
||||
emitWatchersList(watchers.map(toListItem));
|
||||
return watchers[index];
|
||||
};
|
||||
|
||||
const persistWatcherProgress = async (payload: {
|
||||
watcherId: string;
|
||||
item: WatcherItem;
|
||||
importedDelta?: number;
|
||||
seenDelta?: number;
|
||||
lastError?: string;
|
||||
}) => {
|
||||
const db = await readDb();
|
||||
db.watcherItems = upsertWatcherItem(db.watcherItems ?? [], payload.item);
|
||||
db.watchers = (db.watchers ?? []).map((watcher) =>
|
||||
watcher.id === payload.watcherId
|
||||
? {
|
||||
...watcher,
|
||||
totalImported: watcher.totalImported + (payload.importedDelta ?? 0),
|
||||
totalSeen: watcher.totalSeen + (payload.seenDelta ?? 0),
|
||||
lastError: payload.lastError,
|
||||
updatedAt: nowIso(),
|
||||
}
|
||||
: watcher
|
||||
);
|
||||
await writeDb(db);
|
||||
emitWatchersList((db.watchers ?? []).map(toListItem));
|
||||
emitWatcherItems(await getWatcherItems());
|
||||
await refreshWatcherSummary();
|
||||
};
|
||||
|
||||
export const listTrackers = () => trackerRegistry;
|
||||
|
||||
export const listWatchers = async () => {
|
||||
const db = await readDb();
|
||||
return (db.watchers ?? []).map(toListItem);
|
||||
};
|
||||
|
||||
export const deleteWatcher = async (watcherId: string) => {
|
||||
const db = await readDb();
|
||||
const watcher = (db.watchers ?? []).find((entry) => entry.id === watcherId);
|
||||
if (!watcher) {
|
||||
throw new Error("Watcher not found");
|
||||
}
|
||||
db.watchers = (db.watchers ?? []).filter((entry) => entry.id !== watcherId);
|
||||
db.watcherItems = (db.watcherItems ?? []).filter((item) => item.watcherId !== watcherId);
|
||||
db.watcherRuns = (db.watcherRuns ?? []).filter((run) => run.watcherId !== watcherId);
|
||||
await writeDb(db);
|
||||
emitWatchersList((db.watchers ?? []).map(toListItem));
|
||||
emitWatcherItems(await getWatcherItems());
|
||||
await refreshWatcherSummary();
|
||||
return { ok: true };
|
||||
};
|
||||
|
||||
export const createWatcher = async (payload: {
|
||||
tracker: string;
|
||||
cookie: string;
|
||||
intervalMinutes: number;
|
||||
category?: string;
|
||||
enabled?: boolean;
|
||||
}) => {
|
||||
const tracker = getTrackerDefinition(payload.tracker);
|
||||
if (!tracker) {
|
||||
throw new Error("Unsupported tracker");
|
||||
}
|
||||
if (!payload.cookie.trim()) {
|
||||
throw new Error("Cookie is required");
|
||||
}
|
||||
const db = await readDb();
|
||||
const watcher: Watcher = {
|
||||
id: randomUUID(),
|
||||
tracker: tracker.key,
|
||||
trackerLabel: tracker.label,
|
||||
category: payload.category?.trim() || undefined,
|
||||
cookieEncrypted: encryptWatcherCookie(payload.cookie.trim()),
|
||||
cookieHint: buildCookieHint(payload.cookie),
|
||||
intervalMinutes: payload.intervalMinutes,
|
||||
enabled: payload.enabled ?? true,
|
||||
nextRunAt: computeNextRunAt(payload.intervalMinutes),
|
||||
totalImported: 0,
|
||||
totalSeen: 0,
|
||||
createdAt: nowIso(),
|
||||
updatedAt: nowIso(),
|
||||
};
|
||||
db.watchers = [watcher, ...(db.watchers ?? [])];
|
||||
await writeDb(db);
|
||||
emitWatchersList((db.watchers ?? []).map(toListItem));
|
||||
await refreshWatcherSummary();
|
||||
return toListItem(watcher);
|
||||
};
|
||||
|
||||
export const updateWatcher = async (
|
||||
watcherId: string,
|
||||
payload: { cookie?: string; intervalMinutes?: number; category?: string; enabled?: boolean }
|
||||
) => {
|
||||
const updated = await persistWatcher(watcherId, (watcher) => {
|
||||
const next: Watcher = {
|
||||
...watcher,
|
||||
updatedAt: nowIso(),
|
||||
};
|
||||
if (typeof payload.intervalMinutes === "number") {
|
||||
next.intervalMinutes = payload.intervalMinutes;
|
||||
next.nextRunAt = computeNextRunAt(payload.intervalMinutes);
|
||||
}
|
||||
if (typeof payload.enabled === "boolean") {
|
||||
next.enabled = payload.enabled;
|
||||
}
|
||||
if (typeof payload.category === "string") {
|
||||
next.category = payload.category.trim() || undefined;
|
||||
}
|
||||
if (payload.cookie && payload.cookie.trim()) {
|
||||
next.cookieEncrypted = encryptWatcherCookie(payload.cookie.trim());
|
||||
next.cookieHint = buildCookieHint(payload.cookie);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
emitWatchersList((await readDb()).watchers?.map(toListItem) ?? []);
|
||||
await refreshWatcherSummary();
|
||||
return toListItem(updated);
|
||||
};
|
||||
|
||||
export const getWatcherSummary = async (): Promise<WatcherSummaryResponse> => {
|
||||
const summary = await refreshWatcherSummary();
|
||||
const watchers = await listWatchers();
|
||||
return {
|
||||
...summary,
|
||||
watchers: watchers.map((watcher) => ({
|
||||
id: watcher.id,
|
||||
tracker: watcher.tracker,
|
||||
trackerLabel: watcher.trackerLabel,
|
||||
enabled: watcher.enabled,
|
||||
lastRunAt: watcher.lastRunAt,
|
||||
lastSuccessAt: watcher.lastSuccessAt,
|
||||
nextRunAt: watcher.nextRunAt,
|
||||
totalImported: watcher.totalImported,
|
||||
})),
|
||||
};
|
||||
};
|
||||
|
||||
export const getWatcherItems = async () => {
|
||||
const db = await readDb();
|
||||
const torrents = await getQbitClient().getTorrentsInfo().catch(() => []);
|
||||
return (db.watcherItems ?? []).map((item) => mergeQbitState(item, torrents));
|
||||
};
|
||||
|
||||
export const getQbitCategories = async () => {
|
||||
const qbit = getQbitClient();
|
||||
const categories = await qbit.getCategories().catch(async () => {
|
||||
const torrents = await qbit.getTorrentsInfo();
|
||||
return torrents.reduce<Record<string, { name?: string }>>((acc, torrent) => {
|
||||
if (torrent.category?.trim()) {
|
||||
acc[torrent.category] = { name: torrent.category };
|
||||
}
|
||||
return acc;
|
||||
}, {});
|
||||
});
|
||||
return Object.keys(categories).sort((a, b) => a.localeCompare(b, "tr"));
|
||||
};
|
||||
|
||||
export const fetchWatcherImage = async (watcherId: string, imageUrl: string) => {
|
||||
const db = await readDb();
|
||||
const watcher = (db.watchers ?? []).find((entry) => entry.id === watcherId);
|
||||
if (!watcher) {
|
||||
throw new Error("Watcher not found");
|
||||
}
|
||||
const cookie = toCookieHeader(
|
||||
decryptWatcherCookie(watcher.cookieEncrypted),
|
||||
imageUrl
|
||||
);
|
||||
const response = await axios.get<ArrayBuffer>(imageUrl, {
|
||||
responseType: "arraybuffer",
|
||||
headers: {
|
||||
...(cookie ? { Cookie: cookie } : {}),
|
||||
Referer: "https://www.happyfappy.net/",
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
|
||||
},
|
||||
});
|
||||
return {
|
||||
contentType: response.headers["content-type"] || "image/jpeg",
|
||||
data: Buffer.from(response.data),
|
||||
};
|
||||
};
|
||||
|
||||
const recordRun = async (run: WatcherRun) => {
|
||||
const db = await readDb();
|
||||
db.watcherRuns = [run, ...(db.watcherRuns ?? [])].slice(0, MAX_WATCHER_RUNS);
|
||||
await writeDb(db);
|
||||
};
|
||||
|
||||
const updateRun = async (runId: string, patch: Partial<WatcherRun>) => {
|
||||
const db = await readDb();
|
||||
const runs = db.watcherRuns ?? [];
|
||||
const index = runs.findIndex((run) => run.id === runId);
|
||||
if (index >= 0) {
|
||||
runs[index] = { ...runs[index], ...patch };
|
||||
db.watcherRuns = runs;
|
||||
await writeDb(db);
|
||||
}
|
||||
};
|
||||
|
||||
const findExistingItem = (items: WatcherItem[], watcherId: string, sourceKey: string) => {
|
||||
return items.find((item) => item.watcherId === watcherId && item.sourceKey === sourceKey);
|
||||
};
|
||||
|
||||
const verifyQbitImport = async (
|
||||
torrentPath: string,
|
||||
bookmark: BookmarkRecord,
|
||||
options: { category?: string } = {}
|
||||
) => {
|
||||
const qbit = getQbitClient();
|
||||
const fs = await import("node:fs/promises");
|
||||
const { default: parseTorrent } = await import("parse-torrent");
|
||||
const parsed = parseTorrent(await fs.readFile(torrentPath));
|
||||
const expectedHash = String(parsed.infoHash ?? "").toLowerCase();
|
||||
await qbit.addTorrentByFile(
|
||||
torrentPath,
|
||||
options.category ? { category: options.category } : {}
|
||||
);
|
||||
let lastSeen: QbitTorrentInfo | undefined;
|
||||
for (let attempt = 0; attempt < QBIT_VERIFY_ATTEMPTS; attempt += 1) {
|
||||
const torrents = await qbit.getTorrentsInfo();
|
||||
lastSeen = torrents.find((torrent) => {
|
||||
const sameHash = Boolean(expectedHash) && torrent.hash.toLowerCase() === expectedHash;
|
||||
const sameName =
|
||||
torrent.name === path.basename(torrentPath, ".torrent") || torrent.name === bookmark.title;
|
||||
return sameHash || sameName;
|
||||
});
|
||||
if (lastSeen) {
|
||||
return {
|
||||
ok: true,
|
||||
duplicate: false,
|
||||
torrent: lastSeen,
|
||||
};
|
||||
}
|
||||
await delay(QBIT_VERIFY_DELAY_MS);
|
||||
}
|
||||
const torrents = await qbit.getTorrentsInfo();
|
||||
const duplicate = torrents.find(
|
||||
(torrent) =>
|
||||
(Boolean(expectedHash) && torrent.hash.toLowerCase() === expectedHash) ||
|
||||
torrent.name === bookmark.title
|
||||
);
|
||||
if (duplicate) {
|
||||
return {
|
||||
ok: true,
|
||||
duplicate: true,
|
||||
torrent: duplicate,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
duplicate: false,
|
||||
torrent: lastSeen,
|
||||
};
|
||||
};
|
||||
|
||||
const processBookmark = async (watcher: Watcher, bookmark: BookmarkRecord) => {
|
||||
const tracker = getTrackerDefinition(watcher.tracker);
|
||||
if (!tracker) {
|
||||
throw new Error("Unsupported tracker");
|
||||
}
|
||||
const runPaths = await createScraperRunPaths(watcher.id);
|
||||
const itemId = randomUUID();
|
||||
const seenAt = nowIso();
|
||||
let item: WatcherItem = {
|
||||
id: itemId,
|
||||
watcherId: watcher.id,
|
||||
tracker: watcher.tracker,
|
||||
trackerLabel: watcher.trackerLabel,
|
||||
sourceKey: bookmark.pageURL,
|
||||
pageUrl: bookmark.pageURL,
|
||||
title: bookmark.title,
|
||||
imageUrl: normalizeImageUrl(bookmark.pageURL, bookmark.backgroundImage),
|
||||
status: "downloading_torrent",
|
||||
statusLabel: statusLabel("downloading_torrent"),
|
||||
trackerTorrentId: deriveTrackerTorrentId(bookmark.pageURL),
|
||||
seenAt,
|
||||
lastSyncAt: seenAt,
|
||||
};
|
||||
try {
|
||||
const cookie = decryptWatcherCookie(watcher.cookieEncrypted);
|
||||
const torrentPath = await runTorrentDownload(
|
||||
tracker.cliSiteKey,
|
||||
cookie,
|
||||
bookmark.pageURL,
|
||||
runPaths.torrentDir
|
||||
);
|
||||
item = {
|
||||
...item,
|
||||
status: "sending_to_qbit",
|
||||
statusLabel: statusLabel("sending_to_qbit"),
|
||||
downloadedAt: nowIso(),
|
||||
lastSyncAt: nowIso(),
|
||||
};
|
||||
const qbitResult = await verifyQbitImport(torrentPath, bookmark, {
|
||||
category: watcher.category,
|
||||
});
|
||||
if (!qbitResult.ok || !qbitResult.torrent) {
|
||||
throw new Error("Torrent qBittorrent listesinde dogrulanamadi");
|
||||
}
|
||||
item = {
|
||||
...item,
|
||||
status: deriveLiveStatus("sent_to_qbit", qbitResult.torrent),
|
||||
statusLabel: statusLabel(deriveLiveStatus("sent_to_qbit", qbitResult.torrent)),
|
||||
importedAt: nowIso(),
|
||||
qbitHash: qbitResult.torrent.hash,
|
||||
qbitName: qbitResult.torrent.name,
|
||||
qbitProgress: qbitResult.torrent.progress,
|
||||
qbitState: qbitResult.torrent.state,
|
||||
qbitCategory: qbitResult.torrent.category,
|
||||
sizeBytes: qbitResult.torrent.size,
|
||||
seeders: qbitResult.torrent.num_seeds,
|
||||
leechers: qbitResult.torrent.num_leechs,
|
||||
lastSyncAt: nowIso(),
|
||||
};
|
||||
return item;
|
||||
} finally {
|
||||
await cleanupRunPaths(runPaths);
|
||||
}
|
||||
};
|
||||
|
||||
export const runWatcherById = async (watcherId: string) => {
|
||||
if (activeWatcherRuns.has(watcherId)) {
|
||||
logger.info({ watcherId }, "Watcher run skipped because another run is already active");
|
||||
return;
|
||||
}
|
||||
activeWatcherRuns.add(watcherId);
|
||||
const db = await readDb();
|
||||
const watcher = (db.watchers ?? []).find((entry) => entry.id === watcherId);
|
||||
if (!watcher || !watcher.enabled) {
|
||||
logger.info({ watcherId }, "Watcher skipped because it is missing or disabled");
|
||||
activeWatcherRuns.delete(watcherId);
|
||||
return;
|
||||
}
|
||||
const tracker = getTrackerDefinition(watcher.tracker);
|
||||
if (!tracker) {
|
||||
throw new Error("Unsupported tracker");
|
||||
}
|
||||
logger.info({ watcherId, tracker: watcher.tracker, nextRunAt: watcher.nextRunAt }, "Watcher run started");
|
||||
const runId = randomUUID();
|
||||
await recordRun({
|
||||
id: runId,
|
||||
watcherId: watcher.id,
|
||||
startedAt: nowIso(),
|
||||
status: "RUNNING",
|
||||
newBookmarks: 0,
|
||||
importedCount: 0,
|
||||
failedCount: 0,
|
||||
});
|
||||
const runPaths = await createScraperRunPaths(watcher.id);
|
||||
let newBookmarks = 0;
|
||||
let importedCount = 0;
|
||||
let failedCount = 0;
|
||||
try {
|
||||
await persistWatcher(watcher.id, (current) => ({
|
||||
...current,
|
||||
lastRunAt: nowIso(),
|
||||
lastError: undefined,
|
||||
nextRunAt: computeNextRunAt(current.intervalMinutes),
|
||||
updatedAt: nowIso(),
|
||||
}));
|
||||
const cookie = decryptWatcherCookie(watcher.cookieEncrypted);
|
||||
const records = await runBookmarkFetch(
|
||||
tracker.cliSiteKey,
|
||||
cookie
|
||||
);
|
||||
logger.info({ watcherId, count: records.length }, "Watcher bookmarks fetched");
|
||||
newBookmarks = records.length;
|
||||
const freshDb = await readDb();
|
||||
const items = freshDb.watcherItems ?? [];
|
||||
const processedSourceKeys = new Set(items.map((item) => item.sourceKey));
|
||||
for (const bookmark of records) {
|
||||
if (processedSourceKeys.has(bookmark.pageURL)) {
|
||||
logger.info({ watcherId, sourceKey: bookmark.pageURL }, "Watcher bookmark skipped due to dedupe set");
|
||||
continue;
|
||||
}
|
||||
const existing = findExistingItem(items, watcher.id, bookmark.pageURL);
|
||||
if (existing) {
|
||||
existing.lastSyncAt = nowIso();
|
||||
await persistWatcherProgress({
|
||||
watcherId: watcher.id,
|
||||
item: existing,
|
||||
});
|
||||
processedSourceKeys.add(bookmark.pageURL);
|
||||
logger.info({ watcherId, sourceKey: bookmark.pageURL }, "Watcher bookmark already processed");
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const nextItem = await processBookmark(watcher, bookmark);
|
||||
importedCount += 1;
|
||||
await persistWatcherProgress({
|
||||
watcherId: watcher.id,
|
||||
item: nextItem,
|
||||
importedDelta: 1,
|
||||
seenDelta: 1,
|
||||
});
|
||||
processedSourceKeys.add(bookmark.pageURL);
|
||||
logger.info(
|
||||
{
|
||||
watcherId,
|
||||
sourceKey: bookmark.pageURL,
|
||||
qbitHash: nextItem.qbitHash,
|
||||
status: nextItem.status,
|
||||
},
|
||||
"Watcher bookmark imported"
|
||||
);
|
||||
} catch (error) {
|
||||
failedCount += 1;
|
||||
logger.error(
|
||||
{
|
||||
watcherId,
|
||||
sourceKey: bookmark.pageURL,
|
||||
error: error instanceof Error ? error.message : error,
|
||||
},
|
||||
"Watcher bookmark import failed"
|
||||
);
|
||||
const failedItem: WatcherItem = {
|
||||
id: randomUUID(),
|
||||
watcherId: watcher.id,
|
||||
tracker: watcher.tracker,
|
||||
trackerLabel: watcher.trackerLabel,
|
||||
sourceKey: bookmark.pageURL,
|
||||
pageUrl: bookmark.pageURL,
|
||||
title: bookmark.title,
|
||||
imageUrl: normalizeImageUrl(bookmark.pageURL, bookmark.backgroundImage),
|
||||
status: "failed",
|
||||
statusLabel: statusLabel("failed"),
|
||||
trackerTorrentId: deriveTrackerTorrentId(bookmark.pageURL),
|
||||
seenAt: nowIso(),
|
||||
lastSyncAt: nowIso(),
|
||||
errorMessage: error instanceof Error ? error.message : "Unknown watcher error",
|
||||
};
|
||||
await persistWatcherProgress({
|
||||
watcherId: watcher.id,
|
||||
item: failedItem,
|
||||
seenDelta: 1,
|
||||
lastError: failedItem.errorMessage,
|
||||
});
|
||||
processedSourceKeys.add(bookmark.pageURL);
|
||||
}
|
||||
}
|
||||
const finalDb = await readDb();
|
||||
finalDb.watchers = (finalDb.watchers ?? []).map((entry) =>
|
||||
entry.id === watcher.id
|
||||
? {
|
||||
...entry,
|
||||
lastSuccessAt: nowIso(),
|
||||
lastError: failedCount > 0 ? `${failedCount} item hata verdi` : undefined,
|
||||
updatedAt: nowIso(),
|
||||
}
|
||||
: entry
|
||||
);
|
||||
await writeDb(finalDb);
|
||||
await refreshWatcherSummary();
|
||||
logger.info({ watcherId, importedCount, failedCount }, "Watcher run completed");
|
||||
await updateRun(runId, {
|
||||
finishedAt: nowIso(),
|
||||
status: failedCount > 0 ? "FAILED" : "SUCCESS",
|
||||
newBookmarks,
|
||||
importedCount,
|
||||
failedCount,
|
||||
message: failedCount > 0 ? `${failedCount} item hata verdi` : `${importedCount} item qBittorrent'a gonderildi`,
|
||||
});
|
||||
await appendAuditLog({
|
||||
level: failedCount > 0 ? "WARN" : "INFO",
|
||||
event: "ARCHIVE_SUCCESS",
|
||||
message: `Watcher ${watcher.trackerLabel}: ${importedCount} imported, ${failedCount} failed`,
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Watcher run failed";
|
||||
await persistWatcher(watcher.id, (current) => ({
|
||||
...current,
|
||||
lastError: message,
|
||||
updatedAt: nowIso(),
|
||||
}));
|
||||
await updateRun(runId, {
|
||||
finishedAt: nowIso(),
|
||||
status: "FAILED",
|
||||
newBookmarks,
|
||||
importedCount,
|
||||
failedCount: failedCount + 1,
|
||||
message,
|
||||
});
|
||||
logger.error(
|
||||
{
|
||||
watcherId,
|
||||
message,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
error: error instanceof Error ? { name: error.name, message: error.message } : error,
|
||||
},
|
||||
"Watcher run failed"
|
||||
);
|
||||
} finally {
|
||||
await cleanupRunPaths(runPaths);
|
||||
activeWatcherRuns.delete(watcherId);
|
||||
}
|
||||
};
|
||||
|
||||
export const runDueWatchers = async () => {
|
||||
const db = await readDb();
|
||||
const dueWatchers = (db.watchers ?? []).filter((watcher) => {
|
||||
if (!watcher.enabled) {
|
||||
return false;
|
||||
}
|
||||
if (!watcher.nextRunAt) {
|
||||
return true;
|
||||
}
|
||||
return new Date(watcher.nextRunAt).getTime() <= Date.now();
|
||||
});
|
||||
logger.info({ watcherIds: dueWatchers.map((watcher) => watcher.id) }, "Watcher due check completed");
|
||||
return dueWatchers.map((watcher) => watcher.id);
|
||||
};
|
||||
35
apps/server/src/watcher/watcher.types.ts
Normal file
35
apps/server/src/watcher/watcher.types.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { Watcher, WatcherItem, WatcherSummary, WatcherTracker } from "../types";
|
||||
|
||||
export interface TrackerDefinition {
|
||||
key: WatcherTracker;
|
||||
label: string;
|
||||
cliSiteKey: string;
|
||||
supportsRemoveBookmark: boolean;
|
||||
}
|
||||
|
||||
export interface BookmarkRecord {
|
||||
pageURL: string;
|
||||
isVR?: boolean;
|
||||
title: string;
|
||||
backgroundImage?: string | null;
|
||||
}
|
||||
|
||||
export interface ScraperRunPaths {
|
||||
runDir: string;
|
||||
cookiesPath: string;
|
||||
bookmarksPath: string;
|
||||
torrentDir: string;
|
||||
}
|
||||
|
||||
export interface WatcherListItem extends Omit<Watcher, "cookieEncrypted"> {
|
||||
hasCookie: boolean;
|
||||
}
|
||||
|
||||
export interface EnrichedWatcherItem extends WatcherItem {
|
||||
progress: number;
|
||||
state?: string;
|
||||
}
|
||||
|
||||
export interface WatcherSummaryResponse extends WatcherSummary {
|
||||
watchers: Array<Pick<Watcher, "id" | "tracker" | "trackerLabel" | "enabled" | "lastRunAt" | "lastSuccessAt" | "nextRunAt" | "totalImported">>;
|
||||
}
|
||||
34
apps/server/src/watcher/watcher.worker.ts
Normal file
34
apps/server/src/watcher/watcher.worker.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
import { config } from "../config";
|
||||
import { logger } from "../utils/logger";
|
||||
import { runDueWatchers, runWatcherById } from "./watcher.service";
|
||||
|
||||
const activeRuns = new Set<string>();
|
||||
|
||||
export const startWatcherWorker = (intervalMs: number) => {
|
||||
if (!config.watcherEnabled) {
|
||||
logger.info("Watcher worker disabled");
|
||||
return;
|
||||
}
|
||||
setInterval(async () => {
|
||||
try {
|
||||
const dueWatcherIds = await runDueWatchers();
|
||||
logger.info({ count: dueWatcherIds.length, watcherIds: dueWatcherIds }, "Watcher worker tick");
|
||||
for (const watcherId of dueWatcherIds) {
|
||||
if (activeRuns.has(watcherId)) {
|
||||
logger.info({ watcherId }, "Watcher run skipped because another run is active");
|
||||
continue;
|
||||
}
|
||||
activeRuns.add(watcherId);
|
||||
runWatcherById(watcherId)
|
||||
.catch((error) => {
|
||||
logger.error({ error, watcherId }, "Unhandled watcher run failure");
|
||||
})
|
||||
.finally(() => {
|
||||
activeRuns.delete(watcherId);
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error({ error }, "Watcher worker tick failed");
|
||||
}
|
||||
}, intervalMs);
|
||||
};
|
||||
@@ -3,6 +3,7 @@ import { BrowserRouter, Navigate, Route, Routes } from "react-router-dom";
|
||||
import { LoginPage } from "./pages/LoginPage";
|
||||
import { DashboardPage } from "./pages/DashboardPage";
|
||||
import { TimerPage } from "./pages/TimerPage";
|
||||
import { WatcherPage } from "./pages/WatcherPage";
|
||||
import { AppLayout } from "./components/layout/AppLayout";
|
||||
import { useAuthStore } from "./store/useAuthStore";
|
||||
|
||||
@@ -25,6 +26,7 @@ export const App = () => {
|
||||
<Route path="/" element={<Navigate to="/buffer" replace />} />
|
||||
<Route path="/buffer" element={<DashboardPage />} />
|
||||
<Route path="/timer" element={<TimerPage />} />
|
||||
<Route path="/watcher" element={<WatcherPage />} />
|
||||
<Route path="*" element={<Navigate to="/buffer" replace />} />
|
||||
</Routes>
|
||||
</AppLayout>
|
||||
|
||||
@@ -112,6 +112,16 @@ export const AppLayout = ({ children }: { children: React.ReactNode }) => {
|
||||
>
|
||||
Timer
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/watcher"
|
||||
className={({ isActive }) =>
|
||||
`rounded-full px-3 py-1 ${
|
||||
isActive ? "bg-slate-900 text-white" : "hover:bg-white"
|
||||
}`
|
||||
}
|
||||
>
|
||||
Watcher
|
||||
</NavLink>
|
||||
</nav>
|
||||
<Button
|
||||
variant="outline"
|
||||
@@ -164,6 +174,17 @@ export const AppLayout = ({ children }: { children: React.ReactNode }) => {
|
||||
>
|
||||
Timer
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/watcher"
|
||||
onClick={() => setMenuOpen(false)}
|
||||
className={({ isActive }) =>
|
||||
`rounded-lg border px-3 py-2 ${
|
||||
isActive ? "border-slate-900 bg-slate-900 text-white" : "border-slate-200 bg-white"
|
||||
}`
|
||||
}
|
||||
>
|
||||
Watcher
|
||||
</NavLink>
|
||||
</nav>
|
||||
<div className="mt-auto flex items-center gap-3">
|
||||
<Button
|
||||
|
||||
22
apps/web/src/components/ui/Textarea.tsx
Normal file
22
apps/web/src/components/ui/Textarea.tsx
Normal file
@@ -0,0 +1,22 @@
|
||||
import React from "react";
|
||||
import clsx from "clsx";
|
||||
|
||||
export const Textarea = React.forwardRef<
|
||||
HTMLTextAreaElement,
|
||||
React.TextareaHTMLAttributes<HTMLTextAreaElement> & { masked?: boolean }
|
||||
>(({ className, masked = false, style, ...props }, ref) => (
|
||||
<textarea
|
||||
ref={ref}
|
||||
className={clsx(
|
||||
"w-full rounded-xl border border-slate-300 bg-white px-3 py-3 text-sm text-slate-900 focus:border-slate-500 focus:outline-none focus:ring-2 focus:ring-slate-200",
|
||||
className
|
||||
)}
|
||||
style={{
|
||||
...(masked ? ({ WebkitTextSecurity: "disc" } as React.CSSProperties) : {}),
|
||||
...style,
|
||||
}}
|
||||
{...props}
|
||||
/>
|
||||
));
|
||||
|
||||
Textarea.displayName = "Textarea";
|
||||
782
apps/web/src/pages/WatcherPage.tsx
Normal file
782
apps/web/src/pages/WatcherPage.tsx
Normal file
@@ -0,0 +1,782 @@
|
||||
import React, { useEffect, useMemo, useState } from "react";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import {
|
||||
faArrowsRotate,
|
||||
faCheckCircle,
|
||||
faClock,
|
||||
faPlay,
|
||||
faDownload,
|
||||
faEye,
|
||||
faEyeSlash,
|
||||
faLayerGroup,
|
||||
faPlus,
|
||||
faSave,
|
||||
faSatelliteDish,
|
||||
faTrash,
|
||||
faToggleOff,
|
||||
faToggleOn,
|
||||
faTriangleExclamation,
|
||||
faWandMagicSparkles,
|
||||
} from "@fortawesome/free-solid-svg-icons";
|
||||
import { api } from "../api/client";
|
||||
import {
|
||||
Watcher,
|
||||
WatcherItem,
|
||||
WatcherSummary,
|
||||
useAppStore,
|
||||
} from "../store/useAppStore";
|
||||
import { useUiStore } from "../store/useUiStore";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "../components/ui/Card";
|
||||
import { Button } from "../components/ui/Button";
|
||||
import { Input } from "../components/ui/Input";
|
||||
import { Textarea } from "../components/ui/Textarea";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "../components/ui/Select";
|
||||
import { Badge } from "../components/ui/Badge";
|
||||
|
||||
const intervalUnits = [
|
||||
{ label: "Dakika", value: "minutes", multiplier: 1 },
|
||||
{ label: "Saat", value: "hours", multiplier: 60 },
|
||||
] as const;
|
||||
|
||||
const formatDate = (value?: string) => {
|
||||
if (!value) return "—";
|
||||
return new Date(value).toLocaleString("tr-TR");
|
||||
};
|
||||
|
||||
const formatBytes = (value?: number) => {
|
||||
if (!value || !Number.isFinite(value)) return "Boyut bilinmiyor";
|
||||
const units = ["B", "KB", "MB", "GB", "TB"];
|
||||
let current = value;
|
||||
let unit = 0;
|
||||
while (current >= 1024 && unit < units.length - 1) {
|
||||
current /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
return `${current.toFixed(current >= 10 ? 0 : 1)} ${units[unit]}`;
|
||||
};
|
||||
|
||||
const compactStateLabel = (value?: string) => {
|
||||
if (!value) return "Bekliyor";
|
||||
const normalized = value.replace(/_/g, " ");
|
||||
return normalized.length > 18 ? normalized.slice(0, 18) : normalized;
|
||||
};
|
||||
|
||||
const progressLabel = (item: WatcherItem) => `${Math.round((item.progress ?? 0) * 100)}%`;
|
||||
|
||||
const buildWatcherImageUrl = (item: WatcherItem) => {
|
||||
if (!item.imageUrl) {
|
||||
return null;
|
||||
}
|
||||
const base = import.meta.env.VITE_API_BASE || "";
|
||||
const params = new URLSearchParams({
|
||||
watcherId: item.watcherId,
|
||||
url: item.imageUrl,
|
||||
});
|
||||
return `${base}/api/watchers/image?${params.toString()}`;
|
||||
};
|
||||
|
||||
const statusVariant = (status: string): "success" | "warn" | "danger" => {
|
||||
if (status === "completed" || status === "sent_to_qbit") return "success";
|
||||
if (status === "failed") return "danger";
|
||||
return "warn";
|
||||
};
|
||||
|
||||
const placeholderImage =
|
||||
"linear-gradient(135deg, rgba(15,23,42,0.92), rgba(8,47,73,0.78) 55%, rgba(14,165,233,0.44) 100%)";
|
||||
|
||||
export const WatcherPage = () => {
|
||||
const watcherTrackers = useAppStore((s) => s.watcherTrackers);
|
||||
const qbitCategories = useAppStore((s) => s.qbitCategories);
|
||||
const watchers = useAppStore((s) => s.watchers);
|
||||
const watcherItems = useAppStore((s) => s.watcherItems);
|
||||
const watcherSummary = useAppStore((s) => s.watcherSummary);
|
||||
const setWatcherTrackers = useAppStore((s) => s.setWatcherTrackers);
|
||||
const setQbitCategories = useAppStore((s) => s.setQbitCategories);
|
||||
const setWatchers = useAppStore((s) => s.setWatchers);
|
||||
const setWatcherItems = useAppStore((s) => s.setWatcherItems);
|
||||
const setWatcherSummary = useAppStore((s) => s.setWatcherSummary);
|
||||
const pushAlert = useUiStore((s) => s.pushAlert);
|
||||
|
||||
const [selectedWatcherId, setSelectedWatcherId] = useState<string | null>(null);
|
||||
const [tracker, setTracker] = useState("happyfappy");
|
||||
const [cookie, setCookie] = useState("");
|
||||
const [category, setCategory] = useState("__none__");
|
||||
const [showCookie, setShowCookie] = useState(false);
|
||||
const [intervalValue, setIntervalValue] = useState("30");
|
||||
const [intervalUnit, setIntervalUnit] = useState<(typeof intervalUnits)[number]["value"]>("minutes");
|
||||
const [enabled, setEnabled] = useState(true);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [running, setRunning] = useState(false);
|
||||
const [deleting, setDeleting] = useState(false);
|
||||
|
||||
const selectedWatcher = useMemo(
|
||||
() => watchers.find((watcher) => watcher.id === selectedWatcherId) ?? null,
|
||||
[selectedWatcherId, watchers]
|
||||
);
|
||||
|
||||
const load = async () => {
|
||||
const [trackersRes, categoriesRes, watchersRes, summaryRes, itemsRes] =
|
||||
await Promise.allSettled([
|
||||
api.get("/api/watchers/trackers"),
|
||||
api.get("/api/watchers/categories"),
|
||||
api.get("/api/watchers"),
|
||||
api.get("/api/watchers/summary"),
|
||||
api.get("/api/watchers/items"),
|
||||
]);
|
||||
|
||||
const watchersLoaded =
|
||||
watchersRes.status === "fulfilled" ? watchersRes.value.data ?? [] : null;
|
||||
|
||||
if (!watchersLoaded && trackersRes.status !== "fulfilled") {
|
||||
pushAlert({
|
||||
title: "Watcher verileri alinmadi",
|
||||
description: "Sunucu baglantisini kontrol edip tekrar deneyin.",
|
||||
variant: "error",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (trackersRes.status === "fulfilled") {
|
||||
setWatcherTrackers(trackersRes.value.data ?? []);
|
||||
}
|
||||
if (categoriesRes.status === "fulfilled") {
|
||||
setQbitCategories(categoriesRes.value.data ?? []);
|
||||
} else {
|
||||
setQbitCategories([]);
|
||||
}
|
||||
if (watchersLoaded) {
|
||||
setWatchers(watchersLoaded);
|
||||
if (!selectedWatcherId && watchersLoaded[0]?.id) {
|
||||
setSelectedWatcherId(watchersLoaded[0].id);
|
||||
}
|
||||
}
|
||||
if (summaryRes.status === "fulfilled") {
|
||||
setWatcherSummary(summaryRes.value.data ?? null);
|
||||
}
|
||||
if (itemsRes.status === "fulfilled") {
|
||||
setWatcherItems(itemsRes.value.data ?? []);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
const interval = setInterval(load, 15000);
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedWatcher) {
|
||||
return;
|
||||
}
|
||||
setTracker(selectedWatcher.tracker);
|
||||
setCategory(selectedWatcher.category ?? "__none__");
|
||||
setCookie("");
|
||||
setShowCookie(false);
|
||||
setEnabled(selectedWatcher.enabled);
|
||||
if (selectedWatcher.intervalMinutes >= 60 && selectedWatcher.intervalMinutes % 60 === 0) {
|
||||
setIntervalUnit("hours");
|
||||
setIntervalValue(String(selectedWatcher.intervalMinutes / 60));
|
||||
} else {
|
||||
setIntervalUnit("minutes");
|
||||
setIntervalValue(String(selectedWatcher.intervalMinutes));
|
||||
}
|
||||
}, [selectedWatcher]);
|
||||
|
||||
const intervalMinutes = useMemo(() => {
|
||||
const unit = intervalUnits.find((entry) => entry.value === intervalUnit) ?? intervalUnits[0];
|
||||
return Math.max(1, Number(intervalValue || 0) * unit.multiplier);
|
||||
}, [intervalUnit, intervalValue]);
|
||||
|
||||
const resetForm = () => {
|
||||
setSelectedWatcherId(null);
|
||||
setTracker(watcherTrackers[0]?.key ?? "happyfappy");
|
||||
setCategory("__none__");
|
||||
setCookie("");
|
||||
setShowCookie(false);
|
||||
setIntervalValue("30");
|
||||
setIntervalUnit("minutes");
|
||||
setEnabled(true);
|
||||
};
|
||||
|
||||
const submit = async () => {
|
||||
if (!cookie.trim() && !selectedWatcher) {
|
||||
pushAlert({
|
||||
title: "Cookie gerekli",
|
||||
description: "Yeni watcher eklemek icin cookie girin.",
|
||||
variant: "warn",
|
||||
});
|
||||
return;
|
||||
}
|
||||
setSaving(true);
|
||||
try {
|
||||
if (selectedWatcher) {
|
||||
await api.patch(`/api/watchers/${selectedWatcher.id}`, {
|
||||
tracker,
|
||||
category: category === "__none__" ? "" : category,
|
||||
intervalMinutes,
|
||||
enabled,
|
||||
...(cookie.trim() ? { cookie } : {}),
|
||||
});
|
||||
pushAlert({
|
||||
title: "Watcher guncellendi",
|
||||
description: `${selectedWatcher.trackerLabel} ayarlari kaydedildi.`,
|
||||
variant: "success",
|
||||
});
|
||||
} else {
|
||||
const response = await api.post("/api/watchers", {
|
||||
tracker,
|
||||
category: category === "__none__" ? "" : category,
|
||||
cookie,
|
||||
intervalMinutes,
|
||||
enabled,
|
||||
});
|
||||
setWatchers([response.data, ...watchers]);
|
||||
setSelectedWatcherId(response.data.id);
|
||||
pushAlert({
|
||||
title: "Watcher eklendi",
|
||||
description: "Yeni bookmark watcher kaydedildi.",
|
||||
variant: "success",
|
||||
});
|
||||
}
|
||||
void load();
|
||||
setCookie("");
|
||||
setShowCookie(false);
|
||||
} catch (error: any) {
|
||||
pushAlert({
|
||||
title: "Kaydetme basarisiz",
|
||||
description: error?.response?.data?.error ?? "Watcher kaydedilemedi.",
|
||||
variant: "error",
|
||||
});
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
const toggleWatcher = async (watcher: Watcher) => {
|
||||
try {
|
||||
await api.patch(`/api/watchers/${watcher.id}`, {
|
||||
enabled: !watcher.enabled,
|
||||
});
|
||||
await load();
|
||||
} catch (error: any) {
|
||||
pushAlert({
|
||||
title: "Durum degismedi",
|
||||
description: error?.response?.data?.error ?? "Watcher durumu guncellenemedi.",
|
||||
variant: "error",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const runNow = async () => {
|
||||
if (!selectedWatcher) {
|
||||
pushAlert({
|
||||
title: "Once kaydet",
|
||||
description: "Manuel calistirma icin once watcher secin.",
|
||||
variant: "warn",
|
||||
});
|
||||
return;
|
||||
}
|
||||
setRunning(true);
|
||||
try {
|
||||
await api.post(`/api/watchers/${selectedWatcher.id}/run`);
|
||||
await load();
|
||||
pushAlert({
|
||||
title: "Watcher calistirildi",
|
||||
description: "Gercek import akisi bir kez manuel olarak tetiklendi.",
|
||||
variant: "success",
|
||||
});
|
||||
} catch (error: any) {
|
||||
pushAlert({
|
||||
title: "Calistirma basarisiz",
|
||||
description: error?.response?.data?.error ?? "Watcher manuel olarak calistirilamadi.",
|
||||
variant: "error",
|
||||
});
|
||||
} finally {
|
||||
setRunning(false);
|
||||
}
|
||||
};
|
||||
|
||||
const removeWatcher = async (watcherOverride?: Watcher | null) => {
|
||||
const watcherToDelete = watcherOverride ?? selectedWatcher;
|
||||
if (!watcherToDelete) {
|
||||
return;
|
||||
}
|
||||
const confirmed = window.confirm(
|
||||
`${watcherToDelete.trackerLabel} watcher silinsin mi? Ilgili watcher item kayitlari da silinecek.`
|
||||
);
|
||||
if (!confirmed) {
|
||||
return;
|
||||
}
|
||||
setDeleting(true);
|
||||
try {
|
||||
await api.delete(`/api/watchers/${watcherToDelete.id}`);
|
||||
await load();
|
||||
if (!watcherOverride || watcherOverride.id === selectedWatcherId) {
|
||||
resetForm();
|
||||
}
|
||||
pushAlert({
|
||||
title: "Watcher silindi",
|
||||
description: "Kayit ve iliskili item gecmisi kaldirildi.",
|
||||
variant: "success",
|
||||
});
|
||||
} catch (error: any) {
|
||||
pushAlert({
|
||||
title: "Silme basarisiz",
|
||||
description: error?.response?.data?.error ?? "Watcher silinemedi.",
|
||||
variant: "error",
|
||||
});
|
||||
} finally {
|
||||
setDeleting(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="grid w-full grid-cols-1 gap-6 xl:grid-cols-[1.45fr_0.85fr]">
|
||||
<div className="min-w-0 space-y-5">
|
||||
<Card className="overflow-hidden border-slate-200 bg-[radial-gradient(circle_at_top_left,_rgba(15,118,110,0.12),_transparent_42%),linear-gradient(180deg,rgba(255,255,255,0.98),rgba(248,250,252,0.92))]">
|
||||
<CardHeader className="items-start">
|
||||
<div>
|
||||
<CardTitle className="flex items-center gap-3">
|
||||
<span className="inline-flex h-11 w-11 items-center justify-center rounded-2xl bg-slate-900 text-white shadow-lg shadow-slate-900/20">
|
||||
<FontAwesomeIcon icon={faSatelliteDish} />
|
||||
</span>
|
||||
Watcher Feed
|
||||
</CardTitle>
|
||||
<p className="mt-2 max-w-2xl text-sm text-slate-500">
|
||||
Bookmark kaynaklarindan gelen torrentler burada tek akista gorunur. Kartlar,
|
||||
qBittorrent aktarim sonucunu ve canli indirme durumunu bir arada gosterir.
|
||||
</p>
|
||||
</div>
|
||||
<Button variant="outline" onClick={load} className="gap-2">
|
||||
<FontAwesomeIcon icon={faArrowsRotate} />
|
||||
Yenile
|
||||
</Button>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{watcherItems.length === 0 ? (
|
||||
<div className="rounded-2xl border border-dashed border-slate-300 bg-white/70 px-5 py-10 text-center text-sm text-slate-500">
|
||||
Henuz watcher item yok. Sag taraftan bir watcher ekleyip calistirarak akisi
|
||||
baslatabilirsiniz.
|
||||
</div>
|
||||
) : (
|
||||
<div className="grid grid-cols-1 gap-4 md:grid-cols-2 xl:grid-cols-3">
|
||||
{watcherItems.map((item) => {
|
||||
const proxiedImageUrl = buildWatcherImageUrl(item);
|
||||
return (
|
||||
<article
|
||||
key={item.id}
|
||||
className="overflow-hidden rounded-[1.25rem] border border-slate-200 bg-white shadow-[0_20px_60px_-40px_rgba(15,23,42,0.42)]"
|
||||
>
|
||||
<div className="relative h-36 bg-slate-900">
|
||||
{proxiedImageUrl ? (
|
||||
<img
|
||||
src={proxiedImageUrl}
|
||||
alt={item.title}
|
||||
className="absolute inset-0 h-full w-full object-cover"
|
||||
onError={(event) => {
|
||||
event.currentTarget.style.display = "none";
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<div
|
||||
className="absolute inset-0"
|
||||
style={{ backgroundImage: placeholderImage }}
|
||||
/>
|
||||
)}
|
||||
<div className="absolute inset-0 bg-gradient-to-t from-slate-950/46 via-slate-950/10 to-transparent" />
|
||||
<div className="absolute bottom-4 left-4 right-4">
|
||||
<div className="text-xs uppercase tracking-[0.22em] text-slate-200/80">
|
||||
{formatDate(item.lastSyncAt)}
|
||||
</div>
|
||||
<div className="mt-1 truncate text-[0.98rem] font-semibold leading-6 text-white">
|
||||
{item.title}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="space-y-3 p-4">
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Badge variant={statusVariant(item.status)}>{item.statusLabel}</Badge>
|
||||
<Badge variant="default">{item.trackerLabel}</Badge>
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2 text-[11px] font-semibold text-slate-500">
|
||||
<span className="rounded-full bg-slate-100 px-2.5 py-1">
|
||||
{compactStateLabel(item.state)}
|
||||
</span>
|
||||
<span className="rounded-full bg-slate-100 px-2.5 py-1">
|
||||
{item.qbitCategory?.trim() ? item.qbitCategory : "Kategori yok"}
|
||||
</span>
|
||||
<span className="rounded-full bg-slate-100 px-2.5 py-1">
|
||||
Seeds {item.seeders ?? 0}
|
||||
{typeof item.leechers === "number" ? ` / ${item.leechers}` : ""}
|
||||
</span>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-3 text-sm text-slate-600">
|
||||
<div className="rounded-2xl bg-slate-50 px-3 py-2">
|
||||
<div className="text-[10px] uppercase tracking-[0.16em] text-slate-400">
|
||||
Boyut
|
||||
</div>
|
||||
<div className="mt-1 text-[0.92rem] font-semibold text-slate-800">
|
||||
{formatBytes(item.sizeBytes)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="rounded-2xl bg-slate-50 px-3 py-2">
|
||||
<div className="text-[10px] uppercase tracking-[0.16em] text-slate-400">
|
||||
Durum
|
||||
</div>
|
||||
<div className="mt-1 text-[0.92rem] font-semibold text-slate-800">
|
||||
{item.state ?? "Bekleniyor"}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="mb-2 flex items-center justify-between text-[10px] font-semibold uppercase tracking-[0.16em] text-slate-400">
|
||||
<span>Qbit Progress</span>
|
||||
<span>{progressLabel(item)}</span>
|
||||
</div>
|
||||
<div className="h-2 rounded-full bg-slate-100">
|
||||
<div
|
||||
className="h-2 rounded-full bg-gradient-to-r from-teal-500 via-cyan-500 to-slate-900 transition-all"
|
||||
style={{ width: `${Math.min(100, Math.max(4, Math.round((item.progress ?? 0) * 100)))}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center justify-between text-xs text-slate-500">
|
||||
<span>Import: {formatDate(item.importedAt)}</span>
|
||||
<a
|
||||
href={item.pageUrl}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="font-semibold text-slate-700 underline decoration-slate-300 underline-offset-4"
|
||||
>
|
||||
Kaynak sayfa
|
||||
</a>
|
||||
</div>
|
||||
{item.errorMessage ? (
|
||||
<div className="rounded-2xl border border-rose-200 bg-rose-50 px-3 py-3 text-sm text-rose-800">
|
||||
{item.errorMessage}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</article>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
<div className="min-w-0 space-y-4">
|
||||
<Card className="border-slate-200 bg-white/90">
|
||||
<CardHeader className="items-start">
|
||||
<div>
|
||||
<CardTitle className="flex items-center gap-3">
|
||||
<span className="inline-flex h-10 w-10 items-center justify-center rounded-2xl bg-teal-600 text-white">
|
||||
<FontAwesomeIcon icon={selectedWatcher ? faWandMagicSparkles : faPlus} />
|
||||
</span>
|
||||
{selectedWatcher ? "Bookmark Watcher Duzenle" : "Bookmark Watcher Ekle"}
|
||||
</CardTitle>
|
||||
<p className="mt-2 text-sm text-slate-500">
|
||||
Tracker secin, cookie ekleyin ve bookmark izleme araligini belirleyin.
|
||||
</p>
|
||||
</div>
|
||||
{selectedWatcher ? (
|
||||
<Button variant="ghost" onClick={resetForm} className="gap-2">
|
||||
<FontAwesomeIcon icon={faPlus} />
|
||||
Yeni
|
||||
</Button>
|
||||
) : null}
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="space-y-3">
|
||||
<label className="block space-y-2">
|
||||
<span className="text-xs font-semibold uppercase tracking-[0.18em] text-slate-500">
|
||||
Tracker
|
||||
</span>
|
||||
<Select value={tracker} onValueChange={setTracker} disabled={Boolean(selectedWatcher)}>
|
||||
<SelectTrigger className="w-full rounded-xl">
|
||||
<SelectValue placeholder="Tracker sec" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{watcherTrackers.map((entry) => (
|
||||
<SelectItem key={entry.key} value={entry.key}>
|
||||
{entry.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</label>
|
||||
|
||||
<label className="block space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-semibold uppercase tracking-[0.18em] text-slate-500">
|
||||
Cookie
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowCookie((current) => !current)}
|
||||
className="inline-flex items-center gap-2 text-xs font-semibold text-slate-500"
|
||||
>
|
||||
<FontAwesomeIcon icon={showCookie ? faEyeSlash : faEye} />
|
||||
{showCookie ? "Gizle" : "Goster"}
|
||||
</button>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Textarea
|
||||
rows={5}
|
||||
masked={!showCookie}
|
||||
value={cookie}
|
||||
onChange={(event) => setCookie(event.target.value)}
|
||||
placeholder={
|
||||
selectedWatcher
|
||||
? "Yeni cookie yapistirin veya mevcut kaydi korumak icin bos birakin."
|
||||
: "Tracker cookie degerlerini buraya yapistirin."
|
||||
}
|
||||
className="resize-none"
|
||||
/>
|
||||
{selectedWatcher?.hasCookie ? (
|
||||
<div className="text-xs text-slate-500">
|
||||
Kayitli cookie ipucu: <span className="font-semibold">{selectedWatcher.cookieHint}</span>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</label>
|
||||
|
||||
<div className="grid grid-cols-[1fr_140px] gap-3">
|
||||
<label className="block space-y-2">
|
||||
<span className="text-xs font-semibold uppercase tracking-[0.18em] text-slate-500">
|
||||
Kontrol Araligi
|
||||
</span>
|
||||
<Input
|
||||
type="number"
|
||||
min={1}
|
||||
value={intervalValue}
|
||||
onChange={(event) => setIntervalValue(event.target.value)}
|
||||
className="rounded-xl"
|
||||
/>
|
||||
</label>
|
||||
<label className="block space-y-2">
|
||||
<span className="text-xs font-semibold uppercase tracking-[0.18em] text-slate-500">
|
||||
Birim
|
||||
</span>
|
||||
<Select value={intervalUnit} onValueChange={(value) => setIntervalUnit(value as "minutes" | "hours")}>
|
||||
<SelectTrigger className="w-full rounded-xl">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{intervalUnits.map((unit) => (
|
||||
<SelectItem key={unit.value} value={unit.value}>
|
||||
{unit.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<label className="block space-y-2">
|
||||
<span className="text-xs font-semibold uppercase tracking-[0.18em] text-slate-500">
|
||||
qBittorrent Kategori
|
||||
</span>
|
||||
<Select value={category} onValueChange={setCategory}>
|
||||
<SelectTrigger className="w-full rounded-xl">
|
||||
<SelectValue placeholder="Kategori sec" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="__none__">Kategori yok</SelectItem>
|
||||
{qbitCategories.map((entry) => (
|
||||
<SelectItem key={entry} value={entry}>
|
||||
{entry}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</label>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setEnabled((current) => !current)}
|
||||
className="flex w-full items-center justify-between rounded-2xl border border-slate-200 bg-slate-50 px-4 py-3 text-left"
|
||||
>
|
||||
<div>
|
||||
<div className="text-xs font-semibold uppercase tracking-[0.18em] text-slate-500">
|
||||
Watcher Durumu
|
||||
</div>
|
||||
<div className="mt-1 text-sm font-semibold text-slate-900">
|
||||
{enabled ? "Aktif izleme acik" : "Pasif durumda"}
|
||||
</div>
|
||||
</div>
|
||||
<FontAwesomeIcon
|
||||
icon={enabled ? faToggleOn : faToggleOff}
|
||||
className={`text-3xl ${enabled ? "text-teal-600" : "text-slate-300"}`}
|
||||
/>
|
||||
</button>
|
||||
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<Button onClick={submit} disabled={saving} className="gap-2 rounded-xl">
|
||||
<FontAwesomeIcon icon={faSave} />
|
||||
{saving ? "Kaydediliyor" : "Kaydet"}
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={runNow}
|
||||
disabled={!selectedWatcher || running}
|
||||
className="gap-2 rounded-xl"
|
||||
>
|
||||
<FontAwesomeIcon icon={faPlay} />
|
||||
{running ? "Calisiyor" : "Simdi Calistir"}
|
||||
</Button>
|
||||
</div>
|
||||
<div className="rounded-xl border border-dashed border-slate-200 bg-slate-50/70 px-4 py-3 text-xs text-slate-500">
|
||||
Manuel calistirma, scheduler beklemeden watcher akisini aninda tetikler.
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card className="border-slate-200 bg-white/90">
|
||||
<CardHeader>
|
||||
<CardTitle className="flex items-center gap-3">
|
||||
<span className="inline-flex h-10 w-10 items-center justify-center rounded-2xl bg-slate-900 text-white">
|
||||
<FontAwesomeIcon icon={faLayerGroup} />
|
||||
</span>
|
||||
Watcher Listesi
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="space-y-3">
|
||||
{watchers.length === 0 ? (
|
||||
<div className="rounded-2xl border border-dashed border-slate-300 px-4 py-6 text-sm text-slate-500">
|
||||
Henuz kayitli watcher yok.
|
||||
</div>
|
||||
) : (
|
||||
watchers.map((watcher) => (
|
||||
<div
|
||||
key={watcher.id}
|
||||
onClick={() => setSelectedWatcherId(watcher.id)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter" || event.key === " ") {
|
||||
event.preventDefault();
|
||||
setSelectedWatcherId(watcher.id);
|
||||
}
|
||||
}}
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
className={`w-full rounded-2xl border px-4 py-4 text-left transition ${
|
||||
selectedWatcherId === watcher.id
|
||||
? "border-slate-900 bg-slate-900 text-white shadow-lg shadow-slate-900/15"
|
||||
: "border-slate-200 bg-white hover:border-slate-300"
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div>
|
||||
<div className="font-semibold">{watcher.trackerLabel}</div>
|
||||
<div className={`mt-1 text-xs ${selectedWatcherId === watcher.id ? "text-slate-300" : "text-slate-500"}`}>
|
||||
Her {watcher.intervalMinutes} dakikada bir
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={(event) => {
|
||||
event.stopPropagation();
|
||||
toggleWatcher(watcher);
|
||||
}}
|
||||
className={`inline-flex h-10 w-10 items-center justify-center rounded-full ${
|
||||
watcher.enabled
|
||||
? "bg-emerald-500/15 text-emerald-500"
|
||||
: selectedWatcherId === watcher.id
|
||||
? "bg-white/10 text-slate-300"
|
||||
: "bg-slate-100 text-slate-400"
|
||||
}`}
|
||||
>
|
||||
<FontAwesomeIcon icon={watcher.enabled ? faToggleOn : faToggleOff} />
|
||||
</button>
|
||||
</div>
|
||||
<div className={`mt-4 grid grid-cols-[1fr_auto] gap-3 text-xs ${selectedWatcherId === watcher.id ? "text-slate-300" : "text-slate-500"}`}>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<div>Son basari: {formatDate(watcher.lastSuccessAt)}</div>
|
||||
<div>Sonraki tur: {formatDate(watcher.nextRunAt)}</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={(event) => {
|
||||
event.stopPropagation();
|
||||
removeWatcher(watcher);
|
||||
}}
|
||||
disabled={deleting}
|
||||
className={`inline-flex h-8 w-8 items-center justify-center rounded-full border ${
|
||||
selectedWatcherId === watcher.id
|
||||
? "border-white/20 bg-white/10 text-white"
|
||||
: "border-rose-200 bg-rose-50 text-rose-600"
|
||||
}`}
|
||||
title="Watcher sil"
|
||||
>
|
||||
<FontAwesomeIcon icon={faTrash} />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card className="border-slate-200 bg-[linear-gradient(180deg,rgba(255,255,255,0.96),rgba(240,249,255,0.95))]">
|
||||
<CardHeader>
|
||||
<CardTitle className="flex items-center gap-3">
|
||||
<span className="inline-flex h-10 w-10 items-center justify-center rounded-2xl bg-cyan-600 text-white">
|
||||
<FontAwesomeIcon icon={faChartIcon(watcherSummary)} />
|
||||
</span>
|
||||
Watcher Ozeti
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<SummaryTile label="Aktif Watcher" value={String(watcherSummary?.activeWatchers ?? 0)} />
|
||||
<SummaryTile label="Import Edilen" value={String(watcherSummary?.totalImported ?? 0)} />
|
||||
<SummaryTile label="Gorulen" value={String(watcherSummary?.totalSeen ?? 0)} />
|
||||
<SummaryTile label="Tracker" value={String(watcherSummary?.trackedLabels.length ?? 0)} />
|
||||
</div>
|
||||
<div className="space-y-2 rounded-2xl bg-white/80 px-4 py-4 text-sm text-slate-600">
|
||||
<div>Son tur: {formatDate(watcherSummary?.lastRunAt)}</div>
|
||||
<div>Son basari: {formatDate(watcherSummary?.lastSuccessAt)}</div>
|
||||
<div>Sonraki kontrol: {formatDate(watcherSummary?.nextRunAt)}</div>
|
||||
<div>
|
||||
Son hata:{" "}
|
||||
<span className="font-medium text-slate-700">
|
||||
{watcherSummary?.lastError ?? "Yok"}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const SummaryTile = ({ label, value }: { label: string; value: string }) => (
|
||||
<div className="rounded-2xl border border-slate-200 bg-white/90 px-4 py-4">
|
||||
<div className="text-[11px] uppercase tracking-[0.18em] text-slate-400">{label}</div>
|
||||
<div className="mt-2 text-2xl font-semibold text-slate-900">{value}</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
const faChartIcon = (summary: WatcherSummary | null) => {
|
||||
if (summary?.lastError) {
|
||||
return faTriangleExclamation;
|
||||
}
|
||||
if ((summary?.totalImported ?? 0) > 0) {
|
||||
return faCheckCircle;
|
||||
}
|
||||
if ((summary?.activeWatchers ?? 0) > 0) {
|
||||
return faClock;
|
||||
}
|
||||
return faDownload;
|
||||
};
|
||||
@@ -56,6 +56,18 @@ export const connectSocket = () => {
|
||||
useAppStore.getState().setTimerSummary(summary);
|
||||
});
|
||||
|
||||
socket.on("watchers:list", (watchers) => {
|
||||
useAppStore.getState().setWatchers(watchers);
|
||||
});
|
||||
|
||||
socket.on("watcher:items", (items) => {
|
||||
useAppStore.getState().setWatcherItems(items);
|
||||
});
|
||||
|
||||
socket.on("watcher:summary", (summary) => {
|
||||
useAppStore.getState().setWatcherSummary(summary);
|
||||
});
|
||||
|
||||
return socket;
|
||||
};
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ export interface TorrentInfo {
|
||||
state: string;
|
||||
magnet_uri?: string;
|
||||
tracker?: string;
|
||||
num_seeds?: number;
|
||||
num_leechs?: number;
|
||||
tags?: string;
|
||||
category?: string;
|
||||
added_on?: number;
|
||||
@@ -78,6 +80,79 @@ export interface TimerSummary {
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface WatcherTracker {
|
||||
key: string;
|
||||
label: string;
|
||||
cliSiteKey: string;
|
||||
supportsRemoveBookmark: boolean;
|
||||
}
|
||||
|
||||
export interface Watcher {
|
||||
id: string;
|
||||
tracker: string;
|
||||
trackerLabel: string;
|
||||
category?: string;
|
||||
cookieHint: string;
|
||||
hasCookie: boolean;
|
||||
intervalMinutes: number;
|
||||
enabled: boolean;
|
||||
lastRunAt?: string;
|
||||
lastSuccessAt?: string;
|
||||
lastError?: string;
|
||||
nextRunAt?: string;
|
||||
totalImported: number;
|
||||
totalSeen: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface WatcherItem {
|
||||
id: string;
|
||||
watcherId: string;
|
||||
tracker: string;
|
||||
trackerLabel: string;
|
||||
sourceKey: string;
|
||||
pageUrl: string;
|
||||
title: string;
|
||||
imageUrl?: string;
|
||||
status: string;
|
||||
statusLabel: string;
|
||||
qbitHash?: string;
|
||||
qbitName?: string;
|
||||
progress: number;
|
||||
state?: string;
|
||||
qbitCategory?: string;
|
||||
sizeBytes?: number;
|
||||
seeders?: number;
|
||||
leechers?: number;
|
||||
seenAt: string;
|
||||
importedAt?: string;
|
||||
lastSyncAt: string;
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
export interface WatcherSummary {
|
||||
activeWatchers: number;
|
||||
totalImported: number;
|
||||
totalSeen: number;
|
||||
trackedLabels: string[];
|
||||
lastRunAt?: string;
|
||||
lastSuccessAt?: string;
|
||||
nextRunAt?: string;
|
||||
lastError?: string;
|
||||
updatedAt: string;
|
||||
watchers: Array<{
|
||||
id: string;
|
||||
tracker: string;
|
||||
trackerLabel: string;
|
||||
enabled: boolean;
|
||||
lastRunAt?: string;
|
||||
lastSuccessAt?: string;
|
||||
nextRunAt?: string;
|
||||
totalImported: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface AppState {
|
||||
qbit: StatusSnapshot["qbit"];
|
||||
torrents: TorrentInfo[];
|
||||
@@ -87,6 +162,11 @@ interface AppState {
|
||||
timerRules: TimerRule[];
|
||||
timerLogs: TimerLog[];
|
||||
timerSummary: TimerSummary | null;
|
||||
watcherTrackers: WatcherTracker[];
|
||||
qbitCategories: string[];
|
||||
watchers: Watcher[];
|
||||
watcherItems: WatcherItem[];
|
||||
watcherSummary: WatcherSummary | null;
|
||||
selectedHash: string | null;
|
||||
loopForm: { allowIp: string; delayMs: number; targetLoops: number };
|
||||
setSnapshot: (snapshot: StatusSnapshot) => void;
|
||||
@@ -97,6 +177,11 @@ interface AppState {
|
||||
setTimerLogs: (logs: TimerLog[]) => void;
|
||||
addTimerLog: (log: TimerLog) => void;
|
||||
setTimerSummary: (summary: TimerSummary) => void;
|
||||
setWatcherTrackers: (trackers: WatcherTracker[]) => void;
|
||||
setQbitCategories: (categories: string[]) => void;
|
||||
setWatchers: (watchers: Watcher[]) => void;
|
||||
setWatcherItems: (items: WatcherItem[]) => void;
|
||||
setWatcherSummary: (summary: WatcherSummary | null) => void;
|
||||
selectHash: (hash: string) => void;
|
||||
setLoopForm: (partial: Partial<AppState["loopForm"]>) => void;
|
||||
}
|
||||
@@ -110,6 +195,11 @@ export const useAppStore = create<AppState>((set) => ({
|
||||
timerRules: [],
|
||||
timerLogs: [],
|
||||
timerSummary: null,
|
||||
watcherTrackers: [],
|
||||
qbitCategories: [],
|
||||
watchers: [],
|
||||
watcherItems: [],
|
||||
watcherSummary: null,
|
||||
selectedHash: null,
|
||||
loopForm: { allowIp: "", delayMs: 3000, targetLoops: 3 },
|
||||
setSnapshot: (snapshot) =>
|
||||
@@ -140,6 +230,11 @@ export const useAppStore = create<AppState>((set) => ({
|
||||
timerLogs: [log, ...state.timerLogs].slice(0, 500),
|
||||
})),
|
||||
setTimerSummary: (summary) => set({ timerSummary: summary }),
|
||||
setWatcherTrackers: (watcherTrackers) => set({ watcherTrackers }),
|
||||
setQbitCategories: (qbitCategories) => set({ qbitCategories }),
|
||||
setWatchers: (watchers) => set({ watchers }),
|
||||
setWatcherItems: (watcherItems) => set({ watcherItems }),
|
||||
setWatcherSummary: (watcherSummary) => set({ watcherSummary }),
|
||||
selectHash: (hash) => set({ selectedHash: hash }),
|
||||
setLoopForm: (partial) =>
|
||||
set((state) => ({
|
||||
|
||||
168
bin/wscraper-service/server.py
Normal file
168
bin/wscraper-service/server.py
Normal file
@@ -0,0 +1,168 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
WSCRAPER_SRC = REPO_ROOT / "bin" / "wscraper" / "src"
|
||||
if str(WSCRAPER_SRC) not in sys.path:
|
||||
sys.path.insert(0, str(WSCRAPER_SRC))
|
||||
|
||||
from wscraper.sites.happyfappy import run_download_torrent_files, run_get_bookmarks
|
||||
|
||||
HOST = os.environ.get("WSCRAPER_SERVICE_HOST", "0.0.0.0")
|
||||
PORT = int(os.environ.get("WSCRAPER_SERVICE_PORT", "8787"))
|
||||
TOKEN = os.environ.get("WSCRAPER_SERVICE_TOKEN", "")
|
||||
|
||||
|
||||
def json_response(handler: BaseHTTPRequestHandler, status: int, payload: dict) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
handler.send_response(status)
|
||||
handler.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
handler.send_header("Content-Length", str(len(body)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(body)
|
||||
|
||||
|
||||
def parse_json_body(handler: BaseHTTPRequestHandler) -> dict:
|
||||
length = int(handler.headers.get("Content-Length", "0"))
|
||||
raw = handler.rfile.read(length) if length > 0 else b"{}"
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
|
||||
|
||||
def require_auth(handler: BaseHTTPRequestHandler) -> bool:
|
||||
if not TOKEN:
|
||||
return True
|
||||
auth_header = handler.headers.get("Authorization", "")
|
||||
if auth_header == f"Bearer {TOKEN}":
|
||||
return True
|
||||
json_response(handler, HTTPStatus.UNAUTHORIZED, {"error": "Unauthorized"})
|
||||
return False
|
||||
|
||||
|
||||
def normalize_tracker(payload: dict) -> str:
|
||||
tracker = str(payload.get("tracker", "")).strip().lower()
|
||||
if tracker not in {"happyfappy", "hf"}:
|
||||
raise ValueError("Unsupported tracker")
|
||||
return "happyfappy"
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "wscraper-service/1.0"
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
parsed = urlparse(self.path)
|
||||
if parsed.path == "/health":
|
||||
if not require_auth(self):
|
||||
return
|
||||
json_response(self, HTTPStatus.OK, {"ok": True, "service": "wscraper-service"})
|
||||
return
|
||||
if parsed.path == "/trackers":
|
||||
if not require_auth(self):
|
||||
return
|
||||
json_response(
|
||||
self,
|
||||
HTTPStatus.OK,
|
||||
{"items": [{"key": "happyfappy", "label": "HappyFappy"}]},
|
||||
)
|
||||
return
|
||||
json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"})
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
if not require_auth(self):
|
||||
return
|
||||
|
||||
parsed = urlparse(self.path)
|
||||
try:
|
||||
payload = parse_json_body(self)
|
||||
if parsed.path == "/bookmarks":
|
||||
tracker = normalize_tracker(payload)
|
||||
cookie = str(payload.get("cookie", "")).strip()
|
||||
if not cookie:
|
||||
raise ValueError("Cookie is required")
|
||||
with tempfile.TemporaryDirectory(prefix="wscraper-bookmarks-") as tmpdir:
|
||||
output_path = Path(tmpdir) / "bookmarks.json"
|
||||
run_get_bookmarks(
|
||||
argparse.Namespace(
|
||||
base_url="https://www.happyfappy.net",
|
||||
cookie=cookie,
|
||||
cookie_file=None,
|
||||
output=str(output_path),
|
||||
delay_min=1.8,
|
||||
delay_max=3.2,
|
||||
retries=3,
|
||||
backoff_base=5.0,
|
||||
max_pages=200,
|
||||
)
|
||||
)
|
||||
items = json.loads(output_path.read_text(encoding="utf-8"))
|
||||
json_response(self, HTTPStatus.OK, {"tracker": tracker, "items": items})
|
||||
return
|
||||
|
||||
if parsed.path == "/download":
|
||||
tracker = normalize_tracker(payload)
|
||||
cookie = str(payload.get("cookie", "")).strip()
|
||||
detail_url = str(payload.get("url", "")).strip()
|
||||
remove_bookmark = bool(payload.get("removeBookmark", True))
|
||||
if not cookie:
|
||||
raise ValueError("Cookie is required")
|
||||
if not detail_url:
|
||||
raise ValueError("Detail url is required")
|
||||
with tempfile.TemporaryDirectory(prefix="wscraper-download-") as tmpdir:
|
||||
output_dir = Path(tmpdir) / "torrent"
|
||||
run_download_torrent_files(
|
||||
argparse.Namespace(
|
||||
url=detail_url,
|
||||
base_url="https://www.happyfappy.net",
|
||||
cookie=cookie,
|
||||
cookie_file=None,
|
||||
output_dir=str(output_dir),
|
||||
rm_bookmark=remove_bookmark,
|
||||
retries=3,
|
||||
backoff_base=5.0,
|
||||
)
|
||||
)
|
||||
files = sorted(output_dir.glob("*.torrent"))
|
||||
if not files:
|
||||
raise RuntimeError("No torrent file produced")
|
||||
torrent_path = files[0]
|
||||
content = base64.b64encode(torrent_path.read_bytes()).decode("ascii")
|
||||
json_response(
|
||||
self,
|
||||
HTTPStatus.OK,
|
||||
{
|
||||
"tracker": tracker,
|
||||
"filename": torrent_path.name,
|
||||
"contentBase64": content,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"})
|
||||
except Exception as error: # noqa: BLE001
|
||||
json_response(
|
||||
self,
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
{"error": str(error)},
|
||||
)
|
||||
|
||||
def log_message(self, fmt: str, *args) -> None:
|
||||
print(f"[wscraper-service] {self.address_string()} - {fmt % args}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
server = ThreadingHTTPServer((HOST, PORT), Handler)
|
||||
print(f"wscraper-service listening on http://{HOST}:{PORT}")
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,10 +1,14 @@
|
||||
version: "3.9"
|
||||
services:
|
||||
server:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: apps/server/Dockerfile
|
||||
working_dir: /app
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
dns:
|
||||
- 8.8.8.8
|
||||
- 1.1.1.1
|
||||
command: sh -c "pnpm install || true && pnpm -C /app/apps/server dev"
|
||||
ports:
|
||||
- "${SERVER_PORT:-3001}:${SERVER_PORT:-3001}"
|
||||
@@ -30,11 +34,21 @@ services:
|
||||
- ENFORCE_INTERVAL_MS=${ENFORCE_INTERVAL_MS}
|
||||
- DEFAULT_DELAY_MS=${DEFAULT_DELAY_MS}
|
||||
- MAX_LOOP_LIMIT=${MAX_LOOP_LIMIT}
|
||||
- WATCHER_SECRET_KEY=${WATCHER_SECRET_KEY}
|
||||
- WATCHER_ENABLED=${WATCHER_ENABLED}
|
||||
- WATCHER_TICK_MS=${WATCHER_TICK_MS}
|
||||
- WATCHER_TIMEOUT_MS=${WATCHER_TIMEOUT_MS}
|
||||
- WATCHER_RUNTIME_DIR=${WATCHER_RUNTIME_DIR}
|
||||
- WSCRAPER_SERVICE_BASE_URL=${WSCRAPER_SERVICE_BASE_URL:-http://host.docker.internal:8787}
|
||||
- WSCRAPER_SERVICE_TOKEN=${WSCRAPER_SERVICE_TOKEN:-}
|
||||
web:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: apps/web/Dockerfile
|
||||
working_dir: /app
|
||||
dns:
|
||||
- 8.8.8.8
|
||||
- 1.1.1.1
|
||||
command: sh -c "pnpm install || true && pnpm -C /app/apps/web dev"
|
||||
ports:
|
||||
- "${WEB_PORT:-5173}:${WEB_PORT:-5173}"
|
||||
@@ -46,4 +60,4 @@ services:
|
||||
- VITE_API_BASE=http://${APP_HOST:-localhost}:${SERVER_PORT:-3001}
|
||||
- WEB_PORT=${WEB_PORT}
|
||||
- WEB_ORIGIN=http://${APP_HOST:-localhost}:${WEB_PORT:-5173}
|
||||
- WEB_ALLOWED_HOSTS=${WEB_ALLOWED_HOSTS}
|
||||
- WEB_ALLOWED_HOSTS=${WEB_ALLOWED_HOSTS:-localhost}
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
version: "3.9"
|
||||
services:
|
||||
q-buffer:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
restart: unless-stopped
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
dns:
|
||||
- 8.8.8.8
|
||||
- 1.1.1.1
|
||||
ports:
|
||||
- "${SERVER_PORT:-3001}:3001"
|
||||
volumes:
|
||||
@@ -24,3 +28,10 @@ services:
|
||||
- ENFORCE_INTERVAL_MS=${ENFORCE_INTERVAL_MS}
|
||||
- DEFAULT_DELAY_MS=${DEFAULT_DELAY_MS}
|
||||
- MAX_LOOP_LIMIT=${MAX_LOOP_LIMIT}
|
||||
- WATCHER_SECRET_KEY=${WATCHER_SECRET_KEY}
|
||||
- WATCHER_ENABLED=${WATCHER_ENABLED}
|
||||
- WATCHER_TICK_MS=${WATCHER_TICK_MS}
|
||||
- WATCHER_TIMEOUT_MS=${WATCHER_TIMEOUT_MS}
|
||||
- WATCHER_RUNTIME_DIR=${WATCHER_RUNTIME_DIR}
|
||||
- WSCRAPER_SERVICE_BASE_URL=${WSCRAPER_SERVICE_BASE_URL:-http://host.docker.internal:8787}
|
||||
- WSCRAPER_SERVICE_TOKEN=${WSCRAPER_SERVICE_TOKEN:-}
|
||||
|
||||
@@ -7,6 +7,10 @@
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "pnpm -C apps/server dev & pnpm -C apps/web dev --host 0.0.0.0",
|
||||
"dev:server": "pnpm -C apps/server dev",
|
||||
"dev:web": "pnpm -C apps/web dev --host 0.0.0.0",
|
||||
"bootstrap:dev": "./scripts/bootstrap.sh --dev-mode",
|
||||
"bootstrap:prod": "./scripts/bootstrap.sh --prod-mode",
|
||||
"build": "pnpm -C apps/server build && pnpm -C apps/web build",
|
||||
"start": "pnpm -C apps/server start"
|
||||
}
|
||||
|
||||
197
scripts/bootstrap.sh
Executable file
197
scripts/bootstrap.sh
Executable file
@@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
RUNTIME_DIR="$ROOT_DIR/.runtime/wscraper-service"
|
||||
VENV_DIR="$RUNTIME_DIR/.venv"
|
||||
PID_FILE="$RUNTIME_DIR/wscraper-service.pid"
|
||||
LOG_FILE="$RUNTIME_DIR/wscraper-service.log"
|
||||
MARKER_FILE="$RUNTIME_DIR/.setup-complete"
|
||||
MODE=""
|
||||
SKIP_INSTALL="false"
|
||||
RESTART_WSCRAPER="false"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
./scripts/bootstrap.sh --dev-mode [--skip-wscraper-install] [--restart-wscraper]
|
||||
./scripts/bootstrap.sh --prod-mode [--skip-wscraper-install] [--restart-wscraper]
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--dev-mode)
|
||||
MODE="dev"
|
||||
;;
|
||||
--prod-mode)
|
||||
MODE="prod"
|
||||
;;
|
||||
--skip-wscraper-install)
|
||||
SKIP_INSTALL="true"
|
||||
;;
|
||||
--restart-wscraper|--restart)
|
||||
RESTART_WSCRAPER="true"
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [[ -z "$MODE" ]]; then
|
||||
echo "--dev-mode veya --prod-mode zorunlu." >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$RUNTIME_DIR"
|
||||
|
||||
if [[ -f "$ROOT_DIR/.env" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source "$ROOT_DIR/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
WSCRAPER_SERVICE_HOST="${WSCRAPER_SERVICE_HOST:-0.0.0.0}"
|
||||
WSCRAPER_SERVICE_PORT="${WSCRAPER_SERVICE_PORT:-8787}"
|
||||
WSCRAPER_SERVICE_TOKEN="${WSCRAPER_SERVICE_TOKEN:-}"
|
||||
WSCRAPER_SERVICE_PYTHON_BIN="${WSCRAPER_SERVICE_PYTHON_BIN:-}"
|
||||
|
||||
detect_python_bin() {
|
||||
if [[ -n "$WSCRAPER_SERVICE_PYTHON_BIN" ]] && command -v "$WSCRAPER_SERVICE_PYTHON_BIN" >/dev/null 2>&1; then
|
||||
echo "$WSCRAPER_SERVICE_PYTHON_BIN"
|
||||
return
|
||||
fi
|
||||
|
||||
local candidate
|
||||
for candidate in python3.12 python3.11 python3.10 python3; do
|
||||
if ! command -v "$candidate" >/dev/null 2>&1; then
|
||||
continue
|
||||
fi
|
||||
if "$candidate" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)' >/dev/null 2>&1; then
|
||||
echo "$candidate"
|
||||
return
|
||||
fi
|
||||
done
|
||||
|
||||
echo "q-buffer watcher icin Python 3.10+ gerekli." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
PYTHON_BIN="$(detect_python_bin)"
|
||||
|
||||
service_running() {
|
||||
if [[ ! -f "$PID_FILE" ]]; then
|
||||
return 1
|
||||
fi
|
||||
local pid
|
||||
pid="$(cat "$PID_FILE")"
|
||||
if [[ -z "$pid" ]]; then
|
||||
return 1
|
||||
fi
|
||||
if kill -0 "$pid" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
rm -f "$PID_FILE"
|
||||
return 1
|
||||
}
|
||||
|
||||
install_wscraper_service() {
|
||||
if [[ "$SKIP_INSTALL" == "true" ]]; then
|
||||
echo "wscraper kurulum kontrolu atlandi."
|
||||
return
|
||||
fi
|
||||
|
||||
local needs_install="false"
|
||||
if [[ ! -x "$VENV_DIR/bin/python3" ]]; then
|
||||
needs_install="true"
|
||||
elif [[ ! -f "$MARKER_FILE" ]]; then
|
||||
needs_install="true"
|
||||
elif ! "$VENV_DIR/bin/python3" -c "import scrapling" >/dev/null 2>&1; then
|
||||
needs_install="true"
|
||||
fi
|
||||
|
||||
if [[ "$needs_install" == "false" ]]; then
|
||||
echo "wscraper kurulumu mevcut, tekrar kurulmayacak."
|
||||
return
|
||||
fi
|
||||
|
||||
echo "wscraper host servisi kuruluyor..."
|
||||
rm -rf "$VENV_DIR"
|
||||
"$PYTHON_BIN" -m venv "$VENV_DIR"
|
||||
"$VENV_DIR/bin/pip" install --upgrade pip
|
||||
"$VENV_DIR/bin/pip" install "scrapling[fetchers]==0.4.1"
|
||||
"$VENV_DIR/bin/scrapling" install
|
||||
touch "$MARKER_FILE"
|
||||
}
|
||||
|
||||
stop_wscraper_service() {
|
||||
if ! service_running; then
|
||||
return
|
||||
fi
|
||||
local pid
|
||||
pid="$(cat "$PID_FILE")"
|
||||
echo "wscraper-service durduruluyor (pid=$pid)..."
|
||||
kill "$pid" >/dev/null 2>&1 || true
|
||||
rm -f "$PID_FILE"
|
||||
}
|
||||
|
||||
start_wscraper_service() {
|
||||
if service_running && [[ "$RESTART_WSCRAPER" != "true" ]]; then
|
||||
echo "wscraper-service zaten calisiyor."
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ "$RESTART_WSCRAPER" == "true" ]]; then
|
||||
stop_wscraper_service
|
||||
fi
|
||||
|
||||
echo "wscraper-service baslatiliyor..."
|
||||
nohup env \
|
||||
WSCRAPER_SERVICE_HOST="$WSCRAPER_SERVICE_HOST" \
|
||||
WSCRAPER_SERVICE_PORT="$WSCRAPER_SERVICE_PORT" \
|
||||
WSCRAPER_SERVICE_TOKEN="$WSCRAPER_SERVICE_TOKEN" \
|
||||
PYTHONPATH="$ROOT_DIR/bin/wscraper/src" \
|
||||
"$VENV_DIR/bin/python3" \
|
||||
"$ROOT_DIR/bin/wscraper-service/server.py" \
|
||||
>>"$LOG_FILE" 2>&1 &
|
||||
echo $! > "$PID_FILE"
|
||||
|
||||
local health_url="http://127.0.0.1:${WSCRAPER_SERVICE_PORT}/health"
|
||||
for _ in $(seq 1 20); do
|
||||
if [[ -n "$WSCRAPER_SERVICE_TOKEN" ]]; then
|
||||
if curl -fsS -H "Authorization: Bearer ${WSCRAPER_SERVICE_TOKEN}" "$health_url" >/dev/null 2>&1; then
|
||||
echo "wscraper-service hazir."
|
||||
return
|
||||
fi
|
||||
elif curl -fsS "$health_url" >/dev/null 2>&1; then
|
||||
echo "wscraper-service hazir."
|
||||
return
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "wscraper-service health-check basarisiz. Log: $LOG_FILE" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
run_docker() {
|
||||
if [[ "$MODE" == "dev" ]]; then
|
||||
docker compose -f "$ROOT_DIR/docker-compose.dev.yml" up --build
|
||||
return
|
||||
fi
|
||||
docker compose -f "$ROOT_DIR/docker-compose.yml" up --build -d
|
||||
}
|
||||
|
||||
install_wscraper_service
|
||||
start_wscraper_service
|
||||
run_docker
|
||||
Reference in New Issue
Block a user