diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfdb8b7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f8cd3b7..c7e47ee 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -2,8 +2,8 @@ name: Build and Publish Docker Image on: push: - tags: - - 'v*.*.*' + branches: + - bridgemill-ch jobs: docker: @@ -24,15 +24,38 @@ jobs: node-version: '18' cache: 'npm' + - name: Determine next version + id: version + run: | + LATEST_TAG=$(git tag --sort=-version:refname | head -1) + if [ -z "$LATEST_TAG" ]; then + echo "version=1.0.0" >> $GITHUB_OUTPUT + else + # Strip leading 'v' if present + CLEAN_TAG="${LATEST_TAG#v}" + IFS='.' read -r MAJOR MINOR PATCH <<< "$CLEAN_TAG" + # If patch isn't a number, restart at .0 + if ! [[ "$PATCH" =~ ^[0-9]+$ ]]; then + PATCH=0 + fi + PATCH=$((PATCH + 1)) + echo "version=${MAJOR}.${MINOR}.${PATCH}" >> $GITHUB_OUTPUT + fi + + - name: Create tag + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + git tag ${{ steps.version.outputs.version }} + git push origin ${{ steps.version.outputs.version }} + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: - images: mrorbitman/subsyncarr + images: bridgemill/subsyncarr tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} + type=raw,value=${{ steps.version.outputs.version }} type=raw,value=latest - name: Set up QEMU @@ -57,6 +80,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + - name: Generate changelog id: changelog run: | @@ -73,7 +97,8 @@ jobs: - name: Create GitHub Release uses: softprops/action-gh-release@v1 with: - name: Release ${{ github.ref_name }} + name: Release ${{ steps.version.outputs.version }} + tag_name: ${{ steps.version.outputs.version }} body: | ## What's Changed ${{ steps.changelog.outputs.changelog }} @@ -82,11 +107,11 @@ jobs: Pull the image using: ```bash - docker pull mrorbitman/subsyncarr:${{ github.ref_name }} + docker pull bridgemill/subsyncarr:${{ steps.version.outputs.version }} # or - docker pull mrorbitman/subsyncarr:latest + docker pull bridgemill/subsyncarr:latest ``` - Docker Hub URL: https://hub.docker.com/r/mrorbitman/subsyncarr/tags + Docker Hub URL: https://hub.docker.com/r/bridgemill/subsyncarr/tags draft: false prerelease: false diff --git a/src/coordinator.ts b/src/coordinator.ts index 9a3b989..2386bc1 100644 --- a/src/coordinator.ts +++ b/src/coordinator.ts @@ -1,7 +1,6 @@ import { ProcessingEngine } from './processingEngine'; import { StateManager } from './stateManager'; import { ScanConfig } from './config'; -import { findMatchingVideoFile } from './findMatchingVideoFile'; import { Run } from './database'; import { once } from 'events'; @@ -33,16 +32,15 @@ export class ProcessingCoordinator { this.engine.on('run:files_found', (files: string[]) => { this.currentRunId = this.stateManager.startRun(files.length, this.enabledEngines); - // Add all files to database as pending - files.forEach((filePath) => { - const videoPath = findMatchingVideoFile(filePath); - this.stateManager.addFile(this.currentRunId!, filePath, videoPath); - }); + // Add all files to database as pending (video matching happens during processing) + for (const filePath of files) { + this.stateManager.addFile(this.currentRunId!, filePath, null); + } }); - this.engine.on('file:started', ({ srtPath }: { srtPath: string }) => { + this.engine.on('file:started', ({ srtPath, videoPath }: { srtPath: string; videoPath: string | null }) => { if (this.currentRunId) { - this.stateManager.updateFileStatus(this.currentRunId, srtPath, 'processing', null); + this.stateManager.updateFileStatus(this.currentRunId, srtPath, 'processing', null, videoPath); } }); diff --git a/src/database.ts b/src/database.ts index f036868..d8165d2 100644 --- a/src/database.ts +++ b/src/database.ts @@ -280,6 +280,22 @@ export class SubsyncarrPlusDatabase { .all(runId) as FileResult[]; } + getFileResult(runId: string, filePath: string): FileResult | null { + return this.db + .prepare('SELECT * FROM file_results WHERE run_id = ? AND file_path = ?') + .get(runId, filePath) as FileResult | null; + } + + getFileResultsPaginated(runId: string, limit: number, offset: number): { files: FileResult[]; total: number } { + const total = ( + this.db.prepare('SELECT COUNT(*) as count FROM file_results WHERE run_id = ?').get(runId) as { count: number } + ).count; + const files = this.db + .prepare('SELECT * FROM file_results WHERE run_id = ? ORDER BY created_at ASC LIMIT ? OFFSET ?') + .all(runId, limit, offset) as FileResult[]; + return { files, total }; + } + // Engine failure tracking methods getEngineFailureTracking(filePath: string, engine: string): EngineFailureTracking | null { return this.db diff --git a/src/findAllSrtFiles.ts b/src/findAllSrtFiles.ts index a1ad54e..5e46e55 100644 --- a/src/findAllSrtFiles.ts +++ b/src/findAllSrtFiles.ts @@ -1,18 +1,25 @@ import { readdir } from 'fs/promises'; -import { basename, dirname, extname, join } from 'path'; +import { extname, join } from 'path'; import { existsSync } from 'fs'; import { ScanConfig } from './config'; +import { getOutputPath, getSubtitleFormat } from './helpers'; function isAlreadySynced(srtPath: string, engines: string[]): boolean { - const directory = dirname(srtPath); - const srtBaseName = basename(srtPath, '.srt'); + const format = getSubtitleFormat(); + if (format === 'overwrite') return false; // Checked lazily in processFile return engines.every((engine) => { - const outputPath = join(directory, `${srtBaseName}.${engine}.srt`); + const outputPath = getOutputPath(srtPath, engine); return existsSync(outputPath); }); } +const ALL_KNOWN_ENGINES = ['ffsubsync', 'autosubsync', 'alass']; + +function isEngineOutput(filename: string): boolean { + return ALL_KNOWN_ENGINES.some((engine) => filename.includes(`.${engine}.`)); +} + export async function findAllSrtFiles(config: ScanConfig): Promise { const engines = process.env.INCLUDE_ENGINES?.split(',') || ['ffsubsync', 'autosubsync', 'alass']; const files: string[] = []; @@ -34,9 +41,7 @@ export async function findAllSrtFiles(config: ScanConfig): Promise { } else if ( entry.isFile() && extname(entry.name).toLowerCase() === '.srt' && - !entry.name.includes('.ffsubsync.') && - !entry.name.includes('.alass.') && - !entry.name.includes('.autosubsync.') + !isEngineOutput(entry.name) ) { if (isAlreadySynced(fullPath, engines)) { skippedCount++; diff --git a/src/generateAlassSubtitles.ts b/src/generateAlassSubtitles.ts index 7cd01f7..c95832f 100644 --- a/src/generateAlassSubtitles.ts +++ b/src/generateAlassSubtitles.ts @@ -1,19 +1,18 @@ -import { basename, dirname, join } from 'path'; -import { execPromise, ProcessingResult } from './helpers'; +import { execPromise, ProcessingResult, getOutputPath, getSubtitleFormat } from './helpers'; import { existsSync } from 'fs'; export async function generateAlassSubtitles(srtPath: string, videoPath: string): Promise { - const directory = dirname(srtPath); - const srtBaseName = basename(srtPath, '.srt'); - const outputPath = join(directory, `${srtBaseName}.alass.srt`); + const outputPath = getOutputPath(srtPath, 'alass'); - const exists = existsSync(outputPath); - if (exists) { - return { - success: true, - message: `Skipping ${outputPath} - already processed`, - skipped: true, - }; + if (getSubtitleFormat() !== 'overwrite') { + const exists = existsSync(outputPath); + if (exists) { + return { + success: true, + message: `Skipping ${outputPath} - already processed`, + skipped: true, + }; + } } try { diff --git a/src/generateAutosubsyncSubtitles.ts b/src/generateAutosubsyncSubtitles.ts index 256eda7..fce1806 100644 --- a/src/generateAutosubsyncSubtitles.ts +++ b/src/generateAutosubsyncSubtitles.ts @@ -1,19 +1,18 @@ -import { basename, dirname, join } from 'path'; -import { execPromise, ProcessingResult } from './helpers'; +import { execPromise, ProcessingResult, getOutputPath, getSubtitleFormat } from './helpers'; import { existsSync } from 'fs'; export async function generateAutosubsyncSubtitles(srtPath: string, videoPath: string): Promise { - const directory = dirname(srtPath); - const srtBaseName = basename(srtPath, '.srt'); - const outputPath = join(directory, `${srtBaseName}.autosubsync.srt`); + const outputPath = getOutputPath(srtPath, 'autosubsync'); - const exists = existsSync(outputPath); - if (exists) { - return { - success: true, - message: `Skipping ${outputPath} - already processed`, - skipped: true, - }; + if (getSubtitleFormat() !== 'overwrite') { + const exists = existsSync(outputPath); + if (exists) { + return { + success: true, + message: `Skipping ${outputPath} - already processed`, + skipped: true, + }; + } } try { diff --git a/src/generateFfsubsyncSubtitles.ts b/src/generateFfsubsyncSubtitles.ts index 3fccf21..7cc5696 100644 --- a/src/generateFfsubsyncSubtitles.ts +++ b/src/generateFfsubsyncSubtitles.ts @@ -1,20 +1,18 @@ -import { basename, dirname, join } from 'path'; -import { execPromise, ProcessingResult } from './helpers'; +import { execPromise, ProcessingResult, getOutputPath, getSubtitleFormat } from './helpers'; import { existsSync } from 'fs'; export async function generateFfsubsyncSubtitles(srtPath: string, videoPath: string): Promise { - const directory = dirname(srtPath); - const srtBaseName = basename(srtPath, '.srt'); - const outputPath = join(directory, `${srtBaseName}.ffsubsync.srt`); + const outputPath = getOutputPath(srtPath, 'ffsubsync'); - // Check if synced subtitle already exists - const exists = existsSync(outputPath); - if (exists) { - return { - success: true, - message: `Skipping ${outputPath} - already processed`, - skipped: true, - }; + if (getSubtitleFormat() !== 'overwrite') { + const exists = existsSync(outputPath); + if (exists) { + return { + success: true, + message: `Skipping ${outputPath} - already processed`, + skipped: true, + }; + } } try { diff --git a/src/helpers.ts b/src/helpers.ts index 908da77..00dc0d9 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1,4 +1,40 @@ import { exec } from 'child_process'; +import { basename, dirname, join } from 'path'; +import { writeFileSync } from 'fs'; +import { open } from 'fs/promises'; + +export type SubtitleFormat = 'standard' | 'overwrite'; + +export function getSubtitleFormat(): SubtitleFormat { + const format = process.env.SUBTITLE_FORMAT || 'standard'; + if (format === 'overwrite' || process.env.OVERWRITE_SUBTITLES === 'true') return 'overwrite'; + return 'standard'; +} + +export function getOutputPath(srtPath: string, engine: string): string { + const directory = dirname(srtPath); + const srtBaseName = basename(srtPath, '.srt'); + return join(directory, `${srtBaseName}.${engine}.srt`); +} + +const SYNC_MARKER = '# synced:'; + +export async function isSyncedSrt(srtPath: string): Promise { + try { + const fd = await open(srtPath, 'r'); + const buf = Buffer.alloc(100); + await fd.read(buf, 0, 100, 0); + await fd.close(); + return buf.toString('utf8').startsWith(SYNC_MARKER); + } catch { + return false; + } +} + +export function markSrtAsSynced(srtPath: string, engine: string, content: string): void { + const marker = `${SYNC_MARKER}${engine} ${Date.now()}\n`; + writeFileSync(srtPath, marker + content, 'utf8'); +} export interface ProcessingResult { success: boolean; diff --git a/src/processingEngine.ts b/src/processingEngine.ts index 215aca7..431eca4 100644 --- a/src/processingEngine.ts +++ b/src/processingEngine.ts @@ -6,6 +6,8 @@ import { generateFfsubsyncSubtitles } from './generateFfsubsyncSubtitles'; import { generateAutosubsyncSubtitles } from './generateAutosubsyncSubtitles'; import { generateAlassSubtitles } from './generateAlassSubtitles'; import { StateManager } from './stateManager'; +import { readFileSync, unlinkSync } from 'fs'; +import { getSubtitleFormat, getOutputPath, markSrtAsSynced, isSyncedSrt } from './helpers'; export class ProcessingEngine extends EventEmitter { private cancelledFiles: Set = new Set(); @@ -22,6 +24,10 @@ export class ProcessingEngine extends EventEmitter { this.maxLogBufferSize = parseInt(process.env.LOG_BUFFER_SIZE || '1000', 10); } + private get subtitleFormat(): string { + return getSubtitleFormat(); + } + private log(message: string): void { console.log(message); @@ -49,7 +55,6 @@ export class ProcessingEngine extends EventEmitter { const srtFiles = await findAllSrtFiles(scanConfig); this.log(`[${new Date().toISOString()}] Found ${srtFiles.length} subtitle files`); - this.emit('run:files_found', srtFiles); // Process in batches @@ -69,7 +74,13 @@ export class ProcessingEngine extends EventEmitter { private async processFile(srtPath: string): Promise { const fileName = srtPath.split('/').pop(); - this.log(`[${new Date().toISOString()}] Processing: ${fileName}`); + + // Skip already-synced files (overwrite mode) + if (this.subtitleFormat === 'overwrite' && (await isSyncedSrt(srtPath))) { + this.log(`[${new Date().toISOString()}] ⊘ Already synced (header): ${fileName}`); + this.emit('file:skipped', { srtPath, reason: 'already_synced' }); + return; + } // Check if cancelled if (this.cancelledFiles.has(srtPath)) { @@ -168,6 +179,20 @@ export class ProcessingEngine extends EventEmitter { if (result.success) { anyEngineSucceeded = true; + + if (this.subtitleFormat === 'overwrite') { + const engineOutputPath = getOutputPath(srtPath, engine); + const engineContent = readFileSync(engineOutputPath, 'utf8'); + markSrtAsSynced(srtPath, engine, engineContent); + unlinkSync(engineOutputPath); + this.log(`[${new Date().toISOString()}] ✓ Synced (header-marked): ${fileName}`); + this.emit('file:engine_completed', { + srtPath, + engine, + result: { ...result, duration }, + }); + break; + } } this.emit('file:engine_completed', { diff --git a/src/server.ts b/src/server.ts index c45fb0b..dcbde61 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3,6 +3,7 @@ import { WebSocketServer, WebSocket } from 'ws'; import { createServer } from 'http'; import { ProcessingCoordinator } from './coordinator'; import { StateManager } from './stateManager'; +import { Run } from './database'; import { join } from 'path'; import { getScanConfig } from './config'; import cronstrue from 'cronstrue'; @@ -68,9 +69,12 @@ export class SubsyncarrPlusServer { this.app.get('/api/status', (req, res) => { console.log(`[${new Date().toISOString()}] GET /api/status`); const currentRun = this.stateManager.getCurrentRun(); + const limit = Math.min(parseInt(req.query.limit as string, 10) || 500, 5000); + const result = currentRun ? this.stateManager.getFileResultsPaginated(currentRun.id, limit, 0) : null; res.json({ currentRun, - files: currentRun ? this.stateManager.getFileResults(currentRun.id) : [], + files: result?.files || [], + totalFiles: result?.total || 0, isRunning: this.coordinator.isRunning(), }); }); @@ -82,31 +86,55 @@ export class SubsyncarrPlusServer { res.json(this.stateManager.getRunHistory(limit)); }); - // Get specific run details + // Get specific run details (latest files only — use paginated endpoint for full list) this.app.get('/api/runs/:id', (req, res) => { console.log(`[${new Date().toISOString()}] GET /api/runs/${req.params.id}`); const currentRun = this.stateManager.getCurrentRun(); const requestedId = req.params.id; - // Check current run first + let run: Run | null = null; if (currentRun && currentRun.id === requestedId) { - return res.json({ - run: currentRun, - files: this.stateManager.getFileResults(currentRun.id), - }); + run = currentRun; + } else { + const history = this.stateManager.getRunHistory(1000); + run = history.find((r) => r.id === requestedId) || null; + } + + if (!run) { + return res.status(404).json({ error: 'Run not found' }); } - // Check history + const limit = Math.min(parseInt(req.query.limit as string, 10) || 500, 5000); + const result = this.stateManager.getFileResultsPaginated(requestedId, limit, 0); + res.json({ + run, + files: result.files, + totalFiles: result.total, + }); + }); + + // Get paginated file results for a run + this.app.get('/api/runs/:id/files', (req, res) => { + console.log(`[${new Date().toISOString()}] GET /api/runs/${req.params.id}/files`); + const requestedId = req.params.id; + const page = Math.max(1, parseInt(req.query.page as string, 10) || 1); + const limit = Math.min(parseInt(req.query.limit as string, 10) || 500, 5000); + const offset = (page - 1) * limit; + + const currentRun = this.stateManager.getCurrentRun(); const history = this.stateManager.getRunHistory(1000); - const run = history.find((r) => r.id === requestedId); + const run = currentRun?.id === requestedId ? currentRun : history.find((r) => r.id === requestedId); if (!run) { return res.status(404).json({ error: 'Run not found' }); } + const result = this.stateManager.getFileResultsPaginated(requestedId, limit, offset); res.json({ - run, - files: this.stateManager.getFileResults(run.id), + ...result, + page, + limit, + totalPages: Math.ceil(result.total / limit), }); }); @@ -197,9 +225,7 @@ export class SubsyncarrPlusServer { type: 'files:cleared', data: { currentRun, - files: currentRun - ? this.stateManager.getFileResults(currentRun.id).filter((f) => f.status === 'processing') - : [], + files: [], }, }); @@ -244,14 +270,13 @@ export class SubsyncarrPlusServer { console.log(`[${new Date().toISOString()}] WebSocket client connected (total: ${this.clients.size + 1})`); this.clients.add(ws); - // Send initial state + // Send initial state (without file list to avoid OOM with large libraries) const currentRun = this.stateManager.getCurrentRun(); ws.send( JSON.stringify({ type: 'state', data: { currentRun, - files: currentRun ? this.stateManager.getFileResults(currentRun.id) : [], isRunning: this.coordinator.isRunning(), }, }), diff --git a/src/stateManager.ts b/src/stateManager.ts index 8fa533e..91fc46c 100644 --- a/src/stateManager.ts +++ b/src/stateManager.ts @@ -105,14 +105,16 @@ export class StateManager extends EventEmitter { // File management addFile(runId: string, filePath: string, videoPath: string | null): void { this.db.createFileResult(runId, filePath, videoPath); - this.emitFileUpdate(runId, filePath); } - updateFileStatus(runId: string, filePath: string, status: FileResult['status'], currentEngine?: string | null): void { + updateFileStatus(runId: string, filePath: string, status: FileResult['status'], currentEngine?: string | null, videoPath?: string | null): void { const updates: Partial = { status }; if (currentEngine !== undefined) { updates.current_engine = currentEngine; } + if (videoPath !== undefined) { + updates.video_path = videoPath; + } this.db.updateFileResult(runId, filePath, updates); this.emitFileUpdate(runId, filePath); @@ -156,8 +158,7 @@ export class StateManager extends EventEmitter { } private emitFileUpdate(runId: string, filePath: string): void { - const files = this.db.getFileResults(runId); - const file = files.find((f) => f.file_path === filePath); + const file = this.db.getFileResult(runId, filePath); if (file) { const run = this.db.getRun(runId); this.emit('file:updated', { file, run }); @@ -190,6 +191,14 @@ export class StateManager extends EventEmitter { return this.db.getFileResults(runId); } + getFileResultsPaginated( + runId: string, + limit: number, + offset: number, + ): { files: FileResult[]; total: number } { + return this.db.getFileResultsPaginated(runId, limit, offset); + } + appendLog(runId: string, logMessage: string): void { // Write to log file instead of database this.logFileManager.appendLog(runId, logMessage);