From 6fbbc65edf429e5d4f6ef3d05c8b31a4ea2bf94b Mon Sep 17 00:00:00 2001 From: mikiher Date: Sat, 29 Jun 2024 20:04:23 +0300 Subject: [PATCH 01/32] Replace tone with ffmpeg for metadata and cover embedding --- server/managers/AudioMetadataManager.js | 66 ++++++------- server/utils/ffmpegHelpers.js | 123 ++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 39 deletions(-) diff --git a/server/managers/AudioMetadataManager.js b/server/managers/AudioMetadataManager.js index 11c82822..8b068223 100644 --- a/server/managers/AudioMetadataManager.js +++ b/server/managers/AudioMetadataManager.js @@ -5,7 +5,7 @@ const Logger = require('../Logger') const fs = require('../libs/fsExtra') -const toneHelpers = require('../utils/toneHelpers') +const ffmpegHelpers = require('../utils/ffmpegHelpers') const TaskManager = require('./TaskManager') @@ -21,22 +21,19 @@ class AudioMetadataMangaer { } /** - * Get queued task data - * @return {Array} - */ + * Get queued task data + * @return {Array} + */ getQueuedTaskData() { - return this.tasksQueued.map(t => t.data) + return this.tasksQueued.map((t) => t.data) } getIsLibraryItemQueuedOrProcessing(libraryItemId) { - return this.tasksQueued.some(t => t.data.libraryItemId === libraryItemId) || this.tasksRunning.some(t => t.data.libraryItemId === libraryItemId) + return this.tasksQueued.some((t) => t.data.libraryItemId === libraryItemId) || this.tasksRunning.some((t) => t.data.libraryItemId === libraryItemId) } getToneMetadataObjectForApi(libraryItem) { - const audioFiles = libraryItem.media.includedAudioFiles - let mimeType = audioFiles[0].mimeType - if (audioFiles.some(a => a.mimeType !== mimeType)) mimeType = null - return toneHelpers.getToneMetadataObject(libraryItem, libraryItem.media.chapters, libraryItem.media.tracks.length, mimeType) + return ffmpegHelpers.getFFMetadataObject(libraryItem, libraryItem.media.includedAudioFiles.length) } handleBatchEmbed(user, libraryItems, options = {}) { @@ -56,29 +53,28 @@ class AudioMetadataMangaer { const itemCachePath = Path.join(this.itemsCacheDir, libraryItem.id) // Only writing chapters for single file audiobooks - const chapters = (audioFiles.length == 1 || forceEmbedChapters) ? libraryItem.media.chapters.map(c => ({ ...c })) : null + const chapters = audioFiles.length == 1 || forceEmbedChapters ? libraryItem.media.chapters.map((c) => ({ ...c })) : null let mimeType = audioFiles[0].mimeType - if (audioFiles.some(a => a.mimeType !== mimeType)) mimeType = null + if (audioFiles.some((a) => a.mimeType !== mimeType)) mimeType = null // Create task const taskData = { libraryItemId: libraryItem.id, libraryItemPath: libraryItem.path, userId: user.id, - audioFiles: audioFiles.map(af => ( - { - index: af.index, - ino: af.ino, - filename: af.metadata.filename, - path: af.metadata.path, - cachePath: Path.join(itemCachePath, af.metadata.filename) - } - )), + audioFiles: audioFiles.map((af) => ({ + index: af.index, + ino: af.ino, + filename: af.metadata.filename, + path: af.metadata.path, + cachePath: Path.join(itemCachePath, af.metadata.filename) + })), coverPath: libraryItem.media.coverPath, - metadataObject: toneHelpers.getToneMetadataObject(libraryItem, chapters, audioFiles.length, mimeType), + metadataObject: ffmpegHelpers.getFFMetadataObject(libraryItem, audioFiles.length), itemCachePath, chapters, + mimeType, options: { forceEmbedChapters, backupFiles @@ -107,18 +103,19 @@ class AudioMetadataMangaer { // Ensure item cache dir exists let cacheDirCreated = false - if (!await fs.pathExists(task.data.itemCachePath)) { + if (!(await fs.pathExists(task.data.itemCachePath))) { await fs.mkdir(task.data.itemCachePath) cacheDirCreated = true } // Create metadata json file - const toneJsonPath = Path.join(task.data.itemCachePath, 'metadata.json') + const ffmetadataPath = Path.join(task.data.itemCachePath, 'ffmetadata.txt') try { - await fs.writeFile(toneJsonPath, JSON.stringify({ meta: task.data.metadataObject }, null, 2)) + await fs.writeFile(ffmetadataPath, ffmpegHelpers.generateFFMetadata(task.data.metadataObject, task.data.chapters)) + Logger.debug(`[AudioMetadataManager] Wrote ${ffmetadataPath}`) } catch (error) { - Logger.error(`[AudioMetadataManager] Write metadata.json failed`, error) - task.setFailed('Failed to write metadata.json') + Logger.error(`[AudioMetadataManager] Write ${ffmetadataPath} failed`, error) + task.setFailed('Failed to write file ffmetadata.txt') this.handleTaskFinished(task) return } @@ -141,16 +138,7 @@ class AudioMetadataMangaer { } } - const _toneMetadataObject = { - 'ToneJsonFile': toneJsonPath, - 'TrackNumber': af.index, - } - - if (task.data.coverPath) { - _toneMetadataObject['CoverFile'] = task.data.coverPath - } - - const success = await toneHelpers.tagAudioFile(af.path, _toneMetadataObject) + const success = await ffmpegHelpers.addCoverAndMetadataToFile(af.path, task.data.coverPath, ffmetadataPath, af.path) if (success) { Logger.info(`[AudioMetadataManager] Successfully tagged audio file "${af.path}"`) } @@ -167,7 +155,7 @@ class AudioMetadataMangaer { if (cacheDirCreated) { await fs.remove(task.data.itemCachePath) } else { - await fs.remove(toneJsonPath) + await fs.remove(ffmetadataPath) } } @@ -177,7 +165,7 @@ class AudioMetadataMangaer { handleTaskFinished(task) { TaskManager.taskFinished(task) - this.tasksRunning = this.tasksRunning.filter(t => t.id !== task.id) + this.tasksRunning = this.tasksRunning.filter((t) => t.id !== task.id) if (this.tasksRunning.length < this.MAX_CONCURRENT_TASKS && this.tasksQueued.length) { Logger.info(`[AudioMetadataManager] Task finished and dequeueing next task. ${this.tasksQueued} tasks queued.`) diff --git a/server/utils/ffmpegHelpers.js b/server/utils/ffmpegHelpers.js index 491f59dc..c4cf43b9 100644 --- a/server/utils/ffmpegHelpers.js +++ b/server/utils/ffmpegHelpers.js @@ -1,6 +1,7 @@ const axios = require('axios') const Ffmpeg = require('../libs/fluentFfmpeg') const fs = require('../libs/fsExtra') +const os = require('os') const Path = require('path') const Logger = require('../Logger') const { filePathToPOSIX } = require('./fileUtils') @@ -184,3 +185,125 @@ module.exports.downloadPodcastEpisode = (podcastEpisodeDownload) => { ffmpeg.run() }) } + +/** + * Generates ffmetadata file content from the provided metadata object and chapters array. + * @param {Object} metadata - The input metadata object. + * @param {Array} chapters - An array of chapter objects. + * @returns {string} - The ffmetadata file content. + */ +function generateFFMetadata(metadata, chapters) { + let ffmetadataContent = ';FFMETADATA1\n' + + // Add global metadata + for (const key in metadata) { + if (metadata[key]) { + ffmetadataContent += `${key}=${escapeFFMetadataValue(metadata[key])}\n` + } + } + + // Add chapters + chapters.forEach((chapter) => { + ffmetadataContent += '\n[CHAPTER]\n' + ffmetadataContent += `TIMEBASE=1/1000\n` + ffmetadataContent += `START=${Math.floor(chapter.start * 1000)}\n` + ffmetadataContent += `END=${Math.floor(chapter.end * 1000)}\n` + if (chapter.title) { + ffmetadataContent += `title=${escapeFFMetadataValue(chapter.title)}\n` + } + }) + + return ffmetadataContent +} + +module.exports.generateFFMetadata = generateFFMetadata + +/** + * Adds an ffmetadata and optionally a cover image to an audio file using fluent-ffmpeg. + * @param {string} audioFilePath - Path to the input audio file. + * @param {string|null} coverFilePath - Path to the cover image file. + * @param {string} metadataFilePath - Path to the ffmetadata file. + */ +async function addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath) { + return new Promise((resolve) => { + const tempFilePath = Path.join(os.tmpdir(), 'temp_output.m4b') + let ffmpeg = Ffmpeg() + ffmpeg.input(audioFilePath).input(metadataFilePath).outputOptions([ + '-map 0:a', // map audio stream from input file + '-map_metadata 1', // map metadata from metadata file + '-map_chapters 1', // map chapters from metadata file + '-c copy', // copy streams + '-f mp4' // force mp4 format + ]) + + if (coverFilePath) { + ffmpeg.input(coverFilePath).outputOptions([ + '-map 2:v', // map video stream from cover image file + '-disposition:v:0 attached_pic', // set cover image as attached picture + '-metadata:s:v', + 'title=Cover', // add title metadata to cover image stream + '-metadata:s:v', + 'comment=Cover' // add comment metadata to cover image stream + ]) + } else { + ffmpeg.outputOptions([ + '-map 0:v?' // retain video stream from input file if exists + ]) + } + + ffmpeg + .output(tempFilePath) + .on('start', function (commandLine) { + Logger.debug('[ffmpegHelpers] Spawned Ffmpeg with command: ' + commandLine) + }) + .on('end', (stdout, stderr) => { + Logger.debug('[ffmpegHelpers] ffmpeg stdout:', stdout) + Logger.debug('[ffmpegHelpers] ffmpeg stderr:', stderr) + fs.copyFileSync(tempFilePath, audioFilePath) + fs.unlinkSync(tempFilePath) + resolve(true) + }) + .on('error', (err, stdout, stderr) => { + Logger.error('Error adding cover image and metadata:', err) + Logger.error('ffmpeg stdout:', stdout) + Logger.error('ffmpeg stderr:', stderr) + resolve(false) + }) + + ffmpeg.run() + }) +} + +module.exports.addCoverAndMetadataToFile = addCoverAndMetadataToFile + +function escapeFFMetadataValue(value) { + return value.replace(/([;=\n\\#])/g, '\\$1') +} + +function getFFMetadataObject(libraryItem, audioFilesLength) { + const metadata = libraryItem.media.metadata + + const ffmetadata = { + title: metadata.title, + artist: metadata.authors?.map((a) => a.name).join(', '), + album_artist: metadata.authors?.map((a) => a.name).join(', '), + album: (metadata.title || '') + (metadata.subtitle ? `: ${metadata.subtitle}` : ''), + genre: metadata.genres?.join('; '), + date: metadata.publishedYear, + comment: metadata.description, + description: metadata.description, + composer: metadata.narratorName, + copyright: metadata.publisher, + grouping: metadata.series?.map((s) => s.name + (s.sequence ? ` #${s.sequence}` : '')).join(', ') + } + + Object.keys(ffmetadata).forEach((key) => { + if (!ffmetadata[key]) { + delete ffmetadata[key] + } + }) + + return ffmetadata +} + +module.exports.getFFMetadataObject = getFFMetadataObject From a21b1f3b168208977fb566a84b76a9f05f8375c1 Mon Sep 17 00:00:00 2001 From: mikiher Date: Sun, 30 Jun 2024 15:45:25 +0300 Subject: [PATCH 02/32] Make required changes for mp3 embedding --- server/managers/AudioMetadataManager.js | 2 +- server/utils/ffmpegHelpers.js | 57 +++++++++++++++++-------- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/server/managers/AudioMetadataManager.js b/server/managers/AudioMetadataManager.js index 8b068223..b9133de8 100644 --- a/server/managers/AudioMetadataManager.js +++ b/server/managers/AudioMetadataManager.js @@ -138,7 +138,7 @@ class AudioMetadataMangaer { } } - const success = await ffmpegHelpers.addCoverAndMetadataToFile(af.path, task.data.coverPath, ffmetadataPath, af.path) + const success = await ffmpegHelpers.addCoverAndMetadataToFile(af.path, task.data.coverPath, ffmetadataPath, task.data.mimeType) if (success) { Logger.info(`[AudioMetadataManager] Successfully tagged audio file "${af.path}"`) } diff --git a/server/utils/ffmpegHelpers.js b/server/utils/ffmpegHelpers.js index c4cf43b9..c9cddbf4 100644 --- a/server/utils/ffmpegHelpers.js +++ b/server/utils/ffmpegHelpers.js @@ -189,7 +189,7 @@ module.exports.downloadPodcastEpisode = (podcastEpisodeDownload) => { /** * Generates ffmetadata file content from the provided metadata object and chapters array. * @param {Object} metadata - The input metadata object. - * @param {Array} chapters - An array of chapter objects. + * @param {Array|null} chapters - An array of chapter objects. * @returns {string} - The ffmetadata file content. */ function generateFFMetadata(metadata, chapters) { @@ -203,15 +203,17 @@ function generateFFMetadata(metadata, chapters) { } // Add chapters - chapters.forEach((chapter) => { - ffmetadataContent += '\n[CHAPTER]\n' - ffmetadataContent += `TIMEBASE=1/1000\n` - ffmetadataContent += `START=${Math.floor(chapter.start * 1000)}\n` - ffmetadataContent += `END=${Math.floor(chapter.end * 1000)}\n` - if (chapter.title) { - ffmetadataContent += `title=${escapeFFMetadataValue(chapter.title)}\n` - } - }) + if (chapters) { + chapters.forEach((chapter) => { + ffmetadataContent += '\n[CHAPTER]\n' + ffmetadataContent += `TIMEBASE=1/1000\n` + ffmetadataContent += `START=${Math.floor(chapter.start * 1000)}\n` + ffmetadataContent += `END=${Math.floor(chapter.end * 1000)}\n` + if (chapter.title) { + ffmetadataContent += `title=${escapeFFMetadataValue(chapter.title)}\n` + } + }) + } return ffmetadataContent } @@ -223,19 +225,37 @@ module.exports.generateFFMetadata = generateFFMetadata * @param {string} audioFilePath - Path to the input audio file. * @param {string|null} coverFilePath - Path to the cover image file. * @param {string} metadataFilePath - Path to the ffmetadata file. + * @param {string} mimeType - The MIME type of the audio file. */ -async function addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath) { +async function addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath, mimeType) { + const isMp4 = mimeType === 'audio/mp4' + const isMp3 = mimeType === 'audio/mpeg' + + const audioFileDir = Path.dirname(audioFilePath) + const audioFileExt = Path.extname(audioFilePath) + const audioFileBaseName = Path.basename(audioFilePath, audioFileExt) + const tempFilePath = Path.join(audioFileDir, `${audioFileBaseName}.tmp${audioFileExt}`) + return new Promise((resolve) => { - const tempFilePath = Path.join(os.tmpdir(), 'temp_output.m4b') let ffmpeg = Ffmpeg() ffmpeg.input(audioFilePath).input(metadataFilePath).outputOptions([ '-map 0:a', // map audio stream from input file - '-map_metadata 1', // map metadata from metadata file + '-map_metadata 1', // map metadata tags from metadata file first + '-map_metadata 0', // add additional metadata tags from input file '-map_chapters 1', // map chapters from metadata file - '-c copy', // copy streams - '-f mp4' // force mp4 format + '-c copy' // copy streams ]) + if (isMp4) { + ffmpeg.outputOptions([ + '-f mp4' // force output format to mp4 + ]) + } else if (isMp3) { + ffmpeg.outputOptions([ + '-id3v2_version 3' // set ID3v2 version to 3 + ]) + } + if (coverFilePath) { ffmpeg.input(coverFilePath).outputOptions([ '-map 2:v', // map video stream from cover image file @@ -285,15 +305,18 @@ function getFFMetadataObject(libraryItem, audioFilesLength) { const ffmetadata = { title: metadata.title, - artist: metadata.authors?.map((a) => a.name).join(', '), - album_artist: metadata.authors?.map((a) => a.name).join(', '), + artist: metadata.authorName, + album_artist: metadata.authorName, album: (metadata.title || '') + (metadata.subtitle ? `: ${metadata.subtitle}` : ''), + TIT3: metadata.subtitle, // mp3 only genre: metadata.genres?.join('; '), date: metadata.publishedYear, comment: metadata.description, description: metadata.description, composer: metadata.narratorName, copyright: metadata.publisher, + publisher: metadata.publisher, // mp3 only + TRACKTOTAL: `${audioFilesLength}`, // mp3 only grouping: metadata.series?.map((s) => s.name + (s.sequence ? ` #${s.sequence}` : '')).join(', ') } From 4732ca811942be63b7dd2e6bafcc45291f5fb75a Mon Sep 17 00:00:00 2001 From: advplyr Date: Mon, 1 Jul 2024 16:57:14 -0500 Subject: [PATCH 03/32] Embed track number --- server/managers/AudioMetadataManager.js | 2 +- server/utils/ffmpegHelpers.js | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/managers/AudioMetadataManager.js b/server/managers/AudioMetadataManager.js index b9133de8..4d90b5df 100644 --- a/server/managers/AudioMetadataManager.js +++ b/server/managers/AudioMetadataManager.js @@ -138,7 +138,7 @@ class AudioMetadataMangaer { } } - const success = await ffmpegHelpers.addCoverAndMetadataToFile(af.path, task.data.coverPath, ffmetadataPath, task.data.mimeType) + const success = await ffmpegHelpers.addCoverAndMetadataToFile(af.path, task.data.coverPath, ffmetadataPath, af.index, task.data.mimeType) if (success) { Logger.info(`[AudioMetadataManager] Successfully tagged audio file "${af.path}"`) } diff --git a/server/utils/ffmpegHelpers.js b/server/utils/ffmpegHelpers.js index c9cddbf4..ef41b46d 100644 --- a/server/utils/ffmpegHelpers.js +++ b/server/utils/ffmpegHelpers.js @@ -225,9 +225,10 @@ module.exports.generateFFMetadata = generateFFMetadata * @param {string} audioFilePath - Path to the input audio file. * @param {string|null} coverFilePath - Path to the cover image file. * @param {string} metadataFilePath - Path to the ffmetadata file. + * @param {number} track - The track number to embed in the audio file. * @param {string} mimeType - The MIME type of the audio file. */ -async function addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath, mimeType) { +async function addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath, track, mimeType) { const isMp4 = mimeType === 'audio/mp4' const isMp3 = mimeType === 'audio/mpeg' @@ -246,6 +247,10 @@ async function addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataF '-c copy' // copy streams ]) + if (track && !isNaN(track)) { + ffmpeg.outputOptions(['-metadata track=' + track]) + } + if (isMp4) { ffmpeg.outputOptions([ '-f mp4' // force output format to mp4 From ebaec2364898c3579f79d5d915cdeafbd0709bcb Mon Sep 17 00:00:00 2001 From: mikiher Date: Tue, 2 Jul 2024 18:25:04 +0300 Subject: [PATCH 04/32] Replace tone with ffmpeg in AbMergeManager --- server/managers/AbMergeManager.js | 86 +++++++++++++------------ server/managers/AudioMetadataManager.js | 12 ++-- server/utils/ffmpegHelpers.js | 13 ++++ 3 files changed, 64 insertions(+), 47 deletions(-) diff --git a/server/managers/AbMergeManager.js b/server/managers/AbMergeManager.js index 8a87df2e..711e8892 100644 --- a/server/managers/AbMergeManager.js +++ b/server/managers/AbMergeManager.js @@ -1,4 +1,3 @@ - const Path = require('path') const fs = require('../libs/fsExtra') @@ -7,7 +6,7 @@ const Logger = require('../Logger') const TaskManager = require('./TaskManager') const Task = require('../objects/Task') const { writeConcatFile } = require('../utils/ffmpegHelpers') -const toneHelpers = require('../utils/toneHelpers') +const ffmpegHelpers = require('../utils/ffmpegHelpers') class AbMergeManager { constructor() { @@ -17,7 +16,7 @@ class AbMergeManager { } getPendingTaskByLibraryItemId(libraryItemId) { - return this.pendingTasks.find(t => t.task.data.libraryItemId === libraryItemId) + return this.pendingTasks.find((t) => t.task.data.libraryItemId === libraryItemId) } cancelEncode(task) { @@ -31,23 +30,27 @@ class AbMergeManager { const targetFilename = audiobookDirname + '.m4b' const itemCachePath = Path.join(this.itemsCacheDir, libraryItem.id) const tempFilepath = Path.join(itemCachePath, targetFilename) + const ffmetadataPath = Path.join(itemCachePath, 'ffmetadata.txt') const taskData = { libraryItemId: libraryItem.id, libraryItemPath: libraryItem.path, userId: user.id, - originalTrackPaths: libraryItem.media.tracks.map(t => t.metadata.path), + originalTrackPaths: libraryItem.media.tracks.map((t) => t.metadata.path), tempFilepath, targetFilename, targetFilepath: Path.join(libraryItem.path, targetFilename), itemCachePath, - toneJsonObject: null + ffmetadataObject: ffmpegHelpers.getFFMetadataObject(libraryItem, 1), + chapters: libraryItem.media.chapters?.map((c) => ({ ...c })), + coverPath: libraryItem.media.coverPath, + ffmetadataPath } const taskDescription = `Encoding audiobook "${libraryItem.media.metadata.title}" into a single m4b file.` task.setData('encode-m4b', 'Encoding M4b', taskDescription, false, taskData) TaskManager.addTask(task) Logger.info(`Start m4b encode for ${libraryItem.id} - TaskId: ${task.id}`) - if (!await fs.pathExists(taskData.itemCachePath)) { + if (!(await fs.pathExists(taskData.itemCachePath))) { await fs.mkdir(taskData.itemCachePath) } @@ -55,6 +58,15 @@ class AbMergeManager { } async runAudiobookMerge(libraryItem, task, encodingOptions) { + // Create ffmetadata file + const success = await ffmpegHelpers.writeFFMetadataFile(task.data.metadataObject, task.data.chapters, task.data.ffmetadataPath) + if (!success) { + Logger.error(`[AudioMetadataManager] Failed to write ffmetadata file for audiobook "${task.data.libraryItemId}"`) + task.setFailed('Failed to write metadata file.') + this.removeTask(task, true) + return + } + const audioBitrate = encodingOptions.bitrate || '128k' const audioCodec = encodingOptions.codec || 'aac' const audioChannels = encodingOptions.channels || 2 @@ -90,12 +102,7 @@ class AbMergeManager { const ffmpegOutputOptions = ['-f mp4'] if (audioRequiresEncode) { - ffmpegOptions = ffmpegOptions.concat([ - '-map 0:a', - `-acodec ${audioCodec}`, - `-ac ${audioChannels}`, - `-b:a ${audioBitrate}` - ]) + ffmpegOptions = ffmpegOptions.concat(['-map 0:a', `-acodec ${audioCodec}`, `-ac ${audioChannels}`, `-b:a ${audioBitrate}`]) } else { ffmpegOptions.push('-max_muxing_queue_size 1000') @@ -106,24 +113,6 @@ class AbMergeManager { } } - let toneJsonPath = null - try { - toneJsonPath = Path.join(task.data.itemCachePath, 'metadata.json') - await toneHelpers.writeToneMetadataJsonFile(libraryItem, libraryItem.media.chapters, toneJsonPath, 1, 'audio/mp4') - } catch (error) { - Logger.error(`[AbMergeManager] Write metadata.json failed`, error) - toneJsonPath = null - } - - task.data.toneJsonObject = { - 'ToneJsonFile': toneJsonPath, - 'TrackNumber': 1, - } - - if (libraryItem.media.coverPath) { - task.data.toneJsonObject['CoverFile'] = libraryItem.media.coverPath - } - const workerData = { inputs: ffmpegInputs, options: ffmpegOptions, @@ -162,7 +151,7 @@ class AbMergeManager { async sendResult(task, result) { // Remove pending task - this.pendingTasks = this.pendingTasks.filter(d => d.id !== task.id) + this.pendingTasks = this.pendingTasks.filter((d) => d.id !== task.id) if (result.isKilled) { task.setFailed('Ffmpeg task killed') @@ -177,7 +166,7 @@ class AbMergeManager { } // Write metadata to merged file - const success = await toneHelpers.tagAudioFile(task.data.tempFilepath, task.data.toneJsonObject) + const success = await ffmpegHelpers.addCoverAndMetadataToFile(task.data.tempFilepath, task.data.coverPath, task.data.ffmetadataPath, 1, 'audio/mp4') if (!success) { Logger.error(`[AbMergeManager] Failed to write metadata to file "${task.data.tempFilepath}"`) task.setFailed('Failed to write metadata to m4b file') @@ -199,6 +188,9 @@ class AbMergeManager { Logger.debug(`[AbMergeManager] Moving m4b from ${task.data.tempFilepath} to ${task.data.targetFilepath}`) await fs.move(task.data.tempFilepath, task.data.targetFilepath) + // Remove ffmetadata file + await fs.remove(task.data.ffmetadataPath) + task.setFinished() await this.removeTask(task, false) Logger.info(`[AbMergeManager] Ab task finished ${task.id}`) @@ -207,9 +199,9 @@ class AbMergeManager { async removeTask(task, removeTempFilepath = false) { Logger.info('[AbMergeManager] Removing task ' + task.id) - const pendingDl = this.pendingTasks.find(d => d.id === task.id) + const pendingDl = this.pendingTasks.find((d) => d.id === task.id) if (pendingDl) { - this.pendingTasks = this.pendingTasks.filter(d => d.id !== task.id) + this.pendingTasks = this.pendingTasks.filter((d) => d.id !== task.id) if (pendingDl.worker) { Logger.warn(`[AbMergeManager] Removing download in progress - stopping worker`) try { @@ -223,13 +215,27 @@ class AbMergeManager { } } - if (removeTempFilepath) { // On failed tasks remove the bad file if it exists + if (removeTempFilepath) { + // On failed tasks remove the bad file if it exists if (await fs.pathExists(task.data.tempFilepath)) { - await fs.remove(task.data.tempFilepath).then(() => { - Logger.info('[AbMergeManager] Deleted target file', task.data.tempFilepath) - }).catch((err) => { - Logger.error('[AbMergeManager] Failed to delete target file', err) - }) + await fs + .remove(task.data.tempFilepath) + .then(() => { + Logger.info('[AbMergeManager] Deleted target file', task.data.tempFilepath) + }) + .catch((err) => { + Logger.error('[AbMergeManager] Failed to delete target file', err) + }) + } + if (await fs.pathExists(task.data.ffmetadataPath)) { + await fs + .remove(task.data.ffmetadataPath) + .then(() => { + Logger.info('[AbMergeManager] Deleted ffmetadata file', task.data.ffmetadataPath) + }) + .catch((err) => { + Logger.error('[AbMergeManager] Failed to delete ffmetadata file', err) + }) } } diff --git a/server/managers/AudioMetadataManager.js b/server/managers/AudioMetadataManager.js index 4d90b5df..44fbdb57 100644 --- a/server/managers/AudioMetadataManager.js +++ b/server/managers/AudioMetadataManager.js @@ -108,14 +108,12 @@ class AudioMetadataMangaer { cacheDirCreated = true } - // Create metadata json file + // Create ffmetadata file const ffmetadataPath = Path.join(task.data.itemCachePath, 'ffmetadata.txt') - try { - await fs.writeFile(ffmetadataPath, ffmpegHelpers.generateFFMetadata(task.data.metadataObject, task.data.chapters)) - Logger.debug(`[AudioMetadataManager] Wrote ${ffmetadataPath}`) - } catch (error) { - Logger.error(`[AudioMetadataManager] Write ${ffmetadataPath} failed`, error) - task.setFailed('Failed to write file ffmetadata.txt') + const success = await ffmpegHelpers.writeFFMetadataFile(task.data.metadataObject, task.data.chapters, ffmetadataPath) + if (!success) { + Logger.error(`[AudioMetadataManager] Failed to write ffmetadata file for audiobook "${task.data.libraryItemId}"`) + task.setFailed('Failed to write metadata file.') this.handleTaskFinished(task) return } diff --git a/server/utils/ffmpegHelpers.js b/server/utils/ffmpegHelpers.js index ef41b46d..28c6e618 100644 --- a/server/utils/ffmpegHelpers.js +++ b/server/utils/ffmpegHelpers.js @@ -220,6 +220,19 @@ function generateFFMetadata(metadata, chapters) { module.exports.generateFFMetadata = generateFFMetadata +async function writeFFMetadataFile(metadata, chapters, ffmetadataPath) { + try { + await fs.writeFile(ffmetadataPath, generateFFMetadata(metadata, chapters)) + Logger.debug(`[ffmpegHelpers] Wrote ${ffmetadataPath}`) + return true + } catch (error) { + Logger.error(`[ffmpegHelpers] Write ${ffmetadataPath} failed`, error) + return false + } +} + +module.exports.writeFFMetadataFile = writeFFMetadataFile + /** * Adds an ffmetadata and optionally a cover image to an audio file using fluent-ffmpeg. * @param {string} audioFilePath - Path to the input audio file. From 1b015beba403621e618b28721a7ccf44334cad61 Mon Sep 17 00:00:00 2001 From: mikiher Date: Tue, 2 Jul 2024 19:00:03 +0300 Subject: [PATCH 05/32] Remove windows restrictions from Tools.vuw --- client/components/modals/item/tabs/Tools.vue | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/client/components/modals/item/tabs/Tools.vue b/client/components/modals/item/tabs/Tools.vue index de19e04c..b05b2187 100644 --- a/client/components/modals/item/tabs/Tools.vue +++ b/client/components/modals/item/tabs/Tools.vue @@ -2,11 +2,8 @@

{{ $strings.HeaderAudiobookTools }}

- - Not supported for the Windows install yet - -
+

{{ $strings.LabelToolsMakeM4b }}

@@ -23,7 +20,7 @@
-
+

{{ $strings.LabelToolsEmbedMetadata }}

@@ -111,12 +108,6 @@ export default { }, isEncodeTaskRunning() { return this.encodeTask && !this.encodeTask?.isFinished - }, - isWindowsInstall() { - return this.Source == 'windows' - }, - Source() { - return this.$store.state.Source } }, methods: { @@ -141,4 +132,4 @@ export default { } } } - \ No newline at end of file + From 8bdee51798731b136a46f98a183860f1622781e2 Mon Sep 17 00:00:00 2001 From: mikiher Date: Wed, 3 Jul 2024 23:50:42 +0300 Subject: [PATCH 06/32] Add unit tests for new ffmpegHelpers functions --- server/utils/ffmpegHelpers.js | 24 ++- test/server/utils/ffmpegHelpers.test.js | 249 ++++++++++++++++++++++++ 2 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 test/server/utils/ffmpegHelpers.test.js diff --git a/server/utils/ffmpegHelpers.js b/server/utils/ffmpegHelpers.js index 28c6e618..2a242504 100644 --- a/server/utils/ffmpegHelpers.js +++ b/server/utils/ffmpegHelpers.js @@ -5,6 +5,7 @@ const os = require('os') const Path = require('path') const Logger = require('../Logger') const { filePathToPOSIX } = require('./fileUtils') +const LibraryItem = require('../objects/LibraryItem') function escapeSingleQuotes(path) { // return path.replace(/'/g, '\'\\\'\'') @@ -220,6 +221,14 @@ function generateFFMetadata(metadata, chapters) { module.exports.generateFFMetadata = generateFFMetadata +/** + * Writes FFmpeg metadata file with the given metadata and chapters. + * + * @param {Object} metadata - The metadata object. + * @param {Array} chapters - The array of chapter objects. + * @param {string} ffmetadataPath - The path to the FFmpeg metadata file. + * @returns {Promise} - A promise that resolves to true if the file was written successfully, false otherwise. + */ async function writeFFMetadataFile(metadata, chapters, ffmetadataPath) { try { await fs.writeFile(ffmetadataPath, generateFFMetadata(metadata, chapters)) @@ -235,23 +244,25 @@ module.exports.writeFFMetadataFile = writeFFMetadataFile /** * Adds an ffmetadata and optionally a cover image to an audio file using fluent-ffmpeg. + * * @param {string} audioFilePath - Path to the input audio file. * @param {string|null} coverFilePath - Path to the cover image file. * @param {string} metadataFilePath - Path to the ffmetadata file. * @param {number} track - The track number to embed in the audio file. * @param {string} mimeType - The MIME type of the audio file. + * @param {Ffmpeg} ffmpeg - The Ffmpeg instance to use (optional). Used for dependency injection in tests. + * @returns {Promise} A promise that resolves to true if the operation is successful, false otherwise. */ -async function addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath, track, mimeType) { +async function addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath, track, mimeType, ffmpeg = Ffmpeg()) { const isMp4 = mimeType === 'audio/mp4' const isMp3 = mimeType === 'audio/mpeg' const audioFileDir = Path.dirname(audioFilePath) const audioFileExt = Path.extname(audioFilePath) const audioFileBaseName = Path.basename(audioFilePath, audioFileExt) - const tempFilePath = Path.join(audioFileDir, `${audioFileBaseName}.tmp${audioFileExt}`) + const tempFilePath = filePathToPOSIX(Path.join(audioFileDir, `${audioFileBaseName}.tmp${audioFileExt}`)) return new Promise((resolve) => { - let ffmpeg = Ffmpeg() ffmpeg.input(audioFilePath).input(metadataFilePath).outputOptions([ '-map 0:a', // map audio stream from input file '-map_metadata 1', // map metadata tags from metadata file first @@ -318,6 +329,13 @@ function escapeFFMetadataValue(value) { return value.replace(/([;=\n\\#])/g, '\\$1') } +/** + * Retrieves the FFmpeg metadata object for a given library item. + * + * @param {LibraryItem} libraryItem - The library item containing the media metadata. + * @param {number} audioFilesLength - The length of the audio files. + * @returns {Object} - The FFmpeg metadata object. + */ function getFFMetadataObject(libraryItem, audioFilesLength) { const metadata = libraryItem.media.metadata diff --git a/test/server/utils/ffmpegHelpers.test.js b/test/server/utils/ffmpegHelpers.test.js new file mode 100644 index 00000000..82daf1ba --- /dev/null +++ b/test/server/utils/ffmpegHelpers.test.js @@ -0,0 +1,249 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const { generateFFMetadata, addCoverAndMetadataToFile } = require('../../../server/utils/ffmpegHelpers') +const fs = require('../../../server/libs/fsExtra') +const EventEmitter = require('events') + +global.isWin = process.platform === 'win32' + +describe('generateFFMetadata', () => { + function createTestSetup() { + const metadata = { + title: 'My Audiobook', + artist: 'John Doe', + album: 'Best Audiobooks' + } + + const chapters = [ + { start: 0, end: 1000, title: 'Chapter 1' }, + { start: 1000, end: 2000, title: 'Chapter 2' } + ] + + return { metadata, chapters } + } + + let metadata = null + let chapters = null + beforeEach(() => { + const input = createTestSetup() + metadata = input.metadata + chapters = input.chapters + }) + + it('should generate ffmetadata content with chapters', () => { + const result = generateFFMetadata(metadata, chapters) + + expect(result).to.equal(';FFMETADATA1\ntitle=My Audiobook\nartist=John Doe\nalbum=Best Audiobooks\n\n[CHAPTER]\nTIMEBASE=1/1000\nSTART=0\nEND=1000000\ntitle=Chapter 1\n\n[CHAPTER]\nTIMEBASE=1/1000\nSTART=1000000\nEND=2000000\ntitle=Chapter 2\n') + }) + + it('should generate ffmetadata content without chapters', () => { + chapters = null + + const result = generateFFMetadata(metadata, chapters) + + expect(result).to.equal(';FFMETADATA1\ntitle=My Audiobook\nartist=John Doe\nalbum=Best Audiobooks\n') + }) + + it('should handle chapters with no title', () => { + chapters = [ + { start: 0, end: 1000 }, + { start: 1000, end: 2000 } + ] + + const result = generateFFMetadata(metadata, chapters) + + expect(result).to.equal(';FFMETADATA1\ntitle=My Audiobook\nartist=John Doe\nalbum=Best Audiobooks\n\n[CHAPTER]\nTIMEBASE=1/1000\nSTART=0\nEND=1000000\n\n[CHAPTER]\nTIMEBASE=1/1000\nSTART=1000000\nEND=2000000\n') + }) + + it('should handle metadata escaping special characters (=, ;, #, and a newline)', () => { + metadata.title = 'My Audiobook; with = special # characters\n' + chapters[0].title = 'Chapter #1' + + const result = generateFFMetadata(metadata, chapters) + + expect(result).to.equal(';FFMETADATA1\ntitle=My Audiobook\\; with \\= special \\# characters\\\n\nartist=John Doe\nalbum=Best Audiobooks\n\n[CHAPTER]\nTIMEBASE=1/1000\nSTART=0\nEND=1000000\ntitle=Chapter \\#1\n\n[CHAPTER]\nTIMEBASE=1/1000\nSTART=1000000\nEND=2000000\ntitle=Chapter 2\n') + }) +}) + +describe('addCoverAndMetadataToFile', () => { + function createTestSetup() { + const audioFilePath = '/path/to/audio/file.mp3' + const coverFilePath = '/path/to/cover/image.jpg' + const metadataFilePath = '/path/to/metadata/file.txt' + const track = 1 + const mimeType = 'audio/mpeg' + + const ffmpegStub = new EventEmitter() + ffmpegStub.input = sinon.stub().returnsThis() + ffmpegStub.outputOptions = sinon.stub().returnsThis() + ffmpegStub.output = sinon.stub().returnsThis() + ffmpegStub.input = sinon.stub().returnsThis() + ffmpegStub.run = sinon.stub().callsFake(() => { + ffmpegStub.emit('end') + }) + const fsCopyFileSyncStub = sinon.stub(fs, 'copyFileSync') + const fsUnlinkSyncStub = sinon.stub(fs, 'unlinkSync') + + return { audioFilePath, coverFilePath, metadataFilePath, track, mimeType, ffmpegStub, fsCopyFileSyncStub, fsUnlinkSyncStub } + } + + let audioFilePath = null + let coverFilePath = null + let metadataFilePath = null + let track = null + let mimeType = null + let ffmpegStub = null + let fsCopyFileSyncStub = null + let fsUnlinkSyncStub = null + beforeEach(() => { + const input = createTestSetup() + audioFilePath = input.audioFilePath + coverFilePath = input.coverFilePath + metadataFilePath = input.metadataFilePath + track = input.track + mimeType = input.mimeType + ffmpegStub = input.ffmpegStub + fsCopyFileSyncStub = input.fsCopyFileSyncStub + fsUnlinkSyncStub = input.fsUnlinkSyncStub + }) + + it('should add cover image and metadata to audio file', async () => { + // Act + const result = await addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath, track, mimeType, ffmpegStub) + + // Assert + expect(result).to.be.true + expect(ffmpegStub.input.calledThrice).to.be.true + expect(ffmpegStub.input.getCall(0).args[0]).to.equal(audioFilePath) + expect(ffmpegStub.input.getCall(1).args[0]).to.equal(metadataFilePath) + expect(ffmpegStub.input.getCall(2).args[0]).to.equal(coverFilePath) + + expect(ffmpegStub.outputOptions.callCount).to.equal(4) + expect(ffmpegStub.outputOptions.getCall(0).args[0]).to.deep.equal(['-map 0:a', '-map_metadata 1', '-map_metadata 0', '-map_chapters 1', '-c copy']) + expect(ffmpegStub.outputOptions.getCall(1).args[0]).to.deep.equal(['-metadata track=1']) + expect(ffmpegStub.outputOptions.getCall(2).args[0]).to.deep.equal(['-id3v2_version 3']) + expect(ffmpegStub.outputOptions.getCall(3).args[0]).to.deep.equal(['-map 2:v', '-disposition:v:0 attached_pic', '-metadata:s:v', 'title=Cover', '-metadata:s:v', 'comment=Cover']) + + expect(ffmpegStub.output.calledOnce).to.be.true + expect(ffmpegStub.output.firstCall.args[0]).to.equal('/path/to/audio/file.tmp.mp3') + + expect(ffmpegStub.run.calledOnce).to.be.true + + expect(fsCopyFileSyncStub.calledOnce).to.be.true + expect(fsCopyFileSyncStub.firstCall.args[0]).to.equal('/path/to/audio/file.tmp.mp3') + expect(fsCopyFileSyncStub.firstCall.args[1]).to.equal('/path/to/audio/file.mp3') + + expect(fsUnlinkSyncStub.calledOnce).to.be.true + expect(fsUnlinkSyncStub.firstCall.args[0]).to.equal('/path/to/audio/file.tmp.mp3') + + // Restore the stub + sinon.restore() + }) + + it('should handle missing cover image', async () => { + // Arrange + coverFilePath = null + + // Act + const result = await addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath, track, mimeType, ffmpegStub) + + // Assert + expect(result).to.be.true + expect(ffmpegStub.input.calledTwice).to.be.true + expect(ffmpegStub.input.getCall(0).args[0]).to.equal(audioFilePath) + expect(ffmpegStub.input.getCall(1).args[0]).to.equal(metadataFilePath) + + expect(ffmpegStub.outputOptions.callCount).to.equal(4) + expect(ffmpegStub.outputOptions.getCall(0).args[0]).to.deep.equal(['-map 0:a', '-map_metadata 1', '-map_metadata 0', '-map_chapters 1', '-c copy']) + expect(ffmpegStub.outputOptions.getCall(1).args[0]).to.deep.equal(['-metadata track=1']) + expect(ffmpegStub.outputOptions.getCall(2).args[0]).to.deep.equal(['-id3v2_version 3']) + expect(ffmpegStub.outputOptions.getCall(3).args[0]).to.deep.equal(['-map 0:v?']) + + expect(ffmpegStub.output.calledOnce).to.be.true + expect(ffmpegStub.output.firstCall.args[0]).to.equal('/path/to/audio/file.tmp.mp3') + + expect(ffmpegStub.run.calledOnce).to.be.true + + expect(fsCopyFileSyncStub.calledOnce).to.be.true + expect(fsCopyFileSyncStub.firstCall.args[0]).to.equal('/path/to/audio/file.tmp.mp3') + expect(fsCopyFileSyncStub.firstCall.args[1]).to.equal('/path/to/audio/file.mp3') + + expect(fsUnlinkSyncStub.calledOnce).to.be.true + expect(fsUnlinkSyncStub.firstCall.args[0]).to.equal('/path/to/audio/file.tmp.mp3') + + // Restore the stub + sinon.restore() + }) + + it('should handle error during ffmpeg execution', async () => { + // Arrange + ffmpegStub.run = sinon.stub().callsFake(() => { + ffmpegStub.emit('error', new Error('FFmpeg error')) + }) + + // Act + const result = await addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath, track, mimeType, ffmpegStub) + + // Assert + expect(result).to.be.false + expect(ffmpegStub.input.calledThrice).to.be.true + expect(ffmpegStub.input.getCall(0).args[0]).to.equal(audioFilePath) + expect(ffmpegStub.input.getCall(1).args[0]).to.equal(metadataFilePath) + expect(ffmpegStub.input.getCall(2).args[0]).to.equal(coverFilePath) + + expect(ffmpegStub.outputOptions.callCount).to.equal(4) + expect(ffmpegStub.outputOptions.getCall(0).args[0]).to.deep.equal(['-map 0:a', '-map_metadata 1', '-map_metadata 0', '-map_chapters 1', '-c copy']) + expect(ffmpegStub.outputOptions.getCall(1).args[0]).to.deep.equal(['-metadata track=1']) + expect(ffmpegStub.outputOptions.getCall(2).args[0]).to.deep.equal(['-id3v2_version 3']) + expect(ffmpegStub.outputOptions.getCall(3).args[0]).to.deep.equal(['-map 2:v', '-disposition:v:0 attached_pic', '-metadata:s:v', 'title=Cover', '-metadata:s:v', 'comment=Cover']) + + expect(ffmpegStub.output.calledOnce).to.be.true + expect(ffmpegStub.output.firstCall.args[0]).to.equal('/path/to/audio/file.tmp.mp3') + + expect(ffmpegStub.run.calledOnce).to.be.true + + expect(fsCopyFileSyncStub.called).to.be.false + + expect(fsUnlinkSyncStub.called).to.be.false + + // Restore the stub + sinon.restore() + }) + + it('should handle m4b embedding', async () => { + // Arrange + mimeType = 'audio/mp4' + audioFilePath = '/path/to/audio/file.m4b' + + // Act + const result = await addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath, track, mimeType, ffmpegStub) + + // Assert + expect(result).to.be.true + expect(ffmpegStub.input.calledThrice).to.be.true + expect(ffmpegStub.input.getCall(0).args[0]).to.equal(audioFilePath) + expect(ffmpegStub.input.getCall(1).args[0]).to.equal(metadataFilePath) + expect(ffmpegStub.input.getCall(2).args[0]).to.equal(coverFilePath) + + expect(ffmpegStub.outputOptions.callCount).to.equal(4) + expect(ffmpegStub.outputOptions.getCall(0).args[0]).to.deep.equal(['-map 0:a', '-map_metadata 1', '-map_metadata 0', '-map_chapters 1', '-c copy']) + expect(ffmpegStub.outputOptions.getCall(1).args[0]).to.deep.equal(['-metadata track=1']) + expect(ffmpegStub.outputOptions.getCall(2).args[0]).to.deep.equal(['-f mp4']) + expect(ffmpegStub.outputOptions.getCall(3).args[0]).to.deep.equal(['-map 2:v', '-disposition:v:0 attached_pic', '-metadata:s:v', 'title=Cover', '-metadata:s:v', 'comment=Cover']) + + expect(ffmpegStub.output.calledOnce).to.be.true + expect(ffmpegStub.output.firstCall.args[0]).to.equal('/path/to/audio/file.tmp.m4b') + + expect(ffmpegStub.run.calledOnce).to.be.true + + expect(fsCopyFileSyncStub.calledOnce).to.be.true + expect(fsCopyFileSyncStub.firstCall.args[0]).to.equal('/path/to/audio/file.tmp.m4b') + expect(fsCopyFileSyncStub.firstCall.args[1]).to.equal('/path/to/audio/file.m4b') + + expect(fsUnlinkSyncStub.calledOnce).to.be.true + expect(fsUnlinkSyncStub.firstCall.args[0]).to.equal('/path/to/audio/file.tmp.m4b') + + // Restore the stub + sinon.restore() + }) +}) From bfc3c7e7c9073a3ee8f8b13b95b93c9c6d7e7ae1 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Thu, 4 Jul 2024 03:18:52 +0000 Subject: [PATCH 07/32] Initial email settings schemas --- docs/objects/setings/EmailSettings.yaml | 77 +++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 docs/objects/setings/EmailSettings.yaml diff --git a/docs/objects/setings/EmailSettings.yaml b/docs/objects/setings/EmailSettings.yaml new file mode 100644 index 00000000..efa2a5d7 --- /dev/null +++ b/docs/objects/setings/EmailSettings.yaml @@ -0,0 +1,77 @@ +components: + schemas: + EreaderDeviceObject: + type: object + description: An e-reader device configured to receive EPUB through e-mail. + properties: + name: + type: string + description: The name of the e-reader device. + email: + type: string + description: The email address associated with the e-reader device. + availabilityOption: + type: string + description: The availability option for the device. + enum: ['adminOrUp', 'userOrUp', 'guestOrUp', 'specificUsers'] + users: + type: array + description: List of specific users allowed to access the device. + items: + type: string + required: + - name + - email + - availabilityOption + EmailSettings: + type: object + description: The email settings configuration for the server. This includes the credentials to send e-books and an array of e-reader devices. + properties: + id: + type: string + description: The unique identifier for the email settings. Currently this is always `email-settings` + example: email-settings + host: + type: string + description: The SMTP host address. + nullable: true + port: + type: integer + format: int32 + description: The port number for the SMTP server. + example: 465 + secure: + type: boolean + description: Indicates if the connection should use SSL/TLS. + example: true + rejectUnauthorized: + type: boolean + description: Indicates if unauthorized SSL/TLS certificates should be rejected. + example: true + user: + type: string + description: The username for SMTP authentication. + nullable: true + pass: + type: string + description: The password for SMTP authentication. + nullable: true + testAddress: + type: string + description: The test email address used for sending test emails. + nullable: true + fromAddress: + type: string + description: The default "from" email address for outgoing emails. + nullable: true + ereaderDevices: + type: array + description: List of configured e-reader devices. + items: + $ref: '#/components/schemas/EreaderDeviceObject' + required: + - id + - port + - secure + - rejectUnauthorized + - ereaderDevices From 046bf52d883b6c98fcfd5236421280db3c04a1f6 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Thu, 4 Jul 2024 03:36:01 +0000 Subject: [PATCH 08/32] Initial EmailController paths --- docs/controllers/EmailController.yaml | 95 +++++++++++++++++++++++++ docs/objects/setings/EmailSettings.yaml | 7 +- 2 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 docs/controllers/EmailController.yaml diff --git a/docs/controllers/EmailController.yaml b/docs/controllers/EmailController.yaml new file mode 100644 index 00000000..e0025c14 --- /dev/null +++ b/docs/controllers/EmailController.yaml @@ -0,0 +1,95 @@ +components: + schemas: + emailSettings: + type: string + description: The field to sort by from the request. + example: 'media.metadata.title' + responses: + email200: + description: Successful response - Email + content: + application/json: + schema: + $ref: '../objects/settings/EmailSettings.yaml#/components/schemas/EmailSettings' + ereader200: + description: Successful response - Ereader + content: + application/json: + schema: + type: object + properties: + ereaderDevices: + type: array + items: + $ref: '../objects/settings/EmailSettings.yaml#/components/schemas/EreaderDeviceObject' +paths: + /api/emails: + get: + description: Get email settings + operationId: getEmailSettings + responses: + 200: + $ref: '#/components/responses/email200' + patch: + summary: Update email settings + operationId: updateEmailSettings + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EmailSettings' + responses: + 200: + $ref: '#/components/responses/email200' + /api/emails/test: + post: + summary: Send test email + operationId: sendTestEmail + responses: + 200: + description: Successful response + /api/emails/ereader-devices: + post: + summary: Update e-reader devices + operationId: updateEReaderDevices + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + ereaderDevices: + type: array + items: + $ref: '#/components/schemas/EreaderDeviceObject' + responses: + 200: + $ref: '#/components/responses/ereader200' + 400: + description: Invalid payload + /api/emails/send-ebook-to-device: + post: + summary: Send ebook to device + operationId: sendEBookToDevice + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + libraryItemId: + $ref: '../objects/LibraryItem.yaml#/components/schemas/libraryItemId' + deviceName: + $ref: '../objects/schemas/EmailSettings.yaml#/components/schemas/ereaderName' + responses: + 200: + description: Successful response + 400: + description: Invalid request + 403: + description: Forbidden + 404: + description: Not found diff --git a/docs/objects/setings/EmailSettings.yaml b/docs/objects/setings/EmailSettings.yaml index efa2a5d7..acce79c7 100644 --- a/docs/objects/setings/EmailSettings.yaml +++ b/docs/objects/setings/EmailSettings.yaml @@ -1,12 +1,14 @@ components: schemas: + ereaderName: + type: string + description: The name of the e-reader device. EreaderDeviceObject: type: object description: An e-reader device configured to receive EPUB through e-mail. properties: name: - type: string - description: The name of the e-reader device. + $ref: '#/components/schemas/ereaderName' email: type: string description: The email address associated with the e-reader device. @@ -73,5 +75,4 @@ components: - id - port - secure - - rejectUnauthorized - ereaderDevices From e60a91379a8adacdd75d2f2958619d0a1e20dd31 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Thu, 4 Jul 2024 03:40:17 +0000 Subject: [PATCH 09/32] Rename folder --- docs/objects/{setings => settings}/EmailSettings.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/objects/{setings => settings}/EmailSettings.yaml (100%) diff --git a/docs/objects/setings/EmailSettings.yaml b/docs/objects/settings/EmailSettings.yaml similarity index 100% rename from docs/objects/setings/EmailSettings.yaml rename to docs/objects/settings/EmailSettings.yaml From b8e17de8b4ed19658093b08da8e96843f9d61f24 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Thu, 4 Jul 2024 03:45:04 +0000 Subject: [PATCH 10/32] Add: EmailController to `root.yaml` --- docs/controllers/EmailController.yaml | 14 ++++++++++++-- docs/root.yaml | 10 ++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/controllers/EmailController.yaml b/docs/controllers/EmailController.yaml index e0025c14..3a7d990e 100644 --- a/docs/controllers/EmailController.yaml +++ b/docs/controllers/EmailController.yaml @@ -27,12 +27,16 @@ paths: get: description: Get email settings operationId: getEmailSettings + tags: + - Email responses: 200: $ref: '#/components/responses/email200' patch: summary: Update email settings operationId: updateEmailSettings + tags: + - Email requestBody: required: true content: @@ -46,6 +50,8 @@ paths: post: summary: Send test email operationId: sendTestEmail + tags: + - Email responses: 200: description: Successful response @@ -53,6 +59,8 @@ paths: post: summary: Update e-reader devices operationId: updateEReaderDevices + tags: + - Email requestBody: required: true content: @@ -63,7 +71,7 @@ paths: ereaderDevices: type: array items: - $ref: '#/components/schemas/EreaderDeviceObject' + $ref: '../objects/settings/EmailSettings.yaml#/components/schemas/EreaderDeviceObject' responses: 200: $ref: '#/components/responses/ereader200' @@ -73,6 +81,8 @@ paths: post: summary: Send ebook to device operationId: sendEBookToDevice + tags: + - Email requestBody: required: true content: @@ -83,7 +93,7 @@ paths: libraryItemId: $ref: '../objects/LibraryItem.yaml#/components/schemas/libraryItemId' deviceName: - $ref: '../objects/schemas/EmailSettings.yaml#/components/schemas/ereaderName' + $ref: '../objects/settings/EmailSettings.yaml#/components/schemas/ereaderName' responses: 200: description: Successful response diff --git a/docs/root.yaml b/docs/root.yaml index ee9d7b9c..e3caedaa 100644 --- a/docs/root.yaml +++ b/docs/root.yaml @@ -21,6 +21,14 @@ paths: $ref: './controllers/AuthorController.yaml#/paths/~1api~1authors~1{id}~1image' /api/authors/{id}/match: $ref: './controllers/AuthorController.yaml#/paths/~1api~1authors~1{id}~1match' + /api/emails: + $ref: './controllers/EmailController.yaml#/paths/~1api~1emails' + /api/emails/test: + $ref: './controllers/EmailController.yaml#/paths/~1api~1emails~1test' + /api/emails/ereader-devices: + $ref: './controllers/EmailController.yaml#/paths/~1api~1emails~1ereader-devices' + /api/emails/send-ebook-to-device: + $ref: './controllers/EmailController.yaml#/paths/~1api~1emails~1send-ebook-to-device' /api/libraries: $ref: './controllers/LibraryController.yaml#/paths/~1api~1libraries' /api/libraries/{id}: @@ -54,5 +62,7 @@ tags: description: Library endpoints - name: Series description: Series endpoints + - name: Email + description: Email endpoints - name: Notification description: Notifications endpoints From 6d14ed8a72237835987ea0d23941bbdd0693f049 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Thu, 4 Jul 2024 03:48:22 +0000 Subject: [PATCH 11/32] Update: bundled spec --- docs/openapi.json | 276 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 269 insertions(+), 7 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 48274bb3..826bf84b 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -29,6 +29,10 @@ "name": "Series", "description": "Series endpoints" }, + { + "name": "Email", + "description": "Email endpoints" + }, { "name": "Notification", "description": "Notifications endpoints" @@ -416,6 +420,132 @@ } } }, + "/api/emails": { + "get": { + "description": "Get email settings", + "operationId": "getEmailSettings", + "tags": [ + "Email" + ], + "responses": { + "200": { + "$ref": "#/components/responses/email200" + } + } + }, + "patch": { + "summary": "Update email settings", + "operationId": "updateEmailSettings", + "tags": [ + "Email" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EmailSettings" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/email200" + } + } + } + }, + "/api/emails/test": { + "post": { + "summary": "Send test email", + "operationId": "sendTestEmail", + "tags": [ + "Email" + ], + "responses": { + "200": { + "description": "Successful response" + } + } + } + }, + "/api/emails/ereader-devices": { + "post": { + "summary": "Update e-reader devices", + "operationId": "updateEReaderDevices", + "tags": [ + "Email" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ereaderDevices": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EreaderDeviceObject" + } + } + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/ereader200" + }, + "400": { + "description": "Invalid payload" + } + } + } + }, + "/api/emails/send-ebook-to-device": { + "post": { + "summary": "Send ebook to device", + "operationId": "sendEBookToDevice", + "tags": [ + "Email" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "libraryItemId": { + "$ref": "#/components/schemas/libraryItemId" + }, + "deviceName": { + "$ref": "#/components/schemas/ereaderName" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Successful response" + }, + "400": { + "description": "Invalid request" + }, + "403": { + "description": "Forbidden" + }, + "404": { + "description": "Not found" + } + } + } + }, "/api/libraries": { "get": { "operationId": "getLibraries", @@ -1114,12 +1244,6 @@ "application/json": { "schema": { "type": "object", - "required": [ - "eventName", - "urls", - "titleTemplate", - "bodyTemplate" - ], "properties": { "libraryId": { "$ref": "#/components/schemas/libraryIdNullable" @@ -1142,7 +1266,13 @@ "type": { "$ref": "#/components/schemas/notificationType" } - } + }, + "required": [ + "eventName", + "urls", + "titleTemplate", + "bodyTemplate" + ] } } } @@ -1942,6 +2072,110 @@ "example": "us", "default": "us" }, + "ereaderName": { + "type": "string", + "description": "The name of the e-reader device." + }, + "EreaderDeviceObject": { + "type": "object", + "description": "An e-reader device configured to receive EPUB through e-mail.", + "properties": { + "name": { + "$ref": "#/components/schemas/ereaderName" + }, + "email": { + "type": "string", + "description": "The email address associated with the e-reader device." + }, + "availabilityOption": { + "type": "string", + "description": "The availability option for the device.", + "enum": [ + "adminOrUp", + "userOrUp", + "guestOrUp", + "specificUsers" + ] + }, + "users": { + "type": "array", + "description": "List of specific users allowed to access the device.", + "items": { + "type": "string" + } + } + }, + "required": [ + "name", + "email", + "availabilityOption" + ] + }, + "EmailSettings": { + "type": "object", + "description": "The email settings configuration for the server. This includes the credentials to send e-books and an array of e-reader devices.", + "properties": { + "id": { + "type": "string", + "description": "The unique identifier for the email settings. Currently this is always `email-settings`", + "example": "email-settings" + }, + "host": { + "type": "string", + "description": "The SMTP host address.", + "nullable": true + }, + "port": { + "type": "integer", + "format": "int32", + "description": "The port number for the SMTP server.", + "example": 465 + }, + "secure": { + "type": "boolean", + "description": "Indicates if the connection should use SSL/TLS.", + "example": true + }, + "rejectUnauthorized": { + "type": "boolean", + "description": "Indicates if unauthorized SSL/TLS certificates should be rejected.", + "example": true + }, + "user": { + "type": "string", + "description": "The username for SMTP authentication.", + "nullable": true + }, + "pass": { + "type": "string", + "description": "The password for SMTP authentication.", + "nullable": true + }, + "testAddress": { + "type": "string", + "description": "The test email address used for sending test emails.", + "nullable": true + }, + "fromAddress": { + "type": "string", + "description": "The default \"from\" email address for outgoing emails.", + "nullable": true + }, + "ereaderDevices": { + "type": "array", + "description": "List of configured e-reader devices.", + "items": { + "$ref": "#/components/schemas/EreaderDeviceObject" + } + } + }, + "required": [ + "id", + "port", + "secure", + "ereaderDevices" + ] + }, "libraryName": { "description": "The name of the library.", "type": "string", @@ -2530,6 +2764,34 @@ } } }, + "email200": { + "description": "Successful response - Email", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EmailSettings" + } + } + } + }, + "ereader200": { + "description": "Successful response - Ereader", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ereaderDevices": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EreaderDeviceObject" + } + } + } + } + } + } + }, "library200": { "description": "Library found.", "content": { From fa1518cb1d257187dd2fa322023ba8e4a296c93a Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Thu, 4 Jul 2024 03:51:54 +0000 Subject: [PATCH 12/32] Fix: wrong settings path --- docs/controllers/EmailController.yaml | 2 +- docs/openapi.json | 2 +- docs/root.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/controllers/EmailController.yaml b/docs/controllers/EmailController.yaml index 3a7d990e..8fb03043 100644 --- a/docs/controllers/EmailController.yaml +++ b/docs/controllers/EmailController.yaml @@ -23,7 +23,7 @@ components: items: $ref: '../objects/settings/EmailSettings.yaml#/components/schemas/EreaderDeviceObject' paths: - /api/emails: + /api/emails/settings: get: description: Get email settings operationId: getEmailSettings diff --git a/docs/openapi.json b/docs/openapi.json index 826bf84b..2f79a422 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -420,7 +420,7 @@ } } }, - "/api/emails": { + "/api/emails/settings": { "get": { "description": "Get email settings", "operationId": "getEmailSettings", diff --git a/docs/root.yaml b/docs/root.yaml index e3caedaa..4ac22abc 100644 --- a/docs/root.yaml +++ b/docs/root.yaml @@ -21,8 +21,8 @@ paths: $ref: './controllers/AuthorController.yaml#/paths/~1api~1authors~1{id}~1image' /api/authors/{id}/match: $ref: './controllers/AuthorController.yaml#/paths/~1api~1authors~1{id}~1match' - /api/emails: - $ref: './controllers/EmailController.yaml#/paths/~1api~1emails' + /api/emails/settings: + $ref: './controllers/EmailController.yaml#/paths/~1api~1emails~1settings' /api/emails/test: $ref: './controllers/EmailController.yaml#/paths/~1api~1emails~1test' /api/emails/ereader-devices: From 37f62d22b624f49b31e7f3b1f7caeff0688deec5 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 5 Jul 2024 17:27:49 +0000 Subject: [PATCH 13/32] Add: report whether backup path environment is set --- server/controllers/BackupController.js | 3 ++- server/managers/BackupManager.js | 4 ++++ server/objects/settings/ServerSettings.js | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/server/controllers/BackupController.js b/server/controllers/BackupController.js index 30defb0e..69eae30a 100644 --- a/server/controllers/BackupController.js +++ b/server/controllers/BackupController.js @@ -10,7 +10,8 @@ class BackupController { getAll(req, res) { res.json({ backups: this.backupManager.backups.map((b) => b.toJSON()), - backupLocation: this.backupManager.backupPath + backupLocation: this.backupManager.backupPath, + backupEnvSet: this.backupManager.backupPathEnvSet }) } diff --git a/server/managers/BackupManager.js b/server/managers/BackupManager.js index 2749cc7c..ae79569c 100644 --- a/server/managers/BackupManager.js +++ b/server/managers/BackupManager.js @@ -29,6 +29,10 @@ class BackupManager { return global.ServerSettings.backupPath } + get backupPathEnvSet() { + return global.ServerSettings.backupPathEnvSet + } + get backupSchedule() { return global.ServerSettings.backupSchedule } diff --git a/server/objects/settings/ServerSettings.js b/server/objects/settings/ServerSettings.js index 6ade11a9..d5519c52 100644 --- a/server/objects/settings/ServerSettings.js +++ b/server/objects/settings/ServerSettings.js @@ -26,6 +26,7 @@ class ServerSettings { this.rateLimitLoginWindow = 10 * 60 * 1000 // 10 Minutes // Backups + this.backupPathEnvSet = false this.backupPath = Path.join(global.MetadataPath, 'backups') this.backupSchedule = false // If false then auto-backups are disabled this.backupsToKeep = 2 @@ -188,6 +189,8 @@ class ServerSettings { Logger.info(`[ServerSettings] Using backup path from environment variable ${process.env.BACKUP_PATH}`) this.backupPath = process.env.BACKUP_PATH } + + this.backupPathEnvSet = !!settings.process.env.BACKUP_PATH || false } toJSON() { @@ -206,6 +209,7 @@ class ServerSettings { rateLimitLoginRequests: this.rateLimitLoginRequests, rateLimitLoginWindow: this.rateLimitLoginWindow, backupPath: this.backupPath, + backupPathEnvSet: this.backupPathEnvSet, backupSchedule: this.backupSchedule, backupsToKeep: this.backupsToKeep, maxBackupSize: this.maxBackupSize, From d46de541d6712246eef89cb18b6bbdc92ea14424 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 5 Jul 2024 17:41:07 +0000 Subject: [PATCH 14/32] Fix: bad variable name --- server/objects/settings/ServerSettings.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/objects/settings/ServerSettings.js b/server/objects/settings/ServerSettings.js index d5519c52..e5700a07 100644 --- a/server/objects/settings/ServerSettings.js +++ b/server/objects/settings/ServerSettings.js @@ -190,7 +190,7 @@ class ServerSettings { this.backupPath = process.env.BACKUP_PATH } - this.backupPathEnvSet = !!settings.process.env.BACKUP_PATH || false + this.backupPathEnvSet = !!process.env.BACKUP_PATH || false } toJSON() { From 4d24817ced6b30d52b38be2a27ff7191330dc189 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 5 Jul 2024 17:44:49 +0000 Subject: [PATCH 15/32] Prevent editing backup path in web interface when env variable set --- client/pages/config/backups.vue | 10 +++++++--- client/strings/en-us.json | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/client/pages/config/backups.vue b/client/pages/config/backups.vue index dc14e4e1..578ae6bd 100644 --- a/client/pages/config/backups.vue +++ b/client/pages/config/backups.vue @@ -16,11 +16,11 @@
- - {{ $strings.ButtonSave }} + + {{ $strings.ButtonSave }} {{ $strings.ButtonCancel }} -

{{ $strings.MessageBackupsLocationEditNote }}

+

{{ canEditBackup ? $strings.MessageBackupsLocationEditNote : $strings.MessageBackupsLocationNoEditNote }}

@@ -115,6 +115,10 @@ export default { timeFormat() { return this.serverSettings.timeFormat }, + canEditBackup() { + // Prevent editing of backup path if an environement variable is set + return !this.serverSettings.backupPathEnvSet + }, scheduleDescription() { if (!this.cronExpression) return '' const parsed = this.$parseCronExpression(this.cronExpression) diff --git a/client/strings/en-us.json b/client/strings/en-us.json index 74064ad5..b9b9b5de 100644 --- a/client/strings/en-us.json +++ b/client/strings/en-us.json @@ -605,6 +605,7 @@ "MessageAppriseDescription": "To use this feature you will need to have an instance of Apprise API running or an api that will handle those same requests.
The Apprise API Url should be the full URL path to send the notification, e.g., if your API instance is served at http://192.168.1.1:8337 then you would put http://192.168.1.1:8337/notify.", "MessageBackupsDescription": "Backups include users, user progress, library item details, server settings, and images stored in /metadata/items & /metadata/authors. Backups do not include any files stored in your library folders.", "MessageBackupsLocationEditNote": "Note: Updating the backup location will not move or modify existing backups", + "MessageBackupsLocationNoEditNote": "Note: The backup location is set through an environment variable and cannot be changed here.", "MessageBackupsLocationPathEmpty": "Backup location path cannot be empty", "MessageBatchQuickMatchDescription": "Quick Match will attempt to add missing covers and metadata for the selected items. Enable the options below to allow Quick Match to overwrite existing covers and/or metadata.", "MessageBookshelfNoCollections": "You haven't made any collections yet", From a1688488e5d3ff324c9aa650b7caa252da53c58f Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 5 Jul 2024 17:58:42 +0000 Subject: [PATCH 16/32] Fix: name of `backupPathEnvSet` variable --- server/controllers/BackupController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/controllers/BackupController.js b/server/controllers/BackupController.js index 69eae30a..df33aa1d 100644 --- a/server/controllers/BackupController.js +++ b/server/controllers/BackupController.js @@ -11,7 +11,7 @@ class BackupController { res.json({ backups: this.backupManager.backups.map((b) => b.toJSON()), backupLocation: this.backupManager.backupPath, - backupEnvSet: this.backupManager.backupPathEnvSet + backupPathEnvSet: this.backupManager.backupPathEnvSet }) } From 7c0b4e35d758c46639a1dddd494d27423ec36caa Mon Sep 17 00:00:00 2001 From: advplyr Date: Fri, 5 Jul 2024 16:10:07 -0500 Subject: [PATCH 17/32] Update backups config page to use backupPathEnvSet returned from endpoint, remove from ServerConfig --- client/components/tables/BackupsTable.vue | 2 +- client/pages/config/backups.vue | 12 +++++++----- server/managers/BackupManager.js | 2 +- server/objects/settings/ServerSettings.js | 4 ---- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/client/components/tables/BackupsTable.vue b/client/components/tables/BackupsTable.vue index 380394bd..a8fbade6 100644 --- a/client/components/tables/BackupsTable.vue +++ b/client/components/tables/BackupsTable.vue @@ -171,7 +171,7 @@ export default { this.$axios .$get('/api/backups') .then((data) => { - this.$emit('loaded', data.backupLocation) + this.$emit('loaded', data) this.setBackups(data.backups || []) }) .catch((error) => { diff --git a/client/pages/config/backups.vue b/client/pages/config/backups.vue index 578ae6bd..7b64cbb2 100644 --- a/client/pages/config/backups.vue +++ b/client/pages/config/backups.vue @@ -92,6 +92,7 @@ export default { newServerSettings: {}, showCronBuilder: false, showEditBackupPath: false, + backupPathEnvSet: false, backupLocation: '', newBackupLocation: '', savingBackupPath: false @@ -116,8 +117,8 @@ export default { return this.serverSettings.timeFormat }, canEditBackup() { - // Prevent editing of backup path if an environement variable is set - return !this.serverSettings.backupPathEnvSet + // Prevent editing of backup path if an environment variable is set + return !this.backupPathEnvSet }, scheduleDescription() { if (!this.cronExpression) return '' @@ -131,9 +132,10 @@ export default { } }, methods: { - backupsLoaded(backupLocation) { - this.backupLocation = backupLocation - this.newBackupLocation = backupLocation + backupsLoaded(data) { + this.backupLocation = data.backupLocation + this.newBackupLocation = data.backupLocation + this.backupPathEnvSet = data.backupPathEnvSet }, cancelEditBackupPath() { this.newBackupLocation = this.backupLocation diff --git a/server/managers/BackupManager.js b/server/managers/BackupManager.js index ae79569c..88772c58 100644 --- a/server/managers/BackupManager.js +++ b/server/managers/BackupManager.js @@ -30,7 +30,7 @@ class BackupManager { } get backupPathEnvSet() { - return global.ServerSettings.backupPathEnvSet + return !!process.env.BACKUP_PATH } get backupSchedule() { diff --git a/server/objects/settings/ServerSettings.js b/server/objects/settings/ServerSettings.js index e5700a07..6ade11a9 100644 --- a/server/objects/settings/ServerSettings.js +++ b/server/objects/settings/ServerSettings.js @@ -26,7 +26,6 @@ class ServerSettings { this.rateLimitLoginWindow = 10 * 60 * 1000 // 10 Minutes // Backups - this.backupPathEnvSet = false this.backupPath = Path.join(global.MetadataPath, 'backups') this.backupSchedule = false // If false then auto-backups are disabled this.backupsToKeep = 2 @@ -189,8 +188,6 @@ class ServerSettings { Logger.info(`[ServerSettings] Using backup path from environment variable ${process.env.BACKUP_PATH}`) this.backupPath = process.env.BACKUP_PATH } - - this.backupPathEnvSet = !!process.env.BACKUP_PATH || false } toJSON() { @@ -209,7 +206,6 @@ class ServerSettings { rateLimitLoginRequests: this.rateLimitLoginRequests, rateLimitLoginWindow: this.rateLimitLoginWindow, backupPath: this.backupPath, - backupPathEnvSet: this.backupPathEnvSet, backupSchedule: this.backupSchedule, backupsToKeep: this.backupsToKeep, maxBackupSize: this.maxBackupSize, From 92aae736c4a7773c0a86b6f3ec141ea84d26db48 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Sat, 6 Jul 2024 15:57:20 +0000 Subject: [PATCH 18/32] Add: current pubdate --- client/plugins/version.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/plugins/version.js b/client/plugins/version.js index 75b948c8..593b29a5 100644 --- a/client/plugins/version.js +++ b/client/plugins/version.js @@ -49,11 +49,11 @@ export async function checkForUpdate() { } if (verObj.version == currVerObj.version) { + currVerObj.pubdate = new Date(release.published_at) currVerObj.changelog = release.body } }) } - }) if (!largestVer) { console.error('No valid version tags to compare with') @@ -65,6 +65,8 @@ export async function checkForUpdate() { latestVersion: largestVer.version, githubTagUrl: `https://github.com/advplyr/audiobookshelf/releases/tag/v${largestVer.version}`, currentVersion: currVerObj.version, + currentTagUrl: `https://github.com/advplyr/audiobookshelf/releases/tag/v${currVerObj.version}`, + currentVersionPubDate: currVerObj.pubdate, currentVersionChangelog: currVerObj.changelog } -} \ No newline at end of file +} From 3764ef14a9726d38e9d09c859754e107f90e603c Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Sat, 6 Jul 2024 16:21:06 +0000 Subject: [PATCH 19/32] Add: publish date of current version to modal --- client/components/app/SideRail.vue | 10 ++++++++-- client/components/modals/changelog/ViewModal.vue | 10 +++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/client/components/app/SideRail.vue b/client/components/app/SideRail.vue index 56207526..b3985fc2 100644 --- a/client/components/app/SideRail.vue +++ b/client/components/app/SideRail.vue @@ -121,7 +121,7 @@

{{ Source }}

- +
@@ -152,6 +152,9 @@ export default { paramId() { return this.$route.params ? this.$route.params.id || '' : '' }, + dateFormat() { + return this.$store.state.serverSettings.dateFormat + }, currentLibraryId() { return this.$store.state.libraries.currentLibraryId }, @@ -222,6 +225,9 @@ export default { currentVersionChangelog() { return this.versionData.currentVersionChangelog || 'No Changelog Available' }, + currentVersionPubDate() { + return `${this.$formatDate(this.versionData.currentVersionPubDate, this.dateFormat)}` || 'Unknown release date' + }, streamLibraryItem() { return this.$store.state.streamLibraryItem }, @@ -245,4 +251,4 @@ export default { #siderail-buttons-container.player-open { max-height: calc(100vh - 64px - 48px - 160px); } - \ No newline at end of file + diff --git a/client/components/modals/changelog/ViewModal.vue b/client/components/modals/changelog/ViewModal.vue index 98b28b21..05503c43 100644 --- a/client/components/modals/changelog/ViewModal.vue +++ b/client/components/modals/changelog/ViewModal.vue @@ -6,7 +6,7 @@
-

Changelog v{{ currentVersionNumber }}

+

Changelog v{{ currentVersionNumber }} ({{ currentVersionPubDate }})

@@ -19,6 +19,7 @@ export default { props: { value: Boolean, changelog: String, + currentPubDate: String, currentVersion: String }, watch: { @@ -43,6 +44,9 @@ export default { compiledMarkedown() { return marked.parse(this.changelog, { gfm: true, breaks: true }) }, + currentVersionPubDate() { + return this.currentPubDate + }, currentVersionNumber() { return this.currentVersion } @@ -57,7 +61,7 @@ export default { \ No newline at end of file + From 928b08067716350fd58963154e2818c5a0e83ca8 Mon Sep 17 00:00:00 2001 From: mikiher Date: Sat, 6 Jul 2024 19:43:55 +0300 Subject: [PATCH 20/32] Replace pkg with @yao-pkg/pkg and target node20 --- .github/workflows/integration-test.yml | 8 ++++---- build/linuxpackager | 2 +- package.json | 2 +- server/Server.js | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index df639ef7..3e499468 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches-ignore: - - 'dependabot/**' # Don't run dependabot branches, as they are already covered by pull requests + - 'dependabot/**' # Don't run dependabot branches, as they are already covered by pull requests jobs: build: @@ -18,8 +18,8 @@ jobs: with: node-version: 20 - - name: install pkg - run: npm install -g pkg + - name: install pkg (using yao-pkg fork for targetting node20) + run: npm install -g @yao-pkg/pkg - name: get client dependencies working-directory: client @@ -33,7 +33,7 @@ jobs: run: npm ci --only=production - name: build binary - run: pkg -t node18-linux-x64 -o audiobookshelf . + run: pkg -t node20-linux-x64 -o audiobookshelf . - name: run audiobookshelf run: | diff --git a/build/linuxpackager b/build/linuxpackager index 5f03a2e8..52a9beba 100755 --- a/build/linuxpackager +++ b/build/linuxpackager @@ -48,7 +48,7 @@ Description: $DESCRIPTION" echo "$controlfile" > dist/debian/DEBIAN/control; # Package debian -pkg -t node18-linux-x64 -o dist/debian/usr/share/audiobookshelf/audiobookshelf . +pkg -t node20-linux-x64 -o dist/debian/usr/share/audiobookshelf/audiobookshelf . fakeroot dpkg-deb -Zxz --build dist/debian diff --git a/package.json b/package.json index 7914c90d..1212bd7e 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "start": "node index.js", "client": "cd client && npm ci && npm run generate", "prod": "npm run client && npm ci && node prod.js", - "build-win": "npm run client && pkg -t node18-win-x64 -o ./dist/win/audiobookshelf -C GZip .", + "build-win": "npm run client && pkg -t node20-win-x64 -o ./dist/win/audiobookshelf -C GZip .", "build-linux": "build/linuxpackager", "docker": "docker buildx build --platform linux/amd64,linux/arm64 --push . -t advplyr/audiobookshelf", "docker-amd64-local": "docker buildx build --platform linux/amd64 --load . -t advplyr/audiobookshelf-amd64-local", diff --git a/server/Server.js b/server/Server.js index 9b164291..76d8466d 100644 --- a/server/Server.js +++ b/server/Server.js @@ -104,6 +104,7 @@ class Server { */ async init() { Logger.info('[Server] Init v' + version) + Logger.info('[Server] Node.js Version:', process.version) await this.playbackSessionManager.removeOrphanStreams() From e6b1acfb449012fea2af57576332910154778a14 Mon Sep 17 00:00:00 2001 From: advplyr Date: Sat, 6 Jul 2024 16:00:48 -0500 Subject: [PATCH 21/32] Remove tone scripts & references, rename tone-object endpoint, remove node-tone dependency, remove TONE_PATH env --- .devcontainer/Dockerfile | 3 - Dockerfile | 2 - build/debian/DEBIAN/preinst | 16 +- client/pages/audiobook/_id/manage.vue | 24 ++- package-lock.json | 6 - package.json | 1 - readme.md | 2 +- server/controllers/LibraryItemController.js | 6 +- server/managers/AudioMetadataManager.js | 2 +- server/objects/metadata/AudioMetaTags.js | 26 +-- server/routers/ApiRouter.js | 2 +- server/scanner/MediaProbeData.js | 12 +- server/utils/toneHelpers.js | 113 ------------- server/utils/toneProber.js | 173 -------------------- 14 files changed, 34 insertions(+), 354 deletions(-) delete mode 100644 server/utils/toneHelpers.js delete mode 100644 server/utils/toneProber.js diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index ddd7c5f3..16ae302c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -10,6 +10,3 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ curl tzdata ffmpeg && \ rm -rf /var/lib/apt/lists/* - -# Move tone executable to appropriate directory -COPY --from=sandreas/tone:v0.1.5 /usr/local/bin/tone /usr/local/bin/ diff --git a/Dockerfile b/Dockerfile index 97bb4732..fe68b304 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,6 @@ RUN npm ci && npm cache clean --force RUN npm run generate ### STAGE 1: Build server ### -FROM sandreas/tone:v0.1.5 AS tone FROM node:20-alpine ENV NODE_ENV=production @@ -21,7 +20,6 @@ RUN apk update && \ g++ \ tini -COPY --from=tone /usr/local/bin/tone /usr/local/bin/ COPY --from=build /client/dist /client/dist COPY index.js package* / COPY server server diff --git a/build/debian/DEBIAN/preinst b/build/debian/DEBIAN/preinst index f43f2683..c4692ed3 100644 --- a/build/debian/DEBIAN/preinst +++ b/build/debian/DEBIAN/preinst @@ -50,7 +50,6 @@ install_ffmpeg() { echo "Starting FFMPEG Install" WGET="wget https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-amd64-static.tar.xz --output-document=ffmpeg-git-amd64-static.tar.xz" - WGET_TONE="wget https://github.com/sandreas/tone/releases/download/v0.1.5/tone-0.1.5-linux-x64.tar.gz --output-document=tone-0.1.5-linux-x64.tar.gz" if ! cd "$FFMPEG_INSTALL_DIR"; then echo "Creating ffmpeg install dir at $FFMPEG_INSTALL_DIR" @@ -63,13 +62,7 @@ install_ffmpeg() { tar xvf ffmpeg-git-amd64-static.tar.xz --strip-components=1 --no-same-owner rm ffmpeg-git-amd64-static.tar.xz - # Temp downloading tone library to the ffmpeg dir - echo "Getting tone.." - $WGET_TONE - tar xvf tone-0.1.5-linux-x64.tar.gz --strip-components=1 --no-same-owner - rm tone-0.1.5-linux-x64.tar.gz - - echo "Good to go on Ffmpeg (& tone)... hopefully" + echo "Good to go on Ffmpeg... hopefully" } setup_config() { @@ -77,12 +70,6 @@ setup_config() { echo "Existing config found." cat $CONFIG_PATH - # TONE_PATH variable added in 2.1.6, if it doesnt exist then add it - if ! grep -q "TONE_PATH" "$CONFIG_PATH"; then - echo "Adding TONE_PATH to existing config" - echo "TONE_PATH=$FFMPEG_INSTALL_DIR/tone" >> "$CONFIG_PATH" - fi - else if [ ! -d "$DEFAULT_DATA_DIR" ]; then @@ -98,7 +85,6 @@ setup_config() { CONFIG_PATH=$DEFAULT_DATA_DIR/config FFMPEG_PATH=$FFMPEG_INSTALL_DIR/ffmpeg FFPROBE_PATH=$FFMPEG_INSTALL_DIR/ffprobe -TONE_PATH=$FFMPEG_INSTALL_DIR/tone PORT=$DEFAULT_PORT HOST=$DEFAULT_HOST" diff --git a/client/pages/audiobook/_id/manage.vue b/client/pages/audiobook/_id/manage.vue index 03c214b4..8863fd5b 100644 --- a/client/pages/audiobook/_id/manage.vue +++ b/client/pages/audiobook/_id/manage.vue @@ -11,10 +11,9 @@
-
+
-

{{ $strings.HeaderMetadataToEmbed }}

-

audiobookshelf uses tone to write metadata.

+

{{ $strings.HeaderMetadataToEmbed }}

@@ -26,7 +25,7 @@
{{ $strings.LabelValue }}
-
-

Changelog v{{ currentVersionNumber }} ({{ currentVersionPubDate }})

+

+ Changelog v{{ currentVersionNumber }} ({{ currentVersionPubDate }}) +

@@ -18,18 +20,9 @@ import { marked } from '@/static/libs/marked/index.js' export default { props: { value: Boolean, - changelog: String, - currentPubDate: String, - currentVersion: String - }, - watch: { - show: { - immediate: true, - handler(newVal) { - if (newVal) { - this.init() - } - } + versionData: { + type: Object, + default: () => {} } }, computed: { @@ -41,19 +34,27 @@ export default { this.$emit('input', val) } }, + dateFormat() { + return this.$store.state.serverSettings.dateFormat + }, + changelog() { + return this.versionData?.currentVersionChangelog || 'No Changelog Available' + }, compiledMarkedown() { return marked.parse(this.changelog, { gfm: true, breaks: true }) }, currentVersionPubDate() { - return this.currentPubDate + if (!this.versionData?.currentVersionPubDate) return 'Unknown release date' + return `${this.$formatDate(this.versionData.currentVersionPubDate, this.dateFormat)}` + }, + currentTagUrl() { + return this.versionData?.currentTagUrl }, currentVersionNumber() { - return this.currentVersion + return this.$config.version } }, - methods: { - init() {} - }, + methods: {}, mounted() {} } From 2f2ec2ec1f3ae7ef5b8dda5470641ce9c622be12 Mon Sep 17 00:00:00 2001 From: advplyr Date: Sun, 7 Jul 2024 15:51:50 -0500 Subject: [PATCH 26/32] Add book item more menu item for Share, restrict share to admin or up, add admin socket events for open/close shares --- client/components/app/LazyBookshelf.vue | 28 ++++++++++++++++++ client/components/cards/LazyBookCard.vue | 10 +++++++ .../controls/LibraryFilterSelect.vue | 16 ++++++---- client/components/modals/ShareModal.vue | 22 ++++++-------- client/layouts/default.vue | 3 +- client/pages/item/_id/index.vue | 29 ++++++++++++------- client/store/globals.js | 9 ++++++ server/controllers/LibraryItemController.js | 2 +- server/managers/ShareManager.js | 8 +++++ server/models/LibraryItem.js | 4 +-- .../utils/queries/libraryItemsBookFilters.js | 8 ++--- 11 files changed, 102 insertions(+), 37 deletions(-) diff --git a/client/components/app/LazyBookshelf.vue b/client/components/app/LazyBookshelf.vue index fa47ce99..6137cdca 100644 --- a/client/components/app/LazyBookshelf.vue +++ b/client/components/app/LazyBookshelf.vue @@ -601,6 +601,30 @@ export default { this.executeRebuild() } }, + shareOpen(mediaItemShare) { + if (this.entityName === 'items' || this.entityName === 'series-books') { + var indexOf = this.entities.findIndex((ent) => ent?.media?.id === mediaItemShare.mediaItemId) + if (indexOf >= 0) { + if (this.entityComponentRefs[indexOf]) { + const libraryItem = { ...this.entityComponentRefs[indexOf].libraryItem } + libraryItem.mediaItemShare = mediaItemShare + this.entityComponentRefs[indexOf].setEntity?.(libraryItem) + } + } + } + }, + shareClosed(mediaItemShare) { + if (this.entityName === 'items' || this.entityName === 'series-books') { + var indexOf = this.entities.findIndex((ent) => ent?.media?.id === mediaItemShare.mediaItemId) + if (indexOf >= 0) { + if (this.entityComponentRefs[indexOf]) { + const libraryItem = { ...this.entityComponentRefs[indexOf].libraryItem } + libraryItem.mediaItemShare = null + this.entityComponentRefs[indexOf].setEntity?.(libraryItem) + } + } + } + }, updatePagesLoaded() { let numPages = Math.ceil(this.totalEntities / this.booksPerFetch) for (let page = 0; page < numPages; page++) { @@ -703,6 +727,8 @@ export default { this.$root.socket.on('playlist_added', this.playlistAdded) this.$root.socket.on('playlist_updated', this.playlistUpdated) this.$root.socket.on('playlist_removed', this.playlistRemoved) + this.$root.socket.on('share_open', this.shareOpen) + this.$root.socket.on('share_closed', this.shareClosed) } else { console.error('Bookshelf - Socket not initialized') } @@ -730,6 +756,8 @@ export default { this.$root.socket.off('playlist_added', this.playlistAdded) this.$root.socket.off('playlist_updated', this.playlistUpdated) this.$root.socket.off('playlist_removed', this.playlistRemoved) + this.$root.socket.off('share_open', this.shareOpen) + this.$root.socket.off('share_closed', this.shareClosed) } else { console.error('Bookshelf - Socket not initialized') } diff --git a/client/components/cards/LazyBookCard.vue b/client/components/cards/LazyBookCard.vue index 5bb406e4..37af853d 100644 --- a/client/components/cards/LazyBookCard.vue +++ b/client/components/cards/LazyBookCard.vue @@ -528,6 +528,12 @@ export default { func: 'openPlaylists', text: this.$strings.LabelAddToPlaylist }) + if (this.userIsAdminOrUp) { + items.push({ + func: 'openShare', + text: this.$strings.LabelShare + }) + } } if (this.ebookFormat && this.store.state.libraries.ereaderDevices?.length) { items.push({ @@ -880,6 +886,10 @@ export default { this.store.commit('globals/setSelectedPlaylistItems', [{ libraryItem: this.libraryItem, episode: this.recentEpisode }]) this.store.commit('globals/setShowPlaylistsModal', true) }, + openShare() { + this.store.commit('setSelectedLibraryItem', this.libraryItem) + this.store.commit('globals/setShareModal', this.mediaItemShare) + }, deleteLibraryItem() { const payload = { message: this.$strings.MessageConfirmDeleteLibraryItem, diff --git a/client/components/controls/LibraryFilterSelect.vue b/client/components/controls/LibraryFilterSelect.vue index e3cb0d16..d8951321 100644 --- a/client/components/controls/LibraryFilterSelect.vue +++ b/client/components/controls/LibraryFilterSelect.vue @@ -89,6 +89,9 @@ export default { this.$emit('input', val) } }, + userIsAdminOrUp() { + return this.$store.getters['user/getIsAdminOrUp'] + }, libraryMediaType() { return this.$store.getters['libraries/getCurrentLibraryMediaType'] }, @@ -148,7 +151,7 @@ export default { ] }, bookItems() { - return [ + const items = [ { text: this.$strings.LabelAll, value: 'all' @@ -229,13 +232,16 @@ export default { text: this.$strings.LabelRSSFeedOpen, value: 'feed-open', sublist: false - }, - { + } + ] + if (this.userIsAdminOrUp) { + items.push({ text: this.$strings.LabelShareOpen, value: 'share-open', sublist: false - } - ] + }) + } + return items }, podcastItems() { return [ diff --git a/client/components/modals/ShareModal.vue b/client/components/modals/ShareModal.vue index dc781a74..42f9c29a 100644 --- a/client/components/modals/ShareModal.vue +++ b/client/components/modals/ShareModal.vue @@ -53,17 +53,7 @@