diff --git a/client/layouts/default.vue b/client/layouts/default.vue
index 25d6b23e..e0fed353 100644
--- a/client/layouts/default.vue
+++ b/client/layouts/default.vue
@@ -104,11 +104,23 @@ export default {
if (payload.serverSettings) {
this.$store.commit('setServerSettings', payload.serverSettings)
}
+
+ // Start scans currently running
if (payload.librariesScanning) {
payload.librariesScanning.forEach((libraryScan) => {
this.scanStart(libraryScan)
})
}
+
+ // Remove any current scans that are no longer running
+ var currentScans = [...this.$store.state.scanners.libraryScans]
+ currentScans.forEach((ls) => {
+ if (!payload.librariesScanning || !payload.librariesScanning.find((_ls) => _ls.id === ls.id)) {
+ this.$toast.dismiss(ls.toastId)
+ this.$store.commit('scanners/remove', ls)
+ }
+ })
+
if (payload.backups && payload.backups.length) {
this.$store.commit('setBackups', payload.backups)
}
@@ -152,6 +164,16 @@ export default {
}
this.$store.commit('audiobooks/remove', audiobook)
},
+ audiobooksAdded(audiobooks) {
+ audiobooks.forEach((ab) => {
+ this.$store.commit('audiobooks/addUpdate', ab)
+ })
+ },
+ audiobooksUpdated(audiobooks) {
+ audiobooks.forEach((ab) => {
+ this.$store.commit('audiobooks/addUpdate', ab)
+ })
+ },
libraryAdded(library) {
this.$store.commit('libraries/addUpdate', library)
},
@@ -162,6 +184,8 @@ export default {
this.$store.commit('libraries/remove', library)
},
scanComplete(data) {
+ console.log('Scan complete received', data)
+
var message = `Scan "${data.name}" complete!`
if (data.results) {
var scanResultMsgs = []
@@ -337,6 +361,8 @@ export default {
this.socket.on('audiobook_updated', this.audiobookUpdated)
this.socket.on('audiobook_added', this.audiobookAdded)
this.socket.on('audiobook_removed', this.audiobookRemoved)
+ this.socket.on('audiobooks_updated', this.audiobooksUpdated)
+ this.socket.on('audiobooks_added', this.audiobooksAdded)
// Library Listeners
this.socket.on('library_updated', this.libraryUpdated)
diff --git a/client/pages/config/index.vue b/client/pages/config/index.vue
index 9b0ff432..49fb8c5d 100644
--- a/client/pages/config/index.vue
+++ b/client/pages/config/index.vue
@@ -21,6 +21,20 @@
+
+
+
+ Scanner prefer audio metadata info_outlined
+
+
+
+
+
+
+ Scanner prefer OPF metadata info_outlined
+
+
+
@@ -83,6 +97,12 @@ export default {
}
},
computed: {
+ scannerPreferAudioMetaTooltip() {
+ return 'Audio file ID3 meta tags will be used for book details over folder & filenames'
+ },
+ scannerPreferOpfMetaTooltip() {
+ return 'OPF file metadata will be used for book details over folder & filenames'
+ },
saveMetadataTooltip() {
return 'This will write a "metadata.nfo" file in all of your audiobook directories.'
},
@@ -127,6 +147,16 @@ export default {
scannerParseSubtitle: !!val
})
},
+ updateScannerPreferAudioMeta(val) {
+ this.updateServerSettings({
+ scannerPreferAudioMetadata: !!val
+ })
+ },
+ updateScannerPreferOpfMeta(val) {
+ this.updateServerSettings({
+ scannerPreferOpfMetadata: !!val
+ })
+ },
updateServerSettings(payload) {
this.updatingServerSettings = true
this.$store
@@ -144,7 +174,6 @@ export default {
this.newServerSettings = this.serverSettings ? { ...this.serverSettings } : {}
this.storeCoversInAudiobookDir = this.newServerSettings.coverDestination === this.$constants.CoverDestination.AUDIOBOOK
- this.storeCoversInAudiobookDir = this.newServerSettings.coverDestination === this.$constants.CoverDestination.AUDIOBOOK
},
resetAudiobooks() {
if (confirm('WARNING! This action will remove all audiobooks from the database including any updates or matches you have made. This does not do anything to your actual files. Shall we continue?')) {
diff --git a/server/Scanner.js b/server/Scanner.js
index 60e2c015..cf2e057e 100644
--- a/server/Scanner.js
+++ b/server/Scanner.js
@@ -196,7 +196,7 @@ class Scanner {
// Sync other files (all files that are not audio files) - Updates cover path
var hasOtherFileUpdates = false
- var otherFilesUpdated = await existingAudiobook.syncOtherFiles(audiobookData.otherFiles, this.MetadataPath, forceAudioFileScan)
+ var otherFilesUpdated = await existingAudiobook.syncOtherFiles(audiobookData.otherFiles, this.MetadataPath, false, forceAudioFileScan)
if (otherFilesUpdated) {
hasOtherFileUpdates = true
}
@@ -250,7 +250,7 @@ class Scanner {
var hasUpdates = hasOtherFileUpdates || hasUpdatedIno || hasUpdatedLibraryOrFolder || removedAudioFiles.length || removedAudioTracks.length || newAudioFiles.length || hasUpdatedAudioFiles
// Check that audio tracks are in sequential order with no gaps
- if (existingAudiobook.checkUpdateMissingParts()) {
+ if (existingAudiobook.checkUpdateMissingTracks()) {
Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
hasUpdates = true
}
@@ -299,7 +299,7 @@ class Scanner {
}
// Look for desc.txt and reader.txt and update
- await audiobook.saveDataFromTextFiles()
+ await audiobook.saveDataFromTextFiles(false)
// Extract embedded cover art if cover is not already in directory
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
@@ -314,7 +314,7 @@ class Scanner {
audiobook.setDetailsFromFileMetadata()
// Check for gaps in track numbers
- audiobook.checkUpdateMissingParts()
+ audiobook.checkUpdateMissingTracks()
// Set chapters from audio files
audiobook.setChapters()
@@ -671,11 +671,6 @@ class Scanner {
var folder = library.getFolderById(folderId)
if (!folder) {
Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`)
-
- Logger.debug(`Looking at folders in library "${library.name}" for folderid ${folderId}`)
- library.folders.forEach((fold) => {
- Logger.debug(`Folder "${fold.id}" "${fold.fullPath}"`)
- })
continue;
}
diff --git a/server/Server.js b/server/Server.js
index 7a6c6710..8d76dd6e 100644
--- a/server/Server.js
+++ b/server/Server.js
@@ -311,19 +311,18 @@ class Server {
async filesChanged(fileUpdates) {
Logger.info('[Server]', fileUpdates.length, 'Files Changed')
- await this.scanner.filesChanged(fileUpdates)
- // Logger.debug('[Server] Files changed result', result)
+ await this.scanner2.scanFilesChanged(fileUpdates)
}
- async scan(libraryId, forceAudioFileScan = false) {
+ async scan(libraryId) {
Logger.info('[Server] Starting Scan')
- // await this.scanner2.scan(libraryId)
- await this.scanner(libraryId, forceAudioFileScan)
+ await this.scanner2.scan(libraryId)
+ // await this.scanner.scan(libraryId)
Logger.info('[Server] Scan complete')
}
async scanAudiobook(socket, audiobookId) {
- var result = await this.scanner.scanAudiobookById(audiobookId)
+ var result = await this.scanner2.scanAudiobookById(audiobookId)
var scanResultName = ''
for (const key in ScanResult) {
if (ScanResult[key] === result) {
@@ -335,7 +334,7 @@ class Server {
cancelScan(id) {
Logger.debug('[Server] Cancel scan', id)
- this.scanner.cancelLibraryScan[id] = true
+ this.scanner2.cancelLibraryScan[id] = true
}
// Generates an NFO metadata file, if no audiobookId is passed then all audiobooks are done
@@ -624,7 +623,7 @@ class Server {
configPath: this.ConfigPath,
user: client.user.toJSONForBrowser(),
stream: client.stream || null,
- librariesScanning: this.scanner.librariesScanning,
+ librariesScanning: this.scanner2.librariesScanning,
backups: (this.backupManager.backups || []).map(b => b.toJSON())
}
if (user.type === 'root') {
diff --git a/server/Watcher.js b/server/Watcher.js
index 0de3bc29..c9fede87 100644
--- a/server/Watcher.js
+++ b/server/Watcher.js
@@ -24,8 +24,11 @@ class FolderWatcher extends EventEmitter {
Logger.warn('[Watcher] Already watching library', library.name)
return
}
- Logger.info(`[Watcher] Initializing watcher for "${library.name}"..`)
+ Logger.info(`[Watcher] Initializing watcher for "${library.name}".`)
var folderPaths = library.folderPaths
+ folderPaths.forEach((fp) => {
+ Logger.debug(`[Watcher] Init watcher for library folder path "${fp}"`)
+ })
var watcher = new Watcher(folderPaths, {
ignored: /(^|[\/\\])\../, // ignore dotfiles
renameDetection: true,
@@ -48,6 +51,8 @@ class FolderWatcher extends EventEmitter {
Logger.error(`[Watcher] ${error}`)
}).on('ready', () => {
Logger.info(`[Watcher] "${library.name}" Ready`)
+ }).on('close', () => {
+ Logger.debug(`[Watcher] "${library.name}" Closed`)
})
this.libraryWatchers.push({
diff --git a/server/objects/AudioFile.js b/server/objects/AudioFile.js
index da7be888..b448d217 100644
--- a/server/objects/AudioFile.js
+++ b/server/objects/AudioFile.js
@@ -165,9 +165,9 @@ class AudioFile {
this.fullPath = fileData.fullPath
this.addedAt = Date.now()
- this.trackNumFromMeta = fileData.trackNumFromMeta || null
- this.trackNumFromFilename = fileData.trackNumFromFilename || null
- this.cdNumFromFilename = fileData.cdNumFromFilename || null
+ this.trackNumFromMeta = fileData.trackNumFromMeta
+ this.trackNumFromFilename = fileData.trackNumFromFilename
+ this.cdNumFromFilename = fileData.cdNumFromFilename
this.format = probeData.format
this.duration = probeData.duration
@@ -180,15 +180,13 @@ class AudioFile {
this.channelLayout = probeData.channelLayout
this.chapters = probeData.chapters || []
this.metadata = probeData.audioFileMetadata
+ this.embeddedCoverArt = probeData.embeddedCoverArt
}
- validateTrackIndex(isSingleTrack) {
+ validateTrackIndex() {
var numFromMeta = isNullOrNaN(this.trackNumFromMeta) ? null : Number(this.trackNumFromMeta)
var numFromFilename = isNullOrNaN(this.trackNumFromFilename) ? null : Number(this.trackNumFromFilename)
- if (isSingleTrack) { // Single audio track audiobook only use metadata tag and default to 1
- return numFromMeta ? numFromMeta : 1
- }
if (numFromMeta !== null) return numFromMeta
if (numFromFilename !== null) return numFromFilename
@@ -284,5 +282,33 @@ class AudioFile {
})
return hasUpdates
}
+
+ updateFromScan(scannedAudioFile) {
+ var hasUpdated = false
+
+ var newjson = scannedAudioFile.toJSON()
+ if (this.manuallyVerified) newjson.manuallyVerified = true
+ if (this.exclude) newjson.exclude = true
+ newjson.addedAt = this.addedAt
+
+ for (const key in newjson) {
+ if (key === 'metadata') {
+ if (!this.metadata || !this.metadata.isEqual(scannedAudioFile.metadata)) {
+ this.metadata = scannedAudioFile.metadata
+ hasUpdated = true
+ // console.log('metadata updated for audio file')
+ }
+ } else if (key === 'chapters') {
+ if (this.syncChapters(newjson.chapters || [])) {
+ hasUpdated = true
+ }
+ } else if (this[key] !== newjson[key]) {
+ this[key] = newjson[key]
+ hasUpdated = true
+ // console.log('key', key, 'updated', this[key], newjson[key])
+ }
+ }
+ return hasUpdated
+ }
}
module.exports = AudioFile
\ No newline at end of file
diff --git a/server/objects/AudioFileMetadata.js b/server/objects/AudioFileMetadata.js
index bd47538c..3a82ceab 100644
--- a/server/objects/AudioFileMetadata.js
+++ b/server/objects/AudioFileMetadata.js
@@ -101,5 +101,13 @@ class AudioFileMetadata {
}
return hasUpdates
}
+
+ isEqual(audioFileMetadata) {
+ if (!audioFileMetadata || !audioFileMetadata.toJSON) return false
+ for (const key in audioFileMetadata.toJSON()) {
+ if (audioFileMetadata[key] !== this[key]) return false
+ }
+ return true
+ }
}
module.exports = AudioFileMetadata
\ No newline at end of file
diff --git a/server/objects/Audiobook.js b/server/objects/Audiobook.js
index 9ffbe455..6a03b1f9 100644
--- a/server/objects/Audiobook.js
+++ b/server/objects/Audiobook.js
@@ -353,6 +353,11 @@ class Audiobook {
this.lastUpdate = Date.now()
}
+ setInvalid() {
+ this.isInvalid = true
+ this.lastUpdate = Date.now()
+ }
+
setBook(data) {
// Use first image file as cover
if (this.otherFiles && this.otherFiles.length) {
@@ -400,6 +405,11 @@ class Audiobook {
}
}
+ updateAudioFile(updatedAudioFile) {
+ var audioFile = this.audioFiles.find(af => af.ino === updatedAudioFile.ino)
+ return audioFile.updateFromScan(updatedAudioFile)
+ }
+
addOtherFile(fileData) {
var file = new AudiobookFile()
file.setData(fileData)
@@ -437,8 +447,8 @@ class Audiobook {
return this.book.updateCover(cover, coverFullPath)
}
- checkHasTrackNum(trackNum) {
- return this.tracks.find(t => t.index === trackNum)
+ checkHasTrackNum(trackNum, excludeIno) {
+ return this._audioFiles.find(t => t.index === trackNum && t.ino !== excludeIno)
}
updateAudioTracks(orderedFileData) {
@@ -473,6 +483,7 @@ class Audiobook {
}
})
this.setChapters()
+ this.checkUpdateMissingTracks()
this.lastUpdate = Date.now()
}
@@ -486,7 +497,7 @@ class Audiobook {
this.audioFiles = this.audioFiles.filter(f => f.ino !== track.ino)
}
- checkUpdateMissingParts() {
+ checkUpdateMissingTracks() {
var currMissingParts = (this.missingParts || []).join(',') || ''
var current_index = 1
@@ -515,13 +526,14 @@ class Audiobook {
}
// On scan check other files found with other files saved
- async syncOtherFiles(newOtherFiles, metadataPath, forceRescan = false) {
+ async syncOtherFiles(newOtherFiles, metadataPath, opfMetadataOverrideDetails, forceRescan = false) {
var hasUpdates = false
var currOtherFileNum = this.otherFiles.length
- var alreadyHasDescTxt = this.otherFiles.find(of => of.filename === 'desc.txt')
- var alreadyHasReaderTxt = this.otherFiles.find(of => of.filename === 'reader.txt')
+ var otherFilenamesAlreadyInBook = this.otherFiles.map(ofile => ofile.filename)
+ var alreadyHasDescTxt = otherFilenamesAlreadyInBook.includes('desc.txt')
+ var alreadyHasReaderTxt = otherFilenamesAlreadyInBook.includes('reader.txt')
var newOtherFilePaths = newOtherFiles.map(f => f.path)
this.otherFiles = this.otherFiles.filter(f => newOtherFilePaths.includes(f.path))
@@ -553,21 +565,22 @@ class Audiobook {
}
}
+ // If OPF file and was not already there
var metadataOpf = newOtherFiles.find(file => file.ext === '.opf' || file.filename === 'metadata.xml')
- if (metadataOpf) {
+ if (metadataOpf && (!otherFilenamesAlreadyInBook.includes(metadataOpf.filename) || opfMetadataOverrideDetails)) {
var xmlText = await readTextFile(metadataOpf.fullPath)
if (xmlText) {
var opfMetadata = await parseOpfMetadataXML(xmlText)
- Logger.debug(`[Audiobook] Sync Other File "${metadataOpf.filename}" parsed:`, opfMetadata)
+ // Logger.debug(`[Audiobook] Sync Other File "${metadataOpf.filename}" parsed:`, opfMetadata)
if (opfMetadata) {
const bookUpdatePayload = {}
for (const key in opfMetadata) {
// Add genres only if genres are empty
if (key === 'genres') {
- if (opfMetadata.genres.length && !this.book._genres.length) {
+ if (opfMetadata.genres.length && (!this.book._genres.length || opfMetadataOverrideDetails)) {
bookUpdatePayload[key] = opfMetadata.genres
}
- } else if (opfMetadata[key] && !this.book[key]) {
+ } else if (opfMetadata[key] && (!this.book[key] || opfMetadataOverrideDetails)) {
bookUpdatePayload[key] = opfMetadata[key]
}
}
@@ -789,7 +802,7 @@ class Audiobook {
}
// Look for desc.txt and reader.txt and update details if found
- async saveDataFromTextFiles() {
+ async saveDataFromTextFiles(opfMetadataOverrideDetails) {
var bookUpdatePayload = {}
var descriptionText = await this.fetchTextFromTextFile('desc.txt')
if (descriptionText) {
@@ -807,15 +820,15 @@ class Audiobook {
var xmlText = await readTextFile(metadataOpf.fullPath)
if (xmlText) {
var opfMetadata = await parseOpfMetadataXML(xmlText)
- Logger.debug(`[Audiobook] "${this.title}" found "${metadataOpf.filename}" parsed:`, opfMetadata)
+ // Logger.debug(`[Audiobook] "${this.title}" found "${metadataOpf.filename}" parsed:`, opfMetadata)
if (opfMetadata) {
for (const key in opfMetadata) {
// Add genres only if genres are empty
if (key === 'genres') {
- if (opfMetadata.genres.length && !this.book._genres.length) {
+ if (opfMetadata.genres.length && (!this.book._genres.length || opfMetadataOverrideDetails)) {
bookUpdatePayload[key] = opfMetadata.genres
}
- } else if (opfMetadata[key] && !this.book[key] && !bookUpdatePayload[key]) {
+ } else if (opfMetadata[key] && ((!this.book[key] && !bookUpdatePayload[key]) || opfMetadataOverrideDetails)) {
bookUpdatePayload[key] = opfMetadata[key]
}
}
@@ -836,10 +849,10 @@ class Audiobook {
}
// Audio file metadata tags map to book details (will not overwrite)
- setDetailsFromFileMetadata() {
+ setDetailsFromFileMetadata(overrideExistingDetails = false) {
if (!this.audioFiles.length) return false
var audioFile = this.audioFiles[0]
- return this.book.setDetailsFromFileMetadata(audioFile.metadata)
+ return this.book.setDetailsFromFileMetadata(audioFile.metadata, overrideExistingDetails)
}
// Returns null if file not found, true if file was updated, false if up to date
@@ -884,9 +897,15 @@ class Audiobook {
return hasUpdated
}
- checkScanData(dataFound) {
+ checkScanData(dataFound, version) {
var hasUpdated = false
+ if (this.isMissing) {
+ // Audiobook no longer missing
+ this.isMissing = false
+ hasUpdated = true
+ }
+
if (dataFound.ino !== this.ino) {
this.ino = dataFound.ino
hasUpdated = true
@@ -916,7 +935,7 @@ class Audiobook {
var audioFileFoundCheck = this.checkFileFound(af, true)
if (audioFileFoundCheck === null) {
newAudioFileData.push(af)
- } else if (audioFileFoundCheck === true) {
+ } else if (audioFileFoundCheck) {
hasUpdated = true
}
})
@@ -925,7 +944,7 @@ class Audiobook {
var fileFoundCheck = this.checkFileFound(otherFileData, false)
if (fileFoundCheck === null) {
newOtherFileData.push(otherFileData)
- } else if (fileFoundCheck === true) {
+ } else if (fileFoundCheck) {
hasUpdated = true
}
})
@@ -933,7 +952,7 @@ class Audiobook {
const audioFilesRemoved = []
const otherFilesRemoved = []
- // inodes will all be up to date at this point
+ // Remove audio files not found (inodes will all be up to date at this point)
this.audioFiles = this.audioFiles.filter(af => {
if (!dataFound.audioFiles.find(_af => _af.ino === af.ino)) {
audioFilesRemoved.push(af.toJSON())
@@ -946,10 +965,11 @@ class Audiobook {
if (audioFilesRemoved.length) {
const audioFilesRemovedInodes = audioFilesRemoved.map(afr => afr.ino)
this.tracks = this.tracks.filter(t => !audioFilesRemovedInodes.includes(t.ino))
- this.checkUpdateMissingParts()
+ this.checkUpdateMissingTracks()
hasUpdated = true
}
+ // Remove other files not found
this.otherFiles = this.otherFiles.filter(otherFile => {
if (!dataFound.otherFiles.find(_otherFile => _otherFile.ino === otherFile.ino)) {
otherFilesRemoved.push(otherFile.toJSON())
@@ -969,6 +989,15 @@ class Audiobook {
hasUpdated = true
}
+ // Check if invalid (has no audio files or ebooks)
+ if (!this.audioFilesToInclude.length && !this.ebooks.length && !newAudioFileData.length && !newOtherFileData.length) {
+ this.isInvalid = true
+ }
+
+ if (hasUpdated) {
+ this.setLastScan(version)
+ }
+
return {
updated: hasUpdated,
newAudioFileData,
diff --git a/server/objects/Book.js b/server/objects/Book.js
index 63fbf2e9..97e6bb3f 100644
--- a/server/objects/Book.js
+++ b/server/objects/Book.js
@@ -43,6 +43,7 @@ class Book {
get _genres() { return this.genres || [] }
get shouldSearchForCover() {
+ if (this.cover) return false
if (this.authorFL !== this.lastCoverSearchAuthor || this.title !== this.lastCoverSearchTitle || !this.lastCoverSearch) return true
var timeSinceLastSearch = Date.now() - this.lastCoverSearch
return timeSinceLastSearch > 1000 * 60 * 60 * 24 * 7 // every 7 days do another lookup
@@ -297,7 +298,7 @@ class Book {
return [genreTag]
}
- setDetailsFromFileMetadata(audioFileMetadata) {
+ setDetailsFromFileMetadata(audioFileMetadata, overrideExistingDetails = false) {
const MetadataMapArray = [
{
tag: 'tagComposer',
@@ -319,6 +320,10 @@ class Book {
tag: 'tagSubtitle',
key: 'subtitle'
},
+ {
+ tag: 'tagAlbum',
+ key: 'title',
+ },
{
tag: 'tagArtist',
key: 'author'
@@ -342,12 +347,12 @@ class Book {
MetadataMapArray.forEach((mapping) => {
if (audioFileMetadata[mapping.tag]) {
// Genres can contain multiple
- if (mapping.key === 'genres' && (!this[mapping.key].length || !this[mapping.key])) {
+ if (mapping.key === 'genres' && (!this[mapping.key].length || !this[mapping.key] || overrideExistingDetails)) {
updatePayload[mapping.key] = this.parseGenresTag(audioFileMetadata[mapping.tag])
- Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key].join(',')}`)
- } else if (!this[mapping.key]) {
+ // Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key].join(',')}`)
+ } else if (!this[mapping.key] || overrideExistingDetails) {
updatePayload[mapping.key] = audioFileMetadata[mapping.tag]
- Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key]}`)
+ // Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key]}`)
}
}
})
diff --git a/server/objects/ServerSettings.js b/server/objects/ServerSettings.js
index 70e04122..1c805316 100644
--- a/server/objects/ServerSettings.js
+++ b/server/objects/ServerSettings.js
@@ -46,6 +46,9 @@ class ServerSettings {
this.newTagExpireDays = settings.newTagExpireDays
this.scannerFindCovers = !!settings.scannerFindCovers
this.scannerParseSubtitle = settings.scannerParseSubtitle
+ this.scannerPreferAudioMetadata = !!settings.scannerPreferAudioMetadata
+ this.scannerPreferOpfMetadata = !!settings.scannerPreferOpfMetadata
+
this.coverDestination = settings.coverDestination || CoverDestination.METADATA
this.saveMetadataFile = !!settings.saveMetadataFile
this.rateLimitLoginRequests = !isNaN(settings.rateLimitLoginRequests) ? Number(settings.rateLimitLoginRequests) : 10
@@ -73,6 +76,8 @@ class ServerSettings {
newTagExpireDays: this.newTagExpireDays,
scannerFindCovers: this.scannerFindCovers,
scannerParseSubtitle: this.scannerParseSubtitle,
+ scannerPreferAudioMetadata: this.scannerPreferAudioMetadata,
+ scannerPreferOpfMetadata: this.scannerPreferOpfMetadata,
coverDestination: this.coverDestination,
saveMetadataFile: !!this.saveMetadataFile,
rateLimitLoginRequests: this.rateLimitLoginRequests,
diff --git a/server/scanner/AudioFileScanner.js b/server/scanner/AudioFileScanner.js
index e73cbcfc..b52a1415 100644
--- a/server/scanner/AudioFileScanner.js
+++ b/server/scanner/AudioFileScanner.js
@@ -4,7 +4,7 @@ const AudioFile = require('../objects/AudioFile')
const prober = require('../utils/prober')
const Logger = require('../Logger')
-const { msToTimestamp } = require('../utils')
+const { LogLevel } = require('../utils/constants')
class AudioFileScanner {
constructor() { }
@@ -80,6 +80,9 @@ class AudioFileScanner {
audioFileData.trackNumFromFilename = this.getTrackNumberFromFilename(bookScanData, audioFileData.filename)
audioFileData.cdNumFromFilename = this.getCdNumberFromFilename(bookScanData, audioFileData.filename)
audioFile.setDataFromProbe(audioFileData, probeData)
+ if (audioFile.embeddedCoverArt) {
+
+ }
return {
audioFile,
elapsed: Date.now() - probeStart
@@ -87,12 +90,11 @@ class AudioFileScanner {
}
- // Returns array of { AudioFile, elapsed } from audio file scan objects
- async scanAudioFiles(audioFileDataArray, bookScanData) {
+ // Returns array of { AudioFile, elapsed, averageScanDuration } from audio file scan objects
+ async executeAudioFileScans(audioFileDataArray, bookScanData) {
var proms = []
for (let i = 0; i < audioFileDataArray.length; i++) {
- var prom = this.scan(audioFileDataArray[i], bookScanData)
- proms.push(prom)
+ proms.push(this.scan(audioFileDataArray[i], bookScanData))
}
var scanStart = Date.now()
var results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr))
@@ -102,5 +104,62 @@ class AudioFileScanner {
averageScanDuration: this.getAverageScanDurationMs(results)
}
}
+
+ async scanAudioFiles(audioFileDataArray, bookScanData, audiobook, preferAudioMetadata, libraryScan = null) {
+ var hasUpdated = false
+
+ var audioScanResult = await this.executeAudioFileScans(audioFileDataArray, bookScanData)
+ if (audioScanResult.audioFiles.length) {
+ if (libraryScan) {
+ libraryScan.addLog(LogLevel.DEBUG, `Book "${bookScanData.path}" Audio file scan took ${audioScanResult.elapsed}ms for ${audioScanResult.audioFiles.length} with average time of ${audioScanResult.averageScanDuration}ms`)
+ }
+
+ var totalAudioFilesToInclude = audiobook.audioFilesToInclude.filter(af => !audioScanResult.audioFiles.find(_af => _af.ino === af.ino)).length + audioScanResult.audioFiles.length
+
+ // validate & add/update audio files to audiobook
+ for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
+ var newAF = audioScanResult.audioFiles[i]
+ var existingAF = audiobook.getAudioFileByIno(newAF.ino)
+
+ var trackIndex = null
+ if (totalAudioFilesToInclude === 1) { // Single track audiobooks
+ trackIndex = 1
+ } else if (existingAF && existingAF.manuallyVerified) { // manually verified audio files use existing index
+ trackIndex = existingAF.index
+ } else {
+ trackIndex = newAF.validateTrackIndex()
+ }
+
+ if (trackIndex !== null) {
+ if (audiobook.checkHasTrackNum(trackIndex, newAF.ino)) {
+ newAF.setDuplicateTrackNumber(trackIndex)
+ } else {
+ newAF.index = trackIndex
+ }
+ }
+ if (existingAF) {
+ if (audiobook.updateAudioFile(newAF)) {
+ // console.log('update dauido file')
+ hasUpdated = true
+ }
+ } else {
+ audiobook.addAudioFile(newAF)
+ // console.log('added auido file')
+ hasUpdated = true
+ }
+ }
+
+ if (hasUpdated) {
+ audiobook.rebuildTracks()
+ }
+
+ // Set book details from audio file ID3 tags, optional prefer
+ if (audiobook.setDetailsFromFileMetadata(preferAudioMetadata)) {
+ hasUpdated = true
+ }
+
+ }
+ return hasUpdated
+ }
}
module.exports = new AudioFileScanner()
\ No newline at end of file
diff --git a/server/scanner/AudioProbeData.js b/server/scanner/AudioProbeData.js
index 758cb47a..c6513ace 100644
--- a/server/scanner/AudioProbeData.js
+++ b/server/scanner/AudioProbeData.js
@@ -35,7 +35,6 @@ class AudioProbeData {
setData(data) {
var audioStream = this.getDefaultAudioStream(data.audio_streams)
-
this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false
this.format = data.format
this.duration = data.duration
diff --git a/server/scanner/LibraryScan.js b/server/scanner/LibraryScan.js
index 5e32876b..a7a0809d 100644
--- a/server/scanner/LibraryScan.js
+++ b/server/scanner/LibraryScan.js
@@ -1,6 +1,10 @@
-const Folder = require('../objects/Folder')
-const Constants = require('../utils/constants')
+const Path = require('path')
+const fs = require('fs-extra')
+const date = require('date-and-time')
+const Logger = require('../Logger')
+const Folder = require('../objects/Folder')
+const { LogLevel } = require('../utils/constants')
const { getId, secondsToTimestamp } = require('../utils/index')
class LibraryScan {
@@ -9,6 +13,7 @@ class LibraryScan {
this.libraryId = null
this.libraryName = null
this.folders = null
+ this.verbose = false
this.scanOptions = null
@@ -16,14 +21,21 @@ class LibraryScan {
this.finishedAt = null
this.elapsed = null
- this.status = Constants.ScanStatus.NOTHING
this.resultsMissing = 0
this.resultsAdded = 0
this.resultsUpdated = 0
+
+ this.logs = []
}
get _scanOptions() { return this.scanOptions || {} }
get forceRescan() { return !!this._scanOptions.forceRescan }
+ get preferAudioMetadata() { return !!this._scanOptions.preferAudioMetadata }
+ get preferOpfMetadata() { return !!this._scanOptions.preferOpfMetadata }
+ get findCovers() { return !!this._scanOptions.findCovers }
+ get timestamp() {
+ return (new Date()).toISOString()
+ }
get resultStats() {
return `${this.resultsAdded} Added | ${this.resultsUpdated} Updated | ${this.resultsMissing} Missing`
@@ -42,6 +54,28 @@ class LibraryScan {
}
}
}
+ get totalResults() {
+ return this.resultsAdded + this.resultsUpdated + this.resultsMissing
+ }
+ get getLogFilename() {
+ return date.format(new Date(), 'YYYY-MM-DD') + '_' + this.id + '.txt'
+ }
+
+ toJSON() {
+ return {
+ id: this.id,
+ libraryId: this.libraryId,
+ libraryName: this.libraryName,
+ folders: this.folders.map(f => f.toJSON()),
+ scanOptions: this.scanOptions.toJSON(),
+ startedAt: this.startedAt,
+ finishedAt: this.finishedAt,
+ elapsed: this.elapsed,
+ resultsAdded: this.resultsAdded,
+ resultsUpdated: this.resultsUpdated,
+ resultsMissing: this.resultsMissing
+ }
+ }
setData(library, scanOptions) {
this.id = getId('lscan')
@@ -58,5 +92,39 @@ class LibraryScan {
this.finishedAt = Date.now()
this.elapsed = this.finishedAt - this.startedAt
}
+
+ getLogLevelString(level) {
+ for (const key in LogLevel) {
+ if (LogLevel[key] === level) {
+ return key
+ }
+ }
+ return 'UNKNOWN'
+ }
+
+ addLog(level, ...args) {
+ const logObj = {
+ timestamp: this.timestamp,
+ message: args.join(' '),
+ levelName: this.getLogLevelString(level),
+ level
+ }
+
+ if (this.verbose) {
+ Logger.debug(`[LibraryScan] "${this.libraryName}":`, args)
+ }
+ this.logs.push(logObj)
+ }
+
+ async saveLog(logDir) {
+ await fs.ensureDir(logDir)
+ var outputPath = Path.join(logDir, this.getLogFilename)
+ var logLines = [JSON.stringify(this.toJSON())]
+ this.logs.forEach(l => {
+ logLines.push(JSON.stringify(l))
+ })
+ await fs.writeFile(outputPath, logLines.join('\n') + '\n')
+ Logger.info(`[LibraryScan] Scan log saved "${outputPath}"`)
+ }
}
module.exports = LibraryScan
\ No newline at end of file
diff --git a/server/scanner/ScanOptions.js b/server/scanner/ScanOptions.js
index 851cff01..40b09fbb 100644
--- a/server/scanner/ScanOptions.js
+++ b/server/scanner/ScanOptions.js
@@ -4,29 +4,6 @@ class ScanOptions {
constructor(options) {
this.forceRescan = false
- // this.metadataPrecedence = [
- // {
- // id: 'directory',
- // include: true
- // },
- // {
- // id: 'reader-desc-txt',
- // include: true
- // },
- // {
- // id: 'audio-file-metadata',
- // include: true
- // },
- // {
- // id: 'metadata-opf',
- // include: true
- // },
- // {
- // id: 'external-source',
- // include: false
- // }
- // ]
-
// Server settings
this.parseSubtitles = false
this.findCovers = false
diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js
index ddd4c21f..a3203087 100644
--- a/server/scanner/Scanner.js
+++ b/server/scanner/Scanner.js
@@ -4,10 +4,9 @@ const Path = require('path')
// Utils
const Logger = require('../Logger')
const { version } = require('../../package.json')
-const audioFileScanner = require('../utils/audioFileScanner')
const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir')
const { comparePaths, getIno, getId, msToTimestamp } = require('../utils/index')
-const { ScanResult, CoverDestination } = require('../utils/constants')
+const { ScanResult, CoverDestination, LogLevel } = require('../utils/constants')
const AudioFileScanner = require('./AudioFileScanner')
const BookFinder = require('../BookFinder')
@@ -20,6 +19,8 @@ class Scanner {
this.AudiobookPath = AUDIOBOOK_PATH
this.MetadataPath = METADATA_PATH
this.BookMetadataPath = Path.posix.join(this.MetadataPath.replace(/\\/g, '/'), 'books')
+ var LogDirPath = Path.join(this.MetadataPath, 'logs')
+ this.ScanLogPath = Path.join(LogDirPath, 'scans')
this.db = db
this.coverController = coverController
@@ -46,8 +47,82 @@ class Scanner {
}
}
+ isLibraryScanning(libraryId) {
+ return this.librariesScanning.find(ls => ls.id === libraryId)
+ }
+
+ async scanAudiobookById(audiobookId) {
+ var audiobook = this.db.audiobooks.find(ab => ab.id === audiobookId)
+ if (!audiobook) {
+ Logger.error(`[Scanner] Scan audiobook by id not found ${audiobookId}`)
+ return ScanResult.NOTHING
+ }
+ const library = this.db.libraries.find(lib => lib.id === audiobook.libraryId)
+ if (!library) {
+ Logger.error(`[Scanner] Scan audiobook by id library not found "${audiobook.libraryId}"`)
+ return ScanResult.NOTHING
+ }
+ const folder = library.folders.find(f => f.id === audiobook.folderId)
+ if (!folder) {
+ Logger.error(`[Scanner] Scan audiobook by id folder not found "${audiobook.folderId}" in library "${library.name}"`)
+ return ScanResult.NOTHING
+ }
+ Logger.info(`[Scanner] Scanning Audiobook "${audiobook.title}"`)
+ return this.scanAudiobook(folder, audiobook)
+ }
+
+ async scanAudiobook(folder, audiobook) {
+ var audiobookData = await getAudiobookFileData(folder, audiobook.fullPath, this.db.serverSettings)
+ if (!audiobookData) {
+ return ScanResult.NOTHING
+ }
+ var hasUpdated = false
+
+ var checkRes = audiobook.checkScanData(audiobookData, version)
+ if (checkRes.updated) hasUpdated = true
+
+ // Sync other files first so that local images are used as cover art
+ // TODO: Cleanup other file sync
+ var allOtherFiles = checkRes.newOtherFileData.concat(audiobook._otherFiles)
+ if (await audiobook.syncOtherFiles(allOtherFiles, this.MetadataPath, this.db.serverSettings.scannerPreferOpfMetadata)) {
+ hasUpdated = true
+ }
+
+ // Scan all audio files
+ if (audiobookData.audioFiles.length) {
+ if (await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData, audiobook, this.db.serverSettings.scannerPreferAudioMetadata)) {
+ hasUpdated = true
+ }
+
+ // Extract embedded cover art if cover is not already in directory
+ if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
+ var outputCoverDirs = this.getCoverDirectory(audiobook)
+ var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
+ if (relativeDir) {
+ Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
+ hasUpdated = true
+ }
+ }
+ }
+
+ if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { // Audiobook is invalid
+ audiobook.setInvalid()
+ hasUpdated = true
+ } else if (audiobook.isInvalid) {
+ audiobook.isInvalid = false
+ hasUpdated = true
+ }
+
+ if (hasUpdated) {
+ this.emitter('audiobook_updated', audiobook.toJSONExpanded())
+ await this.db.updateEntity('audiobook', audiobook)
+ return ScanResult.UPDATED
+ }
+ return ScanResult.UPTODATE
+ }
+
async scan(libraryId, options = {}) {
- if (this.librariesScanning.includes(libraryId)) {
+ if (this.isLibraryScanning(libraryId)) {
Logger.error(`[Scanner] Already scanning ${libraryId}`)
return
}
@@ -66,172 +141,380 @@ class Scanner {
var libraryScan = new LibraryScan()
libraryScan.setData(library, scanOptions)
- this.librariesScanning.push(libraryScan)
+ libraryScan.verbose = false
+ this.librariesScanning.push(libraryScan.getScanEmitData)
this.emitter('scan_start', libraryScan.getScanEmitData)
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
- await this.scanLibrary(libraryScan)
+ var canceled = await this.scanLibrary(libraryScan)
+
+ if (canceled) {
+ Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
+ delete this.cancelLibraryScan[libraryScan.libraryId]
+ }
libraryScan.setComplete()
- Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp}. ${libraryScan.resultStats}`)
+ Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
+
+ if (canceled && !libraryScan.totalResults) {
+ var emitData = libraryScan.getScanEmitData
+ emitData.results = null
+ this.emitter('scan_complete', emitData)
+ return
+ }
+
this.emitter('scan_complete', libraryScan.getScanEmitData)
+
+ if (libraryScan.totalResults) {
+ libraryScan.saveLog(this.ScanLogPath)
+ }
}
async scanLibrary(libraryScan) {
var audiobookDataFound = []
+
+ // Scan each library
for (let i = 0; i < libraryScan.folders.length; i++) {
var folder = libraryScan.folders[i]
var abDataFoundInFolder = await scanRootDir(folder, this.db.serverSettings)
- Logger.debug(`[Scanner] ${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`)
+ libraryScan.addLog(LogLevel.INFO, `${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`)
audiobookDataFound = audiobookDataFound.concat(abDataFoundInFolder)
}
+ if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+
// Remove audiobooks with no inode
audiobookDataFound = audiobookDataFound.filter(abd => abd.ino)
var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId)
+ const NumScansPerChunk = 25
+ const audiobooksToUpdateChunks = []
+ const audiobookDataToRescanChunks = []
+ const newAudiobookDataToScanChunks = []
var audiobooksToUpdate = []
- var audiobookRescans = []
- var newAudiobookScans = []
+ var audiobookDataToRescan = []
+ var newAudiobookDataToScan = []
+ var audiobooksToFindCovers = []
// Check for existing & removed audiobooks
for (let i = 0; i < audiobooksInLibrary.length; i++) {
var audiobook = audiobooksInLibrary[i]
var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path))
if (!dataFound) {
- Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`)
+ libraryScan.addLog(LogLevel.WARN, `Audiobook "${audiobook.title}" is missing`)
+ libraryScan.resultsMissing++
audiobook.setMissing()
audiobooksToUpdate.push(audiobook)
+ if (audiobooksToUpdate.length === NumScansPerChunk) {
+ audiobooksToUpdateChunks.push(audiobooksToUpdate)
+ audiobooksToUpdate = []
+ }
} else {
- var checkRes = audiobook.checkScanData(dataFound)
- if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) {
- // existing audiobook has new files
+ var checkRes = audiobook.checkScanData(dataFound, version)
+ if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) { // Audiobook has new files
checkRes.audiobook = audiobook
checkRes.bookScanData = dataFound
- audiobookRescans.push(this.rescanAudiobook(checkRes, libraryScan))
- libraryScan.resultsMissing++
- } else if (checkRes.updated) {
- audiobooksToUpdate.push(audiobook)
+ audiobookDataToRescan.push(checkRes)
+ if (audiobookDataToRescan.length === NumScansPerChunk) {
+ audiobookDataToRescanChunks.push(audiobookDataToRescan)
+ audiobookDataToRescan = []
+ }
+ } else if (libraryScan.findCovers && audiobook.book.shouldSearchForCover) {
libraryScan.resultsUpdated++
+ audiobooksToFindCovers.push(audiobook)
+ audiobooksToUpdate.push(audiobook)
+ if (audiobooksToUpdate.length === NumScansPerChunk) {
+ audiobooksToUpdateChunks.push(audiobooksToUpdate)
+ audiobooksToUpdate = []
+ }
+ } else if (checkRes.updated) { // Updated but no scan required
+ libraryScan.resultsUpdated++
+ audiobooksToUpdate.push(audiobook)
+ if (audiobooksToUpdate.length === NumScansPerChunk) {
+ audiobooksToUpdateChunks.push(audiobooksToUpdate)
+ audiobooksToUpdate = []
+ }
}
audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino)
}
}
+ if (audiobooksToUpdate.length) audiobooksToUpdateChunks.push(audiobooksToUpdate)
+ if (audiobookDataToRescan.length) audiobookDataToRescanChunks.push(audiobookDataToRescan)
// Potential NEW Audiobooks
for (let i = 0; i < audiobookDataFound.length; i++) {
var dataFound = audiobookDataFound[i]
var hasEbook = dataFound.otherFiles.find(otherFile => otherFile.filetype === 'ebook')
if (!hasEbook && !dataFound.audioFiles.length) {
- Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
+ libraryScan.addLog(LogLevel.WARN, `Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
} else {
- newAudiobookScans.push(this.scanNewAudiobook(dataFound, libraryScan))
+ newAudiobookDataToScan.push(dataFound)
+ if (newAudiobookDataToScan.length === NumScansPerChunk) {
+ newAudiobookDataToScanChunks.push(newAudiobookDataToScan)
+ newAudiobookDataToScan = []
+ }
}
}
+ if (newAudiobookDataToScan.length) newAudiobookDataToScanChunks.push(newAudiobookDataToScan)
+
+ // console.log('Num chunks to update', audiobooksToUpdateChunks.length)
+ // console.log('Num chunks to rescan', audiobookDataToRescanChunks.length)
+ // console.log('Num chunks to new scan', newAudiobookDataToScanChunks.length)
+
+ // Audiobooks not requiring a scan but require a search for cover
+ for (let i = 0; i < audiobooksToFindCovers.length; i++) {
+ var audiobook = audiobooksToFindCovers[i]
+ var updatedCover = await this.searchForCover(audiobook, libraryScan)
+ audiobook.book.updateLastCoverSearch(updatedCover)
+ }
- if (audiobookRescans.length) {
- var updatedAudiobooks = (await Promise.all(audiobookRescans)).filter(ab => !!ab)
- if (updatedAudiobooks.length) {
- audiobooksToUpdate = audiobooksToUpdate.concat(updatedAudiobooks)
- libraryScan.resultsUpdated += updatedAudiobooks.length
- }
+ for (let i = 0; i < audiobooksToUpdateChunks.length; i++) {
+ await this.updateAudiobooksChunk(audiobooksToUpdateChunks[i])
+ if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+ // console.log('Update chunk done', i, 'of', audiobooksToUpdateChunks.length)
}
- if (audiobooksToUpdate.length) {
- Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" updating ${audiobooksToUpdate.length} books`)
- await this.db.updateEntities('audiobook', audiobooksToUpdate)
+ for (let i = 0; i < audiobookDataToRescanChunks.length; i++) {
+ await this.rescanAudiobookDataChunk(audiobookDataToRescanChunks[i], libraryScan)
+ if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+ // console.log('Rescan chunk done', i, 'of', audiobookDataToRescanChunks.length)
}
+ for (let i = 0; i < newAudiobookDataToScanChunks.length; i++) {
+ await this.scanNewAudiobookDataChunk(newAudiobookDataToScanChunks[i], libraryScan)
+ // console.log('New scan chunk done', i, 'of', newAudiobookDataToScanChunks.length)
+ if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+ }
+ }
- if (newAudiobookScans.length) {
- var newAudiobooks = (await Promise.all(newAudiobookScans)).filter(ab => !!ab)
- if (newAudiobooks.length) {
- Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" inserting ${newAudiobooks.length} books`)
- await this.db.insertEntities('audiobook', newAudiobooks)
- libraryScan.resultsAdded = newAudiobooks.length
- }
- }
+ async updateAudiobooksChunk(audiobooksToUpdate) {
+ await this.db.updateEntities('audiobook', audiobooksToUpdate)
+ this.emitter('audiobooks_updated', audiobooksToUpdate.map(ab => ab.toJSONExpanded()))
+ }
+
+ async rescanAudiobookDataChunk(audiobookDataToRescan, libraryScan) {
+ var audiobooksUpdated = await Promise.all(audiobookDataToRescan.map((abd) => {
+ return this.rescanAudiobook(abd, libraryScan)
+ }))
+ audiobooksUpdated = audiobooksUpdated.filter(ab => ab) // Filter out nulls
+ libraryScan.resultsUpdated += audiobooksUpdated.length
+ await this.db.updateEntities('audiobook', audiobooksUpdated)
+ this.emitter('audiobooks_updated', audiobooksUpdated.map(ab => ab.toJSONExpanded()))
+ }
+
+ async scanNewAudiobookDataChunk(newAudiobookDataToScan, libraryScan) {
+ var newAudiobooks = await Promise.all(newAudiobookDataToScan.map((abd) => {
+ return this.scanNewAudiobook(abd, libraryScan.preferAudioMetadata, libraryScan.preferOpfMetadata, libraryScan.findCovers, libraryScan)
+ }))
+ newAudiobooks = newAudiobooks.filter(ab => ab) // Filter out nulls
+ libraryScan.resultsAdded += newAudiobooks.length
+ await this.db.insertEntities('audiobook', newAudiobooks)
+ this.emitter('audiobooks_added', newAudiobooks.map(ab => ab.toJSONExpanded()))
}
async rescanAudiobook(audiobookCheckData, libraryScan) {
const { newAudioFileData, newOtherFileData, audiobook, bookScanData } = audiobookCheckData
- Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`)
+ libraryScan.addLog(LogLevel.DEBUG, `Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`)
+
+ // Sync other files first to use local images as cover before extracting audio file cover
+ if (newOtherFileData.length) {
+ // TODO: Cleanup other file sync
+ var allOtherFiles = newOtherFileData.concat(audiobook._otherFiles)
+ await audiobook.syncOtherFiles(allOtherFiles, this.MetadataPath, libraryScan.preferOpfMetadata)
+ }
if (newAudioFileData.length) {
- var audioScanResult = await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData)
- Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobook.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
- if (audioScanResult.audioFiles.length) {
- var totalAudioFilesToInclude = audiobook.audioFilesToInclude.length + audioScanResult.audioFiles.length
+ await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData, audiobook, libraryScan.preferAudioMetadata, libraryScan)
- // validate & add audio files to audiobook
- for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
- var newAF = audioScanResult.audioFiles[i]
- var trackIndex = newAF.validateTrackIndex(totalAudioFilesToInclude === 1)
- if (trackIndex !== null) {
- if (audiobook.checkHasTrackNum(trackIndex)) {
- newAF.setDuplicateTrackNumber(trackIndex)
- } else {
- newAF.index = trackIndex
- }
- }
- audiobook.addAudioFile(newAF)
+ // Extract embedded cover art if cover is not already in directory
+ if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
+ var outputCoverDirs = this.getCoverDirectory(audiobook)
+ var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
+ if (relativeDir) {
+ libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${relativeDir}"`)
}
-
- audiobook.rebuildTracks()
}
}
- if (newOtherFileData.length) {
- await audiobook.syncOtherFiles(newOtherFileData, this.MetadataPath)
+
+ if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { // Audiobook is invalid
+ audiobook.setInvalid()
+ } else if (audiobook.isInvalid) {
+ audiobook.isInvalid = false
}
+
+ // Scan for cover if enabled and has no cover
+ if (audiobook && libraryScan.findCovers && !audiobook.cover && audiobook.book.shouldSearchForCover) {
+ var updatedCover = await this.searchForCover(audiobook, libraryScan)
+ audiobook.book.updateLastCoverSearch(updatedCover)
+ }
+
return audiobook
}
- async scanNewAudiobook(audiobookData, libraryScan) {
- Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Scanning new "${audiobookData.path}"`)
+ async scanNewAudiobook(audiobookData, preferAudioMetadata, preferOpfMetadata, findCovers, libraryScan = null) {
+ if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new book "${audiobookData.path}"`)
+ else Logger.debug(`[Scanner] Scanning new book "${audiobookData.path}"`)
+
var audiobook = new Audiobook()
audiobook.setData(audiobookData)
if (audiobookData.audioFiles.length) {
- var audioScanResult = await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData)
- Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobookData.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
- if (audioScanResult.audioFiles.length) {
- // validate & add audio files to audiobook
- for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
- var newAF = audioScanResult.audioFiles[i]
- var trackIndex = newAF.validateTrackIndex(audioScanResult.audioFiles.length === 1)
- if (trackIndex !== null) {
- if (audiobook.checkHasTrackNum(trackIndex)) {
- newAF.setDuplicateTrackNumber(trackIndex)
- } else {
- newAF.index = trackIndex
- }
- }
- audiobook.addAudioFile(newAF)
- }
- audiobook.rebuildTracks()
- } else if (!audiobook.ebooks.length) {
- // Audiobook has no ebooks and no valid audio tracks do not continue
- Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`)
- return null
- }
+ await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData, audiobook, preferAudioMetadata, libraryScan)
+ }
+
+ if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) {
+ // Audiobook has no ebooks and no valid audio tracks do not continue
+ Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`)
+ return null
}
// Look for desc.txt and reader.txt and update
- await audiobook.saveDataFromTextFiles()
+ await audiobook.saveDataFromTextFiles(preferOpfMetadata)
// Extract embedded cover art if cover is not already in directory
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
var outputCoverDirs = this.getCoverDirectory(audiobook)
var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
if (relativeDir) {
- Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
+ if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${relativeDir}"`)
+ else Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
}
}
+ // Scan for cover if enabled and has no cover
+ if (audiobook && findCovers && !audiobook.cover && audiobook.book.shouldSearchForCover) {
+ var updatedCover = await this.searchForCover(audiobook, libraryScan)
+ audiobook.book.updateLastCoverSearch(updatedCover)
+ }
+
return audiobook
}
+
+ getFileUpdatesGrouped(fileUpdates) {
+ var folderGroups = {}
+ fileUpdates.forEach((file) => {
+ if (folderGroups[file.folderId]) {
+ folderGroups[file.folderId].fileUpdates.push(file)
+ } else {
+ folderGroups[file.folderId] = {
+ libraryId: file.libraryId,
+ folderId: file.folderId,
+ fileUpdates: [file]
+ }
+ }
+ })
+ return folderGroups
+ }
+
+ async scanFilesChanged(fileUpdates) {
+ if (!fileUpdates.length) return
+ // files grouped by folder
+ var folderGroups = this.getFileUpdatesGrouped(fileUpdates)
+
+ for (const folderId in folderGroups) {
+ var libraryId = folderGroups[folderId].libraryId
+ var library = this.db.libraries.find(lib => lib.id === libraryId)
+ if (!library) {
+ Logger.error(`[Scanner] Library not found in files changed ${libraryId}`)
+ continue;
+ }
+ var folder = library.getFolderById(folderId)
+ if (!folder) {
+ Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`)
+ continue;
+ }
+ var relFilePaths = folderGroups[folderId].fileUpdates.map(fileUpdate => fileUpdate.relPath)
+ var fileUpdateBookGroup = groupFilesIntoAudiobookPaths(relFilePaths, true)
+ var folderScanResults = await this.scanFolderUpdates(library, folder, fileUpdateBookGroup)
+ Logger.debug(`[Scanner] Folder scan results`, folderScanResults)
+ }
+ }
+
+ async scanFolderUpdates(library, folder, fileUpdateBookGroup) {
+ Logger.debug(`[Scanner] Scanning file update groups in folder "${folder.id}" of library "${library.name}"`)
+
+ var bookGroupingResults = {}
+ for (const bookDir in fileUpdateBookGroup) {
+ var fullPath = Path.posix.join(folder.fullPath.replace(/\\/g, '/'), bookDir)
+
+ // Check if book dir group is already an audiobook or in a subdir of an audiobook
+ var existingAudiobook = this.db.audiobooks.find(ab => fullPath.startsWith(ab.fullPath))
+ if (existingAudiobook) {
+
+ // Is the audiobook exactly - check if was deleted
+ if (existingAudiobook.fullPath === fullPath) {
+ var exists = await fs.pathExists(fullPath)
+ if (!exists) {
+ Logger.info(`[Scanner] Scanning file update group and audiobook was deleted "${existingAudiobook.title}" - marking as missing`)
+ existingAudiobook.setMissing()
+ await this.db.updateAudiobook(existingAudiobook)
+ this.emitter('audiobook_updated', existingAudiobook.toJSONExpanded())
+
+ bookGroupingResults[bookDir] = ScanResult.REMOVED
+ continue;
+ }
+ }
+
+ // Scan audiobook for updates
+ Logger.debug(`[Scanner] Folder update for relative path "${bookDir}" is in audiobook "${existingAudiobook.title}" - scan for updates`)
+ bookGroupingResults[bookDir] = await this.scanAudiobook(folder, existingAudiobook)
+ continue;
+ }
+
+ // Check if an audiobook is a subdirectory of this dir
+ var childAudiobook = this.db.audiobooks.find(ab => ab.fullPath.startsWith(fullPath))
+ if (childAudiobook) {
+ Logger.warn(`[Scanner] Files were modified in a parent directory of an audiobook "${childAudiobook.title}" - ignoring`)
+ bookGroupingResults[bookDir] = ScanResult.NOTHING
+ continue;
+ }
+
+ Logger.debug(`[Scanner] Folder update group must be a new book "${bookDir}" in library "${library.name}"`)
+ var newAudiobook = await this.scanPotentialNewAudiobook(folder, fullPath)
+ if (newAudiobook) {
+ await this.db.insertEntity('audiobook', newAudiobook)
+ this.emitter('audiobook_added', newAudiobook.toJSONExpanded())
+ }
+ bookGroupingResults[bookDir] = newAudiobook ? ScanResult.ADDED : ScanResult.NOTHING
+ }
+
+ return bookGroupingResults
+ }
+
+ async scanPotentialNewAudiobook(folder, fullPath) {
+ var audiobookData = await getAudiobookFileData(folder, fullPath, this.db.serverSettings)
+ if (!audiobookData) return null
+ var serverSettings = this.db.serverSettings
+ return this.scanNewAudiobook(audiobookData, serverSettings.scannerPreferAudioMetadata, serverSettings.scannerPreferOpfMetadata, serverSettings.scannerFindCovers)
+ }
+
+ async searchForCover(audiobook, libraryScan = null) {
+ var options = {
+ titleDistance: 2,
+ authorDistance: 2
+ }
+ var results = await this.bookFinder.findCovers('google', audiobook.title, audiobook.authorFL, options)
+ if (results.length) {
+ if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Found best cover for "${audiobook.title}"`)
+ else Logger.debug(`[Scanner] Found best cover for "${audiobook.title}"`)
+
+ // If the first cover result fails, attempt to download the second
+ for (let i = 0; i < results.length && i < 2; i++) {
+
+ // Downloads and updates the book cover
+ var result = await this.coverController.downloadCoverFromUrl(audiobook, results[i])
+
+ if (result.error) {
+ Logger.error(`[Scanner] Failed to download cover from url "${results[i]}" | Attempt ${i + 1}`, result.error)
+ } else {
+ return true
+ }
+ }
+ }
+ return false
+ }
}
module.exports = Scanner
\ No newline at end of file
diff --git a/server/utils/constants.js b/server/utils/constants.js
index 446a6778..d36ce925 100644
--- a/server/utils/constants.js
+++ b/server/utils/constants.js
@@ -6,14 +6,6 @@ module.exports.ScanResult = {
UPTODATE: 4
}
-module.exports.ScanStatus = {
- NOTHING: 0,
- ADDED: 1,
- UPDATED: 2,
- REMOVED: 3,
- UPTODATE: 4
-}
-
module.exports.CoverDestination = {
METADATA: 0,
AUDIOBOOK: 1
diff --git a/server/utils/scandir.js b/server/utils/scandir.js
index 481b8ffa..d7854e28 100644
--- a/server/utils/scandir.js
+++ b/server/utils/scandir.js
@@ -16,7 +16,7 @@ function isBookFile(path) {
// TODO: Function needs to be re-done
// Input: array of relative file paths
// Output: map of files grouped into potential audiobook dirs
-function groupFilesIntoAudiobookPaths(paths, useAllFileTypes = false) {
+function groupFilesIntoAudiobookPaths(paths) {
// Step 1: Clean path, Remove leading "/", Filter out files in root dir
var pathsFiltered = paths.map(path => {
return path.startsWith('/') ? path.slice(1) : path