Cutlery/Kavita
1
0
Fork 0
mirror of https://github.com/Kareadita/Kavita.git synced 2025-07-09 03:04:19 -04:00
Code Issues Packages Projects Releases Wiki Activity

Scanner Stuff Again (#3003)

Browse Source
This commit is contained in:
Joe Milazzo 2024-06-14 08:01:26 -05:00 committed by GitHub
parent 59699e17e2
commit 12ec980204
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 99 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
API/Controllers/MetadataController.cs
View File

@ -37,12 +37,14 @@ public class MetadataController(IUnitOfWork unitOfWork, ILocalizationService loc
public async Task<ActionResult<IList<GenreTagDto>>> GetAllGenres(string? libraryIds)
{
var ids = libraryIds?.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToList();
// NOTE: libraryIds isn't hooked up in the frontend
if (ids is {Count: > 0})
{
return Ok(await unitOfWork.GenreRepository.GetAllGenreDtosForLibrariesAsync(ids, User.GetUserId()));
return Ok(await unitOfWork.GenreRepository.GetAllGenreDtosForLibrariesAsync(User.GetUserId(), ids));
}
return Ok(await unitOfWork.GenreRepository.GetAllGenreDtosAsync(User.GetUserId()));
return Ok(await unitOfWork.GenreRepository.GetAllGenreDtosForLibrariesAsync(User.GetUserId()));
}
/// <summary>
@ -71,9 +73,9 @@ public class MetadataController(IUnitOfWork unitOfWork, ILocalizationService loc
var ids = libraryIds?.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToList();
if (ids is {Count: > 0})
{
return Ok(await unitOfWork.PersonRepository.GetAllPeopleDtosForLibrariesAsync(ids, User.GetUserId()));
return Ok(await unitOfWork.PersonRepository.GetAllPeopleDtosForLibrariesAsync(User.GetUserId(), ids));
}
return Ok(await unitOfWork.PersonRepository.GetAllPersonDtosAsync(User.GetUserId()));
return Ok(await unitOfWork.PersonRepository.GetAllPeopleDtosForLibrariesAsync(User.GetUserId()));
}
/// <summary>
@ -88,9 +90,9 @@ public class MetadataController(IUnitOfWork unitOfWork, ILocalizationService loc
var ids = libraryIds?.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToList();
if (ids is {Count: > 0})
{
return Ok(await unitOfWork.TagRepository.GetAllTagDtosForLibrariesAsync(ids, User.GetUserId()));
return Ok(await unitOfWork.TagRepository.GetAllTagDtosForLibrariesAsync(User.GetUserId(), ids));
}
return Ok(await unitOfWork.TagRepository.GetAllTagDtosAsync(User.GetUserId()));
return Ok(await unitOfWork.TagRepository.GetAllTagDtosForLibrariesAsync(User.GetUserId()));
}
/// <summary>

53
API/Data/Repositories/GenreRepository.cs
View File

@ -19,9 +19,8 @@ public interface IGenreRepository
Task<Genre?> FindByNameAsync(string genreName);
Task<IList<Genre>> GetAllGenresAsync();
Task<IList<Genre>> GetAllGenresByNamesAsync(IEnumerable<string> normalizedNames);
Task<IList<GenreTagDto>> GetAllGenreDtosAsync(int userId);
Task RemoveAllGenreNoLongerAssociated(bool removeExternal = false);
Task<IList<GenreTagDto>> GetAllGenreDtosForLibrariesAsync(IList<int> libraryIds, int userId);
Task<IList<GenreTagDto>> GetAllGenreDtosForLibrariesAsync(int userId, IList<int>? libraryIds = null);
Task<int> GetCountAsync();
Task<GenreTagDto> GetRandomGenre();
Task<GenreTagDto> GetGenreById(int id);
@ -69,27 +68,6 @@ public class GenreRepository : IGenreRepository
await _context.SaveChangesAsync();
}
/// <summary>
/// Returns a set of Genre tags for a set of library Ids. UserId will restrict returned Genres based on user's age restriction.
/// </summary>
/// <param name="libraryIds"></param>
/// <param name="userId"></param>
/// <returns></returns>
public async Task<IList<GenreTagDto>> GetAllGenreDtosForLibrariesAsync(IList<int> libraryIds, int userId)
{
var userRating = await _context.AppUser.GetUserAgeRestriction(userId);
return await _context.Series
.Where(s => libraryIds.Contains(s.LibraryId))
.RestrictAgainstAgeRestriction(userRating)
.SelectMany(s => s.Metadata.Genres)
.AsSplitQuery()
.Distinct()
.OrderBy(p => p.NormalizedTitle)
.ProjectTo<GenreTagDto>(_mapper.ConfigurationProvider)
.ToListAsync();
}
public async Task<int> GetCountAsync()
{
return await _context.Genre.CountAsync();
@ -128,13 +106,30 @@ public class GenreRepository : IGenreRepository
.ToListAsync();
}
public async Task<IList<GenreTagDto>> GetAllGenreDtosAsync(int userId)
/// <summary>
/// Returns a set of Genre tags for a set of library Ids.
/// UserId will restrict returned Genres based on user's age restriction and library access.
/// </summary>
/// <param name="userId"></param>
/// <param name="libraryIds"></param>
/// <returns></returns>
public async Task<IList<GenreTagDto>> GetAllGenreDtosForLibrariesAsync(int userId, IList<int>? libraryIds = null)
{
var ageRating = await _context.AppUser.GetUserAgeRestriction(userId);
return await _context.Genre
.RestrictAgainstAgeRestriction(ageRating)
.OrderBy(g => g.NormalizedTitle)
.AsNoTracking()
var userRating = await _context.AppUser.GetUserAgeRestriction(userId);
var userLibs = await _context.Library.GetUserLibraries(userId).ToListAsync();
if (libraryIds is {Count: > 0})
{
userLibs = userLibs.Where(libraryIds.Contains).ToList();
}
return await _context.Series
.Where(s => userLibs.Contains(s.LibraryId))
.RestrictAgainstAgeRestriction(userRating)
.SelectMany(s => s.Metadata.Genres)
.AsSplitQuery()
.Distinct()
.OrderBy(p => p.NormalizedTitle)
.ProjectTo<GenreTagDto>(_mapper.ConfigurationProvider)
.ToListAsync();
}

14
API/Data/Repositories/PersonRepository.cs
View File

@ -20,7 +20,7 @@ public interface IPersonRepository
Task<IList<PersonDto>> GetAllPersonDtosAsync(int userId);
Task<IList<PersonDto>> GetAllPersonDtosByRoleAsync(int userId, PersonRole role);
Task RemoveAllPeopleNoLongerAssociated();
Task<IList<PersonDto>> GetAllPeopleDtosForLibrariesAsync(List<int> libraryIds, int userId);
Task<IList<PersonDto>> GetAllPeopleDtosForLibrariesAsync(int userId, List<int>? libraryIds = null);
Task<int> GetCountAsync();
Task<IList<Person>> GetAllPeopleByRoleAndNames(PersonRole role, IEnumerable<string> normalizeNames);
@ -61,11 +61,18 @@ public class PersonRepository : IPersonRepository
await _context.SaveChangesAsync();
}
public async Task<IList<PersonDto>> GetAllPeopleDtosForLibrariesAsync(List<int> libraryIds, int userId)
public async Task<IList<PersonDto>> GetAllPeopleDtosForLibrariesAsync(int userId, List<int>? libraryIds = null)
{
var ageRating = await _context.AppUser.GetUserAgeRestriction(userId);
var userLibs = await _context.Library.GetUserLibraries(userId).ToListAsync();
if (libraryIds is {Count: > 0})
{
userLibs = userLibs.Where(libraryIds.Contains).ToList();
}
return await _context.Series
.Where(s => libraryIds.Contains(s.LibraryId))
.Where(s => userLibs.Contains(s.LibraryId))
.RestrictAgainstAgeRestriction(ageRating)
.SelectMany(s => s.Metadata.People)
.Distinct()
@ -99,6 +106,7 @@ public class PersonRepository : IPersonRepository
public async Task<IList<PersonDto>> GetAllPersonDtosAsync(int userId)
{
var ageRating = await _context.AppUser.GetUserAgeRestriction(userId);
var libraryIds = await _context.Library.GetUserLibraries(userId).ToListAsync();
return await _context.Person
.OrderBy(p => p.Name)
.RestrictAgainstAgeRestriction(ageRating)

13
API/Data/Repositories/TagRepository.cs
View File

@ -19,7 +19,7 @@ public interface ITagRepository
Task<IList<Tag>> GetAllTagsByNameAsync(IEnumerable<string> normalizedNames);
Task<IList<TagDto>> GetAllTagDtosAsync(int userId);
Task RemoveAllTagNoLongerAssociated();
Task<IList<TagDto>> GetAllTagDtosForLibrariesAsync(IList<int> libraryIds, int userId);
Task<IList<TagDto>> GetAllTagDtosForLibrariesAsync(int userId, IList<int>? libraryIds = null);
}
public class TagRepository : ITagRepository
@ -57,11 +57,18 @@ public class TagRepository : ITagRepository
await _context.SaveChangesAsync();
}
public async Task<IList<TagDto>> GetAllTagDtosForLibrariesAsync(IList<int> libraryIds, int userId)
public async Task<IList<TagDto>> GetAllTagDtosForLibrariesAsync(int userId, IList<int>? libraryIds = null)
{
var userRating = await _context.AppUser.GetUserAgeRestriction(userId);
var userLibs = await _context.Library.GetUserLibraries(userId).ToListAsync();
if (libraryIds is {Count: > 0})
{
userLibs = userLibs.Where(libraryIds.Contains).ToList();
}
return await _context.Series
.Where(s => libraryIds.Contains(s.LibraryId))
.Where(s => userLibs.Contains(s.LibraryId))
.RestrictAgainstAgeRestriction(userRating)
.SelectMany(s => s.Metadata.Tags)
.AsSplitQuery()

2
API/Services/DirectoryService.cs
View File

@ -605,7 +605,7 @@ public class DirectoryService : IDirectoryService
{
if (!file.Contains(folder)) continue;
var lowestPath = Path.GetDirectoryName(file)?.Replace(folder, string.Empty);
var lowestPath = Path.GetDirectoryName(file);
if (!string.IsNullOrEmpty(lowestPath))
{
dirs.TryAdd(Parser.NormalizePath(lowestPath), string.Empty);

18
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View File

@ -137,19 +137,6 @@ public class ParseScannedFiles
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent(directory, library.Name, ProgressEventType.Updated));
// This is debug code to help understand why some installs aren't working correctly
if (!forceCheck && seriesPaths.TryGetValue(directory, out var series2) && series2.Count > 1 && series2.All(s => !string.IsNullOrEmpty(s.LowestFolderPath)))
{
_logger.LogDebug("[ProcessFiles] Dirty check passed, series list: {@SeriesModified}", series2);
foreach (var s in series2)
{
_logger.LogDebug("[ProcessFiles] Last Scanned: {LastScanned} vs Directory Check: {DirectoryLastScanned}",
s.LastScanned, _directoryService
.GetLastWriteTime(s.LowestFolderPath!)
.Truncate(TimeSpan.TicksPerSecond));
}
}
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, directory, forceCheck))
{
@ -177,12 +164,12 @@ public class ParseScannedFiles
if (!hasFolderChangedSinceLastScan)
{
_logger.LogDebug("[ProcessFiles] {Directory} subfolder {Folder} did not change since last scan, adding entry to skip", directory, seriesModified.LowestFolderPath);
_logger.LogTrace("[ProcessFiles] {Directory} subfolder {Folder} did not change since last scan, adding entry to skip", directory, seriesModified.LowestFolderPath);
result.Add(CreateScanResult(seriesModified.LowestFolderPath!, folderPath, false, ArraySegment<string>.Empty));
}
else
{
_logger.LogDebug("[ProcessFiles] {Directory} subfolder {Folder} changed for Series {SeriesName}", directory, seriesModified.LowestFolderPath, seriesModified.SeriesName);
_logger.LogTrace("[ProcessFiles] {Directory} subfolder {Folder} changed for Series {SeriesName}", directory, seriesModified.LowestFolderPath, seriesModified.SeriesName);
result.Add(CreateScanResult(directory, folderPath, true,
_directoryService.ScanFiles(seriesModified.LowestFolderPath!, fileExtensions, matcher)));
}
@ -190,7 +177,6 @@ public class ParseScannedFiles
}
else
{
// For a scan, this is doing everything in the directory loop before the folder Action is called...which leads to no progress indication
result.Add(CreateScanResult(directory, folderPath, true,
_directoryService.ScanFiles(directory, fileExtensions, matcher)));
}

81
API/Services/Tasks/Scanner/ProcessSeries.cs
View File

@ -355,44 +355,7 @@ public class ProcessSeries : IProcessSeries
// Set the AgeRating as highest in all the comicInfos
if (!series.Metadata.AgeRatingLocked) series.Metadata.AgeRating = chapters.Max(chapter => chapter.AgeRating);
// Count (aka expected total number of chapters or volumes from metadata) across all chapters
series.Metadata.TotalCount = chapters.Max(chapter => chapter.TotalCount);
// The actual number of count's defined across all chapter's metadata
series.Metadata.MaxCount = chapters.Max(chapter => chapter.Count);
var maxVolume = (int) series.Volumes.Max(v => v.MaxNumber);
var maxChapter = (int) chapters.Max(c => c.MaxNumber);
// Single books usually don't have a number in their Range (filename)
if (series.Format == MangaFormat.Epub || series.Format == MangaFormat.Pdf && chapters.Count == 1)
{
series.Metadata.MaxCount = 1;
} else if (series.Metadata.TotalCount <= 1 && chapters.Count == 1 && chapters[0].IsSpecial)
{
// If a series has a TotalCount of 1 (or no total count) and there is only a Special, mark it as Complete
series.Metadata.MaxCount = series.Metadata.TotalCount;
} else if ((maxChapter == Parser.Parser.DefaultChapterNumber || maxChapter > series.Metadata.TotalCount) && maxVolume <= series.Metadata.TotalCount)
{
series.Metadata.MaxCount = maxVolume;
} else if (maxVolume == series.Metadata.TotalCount)
{
series.Metadata.MaxCount = maxVolume;
} else
{
series.Metadata.MaxCount = maxChapter;
}
if (!series.Metadata.PublicationStatusLocked)
{
series.Metadata.PublicationStatus = PublicationStatus.OnGoing;
if (series.Metadata.MaxCount == series.Metadata.TotalCount && series.Metadata.TotalCount > 0)
{
series.Metadata.PublicationStatus = PublicationStatus.Completed;
} else if (series.Metadata.TotalCount > 0 && series.Metadata.MaxCount > 0)
{
series.Metadata.PublicationStatus = PublicationStatus.Ended;
}
}
DeterminePublicationStatus(series, chapters);
if (!string.IsNullOrEmpty(firstChapter?.Summary) && !series.Metadata.SummaryLocked)
{
@ -616,6 +579,48 @@ public class ProcessSeries : IProcessSeries
}
private static void DeterminePublicationStatus(Series series, List<Chapter> chapters)
{
// Count (aka expected total number of chapters or volumes from metadata) across all chapters
series.Metadata.TotalCount = chapters.Max(chapter => chapter.TotalCount);
// The actual number of count's defined across all chapter's metadata
series.Metadata.MaxCount = chapters.Max(chapter => chapter.Count);
var maxVolume = (int) series.Volumes.Where(v => v.MaxNumber.IsNot(Parser.Parser.SpecialVolumeNumber)).Max(v => v.MaxNumber);
var maxChapter = (int) chapters.Max(c => c.MaxNumber);
// Single books usually don't have a number in their Range (filename)
if (series.Format == MangaFormat.Epub || series.Format == MangaFormat.Pdf && chapters.Count == 1)
{
series.Metadata.MaxCount = 1;
} else if (series.Metadata.TotalCount <= 1 && chapters.Count == 1 && chapters[0].IsSpecial)
{
// If a series has a TotalCount of 1 (or no total count) and there is only a Special, mark it as Complete
series.Metadata.MaxCount = series.Metadata.TotalCount;
} else if ((maxChapter == Parser.Parser.DefaultChapterNumber || maxChapter > series.Metadata.TotalCount) && maxVolume <= series.Metadata.TotalCount)
{
series.Metadata.MaxCount = maxVolume;
} else if (maxVolume == series.Metadata.TotalCount)
{
series.Metadata.MaxCount = maxVolume;
} else
{
series.Metadata.MaxCount = maxChapter;
}
if (!series.Metadata.PublicationStatusLocked)
{
series.Metadata.PublicationStatus = PublicationStatus.OnGoing;
if (series.Metadata.MaxCount == series.Metadata.TotalCount && series.Metadata.TotalCount > 0)
{
series.Metadata.PublicationStatus = PublicationStatus.Completed;
} else if (series.Metadata.TotalCount > 0 && series.Metadata.MaxCount > 0)
{
series.Metadata.PublicationStatus = PublicationStatus.Ended;
}
}
}
private async Task UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
{
// Add new volumes and update chapters per volume

1
UI/Web/src/app/library-detail/library-detail.component.ts
View File

@ -241,7 +241,6 @@ export class LibraryDetailComponent implements OnInit {
async handleAction(action: ActionItem<Library>, library: Library) {
let lib: Partial<Library> = library;
if (library === undefined) {
//lib = {id: this.libraryId, name: this.libraryName}; // BUG: We need the whole library for editLibrary
this.libraryService.getLibrary(this.libraryId).subscribe(async library => {
switch (action.action) {
case(Action.Scan):