Cutlery/Kavita
1
0
Fork 0
mirror of https://github.com/Kareadita/Kavita.git synced 2025-07-09 03:04:19 -04:00
Code Issues Packages Projects Releases Wiki Activity

Scan Bugfixes (#177)

Browse Source 
* Added way  more logging for debugging issue #163.

Fixed #175

* Removed some comment that isn't needed

* Fixed a enumeration issue due to removing while enumerating
This commit is contained in:
Joseph Milazzo 2021-04-16 12:21:16 -05:00 committed by GitHub
parent 09a953be8c
commit 2b99c8abfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 47 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
API/Data/AppUserProgressRepository.cs
View File

@ -17,7 +17,7 @@ namespace API.Data
/// <summary>
/// This will remove any entries that have chapterIds that no longer exists. This will execute the save as well.
/// </summary>
public async Task<bool> CleanupAbandonedChapters()
public async Task<int> CleanupAbandonedChapters()
{
var chapterIds = _context.Chapter.Select(c => c.Id);
@ -26,7 +26,7 @@ namespace API.Data
.ToListAsync();
_context.RemoveRange(rowsToRemove);
return (await _context.SaveChangesAsync()) > 0;
return await _context.SaveChangesAsync() > 0 ? rowsToRemove.Count : 0;
}
}
}

2
API/Interfaces/IAppUserProgressRepository.cs
View File

@ -4,6 +4,6 @@ namespace API.Interfaces
{
public interface IAppUserProgressRepository
{
Task<bool> CleanupAbandonedChapters();
Task<int> CleanupAbandonedChapters();
}
}

73
API/Services/Tasks/ScannerService.cs
View File

@ -38,7 +38,6 @@ namespace API.Services.Tasks
[DisableConcurrentExecution(timeoutInSeconds: 360)]
//[AutomaticRetry(Attempts = 0, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public void ScanLibraries()
{
var libraries = Task.Run(() => _unitOfWork.LibraryRepository.GetLibrariesAsync()).Result.ToList();
@ -64,7 +63,6 @@ namespace API.Services.Tasks
}
[DisableConcurrentExecution(360)]
//[AutomaticRetry(Attempts = 0, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public void ScanLibrary(int libraryId, bool forceUpdate)
{
var sw = Stopwatch.StartNew();
@ -144,14 +142,7 @@ namespace API.Services.Tasks
// Cleanup any user progress that doesn't exist
var cleanedUp = Task.Run(() => _unitOfWork.AppUserProgressRepository.CleanupAbandonedChapters()).Result;
if (cleanedUp)
{
_logger.LogInformation("Removed all abandoned progress rows");
}
else
{
_logger.LogWarning("There are abandoned user progress entities in the DB. In Progress activity stream will be skewed");
}
_logger.LogInformation("Removed {Count} abandoned progress rows", cleanedUp);
BackgroundJob.Enqueue(() => _metadataService.RefreshMetadata(libraryId, forceUpdate));
}
@ -167,16 +158,17 @@ namespace API.Services.Tasks
_logger.LogInformation("Removed {RemoveMissingSeries} series that are no longer on disk", removeCount);
// Add new series that have parsedInfos
foreach (var (key, _) in parsedSeries)
foreach (var (key, infos) in parsedSeries)
{
var existingSeries = library.Series.SingleOrDefault(s => s.NormalizedName == Parser.Parser.Normalize(key));
if (existingSeries == null)
{
var name = infos.Count > 0 ? infos[0].Series : key;
existingSeries = new Series()
{
Name = key,
OriginalName = key,
LocalizedName = key,
Name = name,
OriginalName = name,
LocalizedName = name,
NormalizedName = Parser.Parser.Normalize(key),
SortName = key,
Summary = "",
@ -193,7 +185,7 @@ namespace API.Services.Tasks
Parallel.ForEach(librarySeries, (series) =>
{
_logger.LogInformation("Processing series {SeriesName}", series.Name);
UpdateVolumes(series, parsedSeries[series.OriginalName].ToArray());
UpdateVolumes(series, parsedSeries[Parser.Parser.Normalize(series.OriginalName)].ToArray());
series.Pages = series.Volumes.Sum(v => v.Pages);
});
}
@ -201,22 +193,24 @@ namespace API.Services.Tasks
public IEnumerable<Series> FindSeriesNotOnDisk(ICollection<Series> existingSeries, Dictionary<string, List<ParserInfo>> parsedSeries)
{
var foundSeries = parsedSeries.Select(s => s.Key).ToList();
var missingSeries = existingSeries.Where(existingSeries => !existingSeries.NameInList(foundSeries)
|| !existingSeries.NameInList(parsedSeries.Keys));
var missingSeries = existingSeries.Where(es => !es.NameInList(foundSeries)
|| !es.NameInList(parsedSeries.Keys));
return missingSeries;
}
public int RemoveMissingSeries(ICollection<Series> existingSeries, IEnumerable<Series> missingSeries)
{
var removeCount = 0;
//library.Series = library.Series.Except(missingSeries).ToList();
if (existingSeries == null || existingSeries.Count == 0) return 0;
foreach (var existing in missingSeries)
{
existingSeries.Remove(existing);
removeCount += 1;
}
var removeCount = existingSeries.Count;
var missingList = missingSeries.ToList();
existingSeries = existingSeries.Except(missingList).ToList();
// if (existingSeries == null || existingSeries.Count == 0) return 0;
// foreach (var existing in missingSeries)
// {
// existingSeries.Remove(existing);
// removeCount += 1;
// }
removeCount -= existingSeries.Count;
return removeCount;
}
@ -245,15 +239,31 @@ namespace API.Services.Tasks
}
// NOTE: I don't think we need this as chapters now handle specials
volume.IsSpecial = volume.Number == 0 && infos.All(p => p.Chapters == "0" || p.IsSpecial);
//volume.IsSpecial = volume.Number == 0 && infos.All(p => p.Chapters == "0" || p.IsSpecial);
_logger.LogDebug("Parsing {SeriesName} - Volume {VolumeNumber}", series.Name, volume.Name);
UpdateChapters(volume, infos);
volume.Pages = volume.Chapters.Sum(c => c.Pages);
}
// BUG: This is causing volumes to be removed when they shouldn't
// Remove existing volumes that aren't in parsedInfos and volumes that have no chapters
var existingVolumeLength = series.Volumes.Count;
// var existingVols = series.Volumes;
// foreach (var v in existingVols)
// {
// // NOTE: I think checking if Chapter count is 0 is enough, we don't need parsedInfos
// if (parsedInfos.All(p => p.Volumes != v.Name)) // || v.Chapters.Count == 0 (this wont work yet because we don't take care of chapters correctly vs parsedInfos)
// {
// _logger.LogDebug("Removed {Series} - {Volume} as there were no chapters", series.Name, v.Name);
// series.Volumes.Remove(v);
// }
// }
series.Volumes = series.Volumes.Where(v => parsedInfos.Any(p => p.Volumes == v.Name)).ToList();
if (existingVolumeLength != series.Volumes.Count)
{
_logger.LogDebug("Removed {Count} volumes from {SeriesName} where parsed infos were not mapping with volume name", (existingVolumeLength - series.Volumes.Count), series.Name);
}
_logger.LogDebug("Updated {SeriesName} volumes from {StartingVolumeCount} to {VolumeCount}",
series.Name, startingVolumeCount, series.Volumes.Count);
@ -269,7 +279,6 @@ namespace API.Services.Tasks
var specialTreatment = (info.IsSpecial || (info.Volumes == "0" && info.Chapters == "0"));
// Specials go into their own chapters with Range being their filename and IsSpecial = True. Non-Specials with Vol and Chap as 0
// also are treated like specials for UI grouping.
_logger.LogDebug("Adding new chapters, {Series} - Vol {Volume} Ch {Chapter} - Needs Special Treatment? {NeedsSpecialTreatment}", info.Series, info.Volumes, info.Chapters, specialTreatment);
// NOTE: If there are duplicate files that parse out to be the same but a different series name (but parses to same normalized name ie History's strongest
// vs Historys strongest), this code will break and the duplicate will be skipped.
Chapter chapter = null;
@ -291,6 +300,7 @@ namespace API.Services.Tasks
if (chapter == null)
{
_logger.LogDebug("Adding new chapter, {Series} - Vol {Volume} Ch {Chapter} - Needs Special Treatment? {NeedsSpecialTreatment}", info.Series, info.Volumes, info.Chapters, specialTreatment);
chapter = new Chapter()
{
Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + string.Empty,
@ -320,14 +330,12 @@ namespace API.Services.Tasks
}
if (chapter == null) continue;
AddOrUpdateFileForChapter(chapter, info);
chapter.Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + "";
chapter.Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + string.Empty;
chapter.Range = specialTreatment ? info.Filename : info.Chapters;
chapter.Pages = chapter.Files.Sum(f => f.Pages);
}
// TODO: Extract to
// Remove chapters that aren't in parsedInfos or have no files linked
var existingChapters = volume.Chapters.ToList();
foreach (var existingChapter in existingChapters)
@ -336,8 +344,9 @@ namespace API.Services.Tasks
var hasInfo = specialTreatment ? parsedInfos.Any(v => v.Filename == existingChapter.Range)
: parsedInfos.Any(v => v.Chapters == existingChapter.Range);
if (!hasInfo || !existingChapter.Files.Any())
if (!hasInfo || existingChapter.Files.Count == 0)
{
_logger.LogDebug("Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName}", existingChapter.Range, volume.Name, parsedInfos[0].Series);
volume.Chapters.Remove(existingChapter);
}
else
@ -366,7 +375,7 @@ namespace API.Services.Tasks
// Check if normalized info.Series already exists and if so, update info to use that name instead
info.Series = MergeName(_scannedSeries, info);
_scannedSeries.AddOrUpdate(info.Series, new List<ParserInfo>() {info}, (_, oldValue) =>
_scannedSeries.AddOrUpdate(Parser.Parser.Normalize(info.Series), new List<ParserInfo>() {info}, (_, oldValue) =>
{
oldValue ??= new List<ParserInfo>();
if (!oldValue.Contains(info))