Release Testing Day 2 (#1937)

* Removed unneded ngModel on password field

* Fixed some bad validation messages on Edit Reading List modal and disabled save button

* Added a lot of trace code to help debug a foreign constraint issue.

* Fixed a bug where after a series is scanned, generate covers for series didn't respect webp cover generation.

* Fixed library last scan being stored in Utc, but expected to be server time.

* Fixed up some of that trace logging being way too verbose. Fixed a case where when a missing storyarc number, the whole pair was dropped. Now, it will default that item to the end of the reading list.

Fixed a bug where Start and End dates weren't being calculated after generating a reading list.

* Fixed a bug where the default admin user for reading list creation from files was selecting the wrong user.

Changed so that when there is a bad pair (aka number missing) and only 1 pair, then we wont constantly reorder the item.

* Fixed unit test

API.Tests/Services/ReadingListServiceTests.cs

@@ -1203,10 +1203,67 @@ public class ReadingListServiceTests
 
     #region CreateReadingListsFromSeries
 
+    private async Task<Tuple<Series, Series>> SetupData()
+    {
+        // Setup 2 series, only do this once tho
+        if (await _unitOfWork.SeriesRepository.DoesSeriesNameExistInLibrary("Series 1", 1, MangaFormat.Archive))
+        {
+            return new Tuple<Series, Series>(await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(1),
+                await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(2));
+        }
+
+        var library =
+            await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
+                LibraryIncludes.Series | LibraryIncludes.AppUser);
+        var user = new AppUserBuilder("majora2007", "majora2007@fake.com").Build();
+        library!.AppUsers.Add(user);
+        library.ManageReadingLists = true;
+
+        // Setup the series for CreateReadingListsFromSeries
+        var series1 = new SeriesBuilder("Series 1")
+            .WithFormat(MangaFormat.Archive)
+            .WithVolume(new VolumeBuilder("1")
+                .WithChapter(new ChapterBuilder("1")
+                    .WithStoryArc("CreateReadingListsFromSeries")
+                    .WithStoryArcNumber("1")
+                    .Build())
+                .WithChapter(new ChapterBuilder("2").Build())
+                .Build())
+            .Build();
+
+        var series2 = new SeriesBuilder("Series 2")
+            .WithFormat(MangaFormat.Archive)
+            .WithVolume(new VolumeBuilder(API.Services.Tasks.Scanner.Parser.Parser.DefaultVolume)
+                .WithChapter(new ChapterBuilder("1").Build())
+                .WithChapter(new ChapterBuilder("2").Build())
+                .Build())
+            .Build();
+
+        library!.Series.Add(series1);
+        library!.Series.Add(series2);
+
+        await _unitOfWork.CommitAsync();
+
+        return new Tuple<Series, Series>(series1, series2);
+    }
+
     // [Fact]
     // public async Task CreateReadingListsFromSeries_ShouldCreateFromSinglePair()
     // {
+    //     //await SetupData();
     //
+    //     var series1 = new SeriesBuilder("Series 1")
+    //         .WithFormat(MangaFormat.Archive)
+    //         .WithVolume(new VolumeBuilder("1")
+    //             .WithChapter(new ChapterBuilder("1")
+    //                 .WithStoryArc("CreateReadingListsFromSeries")
+    //                 .WithStoryArcNumber("1")
+    //                 .Build())
+    //             .WithChapter(new ChapterBuilder("2").Build())
+    //             .Build())
+    //         .Build();
+    //
+    //     _readingListService.CreateReadingListsFromSeries(series.Item1)
     // }
 
     #endregion

API/Data/Repositories/UserRepository.cs

@@ -228,7 +228,7 @@ public class UserRepository : IUserRepository
     public async Task<AppUser> GetDefaultAdminUser()
     {
         return (await _userManager.GetUsersInRoleAsync(PolicyConstants.AdminRole))
-            .OrderByDescending(u => u.Created)
+            .OrderBy(u => u.Created)
             .First();
     }
 

API/Entities/Metadata/SeriesMetadata.cs

@@ -13,7 +13,7 @@ public class SeriesMetadata : IHasConcurrencyToken
 
     public string Summary { get; set; } = string.Empty;
 
-    public ICollection<CollectionTag> CollectionTags { get; set; } = null!;
+    public ICollection<CollectionTag> CollectionTags { get; set; } = new List<CollectionTag>();
 
     public ICollection<Genre> Genres { get; set; } = new List<Genre>();
     public ICollection<Tag> Tags { get; set; } = new List<Tag>();

API/Helpers/Builders/ChapterBuilder.cs

@@ -17,7 +17,7 @@ public class ChapterBuilder : IEntityBuilder<Chapter>
         {
             Range = string.IsNullOrEmpty(range) ? number : range,
             Title = string.IsNullOrEmpty(range) ? number : range,
-            Number = Services.Tasks.Scanner.Parser.Parser.MinNumberFromRange(number) + string.Empty,
+            Number = Parser.MinNumberFromRange(number) + string.Empty,
             Files = new List<MangaFile>(),
             Pages = 1
         };
@@ -42,12 +42,24 @@ public class ChapterBuilder : IEntityBuilder<Chapter>
         return this;
     }
 
-    private ChapterBuilder WithNumber(string number)
+    public ChapterBuilder WithNumber(string number)
     {
         _chapter.Number = number;
         return this;
     }
 
+    public ChapterBuilder WithStoryArc(string arc)
+    {
+        _chapter.StoryArc = arc;
+        return this;
+    }
+
+    public ChapterBuilder WithStoryArcNumber(string number)
+    {
+        _chapter.StoryArcNumber = number;
+        return this;
+    }
+
     private ChapterBuilder WithRange(string range)
     {
         _chapter.Range = range;

API/Helpers/PersonHelper.cs

@@ -103,6 +103,7 @@ public static class PersonHelper
         if (string.IsNullOrEmpty(person.Name)) return;
         var existingPerson = metadataPeople.FirstOrDefault(p =>
             p.NormalizedName == person.Name.ToNormalized() && p.Role == person.Role);
+
         if (existingPerson == null)
         {
             metadataPeople.Add(person);

API/Services/ReadingListService.cs

@@ -13,9 +13,11 @@ using API.Entities;
 using API.Entities.Enums;
 using API.Helpers;
 using API.Helpers.Builders;
+using API.Services.Tasks.Scanner.Parser;
 using API.SignalR;
 using Kavita.Common;
 using Microsoft.Extensions.Logging;
+using Microsoft.IdentityModel.Tokens;
 
 namespace API.Services;
 
@@ -447,7 +449,7 @@ public class ReadingListService : IReadingListService
         series.Metadata ??= new SeriesMetadataBuilder().Build();
         foreach (var chapter in series.Volumes.SelectMany(v => v.Chapters))
         {
-            List<Tuple<string, string>> pairs = new List<Tuple<string, string>>();
+            var pairs = new List<Tuple<string, string>>();
             if (!string.IsNullOrEmpty(chapter.StoryArc))
             {
                 pairs.AddRange(GeneratePairs(chapter.Files.FirstOrDefault()!.FilePath, chapter.StoryArc, chapter.StoryArcNumber));
@@ -459,7 +461,6 @@ public class ReadingListService : IReadingListService
 
             foreach (var arcPair in pairs)
             {
-                var order = int.Parse(arcPair.Item2);
                 var readingList = await _unitOfWork.ReadingListRepository.GetReadingListByTitleAsync(arcPair.Item1, user.Id);
                 if (readingList == null)
                 {
@@ -471,22 +472,39 @@ public class ReadingListService : IReadingListService
                 }
 
                 var items = readingList.Items.ToList();
-                var readingListItem = items.FirstOrDefault(item => item.Order == order);
+                var order = int.Parse(arcPair.Item2);
+                var readingListItem = items.FirstOrDefault(item => item.Order == order || item.ChapterId == chapter.Id);
                 if (readingListItem == null)
                 {
+                    // If no number was provided in the reading list, we default to MaxValue and hence we should insert the item at the end of the list
+                    if (order == int.MaxValue)
+                    {
+                        order = items.Count > 0 ? items.Max(item => item.Order) + 1 : 0;
+                    }
                     items.Add(new ReadingListItemBuilder(order, series.Id, chapter.VolumeId, chapter.Id).Build());
                 }
                 else
+                {
+                    if (order == int.MaxValue)
+                    {
+                        _logger.LogWarning("{Filename} has a missing StoryArcNumber/AlternativeNumber but list already exists with this item. Skipping item", chapter.Files.FirstOrDefault()?.FilePath);
+                    }
+                    else
                     {
                         ReorderItems(items, readingListItem.Id, order);
                     }
+                }
 
                 readingList.Items = items;
                 await CalculateReadingListAgeRating(readingList);
+                await CalculateStartAndEndDates(readingList);
+                if (_unitOfWork.HasChanges())
+                {
                     await _unitOfWork.CommitAsync();
                 }
             }
         }
+    }
 
     private IEnumerable<Tuple<string, string>> GeneratePairs(string filename, string storyArc, string storyArcNumbers)
     {
@@ -495,14 +513,19 @@ public class ReadingListService : IReadingListService
 
         var arcs = storyArc.Split(",");
         var arcNumbers = storyArcNumbers.Split(",");
-        if (arcNumbers.Length != arcs.Length)
+        if (arcNumbers.Count(s => !string.IsNullOrEmpty(s)) != arcs.Length)
         {
-            _logger.LogError("There is a mismatch on StoryArc and StoryArcNumber for {FileName}", filename);
+            _logger.LogWarning("There is a mismatch on StoryArc and StoryArcNumber for {FileName}. Def", filename);
         }
 
         var maxPairs = Math.Min(arcs.Length, arcNumbers.Length);
         for (var i = 0; i < maxPairs; i++)
         {
+            // When there is a mismatch on arcs and arc numbers, then we should default to a high number
+            if (string.IsNullOrEmpty(arcNumbers[i]) && !string.IsNullOrEmpty(arcs[i]))
+            {
+                arcNumbers[i] = int.MaxValue.ToString();
+            }
             if (string.IsNullOrEmpty(arcs[i]) || !int.TryParse(arcNumbers[i], out _)) continue;
             data.Add(new Tuple<string, string>(arcs[i], arcNumbers[i]));
         }

API/Services/Tasks/Scanner/ProcessSeries.cs

@@ -196,8 +196,9 @@ public class ProcessSeries : IProcessSeries
                 {
                     await _unitOfWork.RollbackAsync();
                     _logger.LogCritical(ex,
-                        "[ScannerService] There was an issue writing to the database for series {@SeriesName}",
+                        "[ScannerService] There was an issue writing to the database for series {SeriesName}",
                         series.Name);
+                    _logger.LogTrace("[ScannerService] Full Series Dump: {@Series}", series);
 
                     await _eventHub.SendMessageAsync(MessageFactory.Error,
                         MessageFactory.ErrorEvent($"There was an issue writing to the DB for Series {series}",
@@ -222,7 +223,7 @@ public class ProcessSeries : IProcessSeries
             _logger.LogError(ex, "[ScannerService] There was an exception updating series for {SeriesName}", series.Name);
         }
 
-        await _metadataService.GenerateCoversForSeries(series, false);
+        await _metadataService.GenerateCoversForSeries(series, (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertCoverToWebP);
         EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id);
     }
 
@@ -726,7 +727,18 @@ public class ProcessSeries : IProcessSeries
 
         void AddPerson(Person person)
         {
-            PersonHelper.AddPersonIfNotExists(chapter.People, person);
+            // TODO: Temp have code inlined to help debug foreign key constraint issue
+            //PersonHelper.AddPersonIfNotExists(chapter.People, person);
+            if (string.IsNullOrEmpty(person.Name)) return;
+            var existingPerson = chapter.People.FirstOrDefault(p =>
+                p.NormalizedName == person.Name.ToNormalized() && p.Role == person.Role);
+            _logger.LogTrace("[PersonHelper] Attempting to add {@Person} to {FileName} with ChapterID {ChapterId}, adding if not null: {@ExistingPerson}",
+                person, chapter.Files.FirstOrDefault()?.FilePath, chapter.Id, existingPerson);
+
+            if (existingPerson == null)
+            {
+                chapter.People.Add(person);
+            }
         }
 
         void AddGenre(Genre genre, bool newTag)
@@ -823,15 +835,19 @@ public class ProcessSeries : IProcessSeries
         lock (_peopleLock)
         {
             var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();
+            _logger.LogTrace("[UpdatePeople] for {Role} and Names of {Names}", role, names);
+            _logger.LogTrace("[UpdatePeople] for {Role} found {@People}", role, allPeopleTypeRole.Select(p => new {p.Id, p.Name, SeriesMetadataIds = p.SeriesMetadatas?.Select(m => m.Id).ToList()}));
 
             foreach (var name in names)
             {
                 var normalizedName = name.ToNormalized();
                 var person = allPeopleTypeRole.FirstOrDefault(p =>
                     p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));
+
                 if (person == null)
                 {
                     person = new PersonBuilder(name, role).Build();
+                    _logger.LogTrace("[UpdatePeople] for {Role} no one found, adding to _people", role);
                     _people.Add(person);
                 }
 

API/Services/Tasks/ScannerService.cs

@@ -531,7 +531,7 @@ public class ScannerService : IScannerService
 
         _logger.LogInformation("[ScannerService] Finished file scan in {ScanAndUpdateTime} milliseconds. Updating database", scanElapsedTime);
 
-        var time = DateTime.UtcNow;
+        var time = DateTime.Now;
         foreach (var folderPath in library.Folders)
         {
             folderPath.UpdateLastScanned(time);

UI/Web/src/app/pipe/time-ago.pipe.ts

@@ -38,6 +38,7 @@ export class TimeAgoPipe implements PipeTransform, OnDestroy {
 	transform(value: string) {
 		this.removeTimer();
 		const d = new Date(value);
+		console.log('date: ', d);
 		const now = new Date();
 		const seconds = Math.round(Math.abs((now.getTime() - d.getTime()) / 1000));
 		const timeToUpdate = (Number.isNaN(seconds)) ? 1000 : this.getSecondsUntilUpdate(seconds) * 1000;

UI/Web/src/app/reading-list/_modals/edit-reading-list-modal/edit-reading-list-modal.component.html

@@ -58,7 +58,7 @@
                                 aria-describedby="starting-year-header">
                                 <div id="inviteForm-validations" class="invalid-feedback" *ngIf="reviewGroup.dirty || reviewGroup.touched">
                                     <div *ngIf="formControl.errors?.min || formControl.errors?.max">
-                                        Must be between 1 and 12 or blank
+                                        Must be greater than 1000, 0 or blank
                                     </div>
                                 </div>
                             </div>
@@ -84,7 +84,7 @@
                                 aria-describedby="ending-year-header">
                                 <div id="inviteForm-validations" class="invalid-feedback" *ngIf="reviewGroup.dirty || reviewGroup.touched">
                                     <div *ngIf="formControl.errors?.min || formControl.errors?.max">
-                                        Must be between 1 and 12 or blank
+                                        Must be greater than 1000, 0 or blank
                                     </div>
                                 </div>
                             </div>
@@ -109,7 +109,7 @@
 </div>
 <div class="modal-footer">
     <button type="button" class="btn btn-secondary" (click)="close()">Close</button>
-    <button type="submit" class="btn btn-primary" [disabled]="!reviewGroup.valid" (click)="save()">Save</button>
+    <button type="submit" class="btn btn-primary" (click)="save()">Save</button>
 </div>
 
 

UI/Web/src/app/reading-list/_modals/edit-reading-list-modal/edit-reading-list-modal.component.ts

@@ -116,6 +116,8 @@ export class EditReadingListModalComponent implements OnInit, OnDestroy {
 
   updateSelectedIndex(index: number) {
     this.coverImageIndex = index;
+    console.log(this.coverImageIndex)
+    this.cdRef.detectChanges();
   }
 
   updateSelectedImage(url: string) {

UI/Web/src/app/registration/user-login/user-login.component.html

@@ -12,7 +12,7 @@
                     <div class="mb-2">
                         <label for="password" class="form-label visually-hidden">Password</label>
                         <input class="form-control custom-input" formControlName="password" name="password"
-                         id="password" type="password" ngModel pattern="^.{6,32}$" placeholder="Password">
+                         id="password" type="password" pattern="^.{6,32}$" placeholder="Password">
                         <div id="inviteForm-validations" class="invalid-feedback" *ngIf="loginForm.get('password')?.errors?.pattern"  >
                             <div class="" *ngIf="loginForm.get('password')?.errors?.pattern">
                                 Password must be between 6 and 32 characters in length