ComicInfo Derived Reading Lists (#1929)

* Implemented the ability to generate reading lists from StoryArc and StoryArcNumber ComicInfo fields.

* Refactored to add AlternativeSeries support.

* Fixed up the handling when we need to update reading list where order is already present.

* Refactored how skipping empty reading list pairs works

API.Tests/Services/ProcessSeriesTests.cs

@@ -60,7 +60,7 @@ public class ProcessSeriesTests
             , Substitute.For<ICacheHelper>(), Substitute.For<IReadingItemService>(), Substitute.For<IFileService>(),
             Substitute.For<IMetadataService>(),
             Substitute.For<IWordCountAnalyzerService>(),
-            Substitute.For<ICollectionTagService>());
+            Substitute.For<ICollectionTagService>(), Substitute.For<IReadingListService>());
 
         ps.UpdateChapterFromComicInfo(chapter, new ComicInfo()
         {

API/Data/DataContext.cs

@@ -110,6 +110,9 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
         builder.Entity<Library>()
             .Property(b => b.ManageCollections)
             .HasDefaultValue(true);
+        builder.Entity<Library>()
+            .Property(b => b.ManageReadingLists)
+            .HasDefaultValue(true);
     }
 
 

API/Data/Metadata/ComicInfo.cs

@@ -61,7 +61,7 @@ public class ComicInfo
     public string SeriesGroup { get; set; } = string.Empty;
 
     /// <summary>
-    ///
+    /// Can contain multiple comma separated numbers that match with StoryArcNumber
     /// </summary>
     public string StoryArc { get; set; } = string.Empty;
     /// <summary>

API/Data/Migrations/20230415123449_ManageReadingListOnLibrary.cs

@@ -0,0 +1,29 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace API.Data.Migrations
+{
+    /// <inheritdoc />
+    public partial class ManageReadingListOnLibrary : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<bool>(
+                name: "ManageReadingLists",
+                table: "Library",
+                type: "INTEGER",
+                nullable: false,
+                defaultValue: true);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "ManageReadingLists",
+                table: "Library");
+        }
+    }
+}

API/Data/Migrations/DataContextModelSnapshot.cs

@@ -15,7 +15,7 @@ namespace API.Data.Migrations
         protected override void BuildModel(ModelBuilder modelBuilder)
         {
 #pragma warning disable 612, 618
-            modelBuilder.HasAnnotation("ProductVersion", "7.0.4");
+            modelBuilder.HasAnnotation("ProductVersion", "7.0.5");
 
             modelBuilder.Entity("API.Entities.AppRole", b =>
                 {
@@ -649,6 +649,11 @@ namespace API.Data.Migrations
                         .HasColumnType("INTEGER")
                         .HasDefaultValue(true);
 
+                    b.Property<bool>("ManageReadingLists")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasDefaultValue(true);
+
                     b.Property<string>("Name")
                         .HasColumnType("TEXT");
 

API/Data/Repositories/ReadingListRepository.cs

@@ -48,6 +48,7 @@ public interface IReadingListRepository
     Task<IList<ReadingList>> GetAllWithNonWebPCovers();
     Task<IList<string>> GetFirstFourCoverImagesByReadingListId(int readingListId);
     Task<int> RemoveReadingListsWithoutSeries();
+    Task<ReadingList?> GetReadingListByTitleAsync(string name, int userId, ReadingListIncludes includes = ReadingListIncludes.Items);
 }
 
 public class ReadingListRepository : IReadingListRepository
@@ -145,6 +146,15 @@ public class ReadingListRepository : IReadingListRepository
         return await _context.SaveChangesAsync();
     }
 
+
+    public async Task<ReadingList?> GetReadingListByTitleAsync(string name, int userId, ReadingListIncludes includes = ReadingListIncludes.Items)
+    {
+        var normalized = name.ToNormalized();
+        return await _context.ReadingList
+            .Includes(includes)
+            .FirstOrDefaultAsync(x => x.NormalizedTitle != null && x.NormalizedTitle.Equals(normalized) && x.AppUserId == userId);
+    }
+
     public void Remove(ReadingListItem item)
     {
         _context.ReadingListItem.Remove(item);

API/Data/Repositories/UserRepository.cs

@@ -62,6 +62,7 @@ public interface IUserRepository
     Task<bool> HasAccessToLibrary(int libraryId, int userId);
     Task<IEnumerable<AppUser>> GetAllUsersAsync(AppUserIncludes includeFlags = AppUserIncludes.None);
     Task<AppUser?> GetUserByConfirmationToken(string token);
+    Task<AppUser> GetDefaultAdminUser();
 }
 
 public class UserRepository : IUserRepository
@@ -220,6 +221,17 @@ public class UserRepository : IUserRepository
             .SingleOrDefaultAsync(u => u.ConfirmationToken != null && u.ConfirmationToken.Equals(token));
     }
 
+    /// <summary>
+    /// Returns the first admin account created
+    /// </summary>
+    /// <returns></returns>
+    public async Task<AppUser> GetDefaultAdminUser()
+    {
+        return (await _userManager.GetUsersInRoleAsync(PolicyConstants.AdminRole))
+            .OrderByDescending(u => u.Created)
+            .First();
+    }
+
     public async Task<IEnumerable<AppUser>> GetAdminUsersAsync()
     {
         return await _userManager.GetUsersInRoleAsync(PolicyConstants.AdminRole);

API/Entities/Library.cs

@@ -28,9 +28,13 @@ public class Library : IEntityDate
     /// </summary>
     public bool IncludeInSearch { get; set; } = true;
     /// <summary>
-    /// Should this library create and manage collections from Metadata
+    /// Should this library create collections from Metadata
     /// </summary>
     public bool ManageCollections { get; set; } = true;
+    /// <summary>
+    /// Should this library create reading lists from Metadata
+    /// </summary>
+    public bool ManageReadingLists { get; set; } = true;
     public DateTime Created { get; set; }
     public DateTime LastModified { get; set; }
     public DateTime CreatedUtc { get; set; }

API/Helpers/Builders/ReadingListBuilder.cs

@@ -54,4 +54,10 @@ public class ReadingListBuilder : IEntityBuilder<ReadingList>
         _readingList.CoverImage = coverImage;
         return this;
     }
+
+    public ReadingListBuilder WithAppUserId(int userId)
+    {
+        _readingList.AppUserId = userId;
+        return this;
+    }
 }

API/Services/ReadingListService.cs

@@ -36,6 +36,13 @@ public interface IReadingListService
     Task<CblImportSummaryDto> CreateReadingListFromCbl(int userId, CblReadingList cblReading, bool dryRun = false);
     Task CalculateStartAndEndDates(ReadingList readingListWithItems);
     Task<string> GenerateMergedImage(int readingListId);
+    /// <summary>
+    /// This is expected to be called from ProcessSeries and has the Full Series present. Will generate on the default admin user.
+    /// </summary>
+    /// <param name="series"></param>
+    /// <param name="library"></param>
+    /// <returns></returns>
+    Task CreateReadingListsFromSeries(Series series, Library library);
 }
 
 /// <summary>
@@ -228,21 +235,26 @@ public class ReadingListService : IReadingListService
     public async Task<bool> UpdateReadingListItemPosition(UpdateReadingListPosition dto)
     {
         var items = (await _unitOfWork.ReadingListRepository.GetReadingListItemsByIdAsync(dto.ReadingListId)).ToList();
-        var item = items.Find(r => r.Id == dto.ReadingListItemId);
+        ReorderItems(items, dto.ReadingListItemId, dto.ToPosition);
+
+        if (!_unitOfWork.HasChanges()) return true;
+
+        return await _unitOfWork.CommitAsync();
+    }
+
+    private static void ReorderItems(List<ReadingListItem> items, int readingListItemId, int toPosition)
+    {
+        var item = items.Find(r => r.Id == readingListItemId);
         if (item != null)
         {
             items.Remove(item);
-            items.Insert(dto.ToPosition, item);
+            items.Insert(toPosition, item);
         }
 
         for (var i = 0; i < items.Count; i++)
         {
             items[i].Order = i;
         }
-
-        if (!_unitOfWork.HasChanges()) return true;
-
-        return await _unitOfWork.CommitAsync();
     }
 
     /// <summary>
@@ -420,6 +432,84 @@ public class ReadingListService : IReadingListService
         return index > lastOrder + 1;
     }
 
+    public async Task CreateReadingListsFromSeries(Series series, Library library)
+    {
+        if (!library.ManageReadingLists) return;
+
+        var hasReadingListMarkers = series.Volumes
+            .SelectMany(c => c.Chapters)
+            .Any(c => !string.IsNullOrEmpty(c.StoryArc) || !string.IsNullOrEmpty(c.AlternateSeries));
+
+        if (!hasReadingListMarkers) return;
+
+        _logger.LogInformation("Processing Reading Lists for {SeriesName}", series.Name);
+        var user = await _unitOfWork.UserRepository.GetDefaultAdminUser();
+        series.Metadata ??= new SeriesMetadataBuilder().Build();
+        foreach (var chapter in series.Volumes.SelectMany(v => v.Chapters))
+        {
+            List<Tuple<string, string>> pairs = new List<Tuple<string, string>>();
+            if (!string.IsNullOrEmpty(chapter.StoryArc))
+            {
+                pairs.AddRange(GeneratePairs(chapter.Files.FirstOrDefault()!.FilePath, chapter.StoryArc, chapter.StoryArcNumber));
+            }
+            if (!string.IsNullOrEmpty(chapter.AlternateSeries))
+            {
+                pairs.AddRange(GeneratePairs(chapter.Files.FirstOrDefault()!.FilePath, chapter.AlternateSeries, chapter.AlternateNumber));
+            }
+
+            foreach (var arcPair in pairs)
+            {
+                var order = int.Parse(arcPair.Item2);
+                var readingList = await _unitOfWork.ReadingListRepository.GetReadingListByTitleAsync(arcPair.Item1, user.Id);
+                if (readingList == null)
+                {
+                    readingList = new ReadingListBuilder(arcPair.Item1)
+                        .WithAppUserId(user.Id)
+                        .Build();
+                    _unitOfWork.ReadingListRepository.Add(readingList);
+
+                }
+
+                var items = readingList.Items.ToList();
+                var readingListItem = items.FirstOrDefault(item => item.Order == order);
+                if (readingListItem == null)
+                {
+                    items.Add(new ReadingListItemBuilder(order, series.Id, chapter.VolumeId, chapter.Id).Build());
+                }
+                else
+                {
+                    ReorderItems(items, readingListItem.Id, order);
+                }
+
+                readingList.Items = items;
+                await CalculateReadingListAgeRating(readingList);
+                await _unitOfWork.CommitAsync();
+            }
+        }
+    }
+
+    private IList<Tuple<string, string>> GeneratePairs(string filename, string storyArc, string storyArcNumbers)
+    {
+        var data = new List<Tuple<string, string>>();
+        if (string.IsNullOrEmpty(storyArc)) return data;
+
+        var arcs = storyArc.Split(",");
+        var arcNumbers = storyArcNumbers.Split(",");
+        if (arcNumbers.Length != arcs.Length)
+        {
+            _logger.LogError("There is a mismatch on StoryArc and StoryArcNumber for {FileName}", filename);
+        }
+
+        var maxPairs = Math.Min(arcs.Length, arcNumbers.Length);
+        for (var i = 0; i < maxPairs; i++)
+        {
+            if (string.IsNullOrEmpty(arcs[i]) || !int.TryParse(arcNumbers[i], out _)) continue;
+            data.Add(new Tuple<string, string>(arcs[i], arcNumbers[i]));
+        }
+
+        return data;
+    }
+
     /// <summary>
     /// Check for File issues like: No entries, Reading List Name collision, Duplicate Series across Libraries
     /// </summary>

API/Services/Tasks/Scanner/ProcessSeries.cs

@@ -54,6 +54,7 @@ public class ProcessSeries : IProcessSeries
     private readonly IMetadataService _metadataService;
     private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
     private readonly ICollectionTagService _collectionTagService;
+    private readonly IReadingListService _readingListService;
 
     private Dictionary<string, Genre> _genres;
     private IList<Person> _people;
@@ -66,7 +67,7 @@ public class ProcessSeries : IProcessSeries
     public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
         IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
         IFileService fileService, IMetadataService metadataService, IWordCountAnalyzerService wordCountAnalyzerService,
-        ICollectionTagService collectionTagService)
+        ICollectionTagService collectionTagService, IReadingListService readingListService)
     {
         _unitOfWork = unitOfWork;
         _logger = logger;
@@ -78,6 +79,7 @@ public class ProcessSeries : IProcessSeries
         _metadataService = metadataService;
         _wordCountAnalyzerService = wordCountAnalyzerService;
         _collectionTagService = collectionTagService;
+        _readingListService = readingListService;
 
 
         _genres = new Dictionary<string, Genre>();
@@ -179,8 +181,6 @@ public class ProcessSeries : IProcessSeries
 
             UpdateSeriesMetadata(series, library);
 
-            //CreateReadingListsFromSeries(series, library); This will be implemented later when I solution it
-
             // Update series FolderPath here
             await UpdateSeriesFolderPath(parsedInfos, library, series);
 
@@ -205,6 +205,9 @@ public class ProcessSeries : IProcessSeries
                     return;
                 }
 
+                // Process reading list after commit as we need to commit per list
+                await _readingListService.CreateReadingListsFromSeries(series, library);
+
                 if (seriesAdded)
                 {
                     await _eventHub.SendMessageAsync(MessageFactory.SeriesAdded,
@@ -223,26 +226,6 @@ public class ProcessSeries : IProcessSeries
         EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id);
     }
 
-    private void CreateReadingListsFromSeries(Series series, Library library)
-    {
-        //if (!library.ManageReadingLists) return;
-        _logger.LogInformation("Generating Reading Lists for {SeriesName}", series.Name);
-
-        series.Metadata ??= new SeriesMetadataBuilder().Build();
-        foreach (var chapter in series.Volumes.SelectMany(v => v.Chapters))
-        {
-            if (!string.IsNullOrEmpty(chapter.StoryArc))
-            {
-                var readingLists = chapter.StoryArc.Split(',');
-                var readingListOrders = chapter.StoryArcNumber.Split(',');
-                if (readingListOrders.Length == 0)
-                {
-                    _logger.LogDebug("[ScannerService] There are no StoryArc orders listed, all reading lists fueled from StoryArc will be unordered");
-
-                }
-            }
-        }
-    }
 
     private async Task UpdateSeriesFolderPath(IEnumerable<ParserInfo> parsedInfos, Library library, Series series)
     {
@@ -517,14 +500,11 @@ public class ProcessSeries : IProcessSeries
             }
             catch (Exception ex)
             {
-                if (ex.Message.Equals("Sequence contains more than one matching element"))
+                if (!ex.Message.Equals("Sequence contains more than one matching element")) throw;
-                {
                 _logger.LogCritical("[ScannerService] Kavita found corrupted volume entries on {SeriesName}. Please delete the series from Kavita via UI and rescan", series.Name);
                 throw new KavitaException(
                     $"Kavita found corrupted volume entries on {series.Name}. Please delete the series from Kavita via UI and rescan");
             }
-                throw;
-            }
             if (volume == null)
             {
                 volume = new VolumeBuilder(volumeNumber)

openapi.json

@@ -11837,7 +11837,11 @@
           },
           "manageCollections": {
             "type": "boolean",
-            "description": "Should this library create and manage collections from Metadata"
+            "description": "Should this library create collections from Metadata"
+          },
+          "manageReadingLists": {
+            "type": "boolean",
+            "description": "Should this library create reading lists from Metadata"
           },
           "created": {
             "type": "string",