Refactored ScanLibrary to produce page numbers on the Manga File, Format and to update existing series/volumes rather than always create new entries.

This commit is contained in:
Joseph Milazzo 2021-01-09 15:27:04 -06:00
parent 6b4617bab3
commit 59a4921ba9
12 changed files with 709 additions and 38 deletions

View File

@ -151,7 +151,14 @@ namespace API.Controllers
[HttpDelete("delete")]
public async Task<ActionResult<bool>> DeleteLibrary(int libraryId)
{
return Ok(await _libraryRepository.DeleteLibrary(libraryId));
var result = await _libraryRepository.DeleteLibrary(libraryId);
if (result)
{
// TODO: This should clear out any cache items associated with library
}
return Ok(result);
}
[Authorize(Policy = "RequireAdminRole")]

View File

@ -1,7 +1,4 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Linq;
using System.Threading.Tasks;
using API.DTOs;
using API.Entities;
@ -55,8 +52,6 @@ namespace API.Controllers
file.Page = page;
return Ok(file);
//return File(await _directoryService.ReadImageAsync(path), "image/jpg", filename);
}
}
}

View File

@ -0,0 +1,521 @@
// <auto-generated />
using System;
using API.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace API.Data.Migrations
{
[DbContext(typeof(DataContext))]
[Migration("20210109205034_CacheMetadata")]
partial class CacheMetadata
{
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "5.0.1");
modelBuilder.Entity("API.Entities.AppRole", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("ConcurrencyStamp")
.IsConcurrencyToken()
.HasColumnType("TEXT");
b.Property<string>("Name")
.HasMaxLength(256)
.HasColumnType("TEXT");
b.Property<string>("NormalizedName")
.HasMaxLength(256)
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("NormalizedName")
.IsUnique()
.HasDatabaseName("RoleNameIndex");
b.ToTable("AspNetRoles");
});
modelBuilder.Entity("API.Entities.AppUser", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<int>("AccessFailedCount")
.HasColumnType("INTEGER");
b.Property<string>("ConcurrencyStamp")
.IsConcurrencyToken()
.HasColumnType("TEXT");
b.Property<DateTime>("Created")
.HasColumnType("TEXT");
b.Property<string>("Email")
.HasMaxLength(256)
.HasColumnType("TEXT");
b.Property<bool>("EmailConfirmed")
.HasColumnType("INTEGER");
b.Property<DateTime>("LastActive")
.HasColumnType("TEXT");
b.Property<bool>("LockoutEnabled")
.HasColumnType("INTEGER");
b.Property<DateTimeOffset?>("LockoutEnd")
.HasColumnType("TEXT");
b.Property<string>("NormalizedEmail")
.HasMaxLength(256)
.HasColumnType("TEXT");
b.Property<string>("NormalizedUserName")
.HasMaxLength(256)
.HasColumnType("TEXT");
b.Property<string>("PasswordHash")
.HasColumnType("TEXT");
b.Property<string>("PhoneNumber")
.HasColumnType("TEXT");
b.Property<bool>("PhoneNumberConfirmed")
.HasColumnType("INTEGER");
b.Property<uint>("RowVersion")
.IsConcurrencyToken()
.HasColumnType("INTEGER");
b.Property<string>("SecurityStamp")
.HasColumnType("TEXT");
b.Property<bool>("TwoFactorEnabled")
.HasColumnType("INTEGER");
b.Property<string>("UserName")
.HasMaxLength(256)
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("NormalizedEmail")
.HasDatabaseName("EmailIndex");
b.HasIndex("NormalizedUserName")
.IsUnique()
.HasDatabaseName("UserNameIndex");
b.ToTable("AspNetUsers");
});
modelBuilder.Entity("API.Entities.AppUserRole", b =>
{
b.Property<int>("UserId")
.HasColumnType("INTEGER");
b.Property<int>("RoleId")
.HasColumnType("INTEGER");
b.HasKey("UserId", "RoleId");
b.HasIndex("RoleId");
b.ToTable("AspNetUserRoles");
});
modelBuilder.Entity("API.Entities.FolderPath", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<int>("LibraryId")
.HasColumnType("INTEGER");
b.Property<string>("Path")
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("LibraryId");
b.ToTable("FolderPath");
});
modelBuilder.Entity("API.Entities.Library", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("CoverImage")
.HasColumnType("TEXT");
b.Property<DateTime>("Created")
.HasColumnType("TEXT");
b.Property<DateTime>("LastModified")
.HasColumnType("TEXT");
b.Property<string>("Name")
.HasColumnType("TEXT");
b.Property<int>("Type")
.HasColumnType("INTEGER");
b.HasKey("Id");
b.ToTable("Library");
});
modelBuilder.Entity("API.Entities.MangaFile", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<int>("Chapter")
.HasColumnType("INTEGER");
b.Property<string>("FilePath")
.HasColumnType("TEXT");
b.Property<int>("Format")
.HasColumnType("INTEGER");
b.Property<int>("NumberOfPages")
.HasColumnType("INTEGER");
b.Property<int>("VolumeId")
.HasColumnType("INTEGER");
b.HasKey("Id");
b.HasIndex("VolumeId");
b.ToTable("MangaFile");
});
modelBuilder.Entity("API.Entities.Series", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<byte[]>("CoverImage")
.HasColumnType("BLOB");
b.Property<DateTime>("Created")
.HasColumnType("TEXT");
b.Property<DateTime>("LastModified")
.HasColumnType("TEXT");
b.Property<int>("LibraryId")
.HasColumnType("INTEGER");
b.Property<string>("Name")
.HasColumnType("TEXT");
b.Property<string>("OriginalName")
.HasColumnType("TEXT");
b.Property<string>("SortName")
.HasColumnType("TEXT");
b.Property<string>("Summary")
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("LibraryId");
b.ToTable("Series");
});
modelBuilder.Entity("API.Entities.Volume", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<byte[]>("CoverImage")
.HasColumnType("BLOB");
b.Property<DateTime>("Created")
.HasColumnType("TEXT");
b.Property<DateTime>("LastModified")
.HasColumnType("TEXT");
b.Property<string>("Name")
.HasColumnType("TEXT");
b.Property<int>("Number")
.HasColumnType("INTEGER");
b.Property<int>("SeriesId")
.HasColumnType("INTEGER");
b.HasKey("Id");
b.HasIndex("SeriesId");
b.ToTable("Volume");
});
modelBuilder.Entity("AppUserLibrary", b =>
{
b.Property<int>("AppUsersId")
.HasColumnType("INTEGER");
b.Property<int>("LibrariesId")
.HasColumnType("INTEGER");
b.HasKey("AppUsersId", "LibrariesId");
b.HasIndex("LibrariesId");
b.ToTable("AppUserLibrary");
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<int>", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("ClaimType")
.HasColumnType("TEXT");
b.Property<string>("ClaimValue")
.HasColumnType("TEXT");
b.Property<int>("RoleId")
.HasColumnType("INTEGER");
b.HasKey("Id");
b.HasIndex("RoleId");
b.ToTable("AspNetRoleClaims");
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<int>", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("ClaimType")
.HasColumnType("TEXT");
b.Property<string>("ClaimValue")
.HasColumnType("TEXT");
b.Property<int>("UserId")
.HasColumnType("INTEGER");
b.HasKey("Id");
b.HasIndex("UserId");
b.ToTable("AspNetUserClaims");
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<int>", b =>
{
b.Property<string>("LoginProvider")
.HasColumnType("TEXT");
b.Property<string>("ProviderKey")
.HasColumnType("TEXT");
b.Property<string>("ProviderDisplayName")
.HasColumnType("TEXT");
b.Property<int>("UserId")
.HasColumnType("INTEGER");
b.HasKey("LoginProvider", "ProviderKey");
b.HasIndex("UserId");
b.ToTable("AspNetUserLogins");
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<int>", b =>
{
b.Property<int>("UserId")
.HasColumnType("INTEGER");
b.Property<string>("LoginProvider")
.HasColumnType("TEXT");
b.Property<string>("Name")
.HasColumnType("TEXT");
b.Property<string>("Value")
.HasColumnType("TEXT");
b.HasKey("UserId", "LoginProvider", "Name");
b.ToTable("AspNetUserTokens");
});
modelBuilder.Entity("API.Entities.AppUserRole", b =>
{
b.HasOne("API.Entities.AppRole", "Role")
.WithMany("UserRoles")
.HasForeignKey("RoleId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Entities.AppUser", "User")
.WithMany("UserRoles")
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Role");
b.Navigation("User");
});
modelBuilder.Entity("API.Entities.FolderPath", b =>
{
b.HasOne("API.Entities.Library", "Library")
.WithMany("Folders")
.HasForeignKey("LibraryId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Library");
});
modelBuilder.Entity("API.Entities.MangaFile", b =>
{
b.HasOne("API.Entities.Volume", "Volume")
.WithMany("Files")
.HasForeignKey("VolumeId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Volume");
});
modelBuilder.Entity("API.Entities.Series", b =>
{
b.HasOne("API.Entities.Library", "Library")
.WithMany("Series")
.HasForeignKey("LibraryId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Library");
});
modelBuilder.Entity("API.Entities.Volume", b =>
{
b.HasOne("API.Entities.Series", "Series")
.WithMany("Volumes")
.HasForeignKey("SeriesId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Series");
});
modelBuilder.Entity("AppUserLibrary", b =>
{
b.HasOne("API.Entities.AppUser", null)
.WithMany()
.HasForeignKey("AppUsersId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Entities.Library", null)
.WithMany()
.HasForeignKey("LibrariesId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<int>", b =>
{
b.HasOne("API.Entities.AppRole", null)
.WithMany()
.HasForeignKey("RoleId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<int>", b =>
{
b.HasOne("API.Entities.AppUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<int>", b =>
{
b.HasOne("API.Entities.AppUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<int>", b =>
{
b.HasOne("API.Entities.AppUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("API.Entities.AppRole", b =>
{
b.Navigation("UserRoles");
});
modelBuilder.Entity("API.Entities.AppUser", b =>
{
b.Navigation("UserRoles");
});
modelBuilder.Entity("API.Entities.Library", b =>
{
b.Navigation("Folders");
b.Navigation("Series");
});
modelBuilder.Entity("API.Entities.Series", b =>
{
b.Navigation("Volumes");
});
modelBuilder.Entity("API.Entities.Volume", b =>
{
b.Navigation("Files");
});
#pragma warning restore 612, 618
}
}
}

View File

@ -0,0 +1,46 @@
using Microsoft.EntityFrameworkCore.Migrations;
namespace API.Data.Migrations
{
public partial class CacheMetadata : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<int>(
name: "Chapter",
table: "MangaFile",
type: "INTEGER",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<int>(
name: "Format",
table: "MangaFile",
type: "INTEGER",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<int>(
name: "NumberOfPages",
table: "MangaFile",
type: "INTEGER",
nullable: false,
defaultValue: 0);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "Chapter",
table: "MangaFile");
migrationBuilder.DropColumn(
name: "Format",
table: "MangaFile");
migrationBuilder.DropColumn(
name: "NumberOfPages",
table: "MangaFile");
}
}
}

View File

@ -184,9 +184,18 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<int>("Chapter")
.HasColumnType("INTEGER");
b.Property<string>("FilePath")
.HasColumnType("TEXT");
b.Property<int>("Format")
.HasColumnType("INTEGER");
b.Property<int>("NumberOfPages")
.HasColumnType("INTEGER");
b.Property<int>("VolumeId")
.HasColumnType("INTEGER");

View File

@ -1,5 +1,8 @@
namespace API.Entities
{
/// <summary>
/// Represents the progress a single user has on a given Volume.
/// </summary>
public class AppUserProgress
{

View File

@ -5,7 +5,15 @@ namespace API.Entities
{
public int Id { get; set; }
public string FilePath { get; set; }
// Should I just store information related to FilePath here? Reset it on anytime FilePath changes?
/// <summary>
/// Do not expect this to be set. If this MangaFile represents a volume file, this will be null.
/// </summary>
public int Chapter { get; set; }
/// <summary>
/// Number of pages for the given file
/// </summary>
public int NumberOfPages { get; set; }
public MangaFormat Format { get; set; }
// Relationship Mapping
public Volume Volume { get; set; }

View File

@ -1,7 +1,14 @@
namespace API.Entities
using System.ComponentModel;
namespace API.Entities
{
public enum MangaFormat
{
[Description("Image")]
Image = 0,
[Description("Archive")]
Archive = 1,
[Description("Unknown")]
Unknown = 2
}
}

View File

@ -13,6 +13,9 @@ namespace API.Entities
public DateTime Created { get; set; }
public DateTime LastModified { get; set; }
public byte[] CoverImage { get; set; }
// public string CachePath {get; set;} // Path where cache is located. Default null, resets to null on deletion.
//public ICollection<AppUserProgress> AppUserProgress { get; set; }
// Many-to-One relationships
public Series Series { get; set; }

View File

@ -1,12 +1,14 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using API.Entities;
namespace API.Parser
{
public static class Parser
{
public static readonly string MangaFileExtensions = @"\.cbz|\.cbr|\.png|\.jpeg|\.jpg|\.zip|\.rar";
public static readonly string ImageFileExtensions = @"\.png|\.jpeg|\.jpg|\.gif";
//?: is a non-capturing group in C#, else anything in () will be a group
private static readonly Regex[] MangaVolumeRegex = new[]
@ -100,9 +102,17 @@ namespace API.Parser
Chapters = ParseChapter(filePath),
Series = ParseSeries(filePath),
Volumes = ParseVolume(filePath),
File = filePath
Filename = filePath,
Format = ParseFormat(filePath)
};
}
public static MangaFormat ParseFormat(string filePath)
{
if (IsArchive(filePath)) return MangaFormat.Archive;
if (IsImage(filePath)) return MangaFormat.Image;
return MangaFormat.Unknown;
}
public static string ParseSeries(string filename)
{
@ -231,8 +241,13 @@ namespace API.Parser
public static bool IsArchive(string filePath)
{
var fileInfo = new FileInfo(filePath);
return MangaFileExtensions.Contains(fileInfo.Extension);
}
public static bool IsImage(string filePath)
{
var fileInfo = new FileInfo(filePath);
return ImageFileExtensions.Contains(fileInfo.Extension);
}
}
}

View File

@ -1,4 +1,6 @@

using API.Entities;
namespace API.Parser
{
/// <summary>
@ -11,7 +13,11 @@ namespace API.Parser
public string Series { get; set; }
// This can be multiple
public string Volumes { get; set; }
public string File { get; init; }
public string Filename { get; init; }
public string FullFilePath { get; set; }
/// <summary>
/// Raw (image), Archive
/// </summary>
public MangaFormat Format { get; set; }
}
}

View File

@ -123,17 +123,31 @@ namespace API.Services
Name = seriesName,
OriginalName = seriesName,
SortName = seriesName,
Summary = ""
Summary = "" // TODO: Check if comicInfo.xml in file
};
}
var volumes = UpdateVolumes(series, infos, forceUpdate);
series.Volumes = volumes;
// TODO: Instead of taking first entry, re-calculate without compression
series.CoverImage = volumes.OrderBy(x => x.Number).FirstOrDefault()?.CoverImage;
return series;
}
private MangaFile CreateMangaFile(ParserInfo info)
{
_logger.LogDebug($"Creating File Entry for {info.FullFilePath}");
int chapter;
int.TryParse(info.Chapters, out chapter);
_logger.LogDebug($"Chapter? {chapter}");
return new MangaFile()
{
FilePath = info.FullFilePath,
Chapter = chapter,
NumberOfPages = GetNumberOfPagesFromArchive(info.FullFilePath)
};
}
/// <summary>
/// Creates or Updates volumes for a given series
/// </summary>
@ -145,20 +159,32 @@ namespace API.Services
{
ICollection<Volume> volumes = new List<Volume>();
IList<Volume> existingVolumes = _seriesRepository.GetVolumes(series.Id).ToList();
Volume existingVolume = null;
foreach (var info in infos)
{
var existingVolume = existingVolumes.SingleOrDefault(v => v.Name == info.Volumes);
existingVolume = existingVolumes.SingleOrDefault(v => v.Name == info.Volumes);
if (existingVolume != null)
{
// Temp let's overwrite all files (we need to enhance to update files)
existingVolume.Files = new List<MangaFile>()
var existingFile = existingVolume.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath);
if (existingFile != null)
{
new MangaFile()
{
FilePath = info.FullFilePath
}
};
existingFile.Chapter = Int32.Parse(info.Chapters);
existingFile.Format = info.Format;
existingFile.NumberOfPages = GetNumberOfPagesFromArchive(info.FullFilePath);
}
else
{
existingVolume.Files.Add(CreateMangaFile(info));
}
// existingVolume.Files = new List<MangaFile>()
// {
// new MangaFile()
// {
// FilePath = info.FullFilePath,
// Chapter = Int32.Parse(info.Chapters)
// }
// };
if (forceUpdate || existingVolume.CoverImage == null || existingVolumes.Count == 0)
{
@ -168,23 +194,29 @@ namespace API.Services
}
else
{
var vol = new Volume()
existingVolume = volumes.SingleOrDefault(v => v.Name == info.Volumes);
if (existingVolume != null)
{
Name = info.Volumes,
Number = Int32.Parse(info.Volumes),
CoverImage = ImageProvider.GetCoverImage(info.FullFilePath, true),
Files = new List<MangaFile>()
existingVolume.Files.Add(CreateMangaFile(info));
existingVolume.CoverImage = ImageProvider.GetCoverImage(info.FullFilePath, true);
}
else
{
var vol = new Volume()
{
new MangaFile()
Name = info.Volumes,
Number = Int32.Parse(info.Volumes),
CoverImage = ImageProvider.GetCoverImage(info.FullFilePath, true),
Files = new List<MangaFile>()
{
FilePath = info.File
CreateMangaFile(info)
}
}
};
volumes.Add(vol);
};
volumes.Add(vol);
}
}
Console.WriteLine($"Adding volume {volumes.Last().Number} with File: {info.File}");
Console.WriteLine($"Adding volume {volumes.Last().Number} with File: {info.Filename}");
}
return volumes;
@ -224,9 +256,9 @@ namespace API.Services
library.Series = new List<Series>(); // Temp delete everything until we can mark items Unavailable
foreach (var seriesKey in series.Keys)
{
var s = UpdateSeries(seriesKey, series[seriesKey].ToArray(), forceUpdate);
_logger.LogInformation($"Created/Updated series {s.Name}");
library.Series.Add(s);
var mangaSeries = UpdateSeries(seriesKey, series[seriesKey].ToArray(), forceUpdate);
_logger.LogInformation($"Created/Updated series {mangaSeries.Name}");
library.Series.Add(mangaSeries);
}
@ -277,6 +309,18 @@ namespace API.Services
return extractPath;
}
private int GetNumberOfPagesFromArchive(string archivePath)
{
if (!File.Exists(archivePath) || !Parser.Parser.IsArchive(archivePath))
{
_logger.LogError($"Archive {archivePath} could not be found.");
return 0;
}
using ZipArchive archive = ZipFile.OpenRead(archivePath);
return archive.Entries.Count(e => Parser.Parser.IsImage(e.FullName));
}
public async Task<ImageDto> ReadImageAsync(string imagePath)
{
using var image = Image.NewFromFile(imagePath);
@ -292,6 +336,13 @@ namespace API.Services
};
}
/// <summary>
/// Recursively scans files and applies an action on them. This uses as many cores the underlying PC has to speed
/// up processing.
/// </summary>
/// <param name="root">Directory to scan</param>
/// <param name="action">Action to apply on file path</param>
/// <exception cref="ArgumentException"></exception>
private static void TraverseTreeParallelForEach(string root, Action<string> action)
{
//Count of files traversed and timer for diagnostic output