Koreader Progress Sync (#3823)

Co-authored-by: Joseph Milazzo <josephmajora@gmail.com>
This commit is contained in:
Tyler Kenney 2025-06-20 13:45:56 -04:00 committed by GitHub
parent b6d004614a
commit 3107ca73e4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 4165 additions and 0 deletions

View File

@ -26,5 +26,10 @@
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content> </Content>
</ItemGroup> </ItemGroup>
<ItemGroup>
<None Update="Data\AesopsFables.epub">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project> </Project>

Binary file not shown.

View File

@ -0,0 +1,41 @@
using API.Helpers.Builders;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Order;
using System;
using API.Entities.Enums;
namespace API.Benchmark
{
[StopOnFirstError]
[MemoryDiagnoser]
[RankColumn]
[Orderer(SummaryOrderPolicy.FastestToSlowest)]
[SimpleJob(launchCount: 1, warmupCount: 5, invocationCount: 20)]
public class KoreaderHashBenchmark
{
private const string sourceEpub = "./Data/AesopsFables.epub";
[Benchmark(Baseline = true)]
public void TestBuildManga_baseline()
{
var file = new MangaFileBuilder(sourceEpub, MangaFormat.Epub)
.Build();
if (file == null)
{
throw new Exception("Failed to build manga file");
}
}
[Benchmark]
public void TestBuildManga_withHash()
{
var file = new MangaFileBuilder(sourceEpub, MangaFormat.Epub)
.WithHash()
.Build();
if (file == null)
{
throw new Exception("Failed to build manga file");
}
}
}
}

View File

@ -36,4 +36,10 @@
<None Remove="Extensions\Test Data\modified on run.txt" /> <None Remove="Extensions\Test Data\modified on run.txt" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<None Update="Data\AesopsFables.epub">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project> </Project>

Binary file not shown.

View File

@ -0,0 +1,60 @@
using API.DTOs.Koreader;
using API.DTOs.Progress;
using API.Helpers;
using System.Runtime.CompilerServices;
using Xunit;
namespace API.Tests.Helpers;
public class KoreaderHelperTests
{
[Theory]
[InlineData("/body/DocFragment[11]/body/div/a", 10, null)]
[InlineData("/body/DocFragment[1]/body/div/p[40]", 0, 40)]
[InlineData("/body/DocFragment[8]/body/div/p[28]/text().264", 7, 28)]
public void GetEpubPositionDto(string koreaderPosition, int page, int? pNumber)
{
var expected = EmptyProgressDto();
expected.BookScrollId = pNumber.HasValue ? $"//html[1]/BODY/APP-ROOT[1]/DIV[1]/DIV[1]/DIV[1]/APP-BOOK-READER[1]/DIV[1]/DIV[2]/DIV[1]/DIV[1]/DIV[1]/P[{pNumber}]" : null;
expected.PageNum = page;
var actual = EmptyProgressDto();
KoreaderHelper.UpdateProgressDto(actual, koreaderPosition);
Assert.Equal(expected.BookScrollId, actual.BookScrollId);
Assert.Equal(expected.PageNum, actual.PageNum);
}
[Theory]
[InlineData("//html[1]/BODY/APP-ROOT[1]/DIV[1]/DIV[1]/DIV[1]/APP-BOOK-READER[1]/DIV[1]/DIV[2]/DIV[1]/DIV[1]/DIV[1]/P[20]", 5, "/body/DocFragment[6]/body/div/p[20]")]
[InlineData(null, 10, "/body/DocFragment[11]/body/div/a")]
public void GetKoreaderPosition(string scrollId, int page, string koreaderPosition)
{
var given = EmptyProgressDto();
given.BookScrollId = scrollId;
given.PageNum = page;
Assert.Equal(koreaderPosition, KoreaderHelper.GetKoreaderPosition(given));
}
[Theory]
[InlineData("./Data/AesopsFables.epub", "8795ACA4BF264B57C1EEDF06A0CEE688")]
public void GetKoreaderHash(string filePath, string hash)
{
Assert.Equal(KoreaderHelper.HashContents(filePath), hash);
}
private ProgressDto EmptyProgressDto()
{
return new ProgressDto
{
ChapterId = 0,
PageNum = 0,
VolumeId = 0,
SeriesId = 0,
LibraryId = 0
};
}
}

View File

@ -0,0 +1,118 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using System;
using System.Threading.Tasks;
using API.Data;
using API.Data.Repositories;
using API.DTOs.Koreader;
using API.Entities;
using API.Services;
using Kavita.Common;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.Logging;
using static System.Net.WebRequestMethods;
namespace API.Controllers;
#nullable enable
/// <summary>
/// The endpoint to interface with Koreader's Progress Sync plugin.
/// </summary>
/// <remarks>
/// Koreader uses a different form of authentication. It stores the username and password in headers.
/// https://github.com/koreader/koreader/blob/master/plugins/kosync.koplugin/KOSyncClient.lua
/// </remarks>
[AllowAnonymous]
public class KoreaderController : BaseApiController
{
private readonly IUnitOfWork _unitOfWork;
private readonly ILocalizationService _localizationService;
private readonly IKoreaderService _koreaderService;
private readonly ILogger<KoreaderController> _logger;
public KoreaderController(IUnitOfWork unitOfWork, ILocalizationService localizationService,
IKoreaderService koreaderService, ILogger<KoreaderController> logger)
{
_unitOfWork = unitOfWork;
_localizationService = localizationService;
_koreaderService = koreaderService;
_logger = logger;
}
// We won't allow users to be created from Koreader. Rather, they
// must already have an account.
/*
[HttpPost("/users/create")]
public IActionResult CreateUser(CreateUserRequest request)
{
}
*/
[HttpGet("{apiKey}/users/auth")]
public async Task<IActionResult> Authenticate(string apiKey)
{
var userId = await GetUserId(apiKey);
var user = await _unitOfWork.UserRepository.GetUserByIdAsync(userId);
if (user == null) return Unauthorized();
return Ok(new { username = user.UserName });
}
/// <summary>
/// Syncs book progress with Kavita. Will attempt to save the underlying reader position if possible.
/// </summary>
/// <param name="apiKey"></param>
/// <param name="request"></param>
/// <returns></returns>
[HttpPut("{apiKey}/syncs/progress")]
public async Task<ActionResult<KoreaderProgressUpdateDto>> UpdateProgress(string apiKey, KoreaderBookDto request)
{
try
{
var userId = await GetUserId(apiKey);
await _koreaderService.SaveProgress(request, userId);
return Ok(new KoreaderProgressUpdateDto{ Document = request.Document, Timestamp = DateTime.UtcNow });
}
catch (KavitaException ex)
{
return BadRequest(ex.Message);
}
}
/// <summary>
/// Gets book progress from Kavita, if not found will return a 400
/// </summary>
/// <param name="apiKey"></param>
/// <param name="ebookHash"></param>
/// <returns></returns>
[HttpGet("{apiKey}/syncs/progress/{ebookHash}")]
public async Task<ActionResult<KoreaderBookDto>> GetProgress(string apiKey, string ebookHash)
{
try
{
var userId = await GetUserId(apiKey);
var response = await _koreaderService.GetProgress(ebookHash, userId);
_logger.LogDebug("Koreader response progress: {Progress}", response.Progress);
return Ok(response);
}
catch (KavitaException ex)
{
return BadRequest(ex.Message);
}
}
private async Task<int> GetUserId(string apiKey)
{
try
{
return await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(apiKey);
}
catch
{
throw new KavitaException(await _localizationService.Get("en", "user-doesnt-exist"));
}
}
}

View File

@ -0,0 +1,33 @@
using API.DTOs.Progress;
namespace API.DTOs.Koreader;
/// <summary>
/// This is the interface for receiving and sending updates to Koreader. The only fields
/// that are actually used are the Document and Progress fields.
/// </summary>
public class KoreaderBookDto
{
/// <summary>
/// This is the Koreader hash of the book. It is used to identify the book.
/// </summary>
public string Document { get; set; }
/// <summary>
/// A randomly generated id from the koreader device. Only used to maintain the Koreader interface.
/// </summary>
public string Device_id { get; set; }
/// <summary>
/// The Koreader device name. Only used to maintain the Koreader interface.
/// </summary>
public string Device { get; set; }
/// <summary>
/// Percent progress of the book. Only used to maintain the Koreader interface.
/// </summary>
public float Percentage { get; set; }
/// <summary>
/// An XPath string read by Koreader to determine the location within the epub.
/// Essentially, it is Koreader's equivalent to ProgressDto.BookScrollId.
/// </summary>
/// <seealso cref="ProgressDto.BookScrollId"/>
public string Progress { get; set; }
}

View File

@ -0,0 +1,15 @@
using System;
namespace API.DTOs.Koreader;
public class KoreaderProgressUpdateDto
{
/// <summary>
/// This is the Koreader hash of the book. It is used to identify the book.
/// </summary>
public string Document { get; set; }
/// <summary>
/// UTC Timestamp to return to KOReader
/// </summary>
public DateTime Timestamp { get; set; }
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,28 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class KoreaderHash : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "KoreaderHash",
table: "MangaFile",
type: "TEXT",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "KoreaderHash",
table: "MangaFile");
}
}
}

View File

@ -1408,6 +1408,9 @@ namespace API.Data.Migrations
b.Property<int>("Format") b.Property<int>("Format")
.HasColumnType("INTEGER"); .HasColumnType("INTEGER");
b.Property<string>("KoreaderHash")
.HasColumnType("TEXT");
b.Property<DateTime>("LastFileAnalysis") b.Property<DateTime>("LastFileAnalysis")
.HasColumnType("TEXT"); .HasColumnType("TEXT");

View File

@ -5,11 +5,13 @@ using API.Entities;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
namespace API.Data.Repositories; namespace API.Data.Repositories;
#nullable enable
public interface IMangaFileRepository public interface IMangaFileRepository
{ {
void Update(MangaFile file); void Update(MangaFile file);
Task<IList<MangaFile>> GetAllWithMissingExtension(); Task<IList<MangaFile>> GetAllWithMissingExtension();
Task<MangaFile?> GetByKoreaderHash(string hash);
} }
public class MangaFileRepository : IMangaFileRepository public class MangaFileRepository : IMangaFileRepository
@ -32,4 +34,13 @@ public class MangaFileRepository : IMangaFileRepository
.Where(f => string.IsNullOrEmpty(f.Extension)) .Where(f => string.IsNullOrEmpty(f.Extension))
.ToListAsync(); .ToListAsync();
} }
public async Task<MangaFile?> GetByKoreaderHash(string hash)
{
if (string.IsNullOrEmpty(hash)) return null;
return await _context.MangaFile
.FirstOrDefaultAsync(f => f.KoreaderHash != null &&
f.KoreaderHash.Equals(hash.ToUpper()));
}
} }

View File

@ -21,6 +21,11 @@ public class MangaFile : IEntityDate
/// </summary> /// </summary>
public required string FilePath { get; set; } public required string FilePath { get; set; }
/// <summary> /// <summary>
/// A hash of the document using Koreader's unique hashing algorithm
/// </summary>
/// <remark> KoreaderHash is only available for epub types </remark>
public string? KoreaderHash { get; set; }
/// <summary>
/// Number of pages for the given file /// Number of pages for the given file
/// </summary> /// </summary>
public int Pages { get; set; } public int Pages { get; set; }

View File

@ -55,6 +55,7 @@ public static class ApplicationServiceExtensions
services.AddScoped<IRatingService, RatingService>(); services.AddScoped<IRatingService, RatingService>();
services.AddScoped<IPersonService, PersonService>(); services.AddScoped<IPersonService, PersonService>();
services.AddScoped<IReadingProfileService, ReadingProfileService>(); services.AddScoped<IReadingProfileService, ReadingProfileService>();
services.AddScoped<IKoreaderService, KoreaderService>();
services.AddScoped<IScannerService, ScannerService>(); services.AddScoped<IScannerService, ScannerService>();
services.AddScoped<IProcessSeries, ProcessSeries>(); services.AddScoped<IProcessSeries, ProcessSeries>();

View File

@ -0,0 +1,46 @@
using System;
using System.Security.Cryptography;
using System.Text;
using API.DTOs.Koreader;
namespace API.Helpers.Builders;
public class KoreaderBookDtoBuilder : IEntityBuilder<KoreaderBookDto>
{
private readonly KoreaderBookDto _dto;
public KoreaderBookDto Build() => _dto;
public KoreaderBookDtoBuilder(string documentHash)
{
_dto = new KoreaderBookDto()
{
Document = documentHash,
Device = "Kavita"
};
}
public KoreaderBookDtoBuilder WithDocument(string documentHash)
{
_dto.Document = documentHash;
return this;
}
public KoreaderBookDtoBuilder WithProgress(string progress)
{
_dto.Progress = progress;
return this;
}
public KoreaderBookDtoBuilder WithPercentage(int? pageNum, int pages)
{
_dto.Percentage = (pageNum ?? 0) / (float) pages;
return this;
}
public KoreaderBookDtoBuilder WithDeviceId(string installId, int userId)
{
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(installId + userId));
_dto.Device_id = Convert.ToHexString(hash);
return this;
}
}

View File

@ -60,4 +60,17 @@ public class MangaFileBuilder : IEntityBuilder<MangaFile>
_mangaFile.Id = Math.Max(id, 0); _mangaFile.Id = Math.Max(id, 0);
return this; return this;
} }
/// <summary>
/// Generate the Hash on the underlying file
/// </summary>
/// <remarks>Only applicable to Epubs</remarks>
public MangaFileBuilder WithHash()
{
if (_mangaFile.Format != MangaFormat.Epub) return this;
_mangaFile.KoreaderHash = KoreaderHelper.HashContents(_mangaFile.FilePath);
return this;
}
} }

View File

@ -0,0 +1,113 @@
using API.DTOs.Progress;
using System;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using API.Services.Tasks.Scanner.Parser;
namespace API.Helpers;
/// <summary>
/// All things related to Koreader
/// </summary>
/// <remarks>Original developer: https://github.com/MFDeAngelo</remarks>
public static class KoreaderHelper
{
/// <summary>
/// Hashes the document according to a custom Koreader hashing algorithm.
/// Look at the util.partialMD5 method in the attached link.
/// Note: Only applies to epub files
/// </summary>
/// <remarks>The hashing algorithm is relatively quick as it only hashes ~10,000 bytes for the biggest of files.</remarks>
/// <see href="https://github.com/koreader/koreader/blob/master/frontend/util.lua#L1040"/>
/// <param name="filePath">The path to the file to hash</param>
public static string HashContents(string filePath)
{
if (string.IsNullOrEmpty(filePath) || !File.Exists(filePath) || !Parser.IsEpub(filePath))
{
return null;
}
using var file = File.OpenRead(filePath);
const int step = 1024;
const int size = 1024;
var md5 = MD5.Create();
var buffer = new byte[size];
for (var i = -1; i < 10; i++)
{
file.Position = step << 2 * i;
var bytesRead = file.Read(buffer, 0, size);
if (bytesRead > 0)
{
md5.TransformBlock(buffer, 0, bytesRead, buffer, 0);
}
else
{
break;
}
}
file.Close();
md5.TransformFinalBlock([], 0, 0);
return md5.Hash == null ? null : Convert.ToHexString(md5.Hash).ToUpper();
}
/// <summary>
/// Koreader can identify documents based on contents or title.
/// For now, we only support by contents.
/// </summary>
public static string HashTitle(string filePath)
{
var fileName = Path.GetFileName(filePath);
var fileNameBytes = Encoding.ASCII.GetBytes(fileName);
var bytes = MD5.HashData(fileNameBytes);
return Convert.ToHexString(bytes);
}
public static void UpdateProgressDto(ProgressDto progress, string koreaderPosition)
{
var path = koreaderPosition.Split('/');
if (path.Length < 6)
{
return;
}
var docNumber = path[2].Replace("DocFragment[", string.Empty).Replace("]", string.Empty);
progress.PageNum = int.Parse(docNumber) - 1;
var lastTag = path[5].ToUpper();
if (lastTag == "A")
{
progress.BookScrollId = null;
}
else
{
// The format that Kavita accepts as a progress string. It tells Kavita where Koreader last left off.
progress.BookScrollId = $"//html[1]/BODY/APP-ROOT[1]/DIV[1]/DIV[1]/DIV[1]/APP-BOOK-READER[1]/DIV[1]/DIV[2]/DIV[1]/DIV[1]/DIV[1]/{lastTag}";
}
}
public static string GetKoreaderPosition(ProgressDto progressDto)
{
string lastTag;
var koreaderPageNumber = progressDto.PageNum + 1;
if (string.IsNullOrEmpty(progressDto.BookScrollId))
{
lastTag = "a";
}
else
{
var tokens = progressDto.BookScrollId.Split('/');
lastTag = tokens[^1].ToLower();
}
// The format that Koreader accepts as a progress string. It tells Koreader where Kavita last left off.
return $"/body/DocFragment[{koreaderPageNumber}]/body/div/{lastTag}";
}
}

View File

@ -0,0 +1,90 @@
using System.Threading.Tasks;
using API.Data;
using API.DTOs.Koreader;
using API.DTOs.Progress;
using API.Helpers;
using API.Helpers.Builders;
using Kavita.Common;
using Microsoft.Extensions.Logging;
namespace API.Services;
#nullable enable
public interface IKoreaderService
{
Task SaveProgress(KoreaderBookDto koreaderBookDto, int userId);
Task<KoreaderBookDto> GetProgress(string bookHash, int userId);
}
public class KoreaderService : IKoreaderService
{
private readonly IReaderService _readerService;
private readonly IUnitOfWork _unitOfWork;
private readonly ILocalizationService _localizationService;
private readonly ILogger<KoreaderService> _logger;
public KoreaderService(IReaderService readerService, IUnitOfWork unitOfWork, ILocalizationService localizationService, ILogger<KoreaderService> logger)
{
_readerService = readerService;
_unitOfWork = unitOfWork;
_localizationService = localizationService;
_logger = logger;
}
/// <summary>
/// Given a Koreader hash, locate the underlying file and generate/update a progress event.
/// </summary>
/// <param name="koreaderBookDto"></param>
/// <param name="userId"></param>
public async Task SaveProgress(KoreaderBookDto koreaderBookDto, int userId)
{
_logger.LogDebug("Saving Koreader progress for {UserId}: {KoreaderProgress}", userId, koreaderBookDto.Progress);
var file = await _unitOfWork.MangaFileRepository.GetByKoreaderHash(koreaderBookDto.Document);
if (file == null) throw new KavitaException(await _localizationService.Translate(userId, "file-missing"));
var userProgressDto = await _unitOfWork.AppUserProgressRepository.GetUserProgressDtoAsync(file.ChapterId, userId);
if (userProgressDto == null)
{
var chapterDto = await _unitOfWork.ChapterRepository.GetChapterDtoAsync(file.ChapterId);
if (chapterDto == null) throw new KavitaException(await _localizationService.Translate(userId, "chapter-doesnt-exist"));
var volumeDto = await _unitOfWork.VolumeRepository.GetVolumeByIdAsync(chapterDto.VolumeId);
if (volumeDto == null) throw new KavitaException(await _localizationService.Translate(userId, "volume-doesnt-exist"));
userProgressDto = new ProgressDto()
{
ChapterId = file.ChapterId,
VolumeId = chapterDto.VolumeId,
SeriesId = volumeDto.SeriesId,
};
}
// Update the bookScrollId if possible
KoreaderHelper.UpdateProgressDto(userProgressDto, koreaderBookDto.Progress);
await _readerService.SaveReadingProgress(userProgressDto, userId);
}
/// <summary>
/// Returns a Koreader Dto representing current book and the progress within
/// </summary>
/// <param name="bookHash"></param>
/// <param name="userId"></param>
/// <returns></returns>
public async Task<KoreaderBookDto> GetProgress(string bookHash, int userId)
{
var settingsDto = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
var file = await _unitOfWork.MangaFileRepository.GetByKoreaderHash(bookHash);
if (file == null) throw new KavitaException(await _localizationService.Translate(userId, "file-missing"));
var progressDto = await _unitOfWork.AppUserProgressRepository.GetUserProgressDtoAsync(file.ChapterId, userId);
var koreaderProgress = KoreaderHelper.GetKoreaderPosition(progressDto);
return new KoreaderBookDtoBuilder(bookHash).WithProgress(koreaderProgress)
.WithPercentage(progressDto?.PageNum, file.Pages)
.WithDeviceId(settingsDto.InstallId, userId)
.Build();
}
}

View File

@ -880,6 +880,8 @@ public class ProcessSeries : IProcessSeries
existingFile.FileName = Parser.Parser.RemoveExtensionIfSupported(existingFile.FilePath); existingFile.FileName = Parser.Parser.RemoveExtensionIfSupported(existingFile.FilePath);
existingFile.FilePath = Parser.Parser.NormalizePath(existingFile.FilePath); existingFile.FilePath = Parser.Parser.NormalizePath(existingFile.FilePath);
existingFile.Bytes = fileInfo.Length; existingFile.Bytes = fileInfo.Length;
existingFile.KoreaderHash = KoreaderHelper.HashContents(existingFile.FilePath);
// We skip updating DB here with last modified time so that metadata refresh can do it // We skip updating DB here with last modified time so that metadata refresh can do it
} }
else else
@ -888,6 +890,7 @@ public class ProcessSeries : IProcessSeries
var file = new MangaFileBuilder(info.FullFilePath, info.Format, _readingItemService.GetNumberOfPages(info.FullFilePath, info.Format)) var file = new MangaFileBuilder(info.FullFilePath, info.Format, _readingItemService.GetNumberOfPages(info.FullFilePath, info.Format))
.WithExtension(fileInfo.Extension) .WithExtension(fileInfo.Extension)
.WithBytes(fileInfo.Length) .WithBytes(fileInfo.Length)
.WithHash()
.Build(); .Build();
chapter.Files.Add(file); chapter.Files.Add(file);
} }