Random Changes and Enhancements (#1819)

* When skipping over folders in a scan, inform the ui

* Try out new backout condition for library watcher.

* Tweaked the code for folder watching to be more intense on killing if stuck in inotify loop.

* Streamlined my implementation of enhanced LibraryWatcher

* Added new extension method to make complex where statements cleaner.

* Added an implementation to flatten series and not show them if they have relationships defined. Only the parent would show. Currently disabled until i figure out how to apply it.

* Added the ability to collapse series that are not the primary entry point to reading. Configurable in library settings, only applies when all libraries in a filter have the property to true.

* Exclude from parsing .@_thumb directories, a QNAP system folder.

Show number of items a JumpKey has

* Refactored some time reading to display in days, months, years or minutes.
This commit is contained in:
Joe Milazzo 2023-02-20 17:48:04 -06:00 committed by GitHub
parent 8a62d54c0b
commit df68c50256
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 2038 additions and 48 deletions

View File

@ -75,7 +75,7 @@ public class DirectoryServiceTests
[Fact]
public void TraverseTreeParallelForEach_DontCountExcludedDirectories_ShouldBe28()
{
var testDirectory = "/manga/";
const string testDirectory = "/manga/";
var fileSystem = new MockFileSystem();
for (var i = 0; i < 28; i++)
{
@ -85,6 +85,7 @@ public class DirectoryServiceTests
fileSystem.AddFile($"{Path.Join(testDirectory, "@eaDir")}file_{29}.jpg", new MockFileData(""));
fileSystem.AddFile($"{Path.Join(testDirectory, ".DS_Store")}file_{30}.jpg", new MockFileData(""));
fileSystem.AddFile($"{Path.Join(testDirectory, ".qpkg")}file_{30}.jpg", new MockFileData(""));
fileSystem.AddFile($"{Path.Join(testDirectory, ".@_thumb")}file_{30}.jpg", new MockFileData(""));
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
var files = new List<string>();

View File

@ -335,6 +335,7 @@ public class LibraryController : BaseApiController
library.IncludeInRecommended = dto.IncludeInRecommended;
library.IncludeInSearch = dto.IncludeInSearch;
library.ManageCollections = dto.ManageCollections;
library.CollapseSeriesRelationships = dto.CollapseSeriesRelationships;
_unitOfWork.LibraryRepository.Update(library);

View File

@ -37,5 +37,9 @@ public class LibraryDto
/// Include library series in Search
/// </summary>
public bool IncludeInSearch { get; set; } = true;
/// <summary>
/// When showing series, only parent series or series with no relationships will be returned
/// </summary>
public bool CollapseSeriesRelationships { get; set; } = false;
public ICollection<string> Folders { get; init; }
}

View File

@ -24,5 +24,7 @@ public class UpdateLibraryDto
public bool IncludeInSearch { get; init; }
[Required]
public bool ManageCollections { get; init; }
[Required]
public bool CollapseSeriesRelationships { get; init; }
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,26 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
public partial class CollapseSeriesRelationships : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "CollapseSeriesRelationships",
table: "Library",
type: "INTEGER",
nullable: false,
defaultValue: false);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "CollapseSeriesRelationships",
table: "Library");
}
}
}

View File

@ -598,6 +598,9 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<bool>("CollapseSeriesRelationships")
.HasColumnType("INTEGER");
b.Property<string>("CoverImage")
.HasColumnType("TEXT");

View File

@ -743,8 +743,12 @@ public class SeriesRepository : ISeriesRepository
private async Task<IQueryable<Series>> CreateFilteredSearchQueryable(int userId, int libraryId, FilterDto filter, QueryContext queryContext)
{
// NOTE: Why do we even have libraryId when the filter has the actual libraryIds?
var userLibraries = await GetUserLibrariesForFilteredQuery(libraryId, userId, queryContext);
var userRating = await _context.AppUser.GetUserAgeRestriction(userId);
var onlyParentSeries = await _context.Library.AsNoTracking()
.Where(l => filter.Libraries.Contains(l.Id))
.AllAsync(l => l.CollapseSeriesRelationships);
var formats = ExtractFilters(libraryId, userId, filter, ref userLibraries,
out var allPeopleIds, out var hasPeopleFilter, out var hasGenresFilter,
@ -753,30 +757,33 @@ public class SeriesRepository : ISeriesRepository
out var hasPublicationFilter, out var hasSeriesNameFilter, out var hasReleaseYearMinFilter, out var hasReleaseYearMaxFilter);
var query = _context.Series
.Where(s => userLibraries.Contains(s.LibraryId)
&& formats.Contains(s.Format)
&& (!hasGenresFilter || s.Metadata.Genres.Any(g => filter.Genres.Contains(g.Id)))
&& (!hasPeopleFilter || s.Metadata.People.Any(p => allPeopleIds.Contains(p.Id)))
&& (!hasCollectionTagFilter ||
s.Metadata.CollectionTags.Any(t => filter.CollectionTags.Contains(t.Id)))
&& (!hasRatingFilter || s.Ratings.Any(r => r.Rating >= filter.Rating && r.AppUserId == userId))
&& (!hasProgressFilter || seriesIds.Contains(s.Id))
&& (!hasAgeRating || filter.AgeRating.Contains(s.Metadata.AgeRating))
&& (!hasTagsFilter || s.Metadata.Tags.Any(t => filter.Tags.Contains(t.Id)))
&& (!hasLanguageFilter || filter.Languages.Contains(s.Metadata.Language))
&& (!hasReleaseYearMinFilter || s.Metadata.ReleaseYear >= filter.ReleaseYearRange.Min)
&& (!hasReleaseYearMaxFilter || s.Metadata.ReleaseYear <= filter.ReleaseYearRange.Max)
&& (!hasPublicationFilter || filter.PublicationStatus.Contains(s.Metadata.PublicationStatus)))
.Where(s => !hasSeriesNameFilter ||
EF.Functions.Like(s.Name, $"%{filter.SeriesNameQuery}%")
.AsNoTracking()
.WhereIf(hasGenresFilter, s => s.Metadata.Genres.Any(g => filter.Genres.Contains(g.Id)))
.WhereIf(hasPeopleFilter, s => s.Metadata.People.Any(p => allPeopleIds.Contains(p.Id)))
.WhereIf(hasCollectionTagFilter,
s => s.Metadata.CollectionTags.Any(t => filter.CollectionTags.Contains(t.Id)))
.WhereIf(hasRatingFilter, s => s.Ratings.Any(r => r.Rating >= filter.Rating && r.AppUserId == userId))
.WhereIf(hasProgressFilter, s => seriesIds.Contains(s.Id))
.WhereIf(hasAgeRating, s => filter.AgeRating.Contains(s.Metadata.AgeRating))
.WhereIf(hasTagsFilter, s => s.Metadata.Tags.Any(t => filter.Tags.Contains(t.Id)))
.WhereIf(hasLanguageFilter, s => filter.Languages.Contains(s.Metadata.Language))
.WhereIf(hasReleaseYearMinFilter, s => s.Metadata.ReleaseYear >= filter.ReleaseYearRange.Min)
.WhereIf(hasReleaseYearMaxFilter, s => s.Metadata.ReleaseYear <= filter.ReleaseYearRange.Max)
.WhereIf(hasPublicationFilter, s => filter.PublicationStatus.Contains(s.Metadata.PublicationStatus))
.WhereIf(hasSeriesNameFilter, s => EF.Functions.Like(s.Name, $"%{filter.SeriesNameQuery}%")
|| EF.Functions.Like(s.OriginalName, $"%{filter.SeriesNameQuery}%")
|| EF.Functions.Like(s.LocalizedName, $"%{filter.SeriesNameQuery}%"));
|| EF.Functions.Like(s.LocalizedName, $"%{filter.SeriesNameQuery}%"))
.WhereIf(onlyParentSeries,
s => s.RelationOf.Count == 0 || s.RelationOf.All(p => p.RelationKind == RelationKind.Prequel))
.Where(s => userLibraries.Contains(s.LibraryId))
.Where(s => formats.Contains(s.Format));
if (userRating.AgeRating != AgeRating.NotApplicable)
{
query = query.RestrictAgainstAgeRestriction(userRating);
}
query = query.AsNoTracking();
// If no sort options, default to using SortName
filter.SortOptions ??= new SortOptions()
@ -825,24 +832,23 @@ public class SeriesRepository : ISeriesRepository
out var hasPublicationFilter, out var hasSeriesNameFilter, out var hasReleaseYearMinFilter, out var hasReleaseYearMaxFilter);
var query = sQuery
.Where(s => userLibraries.Contains(s.LibraryId)
&& formats.Contains(s.Format)
&& (!hasGenresFilter || s.Metadata.Genres.Any(g => filter.Genres.Contains(g.Id)))
&& (!hasPeopleFilter || s.Metadata.People.Any(p => allPeopleIds.Contains(p.Id)))
&& (!hasCollectionTagFilter ||
s.Metadata.CollectionTags.Any(t => filter.CollectionTags.Contains(t.Id)))
&& (!hasRatingFilter || s.Ratings.Any(r => r.Rating >= filter.Rating && r.AppUserId == userId))
&& (!hasProgressFilter || seriesIds.Contains(s.Id))
&& (!hasAgeRating || filter.AgeRating.Contains(s.Metadata.AgeRating))
&& (!hasTagsFilter || s.Metadata.Tags.Any(t => filter.Tags.Contains(t.Id)))
&& (!hasLanguageFilter || filter.Languages.Contains(s.Metadata.Language))
&& (!hasReleaseYearMinFilter || s.Metadata.ReleaseYear >= filter.ReleaseYearRange.Min)
&& (!hasReleaseYearMaxFilter || s.Metadata.ReleaseYear <= filter.ReleaseYearRange.Max)
&& (!hasPublicationFilter || filter.PublicationStatus.Contains(s.Metadata.PublicationStatus)))
.Where(s => !hasSeriesNameFilter ||
EF.Functions.Like(s.Name, $"%{filter.SeriesNameQuery}%")
.WhereIf(hasGenresFilter, s => s.Metadata.Genres.Any(g => filter.Genres.Contains(g.Id)))
.WhereIf(hasPeopleFilter, s => s.Metadata.People.Any(p => allPeopleIds.Contains(p.Id)))
.WhereIf(hasCollectionTagFilter,
s => s.Metadata.CollectionTags.Any(t => filter.CollectionTags.Contains(t.Id)))
.WhereIf(hasRatingFilter, s => s.Ratings.Any(r => r.Rating >= filter.Rating && r.AppUserId == userId))
.WhereIf(hasProgressFilter, s => seriesIds.Contains(s.Id))
.WhereIf(hasAgeRating, s => filter.AgeRating.Contains(s.Metadata.AgeRating))
.WhereIf(hasTagsFilter, s => s.Metadata.Tags.Any(t => filter.Tags.Contains(t.Id)))
.WhereIf(hasLanguageFilter, s => filter.Languages.Contains(s.Metadata.Language))
.WhereIf(hasReleaseYearMinFilter, s => s.Metadata.ReleaseYear >= filter.ReleaseYearRange.Min)
.WhereIf(hasReleaseYearMaxFilter, s => s.Metadata.ReleaseYear <= filter.ReleaseYearRange.Max)
.WhereIf(hasPublicationFilter, s => filter.PublicationStatus.Contains(s.Metadata.PublicationStatus))
.WhereIf(hasSeriesNameFilter, s => EF.Functions.Like(s.Name, $"%{filter.SeriesNameQuery}%")
|| EF.Functions.Like(s.OriginalName, $"%{filter.SeriesNameQuery}%")
|| EF.Functions.Like(s.LocalizedName, $"%{filter.SeriesNameQuery}%"))
.Where(s => userLibraries.Contains(s.LibraryId)
&& formats.Contains(s.Format))
.AsNoTracking();
// If no sort options, default to using SortName

View File

@ -31,6 +31,10 @@ public class Library : IEntityDate
/// Should this library create and manage collections from Metadata
/// </summary>
public bool ManageCollections { get; set; } = true;
/// <summary>
/// When showing series, only parent series or series with no relationships will be returned
/// </summary>
public bool CollapseSeriesRelationships { get; set; } = false;
public DateTime Created { get; set; }
public DateTime LastModified { get; set; }
public DateTime CreatedUtc { get; set; }

View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Threading.Tasks;
using API.Data.Misc;
using API.Data.Repositories;
@ -234,4 +235,10 @@ public static class QueryableExtensions
public static IEnumerable<DateTime> Range(this DateTime startDate, int numberOfDays) =>
Enumerable.Range(0, numberOfDays).Select(e => startDate.AddDays(e));
public static IQueryable<T> WhereIf<T>(this IQueryable<T> queryable, bool condition,
Expression<Func<T, bool>> predicate)
{
return condition ? queryable.Where(predicate) : queryable;
}
}

View File

@ -89,7 +89,7 @@ public class DirectoryService : IDirectoryService
private readonly ILogger<DirectoryService> _logger;
private static readonly Regex ExcludeDirectories = new Regex(
@"@eaDir|\.DS_Store|\.qpkg|__MACOSX|@Recently-Snapshot|@recycle",
@"@eaDir|\.DS_Store|\.qpkg|__MACOSX|@Recently-Snapshot|@recycle|\.@__thumb",
RegexOptions.Compiled | RegexOptions.IgnoreCase,
Tasks.Scanner.Parser.Parser.RegexTimeout);
private static readonly Regex FileCopyAppend = new Regex(@"\(\d+\)",

View File

@ -5,6 +5,7 @@ using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Data;
using API.Entities.Enums;
using Hangfire;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@ -55,6 +56,8 @@ public class LibraryWatcher : ILibraryWatcher
/// Counts within a time frame how many times the buffer became full. Is used to reschedule LibraryWatcher to start monitoring much later rather than instantly
/// </summary>
private int _bufferFullCounter;
private int _restartCounter;
private DateTime _lastErrorTime = DateTime.MinValue;
/// <summary>
/// Used to lock buffer Full Counter
/// </summary>
@ -180,12 +183,21 @@ public class LibraryWatcher : ILibraryWatcher
lock (Lock)
{
_bufferFullCounter += 1;
condition = _bufferFullCounter >= 3;
_lastErrorTime = DateTime.Now;
condition = _bufferFullCounter >= 3 && (DateTime.Now - _lastErrorTime).TotalMinutes <= 10;
}
if (_restartCounter >= 3)
{
_logger.LogInformation("[LibraryWatcher] Too many restarts occured, you either have limited inotify or an OS constraint. Kavita will turn off folder watching to prevent high utilization of resources");
Task.Run(TurnOffWatching);
return;
}
if (condition)
{
_logger.LogInformation("[LibraryWatcher] Internal buffer has been overflown multiple times in past 10 minutes. Suspending file watching for an hour");
_logger.LogInformation("[LibraryWatcher] Internal buffer has been overflown multiple times in past 10 minutes. Suspending file watching for an hour. Restart count: {RestartCount}", _restartCounter);
_restartCounter++;
StopWatching();
BackgroundJob.Schedule(() => RestartWatching(), TimeSpan.FromHours(1));
return;
@ -194,6 +206,16 @@ public class LibraryWatcher : ILibraryWatcher
BackgroundJob.Schedule(() => UpdateLastBufferOverflow(), TimeSpan.FromMinutes(10));
}
private async Task TurnOffWatching()
{
var setting = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.EnableFolderWatching);
setting.Value = "false";
_unitOfWork.SettingsRepository.Update(setting);
await _unitOfWork.CommitAsync();
StopWatching();
_logger.LogInformation("[LibraryWatcher] Folder watching has been disabled");
}
/// <summary>
/// Processes the file or folder change. If the change is a file change and not from a supported extension, it will be ignored.

View File

@ -289,6 +289,8 @@ public class ParseScannedFiles
}).ToList();
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
_logger.LogDebug("[ScannerService] Skipped File Scan for {Folder} as it hasn't changed since last scan", folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent("Skipped " + normalizedFolder, libraryName, ProgressEventType.Updated));
return;
}

View File

@ -16,4 +16,5 @@ export interface Library {
includeInRecommended: boolean;
includeInSearch: boolean;
manageCollections: boolean;
collapseSeriesRelationships: boolean;
}

View File

@ -52,7 +52,7 @@
<ng-template #jumpBar>
<div class="jump-bar">
<ng-container *ngFor="let jumpKey of jumpBarKeysToRender; let i = index;">
<button class="btn btn-link" [ngClass]="{'disabled': hasCustomSort()}" (click)="scrollTo(jumpKey)">
<button class="btn btn-link" [ngClass]="{'disabled': hasCustomSort()}" (click)="scrollTo(jumpKey)" [ngbTooltip]="jumpKey.size + ' Series'" placement="left">
{{jumpKey.title}}
</button>
</ng-container>

View File

@ -17,6 +17,7 @@ import { SafeStylePipe } from './safe-style.pipe';
import { DefaultDatePipe } from './default-date.pipe';
import { BytesPipe } from './bytes.pipe';
import { TimeAgoPipe } from './time-ago.pipe';
import { TimeDurationPipe } from './time-duration.pipe';
@ -39,6 +40,7 @@ import { TimeAgoPipe } from './time-ago.pipe';
DefaultDatePipe,
BytesPipe,
TimeAgoPipe,
TimeDurationPipe,
],
imports: [
CommonModule,
@ -60,7 +62,8 @@ import { TimeAgoPipe } from './time-ago.pipe';
SafeStylePipe,
DefaultDatePipe,
BytesPipe,
TimeAgoPipe
TimeAgoPipe,
TimeDurationPipe
]
})
export class PipeModule { }

View File

@ -35,7 +35,7 @@ export class TimeAgoPipe implements PipeTransform, OnDestroy {
private timer: number | null = null;
constructor(private changeDetectorRef: ChangeDetectorRef, private ngZone: NgZone) {}
transform(value:string) {
transform(value: string) {
this.removeTimer();
const d = new Date(value);
const now = new Date();

View File

@ -0,0 +1,26 @@
import { Pipe, PipeTransform } from '@angular/core';
/**
* Converts hours -> days, months, years, etc
*/
@Pipe({
name: 'timeDuration'
})
export class TimeDurationPipe implements PipeTransform {
transform(hours: number): string {
if (hours === 0) return `${hours} hours`;
if (hours < 1) {
return `${(hours * 60).toFixed(1)} minutes`;
} else if (hours < 24) {
return `${hours} hours`;
} else if (hours < 720) {
return `${(hours / 24).toFixed(1)} days`;
} else if (hours < 8760) {
return `${(hours / 720).toFixed(1)} months`;
} else {
return `${(hours / 8760).toFixed(1)} years`;
}
}
}

View File

@ -106,6 +106,20 @@
</div>
</div>
<div class="row">
<div class="col-md-12 col-sm-12 pe-2 mb-2">
<div class="mb-3 mt-1">
<div class="form-check form-switch">
<input type="checkbox" id="collapse-relationships" role="switch" formControlName="collapseSeriesRelationships" class="form-check-input" aria-labelledby="auto-close-label">
<label class="form-check-label" for="collapse-relationships">Collapse Series Relationships</label>
</div>
</div>
<p class="accent">
Experiemental: Should Kavita show Series that have no relationships or is the parent/prequel
</p>
</div>
</div>
<div class="row">
<div class="col-md-12 col-sm-12 pe-2 mb-2">
<div class="mb-3 mt-1">

View File

@ -47,6 +47,7 @@ export class LibrarySettingsModalComponent implements OnInit, OnDestroy {
includeInRecommended: new FormControl<boolean>(true, { nonNullable: true, validators: [Validators.required] }),
includeInSearch: new FormControl<boolean>(true, { nonNullable: true, validators: [Validators.required] }),
manageCollections: new FormControl<boolean>(true, { nonNullable: true, validators: [Validators.required] }),
collapseSeriesRelationships: new FormControl<boolean>(false, { nonNullable: true, validators: [Validators.required] }),
});
selectedFolders: string[] = [];
@ -119,6 +120,7 @@ export class LibrarySettingsModalComponent implements OnInit, OnDestroy {
this.libraryForm.get('includeInRecommended')?.setValue(this.library.includeInRecommended);
this.libraryForm.get('includeInSearch')?.setValue(this.library.includeInSearch);
this.libraryForm.get('manageCollections')?.setValue(this.library.manageCollections);
this.libraryForm.get('collapseSeriesRelationships')?.setValue(this.library.collapseSeriesRelationships);
this.selectedFolders = this.library.folders;
this.madeChanges = false;
this.cdRef.markForCheck();

View File

@ -66,7 +66,7 @@
<ng-container>
<div class="col-auto mb-2">
<app-icon-and-title label="Total Read Time" [clickable]="false" fontClasses="fas fa-eye" title="Total Read Time: {{stats.totalReadingTime | number}}">
{{stats.totalReadingTime | compactNumber}} Hours
{{stats.totalReadingTime | timeDuration}}
</app-icon-and-title>
</div>
</ng-container>

View File

@ -20,7 +20,7 @@
<ng-container >
<div class="col-auto mb-2">
<app-icon-and-title label="Time Spent Reading" [clickable]="false" fontClasses="fas fa-eye" title="Time Spent Reading: {{timeSpentReading | number}}">
{{timeSpentReading | compactNumber}} hours
{{timeSpentReading | timeDuration}}
</app-icon-and-title>
</div>
<div class="vr d-none d-lg-block m-2"></div>
@ -28,8 +28,8 @@
<ng-container>
<div class="col-auto mb-2">
<app-icon-and-title label="Average Hours Read / Week" [clickable]="false" fontClasses="fas fa-eye" title="Average Hours Read / Week: {{avgHoursPerWeekSpentReading | number:'1.0-2'}}">
{{avgHoursPerWeekSpentReading | compactNumber | number: '1.0-2'}} hours
<app-icon-and-title label="Average Reading / Week" [clickable]="false" fontClasses="fas fa-eye">
{{avgHoursPerWeekSpentReading | timeDuration}}
</app-icon-and-title>
</div>
<div class="vr d-none d-lg-block m-2"></div>

View File

@ -7,7 +7,7 @@
"name": "GPL-3.0",
"url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE"
},
"version": "0.7.0.2"
"version": "0.7.1.1"
},
"servers": [
{
@ -11441,6 +11441,10 @@
"type": "boolean",
"description": "Should this library create and manage collections from Metadata"
},
"collapseSeriesRelationships": {
"type": "boolean",
"description": "When showing series, only parent series or series with no relationships will be returned"
},
"created": {
"type": "string",
"format": "date-time"
@ -11530,6 +11534,10 @@
"type": "boolean",
"description": "Include library series in Search"
},
"collapseSeriesRelationships": {
"type": "boolean",
"description": "When showing series, only parent series or series with no relationships will be returned"
},
"folders": {
"type": "array",
"items": {
@ -14070,6 +14078,7 @@
},
"UpdateLibraryDto": {
"required": [
"collapseSeriesRelationships",
"folders",
"folderWatching",
"id",
@ -14113,6 +14122,9 @@
},
"manageCollections": {
"type": "boolean"
},
"collapseSeriesRelationships": {
"type": "boolean"
}
},
"additionalProperties": false