Version Fix and Locale Updates (#3626)

This commit is contained in:
Joe Milazzo 2025-03-13 17:54:55 -05:00 committed by GitHub
parent b644022f30
commit a6ccae5849
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 360 additions and 58 deletions

View File

@ -312,7 +312,6 @@ public class VersionUpdaterServiceTests : IDisposable
Assert.NotEmpty(_httpTest.CallLog); // HTTP call was made
}
[Fact]
public async Task GetNumberOfReleasesBehind_ShouldReturnCorrectCount()
{
// Arrange
@ -353,7 +352,6 @@ public class VersionUpdaterServiceTests : IDisposable
Assert.Equal(2 + 1, result); // Behind 0.7.0 and 0.6.0 - We have to add 1 because the current release is > 0.7.0
}
[Fact]
public async Task GetNumberOfReleasesBehind_ShouldReturnCorrectCount_WithNightlies()
{
// Arrange

View File

@ -27,4 +27,8 @@ public static class EasyCacheProfiles
/// Match Series metadata for Kavita+ metadata download
/// </summary>
public const string KavitaPlusMatchSeries = "kavita+matchSeries";
/// <summary>
/// All Locales on the Server
/// </summary>
public const string LocaleOptions = "locales";
}

View File

@ -193,7 +193,6 @@ public class LibraryController : BaseApiController
var ret = _unitOfWork.LibraryRepository.GetLibraryDtosForUsernameAsync(username).ToList();
await _libraryCacheProvider.SetAsync(CacheKey, ret, TimeSpan.FromHours(24));
_logger.LogDebug("Caching libraries for {Key}", cacheKey);
return Ok(ret.Find(l => l.Id == libraryId));
}

View File

@ -2,9 +2,15 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;
using API.Constants;
using API.DTOs.Filtering;
using API.Services;
using EasyCaching.Core;
using Kavita.Common.EnvironmentInfo;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Hosting;
namespace API.Controllers;
@ -13,43 +19,34 @@ namespace API.Controllers;
public class LocaleController : BaseApiController
{
private readonly ILocalizationService _localizationService;
private readonly IEasyCachingProvider _localeCacheProvider;
public LocaleController(ILocalizationService localizationService)
private static readonly string CacheKey = "locales_" + BuildInfo.Version;
public LocaleController(ILocalizationService localizationService, IEasyCachingProviderFactory cachingProviderFactory)
{
_localizationService = localizationService;
_localeCacheProvider = cachingProviderFactory.GetCachingProvider(EasyCacheProfiles.LocaleOptions);
}
/// <summary>
/// Returns all applicable locales on the server
/// </summary>
/// <remarks>This can be cached as it will not change per version.</remarks>
/// <returns></returns>
[AllowAnonymous]
[HttpGet]
public ActionResult<IEnumerable<string>> GetAllLocales()
public async Task<ActionResult<IEnumerable<KavitaLocale>>> GetAllLocales()
{
// Check if temp/locale_map.json exists
var result = await _localeCacheProvider.GetAsync<IEnumerable<KavitaLocale>>(CacheKey);
if (result.HasValue)
{
return Ok(result.Value);
}
// If not, scan the 2 locale files and calculate empty keys or empty values
var ret = _localizationService.GetLocales().Where(l => l.TranslationCompletion > 0f);
await _localeCacheProvider.SetAsync(CacheKey, ret, TimeSpan.FromDays(7));
// Formulate the Locale object with Percentage
var languages = _localizationService.GetLocales().Select(c =>
{
try
{
var cult = new CultureInfo(c);
return new LanguageDto()
{
Title = cult.DisplayName,
IsoCode = cult.IetfLanguageTag
};
}
catch (Exception ex)
{
// Some OS' don't have all culture codes supported like PT_BR, thus we need to default
return new LanguageDto()
{
Title = c,
IsoCode = c
};
}
})
.Where(l => !string.IsNullOrEmpty(l.IsoCode))
.OrderBy(d => d.Title);
return Ok(languages);
return Ok();
}
}

View File

@ -150,7 +150,7 @@ public class UsersController : BaseApiController
}
if (_localizationService.GetLocales().Contains(preferencesDto.Locale))
if (_localizationService.GetLocales().Select(l => l.FileName).Contains(preferencesDto.Locale))
{
existingPreferences.Locale = preferencesDto.Locale;
}

View File

@ -85,6 +85,7 @@ public static class ApplicationServiceExtensions
options.UseInMemory(EasyCacheProfiles.Favicon);
options.UseInMemory(EasyCacheProfiles.Library);
options.UseInMemory(EasyCacheProfiles.RevokedJwt);
options.UseInMemory(EasyCacheProfiles.LocaleOptions);
// KavitaPlus stuff
options.UseInMemory(EasyCacheProfiles.KavitaPlusExternalSeries);

View File

@ -10,12 +10,21 @@ using Microsoft.Extensions.Hosting;
namespace API.Services;
#nullable enable
public class KavitaLocale
{
public string FileName { get; set; } // Key
public string RenderName { get; set; }
public float TranslationCompletion { get; set; }
public bool IsRtL { get; set; }
public string Hash { get; set; } // ETAG hash so I can run my own localization busting implementation
}
public interface ILocalizationService
{
Task<string> Get(string locale, string key, params object[] args);
Task<string> Translate(int userId, string key, params object[] args);
IEnumerable<string> GetLocales();
IEnumerable<KavitaLocale> GetLocales();
}
public class LocalizationService : ILocalizationService
@ -134,14 +143,260 @@ public class LocalizationService : ILocalizationService
/// Returns all available locales that exist on both the Frontend and the Backend
/// </summary>
/// <returns></returns>
public IEnumerable<string> GetLocales()
public IEnumerable<KavitaLocale> GetLocales()
{
var uiLanguages = _directoryService
.GetFilesWithExtension(_directoryService.FileSystem.Path.GetFullPath(_localizationDirectoryUi), @"\.json")
.Select(f => _directoryService.FileSystem.Path.GetFileName(f).Replace(".json", string.Empty));
.GetFilesWithExtension(_directoryService.FileSystem.Path.GetFullPath(_localizationDirectoryUi), @"\.json");
var backendLanguages = _directoryService
.GetFilesWithExtension(_directoryService.LocalizationDirectory, @"\.json")
.Select(f => _directoryService.FileSystem.Path.GetFileName(f).Replace(".json", string.Empty));
return uiLanguages.Intersect(backendLanguages).Distinct();
.GetFilesWithExtension(_directoryService.LocalizationDirectory, @"\.json");
var locales = new Dictionary<string, KavitaLocale>();
var localeCounts = new Dictionary<string, Tuple<int, int>>(); // fileName -> (nonEmptyValues, totalKeys)
// First pass: collect all files and count non-empty strings
// Process UI language files
foreach (var file in uiLanguages)
{
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(file);
var fileContent = _directoryService.FileSystem.File.ReadAllText(file);
var hash = ComputeHash(fileContent);
var counts = CalculateNonEmptyStrings(fileContent);
if (localeCounts.TryGetValue(fileName, out var existingCount))
{
// Update existing counts
localeCounts[fileName] = Tuple.Create(
existingCount.Item1 + counts.Item1,
existingCount.Item2 + counts.Item2
);
}
else
{
// Add new counts
localeCounts[fileName] = counts;
}
if (!locales.TryGetValue(fileName, out var locale))
{
locales[fileName] = new KavitaLocale
{
FileName = fileName,
RenderName = GetDisplayName(fileName),
TranslationCompletion = 0, // Will be calculated later
IsRtL = IsRightToLeft(fileName),
Hash = hash
};
}
else
{
// Update existing locale hash
locale.Hash = CombineHashes(locale.Hash, hash);
}
}
// Process backend language files
foreach (var file in backendLanguages)
{
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(file);
var fileContent = _directoryService.FileSystem.File.ReadAllText(file);
var hash = ComputeHash(fileContent);
var counts = CalculateNonEmptyStrings(fileContent);
if (localeCounts.TryGetValue(fileName, out var existingCount))
{
// Update existing counts
localeCounts[fileName] = Tuple.Create(
existingCount.Item1 + counts.Item1,
existingCount.Item2 + counts.Item2
);
}
else
{
// Add new counts
localeCounts[fileName] = counts;
}
if (!locales.TryGetValue(fileName, out var locale))
{
locales[fileName] = new KavitaLocale
{
FileName = fileName,
RenderName = GetDisplayName(fileName),
TranslationCompletion = 0, // Will be calculated later
IsRtL = IsRightToLeft(fileName),
Hash = hash
};
}
else
{
// Update existing locale hash
locale.Hash = CombineHashes(locale.Hash, hash);
}
}
// Second pass: calculate completion percentages based on English total
if (localeCounts.TryGetValue("en", out var englishCounts) && englishCounts.Item2 > 0)
{
var englishTotalKeys = englishCounts.Item2;
foreach (var locale in locales.Values)
{
if (localeCounts.TryGetValue(locale.FileName, out var counts))
{
// Calculate percentage based on English total keys
locale.TranslationCompletion = (float)counts.Item1 / englishTotalKeys * 100;
}
}
}
return locales.Values;
}
// Helper methods that would need to be implemented
private static string ComputeHash(string content)
{
// Implement a hashing algorithm (e.g., SHA256, MD5) to generate a hash for the content
using var md5 = System.Security.Cryptography.MD5.Create();
var inputBytes = System.Text.Encoding.UTF8.GetBytes(content);
var hashBytes = md5.ComputeHash(inputBytes);
return Convert.ToBase64String(hashBytes);
}
private static string CombineHashes(string hash1, string hash2)
{
// Combine two hashes, possibly by concatenating and rehashing
return ComputeHash(hash1 + hash2);
}
private static string GetDisplayName(string fileName)
{
// Map the filename to a human-readable display name
// This could use a lookup table or follow a naming convention
try
{
var cultureInfo = new System.Globalization.CultureInfo(fileName);
return cultureInfo.NativeName;
}
catch
{
// Fall back to the file name if the culture isn't recognized
return fileName;
}
}
private static bool IsRightToLeft(string fileName)
{
// Determine if the language is right-to-left
try
{
var cultureInfo = new System.Globalization.CultureInfo(fileName);
return cultureInfo.TextInfo.IsRightToLeft;
}
catch
{
return false; // Default to left-to-right
}
}
private static float CalculateTranslationCompletion(string fileContent)
{
try
{
var jsonObject = System.Text.Json.JsonDocument.Parse(fileContent);
int totalKeys = 0;
int nonEmptyValues = 0;
// Count all keys and non-empty values
CountNonEmptyValues(jsonObject.RootElement, ref totalKeys, ref nonEmptyValues);
return totalKeys > 0 ? (nonEmptyValues * 1f) / totalKeys * 100 : 0;
}
catch (Exception ex)
{
// Consider logging the exception
return 0; // Return 0% completion if there's an error parsing
}
}
private static Tuple<int, int> CalculateNonEmptyStrings(string fileContent)
{
try
{
var jsonObject = JsonDocument.Parse(fileContent);
var totalKeys = 0;
var nonEmptyValues = 0;
// Count all keys and non-empty values
CountNonEmptyValues(jsonObject.RootElement, ref totalKeys, ref nonEmptyValues);
return Tuple.Create(nonEmptyValues, totalKeys);
}
catch (Exception)
{
// Consider logging the exception
return Tuple.Create(0, 0); // Return 0% completion if there's an error parsing
}
}
private static void CountNonEmptyValues(JsonElement element, ref int totalKeys, ref int nonEmptyValues)
{
if (element.ValueKind == JsonValueKind.Object)
{
foreach (var property in element.EnumerateObject())
{
if (property.Value.ValueKind == System.Text.Json.JsonValueKind.String)
{
totalKeys++;
var value = property.Value.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
nonEmptyValues++;
}
}
else
{
// Recursively process nested objects
CountNonEmptyValues(property.Value, ref totalKeys, ref nonEmptyValues);
}
}
}
else if (element.ValueKind == System.Text.Json.JsonValueKind.Array)
{
foreach (var item in element.EnumerateArray())
{
CountNonEmptyValues(item, ref totalKeys, ref nonEmptyValues);
}
}
}
private void CountEntries(System.Text.Json.JsonElement element, ref int total, ref int translated)
{
if (element.ValueKind == System.Text.Json.JsonValueKind.Object)
{
foreach (var property in element.EnumerateObject())
{
CountEntries(property.Value, ref total, ref translated);
}
}
else if (element.ValueKind == System.Text.Json.JsonValueKind.Array)
{
foreach (var item in element.EnumerateArray())
{
CountEntries(item, ref total, ref translated);
}
}
else if (element.ValueKind == System.Text.Json.JsonValueKind.String)
{
total++;
string value = element.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
translated++;
}
}
}
}

View File

@ -399,10 +399,19 @@ public partial class VersionUpdaterService : IVersionUpdaterService
public async Task<int> GetNumberOfReleasesBehind()
{
var updates = await GetAllReleases();
return updates
// If the user is on nightly, then we need to handle releases behind differently
if (updates[0].IsPrerelease)
{
return Math.Min(0, updates
.TakeWhile(update => update.UpdateVersion != update.CurrentVersion)
.Count() - 1);
}
return Math.Min(0, updates
.Where(update => !update.IsPrerelease)
.TakeWhile(update => update.UpdateVersion != update.CurrentVersion)
.Count();
.Count());
}
private UpdateNotificationDto? CreateDto(GithubReleaseMetadata? update)

View File

@ -3,3 +3,10 @@ export interface Language {
title: string;
}
export interface KavitaLocale {
fileName: string; // isoCode aka what maps to the file on disk and what transloco loads
renderName: string;
translationCompletion: number;
isRtL: boolean;
hash: string;
}

View File

@ -18,6 +18,7 @@ import {Action} from "./action-factory.service";
import {CoverImageSize} from "../admin/_models/cover-image-size";
import {LicenseInfo} from "../_models/kavitaplus/license-info";
import {LicenseService} from "./license.service";
import {LocalizationService} from "./localization.service";
export enum Role {
Admin = 'Admin',
@ -48,6 +49,7 @@ export class AccountService {
private readonly destroyRef = inject(DestroyRef);
private readonly licenseService = inject(LicenseService);
private readonly localizationService = inject(LocalizationService);
baseUrl = environment.apiUrl;
userKey = 'kavita-user';
@ -168,6 +170,8 @@ export class AccountService {
}
setCurrentUser(user?: User, refreshConnections = true) {
const isSameUser = this.currentUser === user;
if (user) {
user.roles = [];
const roles = this.getDecodedToken(user.token).role;
@ -197,7 +201,9 @@ export class AccountService {
// But that really messes everything up
this.messageHub.stopHubConnection();
this.messageHub.createHubConnection(this.currentUser);
this.licenseService.hasValidLicense().subscribe();
if (!isSameUser) {
this.licenseService.hasValidLicense().subscribe();
}
this.startRefreshTokenTimer();
}
}
@ -316,6 +322,8 @@ export class AccountService {
// Update the locale on disk (for logout and compact-number pipe)
localStorage.setItem(AccountService.localeKey, this.currentUser.preferences.locale);
this.localizationService.refreshTranslations(this.currentUser.preferences.locale);
}
return settings;
}), takeUntilDestroyed(this.destroyRef));

View File

@ -1,18 +1,36 @@
import { Injectable } from '@angular/core';
import {inject, Injectable} from '@angular/core';
import {environment} from "../../environments/environment";
import { HttpClient } from "@angular/common/http";
import {Language} from "../_models/metadata/language";
import {KavitaLocale, Language} from "../_models/metadata/language";
import {ReplaySubject, tap} from "rxjs";
import {TranslocoService} from "@jsverse/transloco";
@Injectable({
providedIn: 'root'
})
export class LocalizationService {
private readonly translocoService = inject(TranslocoService);
baseUrl = environment.apiUrl;
private readonly localeSubject = new ReplaySubject<KavitaLocale[]>(1);
public readonly locales$ = this.localeSubject.asObservable();
constructor(private httpClient: HttpClient) { }
getLocales() {
return this.httpClient.get<Language[]>(this.baseUrl + 'locale');
return this.httpClient.get<KavitaLocale[]>(this.baseUrl + 'locale').pipe(tap(locales => {
this.localeSubject.next(locales);
}));
}
refreshTranslations(lang: string) {
// Clear the cached translation
localStorage.removeItem(`@@TRANSLOCO_PERSIST_TRANSLATIONS/${lang}`);
// Reload the translation
return this.translocoService.load(lang);
}
}

View File

@ -19,13 +19,12 @@ import {SideNavComponent} from './sidenav/_components/side-nav/side-nav.componen
import {NavHeaderComponent} from "./nav/_components/nav-header/nav-header.component";
import {takeUntilDestroyed} from "@angular/core/rxjs-interop";
import {ServerService} from "./_services/server.service";
import {OutOfDateModalComponent} from "./announcements/_components/out-of-date-modal/out-of-date-modal.component";
import {PreferenceNavComponent} from "./sidenav/preference-nav/preference-nav.component";
import {Breakpoint, UtilityService} from "./shared/_services/utility.service";
import {TranslocoService} from "@jsverse/transloco";
import {User} from "./_models/user";
import {VersionService} from "./_services/version.service";
import {LicenseService} from "./_services/license.service";
import {LocalizationService} from "./_services/localization.service";
@Component({
selector: 'app-root',
@ -36,8 +35,9 @@ import {LicenseService} from "./_services/license.service";
changeDetection: ChangeDetectionStrategy.OnPush
})
export class AppComponent implements OnInit {
protected readonly Breakpoint = Breakpoint;
transitionState$!: Observable<boolean>;
private readonly destroyRef = inject(DestroyRef);
private readonly offcanvas = inject(NgbOffcanvas);
@ -53,8 +53,9 @@ export class AppComponent implements OnInit {
private readonly translocoService = inject(TranslocoService);
private readonly versionService = inject(VersionService); // Needs to be injected to run background job
private readonly licenseService = inject(LicenseService);
private readonly localizationService = inject(LocalizationService);
protected readonly Breakpoint = Breakpoint;
transitionState$!: Observable<boolean>;
constructor(ratingConfig: NgbRatingConfig, modalConfig: NgbModalConfig) {
@ -112,6 +113,7 @@ export class AppComponent implements OnInit {
this.setDocHeight();
this.setCurrentUser();
this.themeService.setColorScape('');
this.localizationService.getLocales().subscribe(); // This will cache the localizations on startup
}

View File

@ -15,8 +15,8 @@
</ng-template>
<ng-template #edit>
<select class="form-select" aria-describedby="global-header" formControlName="locale">
@for(opt of locales; track opt.title) {
<option [value]="opt.isoCode">{{opt.title | titlecase}}</option>
@for(opt of locales; track opt.renderName) {
<option [value]="opt.fileName">{{opt.renderName | titlecase}} ({{opt.translationCompletion | number:'1.0-1'}}%)</option>
}
</select>
</ng-template>

View File

@ -21,7 +21,7 @@ import {LocalizationService} from "../../_services/localization.service";
import {bookColorThemes} from "../../book-reader/_components/reader-settings/reader-settings.component";
import {FormControl, FormGroup, ReactiveFormsModule} from "@angular/forms";
import {User} from "../../_models/user";
import {Language} from "../../_models/metadata/language";
import {KavitaLocale, Language} from "../../_models/metadata/language";
import {takeUntilDestroyed} from "@angular/core/rxjs-interop";
import {debounceTime, distinctUntilChanged, filter, forkJoin, switchMap, tap} from "rxjs";
import {take} from "rxjs/operators";
@ -35,7 +35,7 @@ import {
NgbAccordionDirective, NgbAccordionHeader,
NgbAccordionItem, NgbTooltip
} from "@ng-bootstrap/ng-bootstrap";
import {AsyncPipe, NgStyle, NgTemplateOutlet, TitleCasePipe} from "@angular/common";
import {AsyncPipe, DecimalPipe, NgStyle, NgTemplateOutlet, TitleCasePipe} from "@angular/common";
import {ColorPickerModule} from "ngx-color-picker";
import {SettingTitleComponent} from "../../settings/_components/setting-title/setting-title.component";
import {SettingItemComponent} from "../../settings/_components/setting-item/setting-item.component";
@ -76,7 +76,8 @@ import {LicenseService} from "../../_services/license.service";
PdfSpreadModePipe,
PdfThemePipe,
PdfScrollModePipe,
AsyncPipe
AsyncPipe,
DecimalPipe
],
templateUrl: './manage-user-preferences.component.html',
styleUrl: './manage-user-preferences.component.scss',
@ -112,7 +113,7 @@ export class ManageUserPreferencesComponent implements OnInit {
fontFamilies: Array<string> = [];
locales: Array<Language> = [{title: 'English', isoCode: 'en'}];
locales: Array<KavitaLocale> = [];
settingsForm: FormGroup = new FormGroup({});
user: User | undefined = undefined;
@ -120,7 +121,7 @@ export class ManageUserPreferencesComponent implements OnInit {
get Locale() {
if (!this.settingsForm.get('locale')) return 'English';
return this.locales.filter(l => l.isoCode === this.settingsForm.get('locale')!.value)[0].title;
return this.locales.filter(l => l.fileName === this.settingsForm.get('locale')!.value)[0].renderName;
}
@ -128,7 +129,7 @@ export class ManageUserPreferencesComponent implements OnInit {
this.fontFamilies = this.bookService.getFontFamilies().map(f => f.title);
this.cdRef.markForCheck();
this.localizationService.getLocales().subscribe(res => {
this.localizationService.locales$.subscribe(res => {
this.locales = res;
this.cdRef.markForCheck();

View File

@ -30,6 +30,7 @@ import {LazyLoadImageModule} from "ng-lazyload-image";
import {getSaver, SAVER} from "./app/_providers/saver.provider";
import {distinctUntilChanged} from "rxjs/operators";
import {APP_BASE_HREF, PlatformLocation} from "@angular/common";
import {provideTranslocoDefaultLocale} from "@jsverse/transloco-locale/lib/transloco-locale.providers";
const disableAnimations = !('animate' in document.documentElement);
@ -112,7 +113,9 @@ const translocoOptions = {
missingHandler: {
useFallbackTranslation: true,
allowEmpty: false,
logMissingKey: true
},
failedRetries: 2,
} as TranslocoConfig
};