mirror of
https://github.com/immich-app/immich.git
synced 2025-11-26 16:25:20 -05:00
* feat: add a `maintenance.enabled` config flag
* feat: implement graceful restart
feat: restart when maintenance config is toggled
* feat: boot a stripped down maintenance api if enabled
* feat: cli command to toggle maintenance mode
* chore: fallback IMMICH_SERVER_URL environment variable in process
* chore: add additional routes to maintenance controller
* fix: don't wait for nest application to close to finish request response
* chore: add a failsafe on restart to prevent other exit codes from preventing restart
* feat: redirect into/from maintenance page
* refactor: use system metadata for maintenance status
* refactor: wait on WebSocket connection to refresh
* feat: broadcast websocket event on server restart
refactor: listen to WS instead of polling
* refactor: bubble up maintenance information instead of hijacking in fetch function
feat: show modal when server is restarting
* chore: increase timeout for ungraceful restart
* refactor: deduplicate code between api/maintenance workers
* fix: skip config check if database is not initialised
* fix: add `maintenanceMode` field to system config test
* refactor: move maintenance resolution code to static method in service
* chore: clean up linter issues
* chore: generate dart openapi
* refactor: use try{} block for maintenance mode check
* fix: logic error in server redirect
* chore: include `maintenanceMode` key in e2e test
* chore: add i18n entries for maintenance screens
* chore: remove negated condition from hook
* fix: should set default value not override in service
* fix: minor error in page
* feat: initial draft of maintenance module, repo., worker controller, worker service
* refactor: move broadcast code into notification service
* chore: connect websocket on client if in maintenance
* chore: set maintenance module app name
* refactor: rename repository to include worker
chore: configure websocket adapter
* feat: reimplement maintenance mode exit with new module
* refactor: add a constant enum for ExitCode
* refactor: remove redundant route for maintenance
* refactor: only spin up kysely on boot (rather than a Nest app)
* refactor(web): move redirect logic into +layout file where modal is setup
* feat: add Maintenance permission
* refactor: merge common code between api/maintenance
* fix: propagate changes from the CLI to servers
* feat: maintenance authentication guard
* refactor: unify maintenance code into repository
feat: add a step to generate maintenance mode token
* feat: jwt auth for maintenance
* refactor: switch from nest jwt to just jsonwebtokens
* feat: log into maintenance mode from CLI command
* refactor: use `secret` instead of `token` in jwt terminology
chore: log maintenance mode login URL on boot
chore: don't make CLI actions reload if already in target state
* docs: initial draft for maintenance mode page
* refactor: always validate the maintenance auth on the server
* feat: add a link to maintenance mode documentation
* feat: redirect users back to the last page they were on when exiting maintenance
* refactor: provide closeFn in both maintenance repos.
* refactor: ensure the user is also redirected by the server
* chore: swap jsonwebtoken for jose
* refactor: introduce AppRestartEvent w/o secret passing
* refactor: use navigation goto
* refactor: use `continue` instead of `next`
* chore: lint fixes for server
* chore: lint fixes for web
* test: add mock for maintenance repository
* test: add base service dependency to maintenance
* chore: remove @types/jsonwebtoken
* refactor: close database connection after startup check
* refactor: use `request#auth` key
* refactor: use service instead of repository
chore: read token from cookie if possible
chore: rename client event to AppRestartV1
* refactor: more concise redirect logic on web
* refactor: move redirect check into utils
refactor: update translation strings to be more sensible
* refactor: always validate login (i.e. check cookie)
* refactor: lint, open-api, remove old dto
* refactor: encode at point of usage
* refactor: remove business logic from repositories
* chore: fix server/web lints
* refactor: remove repository mock
* chore: fix formatting
* test: write service mocks for maintenance mode
* test: write cli service tests
* fix: catch errors when closing app
* fix: always report no maintenance when usual API is available
* test: api e2e maintenance spec
* chore: add response builder
* chore: add helper to set maint. auth cookie
* feat: add SSR to maintenance API
* test(e2e): write web spec for maintenance
* chore: clean up lint issues
* chore: format files
* feat: perform 302 redirect at server level during maintenance
* fix: keep trying to stop immich until it succeeds (CLI issue)
* chore: lint/format
* refactor: annotate references to other services in worker service
* chore: lint
* refactor: remove unnecessary await
Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
* refactor: move static methods into util
* refactor: assert secret exists in maintenance worker
* refactor: remove assertion which isn't necessary anymore
* refactor: remove assertion
* refactor: remove outer try {} catch block from loadMaintenanceAuth
* refactor: undo earlier change to vite.config.ts
* chore: update tests due to refactors
* revert: vite.config.ts
* test: expect string jwt
* chore: move blanket exceptions into controllers
* test: update tests according with last change
* refactor: use respondWithCookie
refactor: merge start/end into one route
refactor: rename MaintenanceRepository to AppRepository
chore: use new ApiTag/Endpoint
refactor: apply other requested changes
* chore: regenerate openapi
* chore: lint/format
* chore: remove secureOnly for maint. cookie
* refactor: move maintenance worker code into src/maintenance\nfix: various test fixes
* refactor: use `action` property for setting maint. mode
* refactor: remove Websocket#restartApp in favour of individual methods
* chore: incomplete commit
* chore: remove stray log
* fix: call exitApp from maintenance worker on exit
* fix: add app repository mock
* fix: ensure maintenance cookies are secure
* fix: run playwright tests over secure context (localhost)
* test: update other references to 127.0.0.1
* refactor: use serverSideEmitWithAck
* chore: correct the logic in tryTerminate
* test: juggle cookies ourselves
* chore: fix lint error for e2e spec
* chore: format e2e test
* fix: set cookie secure/non-secure depending on context
* chore: format files
---------
Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
169 lines
4.7 KiB
TypeScript
169 lines
4.7 KiB
TypeScript
import { Kysely } from 'kysely';
|
|
import { CommandFactory } from 'nest-commander';
|
|
import { ChildProcess, fork } from 'node:child_process';
|
|
import { dirname, join } from 'node:path';
|
|
import { Worker } from 'node:worker_threads';
|
|
import { PostgresError } from 'postgres';
|
|
import { ImmichAdminModule } from 'src/app.module';
|
|
import { ExitCode, ImmichWorker, LogLevel, SystemMetadataKey } from 'src/enum';
|
|
import { ConfigRepository } from 'src/repositories/config.repository';
|
|
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
|
|
import { type DB } from 'src/schema';
|
|
import { getKyselyConfig } from 'src/utils/database';
|
|
|
|
/**
|
|
* Manages worker lifecycle
|
|
*/
|
|
class Workers {
|
|
/**
|
|
* Currently running workers
|
|
*/
|
|
workers: Partial<Record<ImmichWorker, { kill: (signal: NodeJS.Signals) => Promise<void> | void }>> = {};
|
|
|
|
/**
|
|
* Fail-safe in case anything dies during restart
|
|
*/
|
|
restarting = false;
|
|
|
|
/**
|
|
* Boot all enabled workers
|
|
*/
|
|
async bootstrap() {
|
|
const isMaintenanceMode = await this.isMaintenanceMode();
|
|
const { workers } = new ConfigRepository().getEnv();
|
|
|
|
if (isMaintenanceMode) {
|
|
this.startWorker(ImmichWorker.Maintenance);
|
|
} else {
|
|
for (const worker of workers) {
|
|
this.startWorker(worker);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Initialise a short-lived Nest application to build configuration
|
|
* @returns System configuration
|
|
*/
|
|
private async isMaintenanceMode(): Promise<boolean> {
|
|
const { database } = new ConfigRepository().getEnv();
|
|
const kysely = new Kysely<DB>(getKyselyConfig(database.config));
|
|
const systemMetadataRepository = new SystemMetadataRepository(kysely);
|
|
|
|
try {
|
|
const value = await systemMetadataRepository.get(SystemMetadataKey.MaintenanceMode);
|
|
return value?.isMaintenanceMode || false;
|
|
} catch (error) {
|
|
// Table doesn't exist (migrations haven't run yet)
|
|
if (error instanceof PostgresError && error.code === '42P01') {
|
|
return false;
|
|
}
|
|
|
|
throw error;
|
|
} finally {
|
|
await kysely.destroy();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start an individual worker
|
|
* @param name Worker
|
|
*/
|
|
private startWorker(name: ImmichWorker) {
|
|
console.log(`Starting ${name} worker`);
|
|
|
|
// eslint-disable-next-line unicorn/prefer-module
|
|
const basePath = dirname(__filename);
|
|
const workerFile = join(basePath, 'workers', `${name}.js`);
|
|
|
|
let anyWorker: Worker | ChildProcess;
|
|
let kill: (signal?: NodeJS.Signals) => Promise<void> | void;
|
|
|
|
if (name === ImmichWorker.Api) {
|
|
const worker = fork(workerFile, [], {
|
|
execArgv: process.execArgv.map((arg) => (arg.startsWith('--inspect') ? '--inspect=0.0.0.0:9231' : arg)),
|
|
});
|
|
|
|
kill = (signal) => void worker.kill(signal);
|
|
anyWorker = worker;
|
|
} else {
|
|
const worker = new Worker(workerFile);
|
|
|
|
kill = async () => void (await worker.terminate());
|
|
anyWorker = worker;
|
|
}
|
|
|
|
anyWorker.on('error', (error) => this.onError(name, error));
|
|
anyWorker.on('exit', (exitCode) => this.onExit(name, exitCode));
|
|
|
|
this.workers[name] = { kill };
|
|
}
|
|
|
|
onError(name: ImmichWorker, error: Error) {
|
|
console.error(`${name} worker error: ${error}, stack: ${error.stack}`);
|
|
}
|
|
|
|
onExit(name: ImmichWorker, exitCode: number | null) {
|
|
// restart immich server
|
|
if (exitCode === ExitCode.AppRestart || this.restarting) {
|
|
this.restarting = true;
|
|
|
|
console.info(`${name} worker shutdown for restart`);
|
|
delete this.workers[name];
|
|
|
|
// once all workers shut down, bootstrap again
|
|
if (Object.keys(this.workers).length === 0) {
|
|
void this.bootstrap();
|
|
this.restarting = false;
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
// shutdown the entire process
|
|
delete this.workers[name];
|
|
|
|
if (exitCode !== 0) {
|
|
console.error(`${name} worker exited with code ${exitCode}`);
|
|
|
|
if (this.workers[ImmichWorker.Api] && name !== ImmichWorker.Api) {
|
|
console.error('Killing api process');
|
|
void this.workers[ImmichWorker.Api].kill('SIGTERM');
|
|
}
|
|
}
|
|
|
|
process.exit(exitCode);
|
|
}
|
|
}
|
|
|
|
function main() {
|
|
const immichApp = process.argv[2];
|
|
if (immichApp) {
|
|
process.argv.splice(2, 1);
|
|
}
|
|
|
|
if (immichApp === 'immich-admin') {
|
|
process.title = 'immich_admin_cli';
|
|
process.env.IMMICH_LOG_LEVEL = LogLevel.Warn;
|
|
|
|
return CommandFactory.run(ImmichAdminModule);
|
|
}
|
|
|
|
if (immichApp === 'immich' || immichApp === 'microservices') {
|
|
console.error(
|
|
`Using "start.sh ${immichApp}" has been deprecated. See https://github.com/immich-app/immich/releases/tag/v1.118.0 for more information.`,
|
|
);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (immichApp) {
|
|
console.error(`Unknown command: "${immichApp}"`);
|
|
process.exit(1);
|
|
}
|
|
|
|
process.title = 'immich';
|
|
void new Workers().bootstrap();
|
|
}
|
|
|
|
void main();
|