From 55363bf31b86e7862f9ae9a6d4166bf4ecf8f73d Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 31 Mar 2023 14:42:58 +0200 Subject: [PATCH] Disabled more maintenance nonsense when running tasks --- maloja/database/dbcache.py | 19 +++++++++-- maloja/database/sqldb.py | 64 ++++++++++++++++++++------------------ 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/maloja/database/dbcache.py b/maloja/database/dbcache.py index 582a9e7..0bd0c9a 100644 --- a/maloja/database/dbcache.py +++ b/maloja/database/dbcache.py @@ -23,11 +23,13 @@ if malojaconfig['USE_GLOBAL_CACHE']: @runhourly def maintenance(): from . import AUX_MODE - if not AUX_MODE: - print_stats() - trim_cache() + if AUX_MODE: return + print_stats() + trim_cache() def print_stats(): + from . import AUX_MODE + if AUX_MODE: return for name,c in (('Cache',cache),('Entity Cache',entitycache)): hits, misses = c.get_stats() log(f"{name}: Size: {len(c)} | Hits: {hits}/{hits+misses} | Estimated Memory: {human_readable_size(c)}") @@ -35,6 +37,8 @@ if malojaconfig['USE_GLOBAL_CACHE']: def cached_wrapper(inner_func): + from . import AUX_MODE + if AUX_MODE: return inner_func def outer_func(*args,**kwargs): @@ -59,6 +63,8 @@ if malojaconfig['USE_GLOBAL_CACHE']: # we don't want a new cache entry for every single combination, but keep a common # cache that's aware of what we're calling def cached_wrapper_individual(inner_func): + from . import AUX_MODE + if AUX_MODE: return def outer_func(set_arg,**kwargs): if 'dbconn' in kwargs: @@ -83,6 +89,9 @@ if malojaconfig['USE_GLOBAL_CACHE']: return outer_func def invalidate_caches(scrobbletime=None): + from . import AUX_MODE + if AUX_MODE: return + cleared, kept = 0, 0 for k in cache.keys(): # VERY BIG TODO: differentiate between None as in 'unlimited timerange' and None as in 'time doesnt matter here'! @@ -95,10 +104,14 @@ if malojaconfig['USE_GLOBAL_CACHE']: def invalidate_entity_cache(): + from . import AUX_MODE + if AUX_MODE: return entitycache.clear() def trim_cache(): + from . import AUX_MODE + if AUX_MODE: return ramprct = psutil.virtual_memory().percent if ramprct > malojaconfig["DB_MAX_MEMORY"]: log(f"{ramprct}% RAM usage, clearing cache!") diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index f511488..c39ba5a 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -1360,40 +1360,40 @@ def search_album(searchterm,dbconn=None): def clean_db(dbconn=None): from . import AUX_MODE + if AUX_MODE: return - if not AUX_MODE: - with SCROBBLE_LOCK: - log(f"Database Cleanup...") + with SCROBBLE_LOCK: + log(f"Database Cleanup...") - to_delete = [ - # tracks with no scrobbles (trackartist entries first) - "from trackartists where track_id in (select id from tracks where id not in (select track_id from scrobbles))", - "from tracks where id not in (select track_id from scrobbles)", - # artists with no tracks AND no albums - "from artists where id not in (select artist_id from trackartists) \ - and id not in (select target_artist from associated_artists) \ - and id not in (select artist_id from albumartists)", - # tracks with no artists (scrobbles first) - "from scrobbles where track_id in (select id from tracks where id not in (select track_id from trackartists))", - "from tracks where id not in (select track_id from trackartists)", - # albums with no tracks (albumartist entries first) - "from albumartists where album_id in (select id from albums where id not in (select album_id from tracks where album_id is not null))", - "from albums where id not in (select album_id from tracks where album_id is not null)", - # albumartist entries that are missing a reference - "from albumartists where album_id not in (select album_id from tracks where album_id is not null)", - "from albumartists where artist_id not in (select id from artists)", - # trackartist entries that mare missing a reference - "from trackartists where track_id not in (select id from tracks)", - "from trackartists where artist_id not in (select id from artists)" - ] + to_delete = [ + # tracks with no scrobbles (trackartist entries first) + "from trackartists where track_id in (select id from tracks where id not in (select track_id from scrobbles))", + "from tracks where id not in (select track_id from scrobbles)", + # artists with no tracks AND no albums + "from artists where id not in (select artist_id from trackartists) \ + and id not in (select target_artist from associated_artists) \ + and id not in (select artist_id from albumartists)", + # tracks with no artists (scrobbles first) + "from scrobbles where track_id in (select id from tracks where id not in (select track_id from trackartists))", + "from tracks where id not in (select track_id from trackartists)", + # albums with no tracks (albumartist entries first) + "from albumartists where album_id in (select id from albums where id not in (select album_id from tracks where album_id is not null))", + "from albums where id not in (select album_id from tracks where album_id is not null)", + # albumartist entries that are missing a reference + "from albumartists where album_id not in (select album_id from tracks where album_id is not null)", + "from albumartists where artist_id not in (select id from artists)", + # trackartist entries that mare missing a reference + "from trackartists where track_id not in (select id from tracks)", + "from trackartists where artist_id not in (select id from artists)" + ] - for d in to_delete: - selection = dbconn.execute(sql.text(f"select * {d}")) - for row in selection.all(): - log(f"Deleting {row}") - deletion = dbconn.execute(sql.text(f"delete {d}")) + for d in to_delete: + selection = dbconn.execute(sql.text(f"select * {d}")) + for row in selection.all(): + log(f"Deleting {row}") + deletion = dbconn.execute(sql.text(f"delete {d}")) - log("Database Cleanup complete!") + log("Database Cleanup complete!") @@ -1407,6 +1407,10 @@ def clean_db(dbconn=None): @runmonthly def renormalize_names(): + + from . import AUX_MODE + if AUX_MODE: return + with SCROBBLE_LOCK: with engine.begin() as conn: rows = conn.execute(DB['artists'].select()).all()