From 98d80d3d1160a3e52e621ad04b065d3ad995e2da Mon Sep 17 00:00:00 2001 From: Marco Aceti Date: Tue, 21 Jan 2020 15:09:52 +0100 Subject: [PATCH 01/40] Add Dockerfile support --- Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6a3e329 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:slim + +WORKDIR /usr/src/app +COPY . . + +RUN apt-get update -y +RUN apt-get install gcc -y + +RUN pip3 install -r requirements.txt + +EXPOSE 42010 + +ENTRYPOINT ./maloja start From 2004f7690f1119c391ee0782c79ead7dd0ce3497 Mon Sep 17 00:00:00 2001 From: Marco Aceti Date: Tue, 21 Jan 2020 15:27:27 +0100 Subject: [PATCH 02/40] More logic Dockerfile ordering --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6a3e329..a1b72b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,14 @@ FROM python:slim WORKDIR /usr/src/app -COPY . . RUN apt-get update -y RUN apt-get install gcc -y +COPY requirements.txt . RUN pip3 install -r requirements.txt EXPOSE 42010 +COPY . . ENTRYPOINT ./maloja start From a97a5e2fbf834586a01a131eb1a5d68136840a9b Mon Sep 17 00:00:00 2001 From: krateng Date: Wed, 26 Feb 2020 15:09:53 +0100 Subject: [PATCH 03/40] Update Dockerfile Updated to python package --- Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index a1b72b1..cb7e6e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,12 @@ -FROM python:slim +FROM python:3.6-alpine WORKDIR /usr/src/app -RUN apt-get update -y -RUN apt-get install gcc -y +RUN apk update +RUN apk add gcc libxml2-dev libxslt-dev py3-pip libc-dev -COPY requirements.txt . -RUN pip3 install -r requirements.txt +RUN pip3 install malojaserver EXPOSE 42010 -COPY . . -ENTRYPOINT ./maloja start +ENTRYPOINT maloja start From fb882a1af4b2673e445122c1ea6732e83ba6cde9 Mon Sep 17 00:00:00 2001 From: krateng Date: Thu, 19 Mar 2020 18:45:35 +0100 Subject: [PATCH 04/40] Adjusted entrypoint command --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cb7e6e6..2a49020 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,4 +9,4 @@ RUN pip3 install malojaserver EXPOSE 42010 -ENTRYPOINT maloja start +ENTRYPOINT maloja run From 85f03e443e4e29f30bb3a48524d8f8b990bd7e55 Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 17 May 2020 11:46:41 +0200 Subject: [PATCH 05/40] Added Docker installation to readme --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 7c4a30c..38528d4 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,11 @@ Also neat: You can use your **custom artist or track images**. * Otherwise, simply run the command `maloja update` or use `pip`s update mechanic. +### Docker + +There is a Dockerfile in the repo that should work by itself. You can also use the unofficial [Dockerhub repository](https://hub.docker.com/r/foxxmd/maloja) kindly provided by FoxxMD. + + ## How to use ### Basic control From b8de507a4f29a70b444a2fb6f51e74f9239f5f17 Mon Sep 17 00:00:00 2001 From: Krateng Date: Thu, 28 May 2020 17:26:41 +0200 Subject: [PATCH 06/40] Updated wrong file reference (GH-16) --- maloja/__pkginfo__.py | 2 +- maloja/database.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 96e0c01..7662666 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -20,7 +20,7 @@ requires = [ "setproctitle>=1.1.10", "wand>=0.5.4", "lesscpy>=0.13", - "jinja2">2.11 + "jinja2>2.11" ] resources = [ "web/*/*", diff --git a/maloja/database.py b/maloja/database.py index c17fe77..458b693 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -719,7 +719,7 @@ def abouttoshutdown(): def newrule(**keys): apikey = keys.pop("key",None) if (checkAPIkey(apikey)): - tsv.add_entry("rules/webmade.tsv",[k for k in keys]) + tsv.add_entry(datadir("rules/webmade.tsv"),[k for k in keys]) #addEntry("rules/webmade.tsv",[k for k in keys]) global db_rulestate db_rulestate = False From 6aa65bf1ce273d9fd36d44f6e24439981b2228a3 Mon Sep 17 00:00:00 2001 From: Krateng Date: Thu, 28 May 2020 19:32:41 +0200 Subject: [PATCH 07/40] Added settings for caching --- maloja/__pkginfo__.py | 2 +- maloja/data_files/settings/default.ini | 2 + maloja/database.py | 53 +++++++++++++++++--------- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 7662666..3182425 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,3,8 +version = 2,4,0 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index b94afa9..658a3e3 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -25,6 +25,8 @@ TRACK_SEARCH_PROVIDER = None [Database] +CACHE_DATABASE = true +CACHE_DATABASE_PERM = false #more permanent cache for old timeranges DB_CACHE_SIZE = 8192 # how many MB on disk each database cache should have available. INVALID_ARTISTS = ["[Unknown Artist]","Unknown Artist","Spotify"] REMOVE_FROM_TITLE = ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"] diff --git a/maloja/database.py b/maloja/database.py index 458b693..28a8e26 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1034,17 +1034,29 @@ def sync(): import copy -cache_query = {} -if doreah.version >= (0,7,1) and settings.get_settings("EXPERIMENTAL_FEATURES"): - cache_query_permanent = DiskDict(name="dbquery",folder=datadir("cache"),maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) +if settings.get_settings("CACHE_DATABASE"): + def db_query(**kwargs): + return db_query_cached(**kwargs) + def db_aggregate(**kwargs): + return db_aggregate_cached(**kwargs) else: - cache_query_permanent = Cache(maxmemory=1024*1024*500) + def db_query(**kwargs): + return db_query_full(**kwargs) + def db_aggregate(**kwargs): + return db_aggregate_full(**kwargs) + cacheday = (0,0,0) -def db_query(**kwargs): + + +cache_query = {} +cache_query_permanent = Cache(maxmemory=1024*1024*500) +def db_query_cached(**kwargs): check_cache_age() global cache_query, cache_query_permanent key = utilities.serialize(kwargs) - if "timerange" in kwargs and not kwargs["timerange"].active(): + + # hit permanent cache for past timeranges + if "timerange" in kwargs and not kwargs["timerange"].active() and settings.get_settings("CACHE_DATABASE_PERM"): if key in cache_query_permanent: #print("Hit") return copy.copy(cache_query_permanent.get(key)) @@ -1052,31 +1064,36 @@ def db_query(**kwargs): result = db_query_full(**kwargs) cache_query_permanent.add(key,copy.copy(result)) #print(cache_query_permanent.cache) + # hit short term cache else: #print("I guess they never miss huh") - if key in cache_query: return copy.copy(cache_query[key]) - result = db_query_full(**kwargs) - cache_query[key] = copy.copy(result) + if key in cache_query: + return copy.copy(cache_query[key]) + else: + result = db_query_full(**kwargs) + cache_query[key] = copy.copy(result) return result cache_aggregate = {} -if doreah.version >= (0,7,1) and settings.get_settings("EXPERIMENTAL_FEATURES"): - cache_aggregate_permanent = DiskDict(name="dbaggregate",folder="cache",maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) -else: - cache_aggregate_permanent = Cache(maxmemory=1024*1024*500) -def db_aggregate(**kwargs): +cache_aggregate_permanent = Cache(maxmemory=1024*1024*500) +def db_aggregate_cached(**kwargs): check_cache_age() global cache_aggregate, cache_aggregate_permanent key = utilities.serialize(kwargs) - if "timerange" in kwargs and not kwargs["timerange"].active(): + + # hit permanent cache for past timeranges + if "timerange" in kwargs and not kwargs["timerange"].active() and settings.get_settings("CACHE_DATABASE_PERM"): if key in cache_aggregate_permanent: return copy.copy(cache_aggregate_permanent.get(key)) result = db_aggregate_full(**kwargs) cache_aggregate_permanent.add(key,copy.copy(result)) + # hit short term cache else: - if key in cache_aggregate: return copy.copy(cache_aggregate[key]) - result = db_aggregate_full(**kwargs) - cache_aggregate[key] = copy.copy(result) + if key in cache_aggregate: + return copy.copy(cache_aggregate[key]) + else: + result = db_aggregate_full(**kwargs) + cache_aggregate[key] = copy.copy(result) return result From b117e6f7ec80afc6210314ce97bac087d5ab7b54 Mon Sep 17 00:00:00 2001 From: Krateng Date: Thu, 28 May 2020 19:37:19 +0200 Subject: [PATCH 08/40] Fixed missing file inclusion --- maloja/__pkginfo__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 3182425..b59447c 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,0 +version = 2,4,1 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", @@ -23,6 +23,7 @@ requires = [ "jinja2>2.11" ] resources = [ + "web/*/*/*", "web/*/*", "web/*", "static/*/*", From d989134e65c20ab33b0ea8e4a132655074057757 Mon Sep 17 00:00:00 2001 From: Krateng Date: Thu, 28 May 2020 19:45:30 +0200 Subject: [PATCH 09/40] Fixed some CSS --- maloja/__pkginfo__.py | 2 +- maloja/static/less/maloja.less | 4 ++-- maloja/web/jinja/artist.jinja | 6 +++--- maloja/web/jinja/track.jinja | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index b59447c..517e393 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,1 +version = 2,4,2 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/static/less/maloja.less b/maloja/static/less/maloja.less index 3c1b60e..9a24ef2 100644 --- a/maloja/static/less/maloja.less +++ b/maloja/static/less/maloja.less @@ -291,13 +291,13 @@ span.stat_selector_pulse,span.stat_selector_topartists,span.stat_selector_toptra -h2 { +h2.headerwithextra { display:inline-block; padding-right:5px; margin-bottom:10px; margin-top:15px; } -h2+span.afterheader { +h2.headerwithextra+span.afterheader { color:@TEXT_COLOR_TERTIARY; } diff --git a/maloja/web/jinja/artist.jinja b/maloja/web/jinja/artist.jinja index 258e62d..acc6cad 100644 --- a/maloja/web/jinja/artist.jinja +++ b/maloja/web/jinja/artist.jinja @@ -45,7 +45,7 @@ {% endif %} -

{{ artist }}

+

{{ artist }}

{% if competes %}#{{ info.position }}{% endif %}
{% if competes and included %} @@ -83,7 +83,7 @@ -

Pulse

+

Pulse


{% for range in xranges %} -

Performance

+

Performance

{% if not competes %}of {{ htmlgenerators.artistLink(credited) }} {% endif %}
diff --git a/maloja/web/jinja/track.jinja b/maloja/web/jinja/track.jinja index 461db2a..0dcefae 100644 --- a/maloja/web/jinja/track.jinja +++ b/maloja/web/jinja/track.jinja @@ -40,7 +40,7 @@ {{ htmlgenerators.artistLinks(track.artists) }}
-

{{ track.title }}

+

{{ track.title }}

{{ awards.certs(track) }} #{{ info.position }}
From 9b787fa3b13d77a9cfbe21061f519defac7fafd0 Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 29 May 2020 04:46:53 +0200 Subject: [PATCH 10/40] Replaced DB caches with LRU dicts, hope this improves memory problems --- maloja/__pkginfo__.py | 2 +- maloja/data_files/settings/default.ini | 7 +- maloja/database.py | 145 ++++++++++++++++--------- maloja/utilities.py | 15 ++- 4 files changed, 107 insertions(+), 62 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 517e393..ac62e7d 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,2 +version = 2,4,3 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index 658a3e3..6c03ff8 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -25,9 +25,10 @@ TRACK_SEARCH_PROVIDER = None [Database] -CACHE_DATABASE = true -CACHE_DATABASE_PERM = false #more permanent cache for old timeranges -DB_CACHE_SIZE = 8192 # how many MB on disk each database cache should have available. +USE_DB_CACHE = yes +CACHE_DATABASE_SHORT = true +CACHE_DATABASE_PERM = true #more permanent cache for old timeranges +DB_CACHE_ENTRIES = 10000 #experiment with this depending on your RAM INVALID_ARTISTS = ["[Unknown Artist]","Unknown Artist","Spotify"] REMOVE_FROM_TITLE = ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"] USE_PARSE_PLUGINS = no diff --git a/maloja/database.py b/maloja/database.py index 28a8e26..81fba66 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1,5 +1,6 @@ # server from bottle import request, response, FormsDict + # rest of the project from .cleanup import CleanerAgent, CollectorAgent from . import utilities @@ -9,6 +10,7 @@ from . import compliant_api from .external import proxy_scrobble from .__pkginfo__ import version from .globalconf import datadir + # doreah toolkit from doreah.logging import log from doreah import tsv @@ -18,9 +20,11 @@ try: from doreah.persistence import DiskDict except: pass import doreah + # nimrodel API from nimrodel import EAPI as API from nimrodel import Multi + # technical import os import datetime @@ -29,6 +33,8 @@ import unicodedata from collections import namedtuple from threading import Lock import yaml +import lru + # url handling from importlib.machinery import SourceFileLoader import urllib @@ -1032,9 +1038,11 @@ def sync(): ### + + import copy -if settings.get_settings("CACHE_DATABASE"): +if settings.get_settings("USE_DB_CACHE"): def db_query(**kwargs): return db_query_cached(**kwargs) def db_aggregate(**kwargs): @@ -1045,75 +1053,108 @@ else: def db_aggregate(**kwargs): return db_aggregate_full(**kwargs) -cacheday = (0,0,0) + +csz = settings.get_settings("DB_CACHE_ENTRIES") + +cache_query = lru.LRU(csz) +cache_query_perm = lru.LRU(csz) +cache_aggregate = lru.LRU(csz) +cache_aggregate_perm = lru.LRU(csz) + +cachestats = { + "cache_query_tmp":{ + "obj":cache_query, + "hits":0, + "misses":0 + }, + "cache_query_perm":{ + "obj":cache_query_perm, + "hits":0, + "misses":0 + }, + "cache_aggregate_tmp":{ + "obj":cache_aggregate, + "hits":0, + "misses":0 + }, + "cache_aggregate_perm":{ + "obj":cache_aggregate_perm, + "hits":0, + "misses":0 + }, +} + +from doreah.regular import runhourly + +@runhourly +def log_stats(): + log({c:{"size":len(cachestats[c]["obj"]),"hits":cachestats[c]["hits"],"misses":cachestats[c]["misses"]} for c in cachestats},module="debug") -cache_query = {} -cache_query_permanent = Cache(maxmemory=1024*1024*500) def db_query_cached(**kwargs): - check_cache_age() - global cache_query, cache_query_permanent + global cache_query, cache_query_perm key = utilities.serialize(kwargs) + eligible_permanent_caching = ( + "timerange" in kwargs and + not kwargs["timerange"].active() and + settings.get_settings("CACHE_DATABASE_PERM") + ) + eligible_temporary_caching = ( + not eligible_permanent_caching and + settings.get_settings("CACHE_DATABASE_SHORT") + ) + # hit permanent cache for past timeranges - if "timerange" in kwargs and not kwargs["timerange"].active() and settings.get_settings("CACHE_DATABASE_PERM"): - if key in cache_query_permanent: - #print("Hit") - return copy.copy(cache_query_permanent.get(key)) - #print("Miss") + if eligible_permanent_caching and key in cache_query_perm: + return copy.copy(cache_query_perm.get(key)) + + # hit short term cache + elif eligible_temporary_caching and key in cache_query: + return copy.copy(cache_query.get(key)) + + else: result = db_query_full(**kwargs) - cache_query_permanent.add(key,copy.copy(result)) - #print(cache_query_permanent.cache) - # hit short term cache - else: - #print("I guess they never miss huh") - if key in cache_query: - return copy.copy(cache_query[key]) - else: - result = db_query_full(**kwargs) - cache_query[key] = copy.copy(result) + if eligible_permanent_caching: cache_query_perm[key] = result + elif eligible_temporary_caching: cache_query[key] = result + return result - return result -cache_aggregate = {} -cache_aggregate_permanent = Cache(maxmemory=1024*1024*500) def db_aggregate_cached(**kwargs): - check_cache_age() - global cache_aggregate, cache_aggregate_permanent + global cache_aggregate, cache_aggregate_perm key = utilities.serialize(kwargs) - # hit permanent cache for past timeranges - if "timerange" in kwargs and not kwargs["timerange"].active() and settings.get_settings("CACHE_DATABASE_PERM"): - if key in cache_aggregate_permanent: return copy.copy(cache_aggregate_permanent.get(key)) - result = db_aggregate_full(**kwargs) - cache_aggregate_permanent.add(key,copy.copy(result)) - # hit short term cache - else: - if key in cache_aggregate: - return copy.copy(cache_aggregate[key]) - else: - result = db_aggregate_full(**kwargs) - cache_aggregate[key] = copy.copy(result) + eligible_permanent_caching = ( + "timerange" in kwargs and + not kwargs["timerange"].active() and + settings.get_settings("CACHE_DATABASE_PERM") + ) + eligible_temporary_caching = ( + not eligible_permanent_caching and + settings.get_settings("CACHE_DATABASE_SHORT") + ) - return result + # hit permanent cache for past timeranges + if eligible_permanent_caching and key in cache_aggregate_perm: + return copy.copy(cache_aggregate_perm.get(key)) + + # hit short term cache + elif eligible_temporary_caching and key in cache_aggregate: + return copy.copy(cache_aggregate.get(key)) + + else: + result = db_aggregate_full(**kwargs) + if eligible_permanent_caching: cache_aggregate_perm[key] = result + elif eligible_temporary_caching: cache_aggregate[key] = result + + return result def invalidate_caches(): global cache_query, cache_aggregate - cache_query = {} - cache_aggregate = {} - - now = datetime.datetime.utcnow() - global cacheday - cacheday = (now.year,now.month,now.day) - + cache_query.clear() + cache_aggregate.clear() log("Database caches invalidated.") -def check_cache_age(): - now = datetime.datetime.utcnow() - global cacheday - if cacheday != (now.year,now.month,now.day): invalidate_caches() - - #### ## Database queries #### diff --git a/maloja/utilities.py b/maloja/utilities.py index dbd36f7..48f7a82 100644 --- a/maloja/utilities.py +++ b/maloja/utilities.py @@ -27,13 +27,16 @@ from .globalconf import datadir def serialize(obj): try: - return json.dumps(obj) + return serialize(obj.hashable()) except: - if isinstance(obj,list) or isinstance(obj,tuple): - return "[" + ",".join(serialize(o) for o in obj) + "]" - elif isinstance(obj,dict): - return "{" + ",".join(serialize(o) + ":" + serialize(obj[o]) for o in obj) + "}" - return json.dumps(obj.hashable()) + try: + return json.dumps(obj) + except: + if isinstance(obj,list) or isinstance(obj,tuple): + return "[" + ",".join(serialize(o) for o in obj) + "]" + elif isinstance(obj,dict): + return "{" + ",".join(serialize(o) + ":" + serialize(obj[o]) for o in obj) + "}" + return json.dumps(obj.hashable()) #if isinstance(obj,list) or if isinstance(obj,tuple): From 65f9c88da4d56df37e4a3f974d7f660502c7a310 Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 29 May 2020 04:51:34 +0200 Subject: [PATCH 11/40] Added missing dependency --- maloja/__pkginfo__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index ac62e7d..07b3858 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,3 +version = 2,4,6 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", @@ -20,7 +20,8 @@ requires = [ "setproctitle>=1.1.10", "wand>=0.5.4", "lesscpy>=0.13", - "jinja2>2.11" + "jinja2>2.11", + "lru-dict>=1.1.6" ] resources = [ "web/*/*/*", From c166620d5f9706e54f9cd67044d42bf8583575d8 Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 29 May 2020 16:45:41 +0200 Subject: [PATCH 12/40] Properly implemented cache debug logging --- maloja/__pkginfo__.py | 2 +- maloja/database.py | 46 +++++++++++++++++++++++-------------------- maloja/server.py | 6 +++--- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 07b3858..1c30f30 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,6 +version = 2,4,7 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/database.py b/maloja/database.py index 81fba66..dffc987 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1062,34 +1062,32 @@ cache_aggregate = lru.LRU(csz) cache_aggregate_perm = lru.LRU(csz) cachestats = { - "cache_query_tmp":{ - "obj":cache_query, - "hits":0, - "misses":0 - }, - "cache_query_perm":{ - "obj":cache_query_perm, - "hits":0, - "misses":0 - }, - "cache_aggregate_tmp":{ - "obj":cache_aggregate, - "hits":0, - "misses":0 - }, - "cache_aggregate_perm":{ - "obj":cache_aggregate_perm, - "hits":0, - "misses":0 + "cache_query":{ + "hits_perm":0, + "hits_tmp":0, + "misses":0, + "objperm":cache_query_perm, + "objtmp":cache_query, + "name":"Query Cache" }, + "cache_aggregate":{ + "hits_perm":0, + "hits_tmp":0, + "misses":0, + "objperm":cache_aggregate_perm, + "objtmp":cache_aggregate, + "name":"Aggregate Cache" + } } from doreah.regular import runhourly @runhourly def log_stats(): - log({c:{"size":len(cachestats[c]["obj"]),"hits":cachestats[c]["hits"],"misses":cachestats[c]["misses"]} for c in cachestats},module="debug") - + logstr = "{name}: {hitsperm} Perm Hits, {hitstmp} Tmp Hits, {misses} Misses; Current Size: {sizeperm}/{sizetmp}" + for s in (cachestats["cache_query"],cachestats["cache_aggregate"]): + log(logstr.format(name=s["name"],hitsperm=s["hits_perm"],hitstmp=s["hits_tmp"],misses=s["misses"], + sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"]))) def db_query_cached(**kwargs): global cache_query, cache_query_perm @@ -1107,13 +1105,16 @@ def db_query_cached(**kwargs): # hit permanent cache for past timeranges if eligible_permanent_caching and key in cache_query_perm: + cachestats["cache_query"]["hits_perm"] += 1 return copy.copy(cache_query_perm.get(key)) # hit short term cache elif eligible_temporary_caching and key in cache_query: + cachestats["cache_query"]["hits_tmp"] += 1 return copy.copy(cache_query.get(key)) else: + cachestats["cache_query"]["misses"] += 1 result = db_query_full(**kwargs) if eligible_permanent_caching: cache_query_perm[key] = result elif eligible_temporary_caching: cache_query[key] = result @@ -1136,13 +1137,16 @@ def db_aggregate_cached(**kwargs): # hit permanent cache for past timeranges if eligible_permanent_caching and key in cache_aggregate_perm: + cachestats["cache_aggregate"]["hits_perm"] += 1 return copy.copy(cache_aggregate_perm.get(key)) # hit short term cache elif eligible_temporary_caching and key in cache_aggregate: + cachestats["cache_aggregate"]["hits_tmp"] += 1 return copy.copy(cache_aggregate.get(key)) else: + cachestats["cache_aggregate"]["misses"] += 1 result = db_aggregate_full(**kwargs) if eligible_permanent_caching: cache_aggregate_perm[key] = result elif eligible_temporary_caching: cache_aggregate[key] = result diff --git a/maloja/server.py b/maloja/server.py index 176a47c..51eb690 100755 --- a/maloja/server.py +++ b/maloja/server.py @@ -245,7 +245,7 @@ def static_html(name): template = jinjaenv.get_template(name + '.jinja') res = template.render(**LOCAL_CONTEXT) - log("Generated page {name} in {time}s (Jinja)".format(name=name,time=clock.stop()),module="debug") + log("Generated page {name} in {time:.5f}s (Jinja)".format(name=name,time=clock.stop()),module="debug") return res # if a pyhp file exists, use this @@ -272,7 +272,7 @@ def static_html(name): #response.set_header("Content-Type","application/xhtml+xml") res = pyhpfile(pthjoin(WEBFOLDER,"pyhp",name + ".pyhp"),environ) - log("Generated page {name} in {time}s (PYHP)".format(name=name,time=clock.stop()),module="debug") + log("Generated page {name} in {time:.5f}s (PYHP)".format(name=name,time=clock.stop()),module="debug") return res # if not, use the old way @@ -316,7 +316,7 @@ def static_html(name): response.set_header("Link",",".join(linkheaders)) - log("Generated page " + name + " in " + str(clock.stop()) + "s (Python+HTML)",module="debug") + log("Generated page {name} in {time:.5f}s (Python+HTML)".format(name=name,time=clock.stop()),module="debug") return html #return static_file("web/" + name + ".html",root="") From 98c1527f778958b1a3322a4f026cfe2c421388aa Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 29 May 2020 17:33:42 +0200 Subject: [PATCH 13/40] Added cache dump for high memory usage --- maloja/__pkginfo__.py | 5 +++-- maloja/data_files/settings/default.ini | 1 + maloja/database.py | 15 ++++++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 1c30f30..8e51d9e 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,7 +version = 2,4,8 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", @@ -21,7 +21,8 @@ requires = [ "wand>=0.5.4", "lesscpy>=0.13", "jinja2>2.11", - "lru-dict>=1.1.6" + "lru-dict>=1.1.6", + "psutil>0.5.0" ] resources = [ "web/*/*/*", diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index 6c03ff8..589933c 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -29,6 +29,7 @@ USE_DB_CACHE = yes CACHE_DATABASE_SHORT = true CACHE_DATABASE_PERM = true #more permanent cache for old timeranges DB_CACHE_ENTRIES = 10000 #experiment with this depending on your RAM +DB_MAX_MEMORY = 75 # percentage of RAM utilization (whole container, not just maloja) that should trigger a flush INVALID_ARTISTS = ["[Unknown Artist]","Unknown Artist","Spotify"] REMOVE_FROM_TITLE = ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"] USE_PARSE_PLUGINS = no diff --git a/maloja/database.py b/maloja/database.py index dffc987..8e01297 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -34,6 +34,7 @@ from collections import namedtuple from threading import Lock import yaml import lru +import psutil # url handling from importlib.machinery import SourceFileLoader @@ -1055,6 +1056,7 @@ else: csz = settings.get_settings("DB_CACHE_ENTRIES") +cmp = settings.get_settings("DB_MAX_MEMORY") cache_query = lru.LRU(csz) cache_query_perm = lru.LRU(csz) @@ -1087,7 +1089,7 @@ def log_stats(): logstr = "{name}: {hitsperm} Perm Hits, {hitstmp} Tmp Hits, {misses} Misses; Current Size: {sizeperm}/{sizetmp}" for s in (cachestats["cache_query"],cachestats["cache_aggregate"]): log(logstr.format(name=s["name"],hitsperm=s["hits_perm"],hitstmp=s["hits_tmp"],misses=s["misses"], - sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"]))) + sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"])),module="debug") def db_query_cached(**kwargs): global cache_query, cache_query_perm @@ -1118,6 +1120,12 @@ def db_query_cached(**kwargs): result = db_query_full(**kwargs) if eligible_permanent_caching: cache_query_perm[key] = result elif eligible_temporary_caching: cache_query[key] = result + + ramprct = psutil.virtual_memory().percent + if ramprct > cmp: + log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug") + invalidate_caches() + return result @@ -1151,6 +1159,11 @@ def db_aggregate_cached(**kwargs): if eligible_permanent_caching: cache_aggregate_perm[key] = result elif eligible_temporary_caching: cache_aggregate[key] = result + ramprct = psutil.virtual_memory().percent + if ramprct > cmp: + log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug") + invalidate_caches() + return result def invalidate_caches(): From b21b27bb6e230901281bb524f84e177c937b48fd Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 29 May 2020 17:39:19 +0200 Subject: [PATCH 14/40] Made psutil optional --- maloja/__pkginfo__.py | 5 ++--- maloja/database.py | 24 +++++++++++++++--------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 8e51d9e..561a9ae 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,8 +version = 2,4,9 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", @@ -21,8 +21,7 @@ requires = [ "wand>=0.5.4", "lesscpy>=0.13", "jinja2>2.11", - "lru-dict>=1.1.6", - "psutil>0.5.0" + "lru-dict>=1.1.6" ] resources = [ "web/*/*/*", diff --git a/maloja/database.py b/maloja/database.py index 8e01297..d56fa0d 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -34,7 +34,6 @@ from collections import namedtuple from threading import Lock import yaml import lru -import psutil # url handling from importlib.machinery import SourceFileLoader @@ -1057,6 +1056,11 @@ else: csz = settings.get_settings("DB_CACHE_ENTRIES") cmp = settings.get_settings("DB_MAX_MEMORY") +try: + import psutil + use_psutil = True +except: + use_psutil = False cache_query = lru.LRU(csz) cache_query_perm = lru.LRU(csz) @@ -1121,10 +1125,11 @@ def db_query_cached(**kwargs): if eligible_permanent_caching: cache_query_perm[key] = result elif eligible_temporary_caching: cache_query[key] = result - ramprct = psutil.virtual_memory().percent - if ramprct > cmp: - log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug") - invalidate_caches() + if use_psutil: + ramprct = psutil.virtual_memory().percent + if ramprct > cmp: + log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug") + invalidate_caches() return result @@ -1159,10 +1164,11 @@ def db_aggregate_cached(**kwargs): if eligible_permanent_caching: cache_aggregate_perm[key] = result elif eligible_temporary_caching: cache_aggregate[key] = result - ramprct = psutil.virtual_memory().percent - if ramprct > cmp: - log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug") - invalidate_caches() + if use_psutil: + ramprct = psutil.virtual_memory().percent + if ramprct > cmp: + log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug") + invalidate_caches() return result From 2df77f89e9d7688e5295969c6a5e881c6e9e9ade Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 29 May 2020 17:43:34 +0200 Subject: [PATCH 15/40] Added psutil to dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2a49020..46f6e41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,8 @@ FROM python:3.6-alpine WORKDIR /usr/src/app RUN apk update -RUN apk add gcc libxml2-dev libxslt-dev py3-pip libc-dev +RUN apk add gcc libxml2-dev libxslt-dev py3-pip libc-dev linux-headers +RUN pip3 install psutil RUN pip3 install malojaserver From 2f67f427f21adcfb8ec5222d3156c3c28f0f425e Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 29 May 2020 22:03:59 +0200 Subject: [PATCH 16/40] Various --- README.md | 6 ++++++ maloja/data_files/settings/default.ini | 2 +- maloja/database.py | 25 +++++++++++++++++-------- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 38528d4..dcba5bb 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,10 @@ You can check [my own Maloja page](https://maloja.krateng.ch) to see what it loo ## Table of Contents * [Why not Last.fm / Libre.fm / GNU FM?](#why-not-lastfm--librefm--gnu-fm) * [How to install](#how-to-install) + * [Environment](#environment) * [New Installation](#new-installation) * [Update](#update) + * [Docker](#docker) * [How to use](#how-to-use) * [Basic control](#basic-control) * [Data](#data) @@ -40,6 +42,10 @@ Also neat: You can use your **custom artist or track images**. ## How to install +### Environment + +I can support you with issues best if you use **Alpine Linux**. In my experience, **2-4 GB RAM** should do nicely. + ### New Installation 1) Make sure you have Python 3.5 or higher installed. You also need some basic packages that should be present on most systems, but I've provided simple shell scripts for Alpine and Ubuntu to get everything you need. diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index 589933c..38f0bed 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -59,7 +59,7 @@ SCROBBLES_GOLD = 250 SCROBBLES_PLATINUM = 500 SCROBBLES_DIAMOND = 1000 # name for comparisons -NAME = None +NAME = "A Maloja User" [Misc] diff --git a/maloja/database.py b/maloja/database.py index d56fa0d..aefe140 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1126,10 +1126,7 @@ def db_query_cached(**kwargs): elif eligible_temporary_caching: cache_query[key] = result if use_psutil: - ramprct = psutil.virtual_memory().percent - if ramprct > cmp: - log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug") - invalidate_caches() + reduce_caches_if_low_ram() return result @@ -1165,10 +1162,7 @@ def db_aggregate_cached(**kwargs): elif eligible_temporary_caching: cache_aggregate[key] = result if use_psutil: - ramprct = psutil.virtual_memory().percent - if ramprct > cmp: - log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug") - invalidate_caches() + reduce_caches_if_low_ram() return result @@ -1178,6 +1172,21 @@ def invalidate_caches(): cache_aggregate.clear() log("Database caches invalidated.") +def reduce_caches(to=0.75): + global cache_query, cache_aggregate + for c in cache_query, cache_aggregate: + currentsize = len(c) + targetsize = int(currentsize * to) + c.set_size(targetsize) + c.set_size(csz) + +def reduce_caches_if_low_ram(): + ramprct = psutil.virtual_memory().percent + if ramprct > cmp: + log("{prct}% RAM usage, reducing temporary caches!".format(prct=ramprct),module="debug") + ratio = (cmp / ramprct) ** 3 + reduce_caches(to=ratio) + #### ## Database queries #### From 31585ec64620bd78fcde87ce79544eb3ba985648 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 30 May 2020 18:10:17 +0200 Subject: [PATCH 17/40] Updated Spotify scrobbler --- scrobblers/chromium/sites/spotify.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrobblers/chromium/sites/spotify.js b/scrobblers/chromium/sites/spotify.js index 8858687..a895d6e 100644 --- a/scrobblers/chromium/sites/spotify.js +++ b/scrobblers/chromium/sites/spotify.js @@ -6,7 +6,7 @@ maloja_scrobbler_selector_metadata = ".//div[@class='now-playing-bar__left']" maloja_scrobbler_selector_title = ".//a[@data-testid='nowplaying-track-link']/text()" maloja_scrobbler_selector_artists = ".//a[contains(@href,'/artist/')]" maloja_scrobbler_selector_artist = "./text()" -maloja_scrobbler_selector_duration = ".//div[@class='playback-bar__progress-time'][2]/text()" +maloja_scrobbler_selector_duration = ".//div[@class='playback-bar']/div[3]/text()" maloja_scrobbler_selector_control = ".//div[contains(@class,'player-controls__buttons')]/div[3]/button/@title" From 813dee8400386885feb9c25246d874a9fe81c364 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sun, 31 May 2020 18:11:28 +0200 Subject: [PATCH 18/40] More incomplete jinja templates --- maloja/web/jinja/charts_artists.jinja | 34 +++++++++++++++ maloja/web/jinja/charts_tracks.jinja | 34 +++++++++++++++ .../web/jinja/partials/charts_artists.jinja | 41 +++++++++++++++++++ maloja/web/jinja/partials/charts_tracks.jinja | 15 +++---- 4 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 maloja/web/jinja/charts_artists.jinja create mode 100644 maloja/web/jinja/charts_tracks.jinja create mode 100644 maloja/web/jinja/partials/charts_artists.jinja diff --git a/maloja/web/jinja/charts_artists.jinja b/maloja/web/jinja/charts_artists.jinja new file mode 100644 index 0000000..4866901 --- /dev/null +++ b/maloja/web/jinja/charts_artists.jinja @@ -0,0 +1,34 @@ +{% extends "base.jinja" %} +{% block title %}Maloja - {{ artist }}{% endblock %} + +{% block scripts %} + +{% endblock %} + +{% set charts = dbp.get_charts_artists(filterkeys,limitkeys) %} + + +{% block content %} + + + + + + +
+
+ 		+

Artist Charts

View #1 Artists
+ {{ limitkeys.timerange.desc(prefix=True) }} +

+ {{ htmlmodules.module_filterselection(_urikeys) }} + +
+ + +{% import 'partials/charts_artists.jinja' as charts_artists %} + +{{ charts_artists.charts_artists(limitkeys,amountkeys,compare=False) }} + + +{% endblock %} diff --git a/maloja/web/jinja/charts_tracks.jinja b/maloja/web/jinja/charts_tracks.jinja new file mode 100644 index 0000000..b25b11e --- /dev/null +++ b/maloja/web/jinja/charts_tracks.jinja @@ -0,0 +1,34 @@ +{% extends "base.jinja" %} +{% block title %}Maloja - Track Charts{% endblock %} + +{% block scripts %} + +{% endblock %} + +{% set charts = dbp.get_charts_tracks(filterkeys,limitkeys) %} + + +{% block content %} + + + + + + +
+
+ 		+

Track Charts

View #1 Tracks
+ {{ limitkeys.timerange.desc(prefix=True) }} +

+ {{ htmlmodules.module_filterselection(_urikeys) }} + +
+ + +{% import 'partials/charts_tracks.jinja' as charts_tracks %} + +{{ charts_tracks.charts_tracks(filterkeys,limitkeys,amountkeys,charts=charts,compare=false) }} + + +{% endblock %} diff --git a/maloja/web/jinja/partials/charts_artists.jinja b/maloja/web/jinja/partials/charts_artists.jinja new file mode 100644 index 0000000..3049384 --- /dev/null +++ b/maloja/web/jinja/partials/charts_artists.jinja @@ -0,0 +1,41 @@ +{% macro charts_artists(limitkeys,amountkeys,charts=None,compare=False) %} + +{% if charts is none %} + {% set charts = dbp.get_charts_artists(limitkeys) %} +{% endif %} + +{% if compare %} +{% endif %} + +{% set firstindex = amountkeys.page * amountkeys.perpage %} +{% set lastindex = firstindex + amountkeys.perpage %} + + +{% set maxbar = charts[0]['scrobbles'] if charts != [] else 0 %} + + + {% for e in charts %} + {% if loop.index0 >= firstindex and loop.index0 < lastindex %} + + + + + {% if false %} + {% if e not in prevcharts %}{% endif %} + + {% endif %} + + + {{ htmlgenerators.entity_column(e['artist']) }} + + + + + + {% endif %} + {% endfor %} +
{%if loop.changed(e.scrobbles) %}#{{ e.rank }}{% endif %}πŸ†•{{ htmlgenerators.scrobblesArtistLink(e['artist'],urihandler.internal_to_uri(limitkeys),amount=e['scrobbles']) }}{{ htmlgenerators.scrobblesArtistLink(e['artist'],urihandler.internal_to_uri(limitkeys),percent=e['scrobbles']*100/maxbar) }}
+ + + +{%- endmacro %} diff --git a/maloja/web/jinja/partials/charts_tracks.jinja b/maloja/web/jinja/partials/charts_tracks.jinja index f090ff3..9bfee8e 100644 --- a/maloja/web/jinja/partials/charts_tracks.jinja +++ b/maloja/web/jinja/partials/charts_tracks.jinja @@ -1,7 +1,8 @@ -{% macro charts_tracks(filterkeys,limitkeys,amountkeys,compare=False) %} +{% macro charts_tracks(filterkeys,limitkeys,amountkeys,charts=None,compare=False) %} - -{% set tracks = dbp.get_charts_tracks(filterkeys,limitkeys) %} +{% if charts is none %} + {% set charts = dbp.get_charts_tracks(filterkeys,limitkeys) %} +{% endif %} {% if compare %} {% if compare is true %} {% set compare = limitkeys.timerange.next(step=-1) %} @@ -9,11 +10,11 @@ {% set prevtracks = dbp.get_charts_tracks(filterkeys,{'timerange':compare}) %} {% set lastrank = {} %} - {% for t in tracks %} + {% for t in charts %} {% if lastrank.update({(t.track.artists,t.track.title):t.rank}) %}{% endif %} {% endfor %} - {% for t in tracks %} + {% for t in charts %} {% if (t.track.artists,t.track.title) in lastrank %} {% if t.update({'lastrank':lastrank[(t.track.artists,t.track.title)]}) %}{% endif %} {% endif %} @@ -24,10 +25,10 @@ {% set lastindex = firstindex + amountkeys.perpage %} -{% set maxbar = tracks[0]['scrobbles'] if tracks != [] else 0 %} +{% set maxbar = charts[0]['scrobbles'] if charts != [] else 0 %} - {% for e in tracks %} + {% for e in charts %} {% if loop.index0 >= firstindex and loop.index0 < lastindex %} From ef2a2c817e3fba22128bb171abd92a5fc76030dc Mon Sep 17 00:00:00 2001 From: krateng Date: Mon, 1 Jun 2020 18:22:16 +0200 Subject: [PATCH 19/40] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dcba5bb..ba6578c 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ I can support you with issues best if you use **Alpine Linux**. In my experience 5) (Recommended) Until I have a proper service implemented, I would recommend setting two cronjobs for maloja: ``` -@reboot maloja start +@reboot sleep 15 && maloja start 42 0 * * * maloja restart ``` From 08fe4695f6d5ef09789688481db478d0decbd5df Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 5 Jun 2020 13:20:54 +0200 Subject: [PATCH 20/40] High RAM usage affects all caches --- maloja/__pkginfo__.py | 2 +- maloja/database.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 561a9ae..8fb703b 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,9 +version = 2,4,10 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/database.py b/maloja/database.py index aefe140..4f62ab5 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1173,8 +1173,8 @@ def invalidate_caches(): log("Database caches invalidated.") def reduce_caches(to=0.75): - global cache_query, cache_aggregate - for c in cache_query, cache_aggregate: + global cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm + for c in cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm: currentsize = len(c) targetsize = int(currentsize * to) c.set_size(targetsize) @@ -1183,7 +1183,7 @@ def reduce_caches(to=0.75): def reduce_caches_if_low_ram(): ramprct = psutil.virtual_memory().percent if ramprct > cmp: - log("{prct}% RAM usage, reducing temporary caches!".format(prct=ramprct),module="debug") + log("{prct}% RAM usage, reducing caches!".format(prct=ramprct),module="debug") ratio = (cmp / ramprct) ** 3 reduce_caches(to=ratio) From e531dc4007657ecdf2b3f11a88054a5b18fa10a0 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 6 Jun 2020 11:53:42 +0200 Subject: [PATCH 21/40] Remove pip3 and dependencies after installing --- Dockerfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 46f6e41..44c6960 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,18 @@ -FROM python:3.6-alpine +FROM python:3-alpine WORKDIR /usr/src/app -RUN apk update -RUN apk add gcc libxml2-dev libxslt-dev py3-pip libc-dev linux-headers -RUN pip3 install psutil - -RUN pip3 install malojaserver +RUN apk add --no-cache --virtual .build-deps \ + gcc \ + libxml2-dev \ + libxslt-dev \ + py3-pip \ + libc-dev \ + linux-headers \ + && \ + pip3 install psutil && \ + pip3 install malojaserver && \ + apk del .build-deps EXPOSE 42010 From e73e047af9160a089216b71d37cb40cd64eb03a4 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 6 Jun 2020 16:38:45 +0200 Subject: [PATCH 22/40] Reduced disk access for cache settings --- maloja/database.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/maloja/database.py b/maloja/database.py index 4f62ab5..d275db6 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1067,6 +1067,9 @@ cache_query_perm = lru.LRU(csz) cache_aggregate = lru.LRU(csz) cache_aggregate_perm = lru.LRU(csz) +perm_caching = settings.get_settings("CACHE_DATABASE_PERM") +temp_caching = settings.get_settings("CACHE_DATABASE_SHORT") + cachestats = { "cache_query":{ "hits_perm":0, @@ -1102,11 +1105,11 @@ def db_query_cached(**kwargs): eligible_permanent_caching = ( "timerange" in kwargs and not kwargs["timerange"].active() and - settings.get_settings("CACHE_DATABASE_PERM") + perm_caching ) eligible_temporary_caching = ( not eligible_permanent_caching and - settings.get_settings("CACHE_DATABASE_SHORT") + temp_caching ) # hit permanent cache for past timeranges @@ -1138,11 +1141,11 @@ def db_aggregate_cached(**kwargs): eligible_permanent_caching = ( "timerange" in kwargs and not kwargs["timerange"].active() and - settings.get_settings("CACHE_DATABASE_PERM") + perm_caching ) eligible_temporary_caching = ( not eligible_permanent_caching and - settings.get_settings("CACHE_DATABASE_SHORT") + temp_caching ) # hit permanent cache for past timeranges From 1828bd35bb8c942169cfd51d8f12b4d156eae7ec Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 6 Jun 2020 16:46:25 +0200 Subject: [PATCH 23/40] Can now use custom data directory with environment variable, close GH-18 --- maloja/__pkginfo__.py | 2 +- maloja/globalconf.py | 30 +++++++++++++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 8fb703b..807f5a6 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,10 +version = 2,4,11 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/globalconf.py b/maloja/globalconf.py index 4118189..3717d35 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -1,23 +1,34 @@ import os +from doreah.settings import get_settings +from doreah.settings import config as settingsconfig -# data folder -# must be determined first because getting settings relies on it -try: - DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] - assert os.path.exists(DATA_DIR) -except: - DATA_DIR = os.path.join(os.environ["HOME"],".local/share/") +# check environment variables for data directory +# otherwise, go with defaults +setting_datadir = get_settings("DATA_DIRECTORY",files=[],environ_prefix="MALOJA_") +if setting_datadir is not None and os.path.exists(setting_datadir): + DATA_DIR = setting_datadir +else: + try: + HOME_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] + assert os.path.exists(HOME_DIR) + except: + HOME_DIR = os.path.join(os.environ["HOME"],".local/share/") + + DATA_DIR = os.path.join(HOME_DIR,"maloja") -DATA_DIR = os.path.join(DATA_DIR,"maloja") os.makedirs(DATA_DIR,exist_ok=True) + + def datadir(*args): return os.path.join(DATA_DIR,*args) + + ### DOREAH CONFIGURATION from doreah import config @@ -44,9 +55,10 @@ config( } ) +# because we loaded a doreah module already before setting the config, we need to to that manually +settingsconfig._readpreconfig() -from doreah.settings import get_settings # thumbor From 5c6a901f5118be54ae44affbd6881b14bc30e04a Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 6 Jun 2020 16:47:59 +0200 Subject: [PATCH 24/40] Lowered recommended restart frequency --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba6578c..185e2df 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ I can support you with issues best if you use **Alpine Linux**. In my experience ``` @reboot sleep 15 && maloja start -42 0 * * * maloja restart +42 0 * * 2 maloja restart ``` From d551513733e8f1d18c8825305be302412ac9b924 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sun, 7 Jun 2020 15:01:05 +0200 Subject: [PATCH 25/40] Update install scripts --- install_alpine.sh | 3 ++- install_ubuntu.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/install_alpine.sh b/install_alpine.sh index 75c9ac3..43705d7 100644 --- a/install_alpine.sh +++ b/install_alpine.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash -apk add python3 python3-dev gcc libxml2-dev libxslt-dev py3-pip libc-dev +apk add python3 python3-dev gcc libxml2-dev libxslt-dev py3-pip libc-dev linux-headers +pip3 install psutil pip3 install malojaserver diff --git a/install_ubuntu.sh b/install_ubuntu.sh index ff0bd66..d30bd7b 100644 --- a/install_ubuntu.sh +++ b/install_ubuntu.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash apt update apt install python3 python3-pip +pip3 install psutil pip3 install malojaserver From 6658165baedeee3939084ba4500de3de06bbc045 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 13 Jun 2020 17:34:30 +0200 Subject: [PATCH 26/40] Added setting for file logging (GH-19) --- maloja/__pkginfo__.py | 4 ++-- maloja/data_files/settings/default.ini | 1 + maloja/globalconf.py | 11 ++++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 807f5a6..299c8fc 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,11 +version = 2,4,12 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", @@ -15,7 +15,7 @@ links = { requires = [ "bottle>=0.12.16", "waitress>=1.3", - "doreah>=1.5.6", + "doreah>=1.6.3", "nimrodel>=0.6.3", "setproctitle>=1.1.10", "wand>=0.5.4", diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index 38f0bed..37d2ab4 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -70,3 +70,4 @@ FEDERATION = yes #does nothing yet UPDATE_AFTER_CRASH = no #update when server is automatically restarted DAILY_RESTART = 2 # hour of day. no / none means no daily restarts SKIP_SETUP = no +LOGGING = true diff --git a/maloja/globalconf.py b/maloja/globalconf.py index 3717d35..5ed516f 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -37,9 +37,6 @@ config( pyhp={ "version": 2 }, - logging={ - "logfolder": datadir("logs") - }, settings={ "files":[ datadir("settings/default.ini"), @@ -58,6 +55,14 @@ config( # because we loaded a doreah module already before setting the config, we need to to that manually settingsconfig._readpreconfig() +config( + logging={ + "logfolder": datadir("logs") if get_settings("LOGGING") else None + } +) + +settingsconfig._readpreconfig() + # thumbor From 4c6b40e42f695a5cf78548796a77a45ad77a48b0 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 13 Jun 2020 17:42:59 +0200 Subject: [PATCH 27/40] Added some sanity checks to cache reduction --- maloja/database.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/maloja/database.py b/maloja/database.py index d275db6..7fb9aa7 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1179,9 +1179,10 @@ def reduce_caches(to=0.75): global cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm for c in cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm: currentsize = len(c) - targetsize = int(currentsize * to) - c.set_size(targetsize) - c.set_size(csz) + if currentsize > 100: + targetsize = max(int(currentsize * to),10) + c.set_size(targetsize) + c.set_size(csz) def reduce_caches_if_low_ram(): ramprct = psutil.virtual_memory().percent From 57403a89ab1d679523341d6a607d0b03e495ff35 Mon Sep 17 00:00:00 2001 From: Krateng Date: Thu, 18 Jun 2020 15:16:24 +0200 Subject: [PATCH 28/40] Updated database rebuild, should fix GH-16 --- maloja/__pkginfo__.py | 2 +- maloja/database.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 299c8fc..2fd02e5 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,12 +version = 2,4,13 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/database.py b/maloja/database.py index 7fb9aa7..84aada3 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -933,6 +933,7 @@ def build_db(): log("Building database...") global SCROBBLES, ARTISTS, TRACKS + global TRACKS_NORMALIZED_SET, TRACKS_NORMALIZED, ARTISTS_NORMALIZED_SET, ARTISTS_NORMALIZED global SCROBBLESDICT, STAMPS SCROBBLES = [] @@ -941,6 +942,11 @@ def build_db(): STAMPS = [] SCROBBLESDICT = {} + TRACKS_NORMALIZED = [] + ARTISTS_NORMALIZED = [] + ARTISTS_NORMALIZED_SET = set() + TRACKS_NORMALIZED_SET = set() + # parse files db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False) From 18352436780592060915ef99bc3bd17cbd0787d6 Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 5 Jun 2020 13:16:02 +0200 Subject: [PATCH 29/40] Reorganized process control --- maloja/controller.py | 202 ------------------------- maloja/data_files/settings/default.ini | 2 +- maloja/proccontrol/control.py | 94 ++++++++++++ maloja/proccontrol/setup.py | 71 +++++++++ maloja/proccontrol/supervisor.py | 37 +++++ maloja/proccontrol/tasks.py | 33 ++++ maloja/supervisor.py | 67 -------- 7 files changed, 236 insertions(+), 270 deletions(-) delete mode 100755 maloja/controller.py create mode 100644 maloja/proccontrol/control.py create mode 100644 maloja/proccontrol/setup.py create mode 100644 maloja/proccontrol/supervisor.py create mode 100644 maloja/proccontrol/tasks.py delete mode 100644 maloja/supervisor.py diff --git a/maloja/controller.py b/maloja/controller.py deleted file mode 100755 index fa61563..0000000 --- a/maloja/controller.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -import sys -import signal -import os -import shutil -from distutils import dir_util -import stat -import pathlib -import pkg_resources -from doreah.control import mainfunction -from doreah.io import col, ask, prompt - -from .globalconf import datadir -from .backup import backup - - - - -def copy_initial_local_files(): - folder = pkg_resources.resource_filename(__name__,"data_files") - #shutil.copy(folder,DATA_DIR) - dir_util.copy_tree(folder,datadir(),update=False) - - -def setup(): - - copy_initial_local_files() - - from doreah import settings - - # EXTERNAL API KEYS - apikeys = { - "LASTFM_API_KEY":"Last.fm API Key", - "FANARTTV_API_KEY":"Fanart.tv API Key", - "SPOTIFY_API_ID":"Spotify Client ID", - "SPOTIFY_API_SECRET":"Spotify Client Secret" - } - - SKIP = settings.get_settings("SKIP_SETUP") - - print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.") - for k in apikeys: - key = settings.get_settings(k) - if key is None: - print("\t" + "Currently not using a " + apikeys[k] + " for image display.") - elif key == "ASK": - print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.") - key = prompt("",types=(str,),default=None,skip=SKIP) - settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True) - else: - print("\t" + apikeys[k] + " found.") - - - # OWN API KEY - if os.path.exists(datadir("clients/authenticated_machines.tsv")): - pass - else: - answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP) - if answer: - import random - key = "" - for i in range(64): - key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) - print("Your API Key: " + col["yellow"](key)) - with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile: - keyfile.write(key + "\t" + "Default Generated Key") - else: - pass - - - if settings.get_settings("NAME") is None: - name = prompt("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.",default="Generic Maloja User",skip=SKIP) - settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True) - - if settings.get_settings("SEND_STATS") is None: - answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP) - if answer: - settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True) - else: - settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True) - - -def getInstance(): - try: - output = subprocess.check_output(["pidof","Maloja"]) - pid = int(output) - return pid - except: - return None - -def getInstanceSupervisor(): - try: - output = subprocess.check_output(["pidof","maloja_supervisor"]) - pid = int(output) - return pid - except: - return None - -def start(): - setup() - try: - #p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - sp = subprocess.Popen(["python3","-m","maloja.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - print(col["green"]("Maloja started!")) - - from doreah import settings - port = settings.get_settings("WEB_PORT") - - print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.") - print("If you're installing this on your local machine, these links should get you there:") - print("\t" + col["blue"]("http://localhost:" + str(port))) - print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup")) - return True - except: - print("Error while starting Maloja.") - return False - -def restart(): - wasrunning = stop() - start() - return wasrunning - -def stop(): - pid_sv = getInstanceSupervisor() - if pid_sv is not None: - os.kill(pid_sv,signal.SIGTERM) -# return True - -# else: -# print("Server is not running") -# return False - - - pid = getInstance() - if pid is not None: -# print("Server is not running") -# return False -# pass -# else: - os.kill(pid,signal.SIGTERM) -# print("Maloja stopped! PID: " + str(pid)) - if pid is not None or pid_sv is not None: - return True - else: - return False - - -def loadlastfm(filename): - - if not os.path.exists(filename): - print("File could not be found.") - return - - if os.path.exists(datadir("scrobbles/lastfmimport.tsv")): - print("Already imported Last.FM data. Overwrite? [y/N]") - if input().lower() in ["y","yes","yea","1","positive","true"]: - pass - else: - return - print("Please wait...") - from .lastfmconverter import convert - convert(filename,datadir("scrobbles/lastfmimport.tsv")) - #os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) - print("Successfully imported your Last.FM scrobbles!") - -def direct(): - setup() - from . import server - -def backuphere(): - backup(folder=os.getcwd()) - -def update(): - os.system("pip3 install malojaserver --upgrade --no-cache-dir") - restart() - -def fixdb(): - from .fixexisting import fix - fix() - -@mainfunction({"l":"level"},shield=True) -def main(action,*args,**kwargs): - actions = { - "start":restart, - "restart":restart, - "stop":stop, - "import":loadlastfm, - "debug":direct, - "backup":backuphere, - "update":update, - "fix":fixdb, - "run":direct - } - if action in actions: actions[action](*args,**kwargs) - else: print("Valid commands: " + " ".join(a for a in actions)) - - return True - -#if __name__ == "__main__": -# main() diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index 38f0bed..589933c 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -59,7 +59,7 @@ SCROBBLES_GOLD = 250 SCROBBLES_PLATINUM = 500 SCROBBLES_DIAMOND = 1000 # name for comparisons -NAME = "A Maloja User" +NAME = None [Misc] diff --git a/maloja/proccontrol/control.py b/maloja/proccontrol/control.py new file mode 100644 index 0000000..30c3a3c --- /dev/null +++ b/maloja/proccontrol/control.py @@ -0,0 +1,94 @@ +import subprocess +from doreah import settings +from doreah.control import mainfunction +from doreah.io import col +import os +import signal + +from .setup import setup + +def getInstance(): + try: + output = subprocess.check_output(["pidof","Maloja"]) + pid = int(output) + return pid + except: + return None + +def getInstanceSupervisor(): + try: + output = subprocess.check_output(["pidof","maloja_supervisor"]) + pid = int(output) + return pid + except: + return None + +def restart(): + stop() + start() + +def start(): + if getInstanceSupervisor() is not None: + print("Maloja is already running.") + else: + setup() + try: + #p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + sp = subprocess.Popen(["python3","-m","maloja.proccontrol.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + print(col["green"]("Maloja started!")) + + port = settings.get_settings("WEB_PORT") + + print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.") + print("If you're installing this on your local machine, these links should get you there:") + print("\t" + col["blue"]("http://localhost:" + str(port))) + print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup")) + return True + except: + print("Error while starting Maloja.") + return False + + +def stop(): + + pid_sv = getInstanceSupervisor() + if pid_sv is not None: + os.kill(pid_sv,signal.SIGTERM) + + pid = getInstance() + if pid is not None: + os.kill(pid,signal.SIGTERM) + + if pid is not None or pid_sv is not None: + print("Maloja stopped!") + return True + else: + return False + + + +def direct(): + setup() + from .. import server + + + +@mainfunction({"l":"level"},shield=True) +def main(action,*args,**kwargs): + actions = { + "start":start, + "restart":restart, + "stop":stop, + "run":direct, + "debug":direct, + + # "import":loadlastfm, + + # "backup":backuphere, + # "update":update, + # "fix":fixdb + } + if action in actions: actions[action](*args,**kwargs) + else: print("Valid commands: " + " ".join(a for a in actions)) + + return True diff --git a/maloja/proccontrol/setup.py b/maloja/proccontrol/setup.py new file mode 100644 index 0000000..ad6752a --- /dev/null +++ b/maloja/proccontrol/setup.py @@ -0,0 +1,71 @@ +import pkg_resources +from distutils import dir_util +from doreah import settings +from doreah.io import col, ask, prompt +import os + +from ..globalconf import datadir + + +# EXTERNAL API KEYS +apikeys = { + "LASTFM_API_KEY":"Last.fm API Key", + "FANARTTV_API_KEY":"Fanart.tv API Key", + "SPOTIFY_API_ID":"Spotify Client ID", + "SPOTIFY_API_SECRET":"Spotify Client Secret" +} + + + +def copy_initial_local_files(): + folder = pkg_resources.resource_filename("maloja","data_files") + #shutil.copy(folder,DATA_DIR) + dir_util.copy_tree(folder,datadir(),update=False) + + + +def setup(): + + copy_initial_local_files() + SKIP = settings.get_settings("SKIP_SETUP") + + print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.") + for k in apikeys: + key = settings.get_settings(k) + if key is None: + print("\t" + "Currently not using a " + apikeys[k] + " for image display.") + elif key == "ASK": + print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.") + key = prompt("",types=(str,),default=None,skip=SKIP) + settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True) + else: + print("\t" + apikeys[k] + " found.") + + + # OWN API KEY + if os.path.exists(datadir("clients/authenticated_machines.tsv")): + pass + else: + answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP) + if answer: + import random + key = "" + for i in range(64): + key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) + print("Your API Key: " + col["yellow"](key)) + with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile: + keyfile.write(key + "\t" + "Default Generated Key") + else: + pass + + + if settings.get_settings("NAME") is None: + name = prompt("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.",default="Generic Maloja User",skip=SKIP) + settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True) + + if settings.get_settings("SEND_STATS") is None: + answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP) + if answer: + settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True) + else: + settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True) diff --git a/maloja/proccontrol/supervisor.py b/maloja/proccontrol/supervisor.py new file mode 100644 index 0000000..9fa05b0 --- /dev/null +++ b/maloja/proccontrol/supervisor.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +import os + +import subprocess +import setproctitle +import signal +from doreah.logging import log +from doreah.settings import get_settings + +from .control import getInstance + + +setproctitle.setproctitle("maloja_supervisor") + +def update(): + log("Updating...",module="supervisor") + try: + os.system("pip3 install maloja --upgrade --no-cache-dir") + except: + log("Could not update.",module="supervisor") + +def start(): + try: + p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + return p + except e: + log("Error starting Maloja: " + str(e),module="supervisor") + + + +while True: + log("Maloja is not running, starting...",module="supervisor") + if get_settings("UPDATE_AFTER_CRASH"): + update() + process = start() + + process.wait() diff --git a/maloja/proccontrol/tasks.py b/maloja/proccontrol/tasks.py new file mode 100644 index 0000000..f67798c --- /dev/null +++ b/maloja/proccontrol/tasks.py @@ -0,0 +1,33 @@ +import os +from ..lastfmconverter import convert +from ..backup import backup +from ..fixexisting import fix +from ..globalconf import datadir +from .control import restart +from doreah.io import ask + +def loadlastfm(filename): + + if not os.path.exists(filename): + print("File could not be found.") + return + + if os.path.exists(datadir("scrobbles/lastfmimport.tsv")): + overwrite = ask("Already imported Last.FM data. Overwrite?",default=False) + if not overwrite: return + print("Please wait...") + + convert(filename,datadir("scrobbles/lastfmimport.tsv")) + #os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) + print("Successfully imported your Last.FM scrobbles!") + + +def backuphere(): + backup(folder=os.getcwd()) + +def update(): + os.system("pip3 install malojaserver --upgrade --no-cache-dir") + restart() + +def fixdb(): + fix() diff --git a/maloja/supervisor.py b/maloja/supervisor.py deleted file mode 100644 index 2846252..0000000 --- a/maloja/supervisor.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -import os - -import subprocess -import time -import setproctitle -import signal -from datetime import datetime -from doreah.logging import log -from doreah.settings import get_settings - - -setproctitle.setproctitle("maloja_supervisor") - -lastrestart = () - -def get_pid(): - try: - output = subprocess.check_output(["pidof","Maloja"]) - return int(output) - except: - return None - -def update(): - log("Updating...",module="supervisor") - try: - os.system("pip3 install maloja --upgrade --no-cache-dir") - except: - log("Could not update.",module="supervisor") - -def start(): - try: - p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - - except e: - log("Error starting Maloja: " + str(e),module="supervisor") - - - -while True: - - - now = datetime.now() - today = now.year, now.month, now.day - - pid = get_pid() - - if pid: - - restart = get_settings("DAILY_RESTART") - if restart not in [None,False]: - if today != lastrestart: - if now.hour == restart: - log("Daily restart...",module="supervisor") - os.kill(pid,signal.SIGTERM) - start() - lastrestart = today - - else: - log("Maloja is not running, starting...",module="supervisor") - if get_settings("UPDATE_AFTER_CRASH"): - update() - start() - lastrestart = today - - - time.sleep(60) From 6c2eac550b751621713e53f4dff596bae040d096 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 20 Jun 2020 17:33:23 +0200 Subject: [PATCH 30/40] Updated console script --- maloja/__pkginfo__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 561a9ae..0d9379c 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -33,5 +33,5 @@ resources = [ ] commands = { - "maloja":"controller:main" + "maloja":"proccontrol.control:main" } From d48ffc964dde6f94687bb8259234d4f0d1d75af7 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 20 Jun 2020 17:34:40 +0200 Subject: [PATCH 31/40] Removed unused settings --- maloja/data_files/settings/default.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index 589933c..427fe05 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -67,6 +67,4 @@ EXPERIMENTAL_FEATURES = no USE_PYHP = no #not recommended at the moment USE_JINJA = no #overwrites pyhp preference FEDERATION = yes #does nothing yet -UPDATE_AFTER_CRASH = no #update when server is automatically restarted -DAILY_RESTART = 2 # hour of day. no / none means no daily restarts SKIP_SETUP = no From be79dc188822a23ca9b50a8fe644094d59a397cf Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 20 Jun 2020 17:52:26 +0200 Subject: [PATCH 32/40] Reorganized tasks --- maloja/proccontrol/control.py | 8 ++++---- maloja/proccontrol/{tasks.py => tasks/__init__.py} | 11 ++++++----- maloja/{ => proccontrol/tasks}/backup.py | 2 +- maloja/{ => proccontrol/tasks}/fixexisting.py | 4 ++-- maloja/{ => proccontrol/tasks}/lastfmconverter.py | 4 ++-- 5 files changed, 15 insertions(+), 14 deletions(-) rename maloja/proccontrol/{tasks.py => tasks/__init__.py} (81%) rename maloja/{ => proccontrol/tasks}/backup.py (95%) rename maloja/{ => proccontrol/tasks}/fixexisting.py (96%) rename maloja/{ => proccontrol/tasks}/lastfmconverter.py (96%) diff --git a/maloja/proccontrol/control.py b/maloja/proccontrol/control.py index 30c3a3c..d0cf53e 100644 --- a/maloja/proccontrol/control.py +++ b/maloja/proccontrol/control.py @@ -6,6 +6,7 @@ import os import signal from .setup import setup +from . import tasks def getInstance(): try: @@ -82,11 +83,10 @@ def main(action,*args,**kwargs): "run":direct, "debug":direct, - # "import":loadlastfm, - - # "backup":backuphere, + "import":tasks.loadlastfm, + "backup":tasks.backuphere, # "update":update, - # "fix":fixdb + "fix":tasks.fixdb } if action in actions: actions[action](*args,**kwargs) else: print("Valid commands: " + " ".join(a for a in actions)) diff --git a/maloja/proccontrol/tasks.py b/maloja/proccontrol/tasks/__init__.py similarity index 81% rename from maloja/proccontrol/tasks.py rename to maloja/proccontrol/tasks/__init__.py index f67798c..daaf86f 100644 --- a/maloja/proccontrol/tasks.py +++ b/maloja/proccontrol/tasks/__init__.py @@ -1,9 +1,10 @@ import os -from ..lastfmconverter import convert -from ..backup import backup -from ..fixexisting import fix -from ..globalconf import datadir -from .control import restart +from .lastfmconverter import convert +from .backup import backup +from .fixexisting import fix + +from ...globalconf import datadir +from ..control import restart from doreah.io import ask def loadlastfm(filename): diff --git a/maloja/backup.py b/maloja/proccontrol/tasks/backup.py similarity index 95% rename from maloja/backup.py rename to maloja/proccontrol/tasks/backup.py index 3a7e9da..9be8281 100644 --- a/maloja/backup.py +++ b/maloja/proccontrol/tasks/backup.py @@ -2,7 +2,7 @@ import tarfile from datetime import datetime import glob import os -from .globalconf import datadir +from ...globalconf import datadir user_files = { diff --git a/maloja/fixexisting.py b/maloja/proccontrol/tasks/fixexisting.py similarity index 96% rename from maloja/fixexisting.py rename to maloja/proccontrol/tasks/fixexisting.py index b0c39a3..35ca8af 100644 --- a/maloja/fixexisting.py +++ b/maloja/proccontrol/tasks/fixexisting.py @@ -1,7 +1,7 @@ import os -from .globalconf import datadir +from ...globalconf import datadir import re -from .cleanup import CleanerAgent +from ...cleanup import CleanerAgent from doreah.logging import log import difflib import datetime diff --git a/maloja/lastfmconverter.py b/maloja/proccontrol/tasks/lastfmconverter.py similarity index 96% rename from maloja/lastfmconverter.py rename to maloja/proccontrol/tasks/lastfmconverter.py index 6a70d80..bb5a6dc 100644 --- a/maloja/lastfmconverter.py +++ b/maloja/proccontrol/tasks/lastfmconverter.py @@ -1,6 +1,6 @@ import os, datetime, re -from .cleanup import * -from .utilities import * +from ...cleanup import * +from ...utilities import * From 398b737781c6e693342e92bf81919ad796581c12 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 20 Jun 2020 18:04:20 +0200 Subject: [PATCH 33/40] Fixed relative file hierarchy for backups --- maloja/proccontrol/tasks/backup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/maloja/proccontrol/tasks/backup.py b/maloja/proccontrol/tasks/backup.py index 9be8281..fe5c443 100644 --- a/maloja/proccontrol/tasks/backup.py +++ b/maloja/proccontrol/tasks/backup.py @@ -3,6 +3,7 @@ from datetime import datetime import glob import os from ...globalconf import datadir +from pathlib import PurePath user_files = { @@ -32,4 +33,6 @@ def backup(folder,level="full"): assert not os.path.exists(archivefile) with tarfile.open(name=archivefile,mode="x:gz") as archive: for f in real_files: - archive.add(f) + p = PurePath(f) + r = p.relative_to(datadir()) + archive.add(f,arcname=r) From daa256fc3b66d75e2b8b331aa5e4aed75d6a9296 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 20 Jun 2020 18:06:08 +0200 Subject: [PATCH 34/40] Fixed db fix from main process --- maloja/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/database.py b/maloja/database.py index aefe140..9730588 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -854,7 +854,7 @@ def rebuild(**keys): global db_rulestate db_rulestate = False sync() - from .fixexisting import fix + from .proccontrol.tasks.fixexisting import fix fix() global cla, coa cla = CleanerAgent() From 4d10276cc11b462023511f8901f4515aabd2d3b9 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 20 Jun 2020 18:15:02 +0200 Subject: [PATCH 35/40] Added some log output --- maloja/proccontrol/tasks/backup.py | 5 +++++ maloja/proccontrol/tasks/fixexisting.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/maloja/proccontrol/tasks/backup.py b/maloja/proccontrol/tasks/backup.py index fe5c443..d3aefce 100644 --- a/maloja/proccontrol/tasks/backup.py +++ b/maloja/proccontrol/tasks/backup.py @@ -5,6 +5,8 @@ import os from ...globalconf import datadir from pathlib import PurePath +from doreah.logging import log + user_files = { "minimal":[ @@ -26,6 +28,8 @@ def backup(folder,level="full"): for g in selected_files: real_files += glob.glob(datadir(g)) + log("Creating backup of " + str(len(real_files)) + " files...") + now = datetime.utcnow() timestr = now.strftime("%Y_%m_%d_%H_%M_%S") filename = "maloja_backup_" + timestr + ".tar.gz" @@ -36,3 +40,4 @@ def backup(folder,level="full"): p = PurePath(f) r = p.relative_to(datadir()) archive.add(f,arcname=r) + log("Backup created!") diff --git a/maloja/proccontrol/tasks/fixexisting.py b/maloja/proccontrol/tasks/fixexisting.py index 35ca8af..76036c4 100644 --- a/maloja/proccontrol/tasks/fixexisting.py +++ b/maloja/proccontrol/tasks/fixexisting.py @@ -35,9 +35,10 @@ def fix(): #with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile: - + log("Fixing database...") for filename in os.listdir(datadir("scrobbles")): if filename.endswith(".tsv"): + log("Fix file " + filename) filename_new = filename + "_new" with open(datadir("scrobbles",filename_new),"w") as newfile: @@ -68,3 +69,5 @@ def fix(): with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile: checkfile.write(wendigo.checksums) + + log("Database fixed!") From 00be885847ed36c3d36b042613008e269032a1ea Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 20 Jun 2020 18:24:30 +0200 Subject: [PATCH 36/40] Updated requirements.txt --- requirements.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8159188..eb5da13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,10 @@ bottle>=0.12.16 waitress>=1.3 -doreah>=1.2.7 -nimrodel>=0.4.9 +doreah>=1.6.3 +nimrodel>=0.6.3 setproctitle>=1.1.10 wand>=0.5.4 lesscpy>=0.13 pip>=19.3 +jinja2>2.11 +lru-dict>=1.1.6 From 990131f546876d1461bac745e5cab3e60c78d038 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 20 Jun 2020 20:27:59 +0200 Subject: [PATCH 37/40] Version bump --- maloja/__pkginfo__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index f1bf847..556be8f 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,13 +version = 2,5,0 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", From bda279d01dab730a21aceaf88b9d1af1080cc528 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 20 Jun 2020 20:30:41 +0200 Subject: [PATCH 38/40] Fixed some issues --- maloja/__pkginfo__.py | 6 ++++-- maloja/proccontrol/__init__.py | 0 maloja/proccontrol/tasks/__init__.py | 12 +++++++----- 3 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 maloja/proccontrol/__init__.py diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 556be8f..2b2677d 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,5,0 +version = 2,5,3 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", @@ -29,7 +29,9 @@ resources = [ "web/*", "static/*/*", "data_files/*/*", - "data_files/*/*/*" + "data_files/*/*/*", + "proccontrol/*", + "proccontrol/*/*" ] commands = { diff --git a/maloja/proccontrol/__init__.py b/maloja/proccontrol/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/maloja/proccontrol/tasks/__init__.py b/maloja/proccontrol/tasks/__init__.py index daaf86f..cc37761 100644 --- a/maloja/proccontrol/tasks/__init__.py +++ b/maloja/proccontrol/tasks/__init__.py @@ -1,11 +1,9 @@ import os -from .lastfmconverter import convert -from .backup import backup -from .fixexisting import fix +from doreah.io import ask from ...globalconf import datadir -from ..control import restart -from doreah.io import ask + + def loadlastfm(filename): @@ -18,17 +16,21 @@ def loadlastfm(filename): if not overwrite: return print("Please wait...") + from .lastfmconverter import convert convert(filename,datadir("scrobbles/lastfmimport.tsv")) #os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) print("Successfully imported your Last.FM scrobbles!") def backuphere(): + from .backup import backup backup(folder=os.getcwd()) def update(): os.system("pip3 install malojaserver --upgrade --no-cache-dir") + from ..control import restart restart() def fixdb(): + from .fixexisting import fix fix() From 8659f98935e0a9a5a64c938f7ebcedbef115e0e8 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 20 Jun 2020 21:00:49 +0200 Subject: [PATCH 39/40] Fixed external link --- maloja/web/pyhp/admin.pyhp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/web/pyhp/admin.pyhp b/maloja/web/pyhp/admin.pyhp index 098b2a0..61ff121 100644 --- a/maloja/web/pyhp/admin.pyhp +++ b/maloja/web/pyhp/admin.pyhp @@ -112,7 +112,7 @@

External

- Report Issue
+ Report Issue
From d911a7a8c4c3c59cd2c8f89adecb6c4ef22d6e90 Mon Sep 17 00:00:00 2001 From: Krateng Date: Thu, 23 Jul 2020 16:27:13 +0200 Subject: [PATCH 40/40] Added rule and bumped doreah version --- maloja/__pkginfo__.py | 2 +- maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv | 3 +++ requirements.txt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 2b2677d..6433cf5 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -15,7 +15,7 @@ links = { requires = [ "bottle>=0.12.16", "waitress>=1.3", - "doreah>=1.6.3", + "doreah>=1.6.7", "nimrodel>=0.6.3", "setproctitle>=1.1.10", "wand>=0.5.4", diff --git a/maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv b/maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv index 863238a..f597e8d 100644 --- a/maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv +++ b/maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv @@ -159,6 +159,9 @@ replacetitle 벌써 12μ‹œ Gotta Go Gotta Go # ITZY replacetitle 달라달라 (DALLA DALLA) Dalla Dalla +# K/DA +belongtogether K/DA + # Popular Remixes artistintitle Areia Remix Areia diff --git a/requirements.txt b/requirements.txt index eb5da13..688b1ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ bottle>=0.12.16 waitress>=1.3 -doreah>=1.6.3 +doreah>=1.6.7 nimrodel>=0.6.3 setproctitle>=1.1.10 wand>=0.5.4