diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..44c6960 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3-alpine + +WORKDIR /usr/src/app + +RUN apk add --no-cache --virtual .build-deps \ + gcc \ + libxml2-dev \ + libxslt-dev \ + py3-pip \ + libc-dev \ + linux-headers \ + && \ + pip3 install psutil && \ + pip3 install malojaserver && \ + apk del .build-deps + +EXPOSE 42010 + +ENTRYPOINT maloja run diff --git a/README.md b/README.md index 7c4a30c..185e2df 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,10 @@ You can check [my own Maloja page](https://maloja.krateng.ch) to see what it loo ## Table of Contents * [Why not Last.fm / Libre.fm / GNU FM?](#why-not-lastfm--librefm--gnu-fm) * [How to install](#how-to-install) + * [Environment](#environment) * [New Installation](#new-installation) * [Update](#update) + * [Docker](#docker) * [How to use](#how-to-use) * [Basic control](#basic-control) * [Data](#data) @@ -40,6 +42,10 @@ Also neat: You can use your **custom artist or track images**. ## How to install +### Environment + +I can support you with issues best if you use **Alpine Linux**. In my experience, **2-4 GB RAM** should do nicely. + ### New Installation 1) Make sure you have Python 3.5 or higher installed. You also need some basic packages that should be present on most systems, but I've provided simple shell scripts for Alpine and Ubuntu to get everything you need. @@ -53,8 +59,8 @@ Also neat: You can use your **custom artist or track images**. 5) (Recommended) Until I have a proper service implemented, I would recommend setting two cronjobs for maloja: ``` -@reboot maloja start -42 0 * * * maloja restart +@reboot sleep 15 && maloja start +42 0 * * 2 maloja restart ``` @@ -64,6 +70,11 @@ Also neat: You can use your **custom artist or track images**. * Otherwise, simply run the command `maloja update` or use `pip`s update mechanic. +### Docker + +There is a Dockerfile in the repo that should work by itself. You can also use the unofficial [Dockerhub repository](https://hub.docker.com/r/foxxmd/maloja) kindly provided by FoxxMD. + + ## How to use ### Basic control diff --git a/install_alpine.sh b/install_alpine.sh index 75c9ac3..43705d7 100644 --- a/install_alpine.sh +++ b/install_alpine.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash -apk add python3 python3-dev gcc libxml2-dev libxslt-dev py3-pip libc-dev +apk add python3 python3-dev gcc libxml2-dev libxslt-dev py3-pip libc-dev linux-headers +pip3 install psutil pip3 install malojaserver diff --git a/install_ubuntu.sh b/install_ubuntu.sh index ff0bd66..d30bd7b 100644 --- a/install_ubuntu.sh +++ b/install_ubuntu.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash apt update apt install python3 python3-pip +pip3 install psutil pip3 install malojaserver diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 7662666..6433cf5 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,3,8 +version = 2,5,3 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", @@ -15,21 +15,25 @@ links = { requires = [ "bottle>=0.12.16", "waitress>=1.3", - "doreah>=1.5.6", + "doreah>=1.6.7", "nimrodel>=0.6.3", "setproctitle>=1.1.10", "wand>=0.5.4", "lesscpy>=0.13", - "jinja2>2.11" + "jinja2>2.11", + "lru-dict>=1.1.6" ] resources = [ + "web/*/*/*", "web/*/*", "web/*", "static/*/*", "data_files/*/*", - "data_files/*/*/*" + "data_files/*/*/*", + "proccontrol/*", + "proccontrol/*/*" ] commands = { - "maloja":"controller:main" + "maloja":"proccontrol.control:main" } diff --git a/maloja/controller.py b/maloja/controller.py deleted file mode 100755 index fa61563..0000000 --- a/maloja/controller.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -import sys -import signal -import os -import shutil -from distutils import dir_util -import stat -import pathlib -import pkg_resources -from doreah.control import mainfunction -from doreah.io import col, ask, prompt - -from .globalconf import datadir -from .backup import backup - - - - -def copy_initial_local_files(): - folder = pkg_resources.resource_filename(__name__,"data_files") - #shutil.copy(folder,DATA_DIR) - dir_util.copy_tree(folder,datadir(),update=False) - - -def setup(): - - copy_initial_local_files() - - from doreah import settings - - # EXTERNAL API KEYS - apikeys = { - "LASTFM_API_KEY":"Last.fm API Key", - "FANARTTV_API_KEY":"Fanart.tv API Key", - "SPOTIFY_API_ID":"Spotify Client ID", - "SPOTIFY_API_SECRET":"Spotify Client Secret" - } - - SKIP = settings.get_settings("SKIP_SETUP") - - print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.") - for k in apikeys: - key = settings.get_settings(k) - if key is None: - print("\t" + "Currently not using a " + apikeys[k] + " for image display.") - elif key == "ASK": - print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.") - key = prompt("",types=(str,),default=None,skip=SKIP) - settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True) - else: - print("\t" + apikeys[k] + " found.") - - - # OWN API KEY - if os.path.exists(datadir("clients/authenticated_machines.tsv")): - pass - else: - answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP) - if answer: - import random - key = "" - for i in range(64): - key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) - print("Your API Key: " + col["yellow"](key)) - with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile: - keyfile.write(key + "\t" + "Default Generated Key") - else: - pass - - - if settings.get_settings("NAME") is None: - name = prompt("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.",default="Generic Maloja User",skip=SKIP) - settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True) - - if settings.get_settings("SEND_STATS") is None: - answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP) - if answer: - settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True) - else: - settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True) - - -def getInstance(): - try: - output = subprocess.check_output(["pidof","Maloja"]) - pid = int(output) - return pid - except: - return None - -def getInstanceSupervisor(): - try: - output = subprocess.check_output(["pidof","maloja_supervisor"]) - pid = int(output) - return pid - except: - return None - -def start(): - setup() - try: - #p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - sp = subprocess.Popen(["python3","-m","maloja.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - print(col["green"]("Maloja started!")) - - from doreah import settings - port = settings.get_settings("WEB_PORT") - - print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.") - print("If you're installing this on your local machine, these links should get you there:") - print("\t" + col["blue"]("http://localhost:" + str(port))) - print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup")) - return True - except: - print("Error while starting Maloja.") - return False - -def restart(): - wasrunning = stop() - start() - return wasrunning - -def stop(): - pid_sv = getInstanceSupervisor() - if pid_sv is not None: - os.kill(pid_sv,signal.SIGTERM) -# return True - -# else: -# print("Server is not running") -# return False - - - pid = getInstance() - if pid is not None: -# print("Server is not running") -# return False -# pass -# else: - os.kill(pid,signal.SIGTERM) -# print("Maloja stopped! PID: " + str(pid)) - if pid is not None or pid_sv is not None: - return True - else: - return False - - -def loadlastfm(filename): - - if not os.path.exists(filename): - print("File could not be found.") - return - - if os.path.exists(datadir("scrobbles/lastfmimport.tsv")): - print("Already imported Last.FM data. Overwrite? [y/N]") - if input().lower() in ["y","yes","yea","1","positive","true"]: - pass - else: - return - print("Please wait...") - from .lastfmconverter import convert - convert(filename,datadir("scrobbles/lastfmimport.tsv")) - #os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) - print("Successfully imported your Last.FM scrobbles!") - -def direct(): - setup() - from . import server - -def backuphere(): - backup(folder=os.getcwd()) - -def update(): - os.system("pip3 install malojaserver --upgrade --no-cache-dir") - restart() - -def fixdb(): - from .fixexisting import fix - fix() - -@mainfunction({"l":"level"},shield=True) -def main(action,*args,**kwargs): - actions = { - "start":restart, - "restart":restart, - "stop":stop, - "import":loadlastfm, - "debug":direct, - "backup":backuphere, - "update":update, - "fix":fixdb, - "run":direct - } - if action in actions: actions[action](*args,**kwargs) - else: print("Valid commands: " + " ".join(a for a in actions)) - - return True - -#if __name__ == "__main__": -# main() diff --git a/maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv b/maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv index 863238a..f597e8d 100644 --- a/maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv +++ b/maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv @@ -159,6 +159,9 @@ replacetitle 벌써 12시 Gotta Go Gotta Go # ITZY replacetitle 달라달라 (DALLA DALLA) Dalla Dalla +# K/DA +belongtogether K/DA + # Popular Remixes artistintitle Areia Remix Areia diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index b94afa9..e6415d7 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -25,7 +25,11 @@ TRACK_SEARCH_PROVIDER = None [Database] -DB_CACHE_SIZE = 8192 # how many MB on disk each database cache should have available. +USE_DB_CACHE = yes +CACHE_DATABASE_SHORT = true +CACHE_DATABASE_PERM = true #more permanent cache for old timeranges +DB_CACHE_ENTRIES = 10000 #experiment with this depending on your RAM +DB_MAX_MEMORY = 75 # percentage of RAM utilization (whole container, not just maloja) that should trigger a flush INVALID_ARTISTS = ["[Unknown Artist]","Unknown Artist","Spotify"] REMOVE_FROM_TITLE = ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"] USE_PARSE_PLUGINS = no @@ -63,6 +67,5 @@ EXPERIMENTAL_FEATURES = no USE_PYHP = no #not recommended at the moment USE_JINJA = no #overwrites pyhp preference FEDERATION = yes #does nothing yet -UPDATE_AFTER_CRASH = no #update when server is automatically restarted -DAILY_RESTART = 2 # hour of day. no / none means no daily restarts SKIP_SETUP = no +LOGGING = true diff --git a/maloja/database.py b/maloja/database.py index 7c0f0b2..c491c9c 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1,5 +1,6 @@ # server from bottle import request, response, FormsDict + # rest of the project from .cleanup import CleanerAgent, CollectorAgent from . import utilities @@ -12,6 +13,7 @@ from .thirdparty import proxy_scrobble_all from .__pkginfo__ import version from .globalconf import datadir + # doreah toolkit from doreah.logging import log from doreah import tsv @@ -21,9 +23,11 @@ try: from doreah.persistence import DiskDict except: pass import doreah + # nimrodel API from nimrodel import EAPI as API from nimrodel import Multi + # technical import os import datetime @@ -32,6 +36,8 @@ import unicodedata from collections import namedtuple from threading import Lock import yaml +import lru + # url handling from importlib.machinery import SourceFileLoader import urllib @@ -722,7 +728,7 @@ def abouttoshutdown(): def newrule(**keys): apikey = keys.pop("key",None) if (checkAPIkey(apikey)): - tsv.add_entry("rules/webmade.tsv",[k for k in keys]) + tsv.add_entry(datadir("rules/webmade.tsv"),[k for k in keys]) #addEntry("rules/webmade.tsv",[k for k in keys]) global db_rulestate db_rulestate = False @@ -851,7 +857,7 @@ def rebuild(**keys): global db_rulestate db_rulestate = False sync() - from .fixexisting import fix + from .proccontrol.tasks.fixexisting import fix fix() global cla, coa cla = CleanerAgent() @@ -930,6 +936,7 @@ def build_db(): log("Building database...") global SCROBBLES, ARTISTS, TRACKS + global TRACKS_NORMALIZED_SET, TRACKS_NORMALIZED, ARTISTS_NORMALIZED_SET, ARTISTS_NORMALIZED global SCROBBLESDICT, STAMPS SCROBBLES = [] @@ -938,6 +945,11 @@ def build_db(): STAMPS = [] SCROBBLESDICT = {} + TRACKS_NORMALIZED = [] + ARTISTS_NORMALIZED = [] + ARTISTS_NORMALIZED_SET = set() + TRACKS_NORMALIZED_SET = set() + # parse files db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False) @@ -1035,70 +1047,158 @@ def sync(): ### + + import copy -cache_query = {} -if doreah.version >= (0,7,1) and settings.get_settings("EXPERIMENTAL_FEATURES"): - cache_query_permanent = DiskDict(name="dbquery",folder=datadir("cache"),maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) +if settings.get_settings("USE_DB_CACHE"): + def db_query(**kwargs): + return db_query_cached(**kwargs) + def db_aggregate(**kwargs): + return db_aggregate_cached(**kwargs) else: - cache_query_permanent = Cache(maxmemory=1024*1024*500) -cacheday = (0,0,0) -def db_query(**kwargs): - check_cache_age() - global cache_query, cache_query_permanent + def db_query(**kwargs): + return db_query_full(**kwargs) + def db_aggregate(**kwargs): + return db_aggregate_full(**kwargs) + + +csz = settings.get_settings("DB_CACHE_ENTRIES") +cmp = settings.get_settings("DB_MAX_MEMORY") +try: + import psutil + use_psutil = True +except: + use_psutil = False + +cache_query = lru.LRU(csz) +cache_query_perm = lru.LRU(csz) +cache_aggregate = lru.LRU(csz) +cache_aggregate_perm = lru.LRU(csz) + +perm_caching = settings.get_settings("CACHE_DATABASE_PERM") +temp_caching = settings.get_settings("CACHE_DATABASE_SHORT") + +cachestats = { + "cache_query":{ + "hits_perm":0, + "hits_tmp":0, + "misses":0, + "objperm":cache_query_perm, + "objtmp":cache_query, + "name":"Query Cache" + }, + "cache_aggregate":{ + "hits_perm":0, + "hits_tmp":0, + "misses":0, + "objperm":cache_aggregate_perm, + "objtmp":cache_aggregate, + "name":"Aggregate Cache" + } +} + +from doreah.regular import runhourly + +@runhourly +def log_stats(): + logstr = "{name}: {hitsperm} Perm Hits, {hitstmp} Tmp Hits, {misses} Misses; Current Size: {sizeperm}/{sizetmp}" + for s in (cachestats["cache_query"],cachestats["cache_aggregate"]): + log(logstr.format(name=s["name"],hitsperm=s["hits_perm"],hitstmp=s["hits_tmp"],misses=s["misses"], + sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"])),module="debug") + +def db_query_cached(**kwargs): + global cache_query, cache_query_perm key = utilities.serialize(kwargs) - if "timerange" in kwargs and not kwargs["timerange"].active(): - if key in cache_query_permanent: - #print("Hit") - return copy.copy(cache_query_permanent.get(key)) - #print("Miss") - result = db_query_full(**kwargs) - cache_query_permanent.add(key,copy.copy(result)) - #print(cache_query_permanent.cache) + + eligible_permanent_caching = ( + "timerange" in kwargs and + not kwargs["timerange"].active() and + perm_caching + ) + eligible_temporary_caching = ( + not eligible_permanent_caching and + temp_caching + ) + + # hit permanent cache for past timeranges + if eligible_permanent_caching and key in cache_query_perm: + cachestats["cache_query"]["hits_perm"] += 1 + return copy.copy(cache_query_perm.get(key)) + + # hit short term cache + elif eligible_temporary_caching and key in cache_query: + cachestats["cache_query"]["hits_tmp"] += 1 + return copy.copy(cache_query.get(key)) + else: - #print("I guess they never miss huh") - if key in cache_query: return copy.copy(cache_query[key]) + cachestats["cache_query"]["misses"] += 1 result = db_query_full(**kwargs) - cache_query[key] = copy.copy(result) + if eligible_permanent_caching: cache_query_perm[key] = result + elif eligible_temporary_caching: cache_query[key] = result - return result + if use_psutil: + reduce_caches_if_low_ram() -cache_aggregate = {} -if doreah.version >= (0,7,1) and settings.get_settings("EXPERIMENTAL_FEATURES"): - cache_aggregate_permanent = DiskDict(name="dbaggregate",folder="cache",maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) -else: - cache_aggregate_permanent = Cache(maxmemory=1024*1024*500) -def db_aggregate(**kwargs): - check_cache_age() - global cache_aggregate, cache_aggregate_permanent + return result + + +def db_aggregate_cached(**kwargs): + global cache_aggregate, cache_aggregate_perm key = utilities.serialize(kwargs) - if "timerange" in kwargs and not kwargs["timerange"].active(): - if key in cache_aggregate_permanent: return copy.copy(cache_aggregate_permanent.get(key)) - result = db_aggregate_full(**kwargs) - cache_aggregate_permanent.add(key,copy.copy(result)) - else: - if key in cache_aggregate: return copy.copy(cache_aggregate[key]) - result = db_aggregate_full(**kwargs) - cache_aggregate[key] = copy.copy(result) - return result + eligible_permanent_caching = ( + "timerange" in kwargs and + not kwargs["timerange"].active() and + perm_caching + ) + eligible_temporary_caching = ( + not eligible_permanent_caching and + temp_caching + ) + + # hit permanent cache for past timeranges + if eligible_permanent_caching and key in cache_aggregate_perm: + cachestats["cache_aggregate"]["hits_perm"] += 1 + return copy.copy(cache_aggregate_perm.get(key)) + + # hit short term cache + elif eligible_temporary_caching and key in cache_aggregate: + cachestats["cache_aggregate"]["hits_tmp"] += 1 + return copy.copy(cache_aggregate.get(key)) + + else: + cachestats["cache_aggregate"]["misses"] += 1 + result = db_aggregate_full(**kwargs) + if eligible_permanent_caching: cache_aggregate_perm[key] = result + elif eligible_temporary_caching: cache_aggregate[key] = result + + if use_psutil: + reduce_caches_if_low_ram() + + return result def invalidate_caches(): global cache_query, cache_aggregate - cache_query = {} - cache_aggregate = {} - - now = datetime.datetime.utcnow() - global cacheday - cacheday = (now.year,now.month,now.day) - + cache_query.clear() + cache_aggregate.clear() log("Database caches invalidated.") -def check_cache_age(): - now = datetime.datetime.utcnow() - global cacheday - if cacheday != (now.year,now.month,now.day): invalidate_caches() +def reduce_caches(to=0.75): + global cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm + for c in cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm: + currentsize = len(c) + if currentsize > 100: + targetsize = max(int(currentsize * to),10) + c.set_size(targetsize) + c.set_size(csz) +def reduce_caches_if_low_ram(): + ramprct = psutil.virtual_memory().percent + if ramprct > cmp: + log("{prct}% RAM usage, reducing caches!".format(prct=ramprct),module="debug") + ratio = (cmp / ramprct) ** 3 + reduce_caches(to=ratio) #### ## Database queries diff --git a/maloja/globalconf.py b/maloja/globalconf.py index 4118189..5ed516f 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -1,23 +1,34 @@ import os +from doreah.settings import get_settings +from doreah.settings import config as settingsconfig -# data folder -# must be determined first because getting settings relies on it -try: - DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] - assert os.path.exists(DATA_DIR) -except: - DATA_DIR = os.path.join(os.environ["HOME"],".local/share/") +# check environment variables for data directory +# otherwise, go with defaults +setting_datadir = get_settings("DATA_DIRECTORY",files=[],environ_prefix="MALOJA_") +if setting_datadir is not None and os.path.exists(setting_datadir): + DATA_DIR = setting_datadir +else: + try: + HOME_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] + assert os.path.exists(HOME_DIR) + except: + HOME_DIR = os.path.join(os.environ["HOME"],".local/share/") + + DATA_DIR = os.path.join(HOME_DIR,"maloja") -DATA_DIR = os.path.join(DATA_DIR,"maloja") os.makedirs(DATA_DIR,exist_ok=True) + + def datadir(*args): return os.path.join(DATA_DIR,*args) + + ### DOREAH CONFIGURATION from doreah import config @@ -26,9 +37,6 @@ config( pyhp={ "version": 2 }, - logging={ - "logfolder": datadir("logs") - }, settings={ "files":[ datadir("settings/default.ini"), @@ -44,9 +52,18 @@ config( } ) +# because we loaded a doreah module already before setting the config, we need to to that manually +settingsconfig._readpreconfig() + +config( + logging={ + "logfolder": datadir("logs") if get_settings("LOGGING") else None + } +) + +settingsconfig._readpreconfig() -from doreah.settings import get_settings # thumbor diff --git a/maloja/proccontrol/__init__.py b/maloja/proccontrol/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/maloja/proccontrol/control.py b/maloja/proccontrol/control.py new file mode 100644 index 0000000..d0cf53e --- /dev/null +++ b/maloja/proccontrol/control.py @@ -0,0 +1,94 @@ +import subprocess +from doreah import settings +from doreah.control import mainfunction +from doreah.io import col +import os +import signal + +from .setup import setup +from . import tasks + +def getInstance(): + try: + output = subprocess.check_output(["pidof","Maloja"]) + pid = int(output) + return pid + except: + return None + +def getInstanceSupervisor(): + try: + output = subprocess.check_output(["pidof","maloja_supervisor"]) + pid = int(output) + return pid + except: + return None + +def restart(): + stop() + start() + +def start(): + if getInstanceSupervisor() is not None: + print("Maloja is already running.") + else: + setup() + try: + #p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + sp = subprocess.Popen(["python3","-m","maloja.proccontrol.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + print(col["green"]("Maloja started!")) + + port = settings.get_settings("WEB_PORT") + + print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.") + print("If you're installing this on your local machine, these links should get you there:") + print("\t" + col["blue"]("http://localhost:" + str(port))) + print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup")) + return True + except: + print("Error while starting Maloja.") + return False + + +def stop(): + + pid_sv = getInstanceSupervisor() + if pid_sv is not None: + os.kill(pid_sv,signal.SIGTERM) + + pid = getInstance() + if pid is not None: + os.kill(pid,signal.SIGTERM) + + if pid is not None or pid_sv is not None: + print("Maloja stopped!") + return True + else: + return False + + + +def direct(): + setup() + from .. import server + + + +@mainfunction({"l":"level"},shield=True) +def main(action,*args,**kwargs): + actions = { + "start":start, + "restart":restart, + "stop":stop, + "run":direct, + "debug":direct, + + "import":tasks.loadlastfm, + "backup":tasks.backuphere, + # "update":update, + "fix":tasks.fixdb + } + if action in actions: actions[action](*args,**kwargs) + else: print("Valid commands: " + " ".join(a for a in actions)) + + return True diff --git a/maloja/proccontrol/setup.py b/maloja/proccontrol/setup.py new file mode 100644 index 0000000..ad6752a --- /dev/null +++ b/maloja/proccontrol/setup.py @@ -0,0 +1,71 @@ +import pkg_resources +from distutils import dir_util +from doreah import settings +from doreah.io import col, ask, prompt +import os + +from ..globalconf import datadir + + +# EXTERNAL API KEYS +apikeys = { + "LASTFM_API_KEY":"Last.fm API Key", + "FANARTTV_API_KEY":"Fanart.tv API Key", + "SPOTIFY_API_ID":"Spotify Client ID", + "SPOTIFY_API_SECRET":"Spotify Client Secret" +} + + + +def copy_initial_local_files(): + folder = pkg_resources.resource_filename("maloja","data_files") + #shutil.copy(folder,DATA_DIR) + dir_util.copy_tree(folder,datadir(),update=False) + + + +def setup(): + + copy_initial_local_files() + SKIP = settings.get_settings("SKIP_SETUP") + + print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.") + for k in apikeys: + key = settings.get_settings(k) + if key is None: + print("\t" + "Currently not using a " + apikeys[k] + " for image display.") + elif key == "ASK": + print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.") + key = prompt("",types=(str,),default=None,skip=SKIP) + settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True) + else: + print("\t" + apikeys[k] + " found.") + + + # OWN API KEY + if os.path.exists(datadir("clients/authenticated_machines.tsv")): + pass + else: + answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP) + if answer: + import random + key = "" + for i in range(64): + key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) + print("Your API Key: " + col["yellow"](key)) + with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile: + keyfile.write(key + "\t" + "Default Generated Key") + else: + pass + + + if settings.get_settings("NAME") is None: + name = prompt("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.",default="Generic Maloja User",skip=SKIP) + settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True) + + if settings.get_settings("SEND_STATS") is None: + answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP) + if answer: + settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True) + else: + settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True) diff --git a/maloja/proccontrol/supervisor.py b/maloja/proccontrol/supervisor.py new file mode 100644 index 0000000..9fa05b0 --- /dev/null +++ b/maloja/proccontrol/supervisor.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +import os + +import subprocess +import setproctitle +import signal +from doreah.logging import log +from doreah.settings import get_settings + +from .control import getInstance + + +setproctitle.setproctitle("maloja_supervisor") + +def update(): + log("Updating...",module="supervisor") + try: + os.system("pip3 install maloja --upgrade --no-cache-dir") + except: + log("Could not update.",module="supervisor") + +def start(): + try: + p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + return p + except e: + log("Error starting Maloja: " + str(e),module="supervisor") + + + +while True: + log("Maloja is not running, starting...",module="supervisor") + if get_settings("UPDATE_AFTER_CRASH"): + update() + process = start() + + process.wait() diff --git a/maloja/proccontrol/tasks/__init__.py b/maloja/proccontrol/tasks/__init__.py new file mode 100644 index 0000000..cc37761 --- /dev/null +++ b/maloja/proccontrol/tasks/__init__.py @@ -0,0 +1,36 @@ +import os +from doreah.io import ask + +from ...globalconf import datadir + + + +def loadlastfm(filename): + + if not os.path.exists(filename): + print("File could not be found.") + return + + if os.path.exists(datadir("scrobbles/lastfmimport.tsv")): + overwrite = ask("Already imported Last.FM data. Overwrite?",default=False) + if not overwrite: return + print("Please wait...") + + from .lastfmconverter import convert + convert(filename,datadir("scrobbles/lastfmimport.tsv")) + #os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) + print("Successfully imported your Last.FM scrobbles!") + + +def backuphere(): + from .backup import backup + backup(folder=os.getcwd()) + +def update(): + os.system("pip3 install malojaserver --upgrade --no-cache-dir") + from ..control import restart + restart() + +def fixdb(): + from .fixexisting import fix + fix() diff --git a/maloja/backup.py b/maloja/proccontrol/tasks/backup.py similarity index 74% rename from maloja/backup.py rename to maloja/proccontrol/tasks/backup.py index 3a7e9da..d3aefce 100644 --- a/maloja/backup.py +++ b/maloja/proccontrol/tasks/backup.py @@ -2,7 +2,10 @@ import tarfile from datetime import datetime import glob import os -from .globalconf import datadir +from ...globalconf import datadir +from pathlib import PurePath + +from doreah.logging import log user_files = { @@ -25,6 +28,8 @@ def backup(folder,level="full"): for g in selected_files: real_files += glob.glob(datadir(g)) + log("Creating backup of " + str(len(real_files)) + " files...") + now = datetime.utcnow() timestr = now.strftime("%Y_%m_%d_%H_%M_%S") filename = "maloja_backup_" + timestr + ".tar.gz" @@ -32,4 +37,7 @@ def backup(folder,level="full"): assert not os.path.exists(archivefile) with tarfile.open(name=archivefile,mode="x:gz") as archive: for f in real_files: - archive.add(f) + p = PurePath(f) + r = p.relative_to(datadir()) + archive.add(f,arcname=r) + log("Backup created!") diff --git a/maloja/fixexisting.py b/maloja/proccontrol/tasks/fixexisting.py similarity index 92% rename from maloja/fixexisting.py rename to maloja/proccontrol/tasks/fixexisting.py index b0c39a3..76036c4 100644 --- a/maloja/fixexisting.py +++ b/maloja/proccontrol/tasks/fixexisting.py @@ -1,7 +1,7 @@ import os -from .globalconf import datadir +from ...globalconf import datadir import re -from .cleanup import CleanerAgent +from ...cleanup import CleanerAgent from doreah.logging import log import difflib import datetime @@ -35,9 +35,10 @@ def fix(): #with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile: - + log("Fixing database...") for filename in os.listdir(datadir("scrobbles")): if filename.endswith(".tsv"): + log("Fix file " + filename) filename_new = filename + "_new" with open(datadir("scrobbles",filename_new),"w") as newfile: @@ -68,3 +69,5 @@ def fix(): with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile: checkfile.write(wendigo.checksums) + + log("Database fixed!") diff --git a/maloja/lastfmconverter.py b/maloja/proccontrol/tasks/lastfmconverter.py similarity index 96% rename from maloja/lastfmconverter.py rename to maloja/proccontrol/tasks/lastfmconverter.py index 6a70d80..bb5a6dc 100644 --- a/maloja/lastfmconverter.py +++ b/maloja/proccontrol/tasks/lastfmconverter.py @@ -1,6 +1,6 @@ import os, datetime, re -from .cleanup import * -from .utilities import * +from ...cleanup import * +from ...utilities import * diff --git a/maloja/server.py b/maloja/server.py index 176a47c..51eb690 100755 --- a/maloja/server.py +++ b/maloja/server.py @@ -245,7 +245,7 @@ def static_html(name): template = jinjaenv.get_template(name + '.jinja') res = template.render(**LOCAL_CONTEXT) - log("Generated page {name} in {time}s (Jinja)".format(name=name,time=clock.stop()),module="debug") + log("Generated page {name} in {time:.5f}s (Jinja)".format(name=name,time=clock.stop()),module="debug") return res # if a pyhp file exists, use this @@ -272,7 +272,7 @@ def static_html(name): #response.set_header("Content-Type","application/xhtml+xml") res = pyhpfile(pthjoin(WEBFOLDER,"pyhp",name + ".pyhp"),environ) - log("Generated page {name} in {time}s (PYHP)".format(name=name,time=clock.stop()),module="debug") + log("Generated page {name} in {time:.5f}s (PYHP)".format(name=name,time=clock.stop()),module="debug") return res # if not, use the old way @@ -316,7 +316,7 @@ def static_html(name): response.set_header("Link",",".join(linkheaders)) - log("Generated page " + name + " in " + str(clock.stop()) + "s (Python+HTML)",module="debug") + log("Generated page {name} in {time:.5f}s (Python+HTML)".format(name=name,time=clock.stop()),module="debug") return html #return static_file("web/" + name + ".html",root="") diff --git a/maloja/static/less/maloja.less b/maloja/static/less/maloja.less index 3c1b60e..9a24ef2 100644 --- a/maloja/static/less/maloja.less +++ b/maloja/static/less/maloja.less @@ -291,13 +291,13 @@ span.stat_selector_pulse,span.stat_selector_topartists,span.stat_selector_toptra -h2 { +h2.headerwithextra { display:inline-block; padding-right:5px; margin-bottom:10px; margin-top:15px; } -h2+span.afterheader { +h2.headerwithextra+span.afterheader { color:@TEXT_COLOR_TERTIARY; } diff --git a/maloja/supervisor.py b/maloja/supervisor.py deleted file mode 100644 index 2846252..0000000 --- a/maloja/supervisor.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -import os - -import subprocess -import time -import setproctitle -import signal -from datetime import datetime -from doreah.logging import log -from doreah.settings import get_settings - - -setproctitle.setproctitle("maloja_supervisor") - -lastrestart = () - -def get_pid(): - try: - output = subprocess.check_output(["pidof","Maloja"]) - return int(output) - except: - return None - -def update(): - log("Updating...",module="supervisor") - try: - os.system("pip3 install maloja --upgrade --no-cache-dir") - except: - log("Could not update.",module="supervisor") - -def start(): - try: - p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - - except e: - log("Error starting Maloja: " + str(e),module="supervisor") - - - -while True: - - - now = datetime.now() - today = now.year, now.month, now.day - - pid = get_pid() - - if pid: - - restart = get_settings("DAILY_RESTART") - if restart not in [None,False]: - if today != lastrestart: - if now.hour == restart: - log("Daily restart...",module="supervisor") - os.kill(pid,signal.SIGTERM) - start() - lastrestart = today - - else: - log("Maloja is not running, starting...",module="supervisor") - if get_settings("UPDATE_AFTER_CRASH"): - update() - start() - lastrestart = today - - - time.sleep(60) diff --git a/maloja/utilities.py b/maloja/utilities.py index dbd36f7..48f7a82 100644 --- a/maloja/utilities.py +++ b/maloja/utilities.py @@ -27,13 +27,16 @@ from .globalconf import datadir def serialize(obj): try: - return json.dumps(obj) + return serialize(obj.hashable()) except: - if isinstance(obj,list) or isinstance(obj,tuple): - return "[" + ",".join(serialize(o) for o in obj) + "]" - elif isinstance(obj,dict): - return "{" + ",".join(serialize(o) + ":" + serialize(obj[o]) for o in obj) + "}" - return json.dumps(obj.hashable()) + try: + return json.dumps(obj) + except: + if isinstance(obj,list) or isinstance(obj,tuple): + return "[" + ",".join(serialize(o) for o in obj) + "]" + elif isinstance(obj,dict): + return "{" + ",".join(serialize(o) + ":" + serialize(obj[o]) for o in obj) + "}" + return json.dumps(obj.hashable()) #if isinstance(obj,list) or if isinstance(obj,tuple): diff --git a/maloja/web/jinja/artist.jinja b/maloja/web/jinja/artist.jinja index 258e62d..acc6cad 100644 --- a/maloja/web/jinja/artist.jinja +++ b/maloja/web/jinja/artist.jinja @@ -45,7 +45,7 @@ {% endif %} -

{{ artist }}

+

{{ artist }}

{% if competes %}#{{ info.position }}{% endif %}
{% if competes and included %} @@ -83,7 +83,7 @@ -

Pulse

+

Pulse


{% for range in xranges %} -

Performance

+

Performance

{% if not competes %}of {{ htmlgenerators.artistLink(credited) }} {% endif %}
diff --git a/maloja/web/jinja/charts_artists.jinja b/maloja/web/jinja/charts_artists.jinja new file mode 100644 index 0000000..4866901 --- /dev/null +++ b/maloja/web/jinja/charts_artists.jinja @@ -0,0 +1,34 @@ +{% extends "base.jinja" %} +{% block title %}Maloja - {{ artist }}{% endblock %} + +{% block scripts %} + +{% endblock %} + +{% set charts = dbp.get_charts_artists(filterkeys,limitkeys) %} + + +{% block content %} + + + + + + +
+
+ 		+

Artist Charts

View #1 Artists
+ {{ limitkeys.timerange.desc(prefix=True) }} +

+ {{ htmlmodules.module_filterselection(_urikeys) }} + +
+ + +{% import 'partials/charts_artists.jinja' as charts_artists %} + +{{ charts_artists.charts_artists(limitkeys,amountkeys,compare=False) }} + + +{% endblock %} diff --git a/maloja/web/jinja/charts_tracks.jinja b/maloja/web/jinja/charts_tracks.jinja new file mode 100644 index 0000000..b25b11e --- /dev/null +++ b/maloja/web/jinja/charts_tracks.jinja @@ -0,0 +1,34 @@ +{% extends "base.jinja" %} +{% block title %}Maloja - Track Charts{% endblock %} + +{% block scripts %} + +{% endblock %} + +{% set charts = dbp.get_charts_tracks(filterkeys,limitkeys) %} + + +{% block content %} + + + + + + +
+
+ 		+

Track Charts

View #1 Tracks
+ {{ limitkeys.timerange.desc(prefix=True) }} +

+ {{ htmlmodules.module_filterselection(_urikeys) }} + +
+ + +{% import 'partials/charts_tracks.jinja' as charts_tracks %} + +{{ charts_tracks.charts_tracks(filterkeys,limitkeys,amountkeys,charts=charts,compare=false) }} + + +{% endblock %} diff --git a/maloja/web/jinja/partials/charts_artists.jinja b/maloja/web/jinja/partials/charts_artists.jinja new file mode 100644 index 0000000..3049384 --- /dev/null +++ b/maloja/web/jinja/partials/charts_artists.jinja @@ -0,0 +1,41 @@ +{% macro charts_artists(limitkeys,amountkeys,charts=None,compare=False) %} + +{% if charts is none %} + {% set charts = dbp.get_charts_artists(limitkeys) %} +{% endif %} + +{% if compare %} +{% endif %} + +{% set firstindex = amountkeys.page * amountkeys.perpage %} +{% set lastindex = firstindex + amountkeys.perpage %} + + +{% set maxbar = charts[0]['scrobbles'] if charts != [] else 0 %} + + + {% for e in charts %} + {% if loop.index0 >= firstindex and loop.index0 < lastindex %} + + + + + {% if false %} + {% if e not in prevcharts %}{% endif %} + + {% endif %} + + + {{ htmlgenerators.entity_column(e['artist']) }} + + + + + + {% endif %} + {% endfor %} +
{%if loop.changed(e.scrobbles) %}#{{ e.rank }}{% endif %}🆕{{ htmlgenerators.scrobblesArtistLink(e['artist'],urihandler.internal_to_uri(limitkeys),amount=e['scrobbles']) }}{{ htmlgenerators.scrobblesArtistLink(e['artist'],urihandler.internal_to_uri(limitkeys),percent=e['scrobbles']*100/maxbar) }}
+ + + +{%- endmacro %} diff --git a/maloja/web/jinja/partials/charts_tracks.jinja b/maloja/web/jinja/partials/charts_tracks.jinja index f090ff3..9bfee8e 100644 --- a/maloja/web/jinja/partials/charts_tracks.jinja +++ b/maloja/web/jinja/partials/charts_tracks.jinja @@ -1,7 +1,8 @@ -{% macro charts_tracks(filterkeys,limitkeys,amountkeys,compare=False) %} +{% macro charts_tracks(filterkeys,limitkeys,amountkeys,charts=None,compare=False) %} - -{% set tracks = dbp.get_charts_tracks(filterkeys,limitkeys) %} +{% if charts is none %} + {% set charts = dbp.get_charts_tracks(filterkeys,limitkeys) %} +{% endif %} {% if compare %} {% if compare is true %} {% set compare = limitkeys.timerange.next(step=-1) %} @@ -9,11 +10,11 @@ {% set prevtracks = dbp.get_charts_tracks(filterkeys,{'timerange':compare}) %} {% set lastrank = {} %} - {% for t in tracks %} + {% for t in charts %} {% if lastrank.update({(t.track.artists,t.track.title):t.rank}) %}{% endif %} {% endfor %} - {% for t in tracks %} + {% for t in charts %} {% if (t.track.artists,t.track.title) in lastrank %} {% if t.update({'lastrank':lastrank[(t.track.artists,t.track.title)]}) %}{% endif %} {% endif %} @@ -24,10 +25,10 @@ {% set lastindex = firstindex + amountkeys.perpage %} -{% set maxbar = tracks[0]['scrobbles'] if tracks != [] else 0 %} +{% set maxbar = charts[0]['scrobbles'] if charts != [] else 0 %} - {% for e in tracks %} + {% for e in charts %} {% if loop.index0 >= firstindex and loop.index0 < lastindex %} diff --git a/maloja/web/jinja/track.jinja b/maloja/web/jinja/track.jinja index 461db2a..0dcefae 100644 --- a/maloja/web/jinja/track.jinja +++ b/maloja/web/jinja/track.jinja @@ -40,7 +40,7 @@
{{ htmlgenerators.artistLinks(track.artists) }}
-

{{ track.title }}

+

{{ track.title }}

{{ awards.certs(track) }} #{{ info.position }}
diff --git a/maloja/web/pyhp/admin.pyhp b/maloja/web/pyhp/admin.pyhp index 098b2a0..61ff121 100644 --- a/maloja/web/pyhp/admin.pyhp +++ b/maloja/web/pyhp/admin.pyhp @@ -112,7 +112,7 @@

External

- Report Issue
+ Report Issue
diff --git a/requirements.txt b/requirements.txt index 8159188..688b1ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,10 @@ bottle>=0.12.16 waitress>=1.3 -doreah>=1.2.7 -nimrodel>=0.4.9 +doreah>=1.6.7 +nimrodel>=0.6.3 setproctitle>=1.1.10 wand>=0.5.4 lesscpy>=0.13 pip>=19.3 +jinja2>2.11 +lru-dict>=1.1.6 diff --git a/scrobblers/chromium/sites/spotify.js b/scrobblers/chromium/sites/spotify.js index 8858687..a895d6e 100644 --- a/scrobblers/chromium/sites/spotify.js +++ b/scrobblers/chromium/sites/spotify.js @@ -6,7 +6,7 @@ maloja_scrobbler_selector_metadata = ".//div[@class='now-playing-bar__left']" maloja_scrobbler_selector_title = ".//a[@data-testid='nowplaying-track-link']/text()" maloja_scrobbler_selector_artists = ".//a[contains(@href,'/artist/')]" maloja_scrobbler_selector_artist = "./text()" -maloja_scrobbler_selector_duration = ".//div[@class='playback-bar__progress-time'][2]/text()" +maloja_scrobbler_selector_duration = ".//div[@class='playback-bar']/div[3]/text()" maloja_scrobbler_selector_control = ".//div[contains(@class,'player-controls__buttons')]/div[3]/button/@title"