diff --git a/.doreah b/.doreah deleted file mode 100644 index 29356ba..0000000 --- a/.doreah +++ /dev/null @@ -1,12 +0,0 @@ -logging: - logfolder: "logs" -settings: - files: - - "settings/default.ini" - - "settings/settings.ini" -caching: - folder: "cache/" -regular: - autostart: false -pyhp: - version: 2 diff --git a/.gitignore b/.gitignore index ff3a5ed..5614825 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,10 @@ # generic temporary / dev files *.pyc *.sh -!/update_requirements.sh +!/install_*.sh *.note *.xcf nohup.out -/.dev - -# user files -*.tsv -*.rulestate -*.log -*.css # currently not using /screenshot*.png diff --git a/README.md b/README.md index c8b0080..8300f19 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # Maloja +[![](https://img.shields.io/pypi/v/malojaserver?style=for-the-badge)](https://pypi.org/project/malojaserver/) +[![](https://img.shields.io/pypi/dm/malojaserver?style=for-the-badge)](https://pypi.org/project/malojaserver/) +[![](https://img.shields.io/github/stars/krateng/maloja?style=for-the-badge&color=purple)](https://github.com/krateng/maloja/stargazers) +[![](https://img.shields.io/pypi/l/malojaserver?style=for-the-badge)](https://github.com/krateng/maloja/blob/master/LICENSE) + Simple self-hosted music scrobble database to create personal listening statistics. No recommendations, no social network, no nonsense. You can check [my own Maloja page](https://maloja.krateng.ch) to see what it looks like. @@ -8,7 +13,7 @@ You can check [my own Maloja page](https://maloja.krateng.ch) to see what it loo **Update to Version 2** -With the update 2.0, Maloja has been refactored into a Python package and the old update script no longer works. I will keep this repository on the old version for a while so that users with regular updates have a chance to load the transition script. If you have any trouble with updating, simply install Maloja as described below, then manually copy all your user data to your `~/.local/share/maloja` folder. +With the update 2.0, Maloja has been refactored into a Python package and the old update script no longer works. If you're still on version 1, simply install Maloja as described below, then manually copy all your user data to your `~/.local/share/maloja` folder. ## Why not Last.fm / Libre.fm / GNU FM? @@ -22,15 +27,12 @@ Also neat: You can use your **custom artist or track images**. ## Requirements -* Python 3 -* Pip packages specified in `requirements.txt` -* If you'd like to display images, you will need API keys for [Last.fm](https://www.last.fm/api/account/create) and [Fanart.tv](https://fanart.tv/get-an-api-key/). These are free of charge! +* Python 3.5 or higher +* If you'd like to display images, you will need API keys for [Last.fm](https://www.last.fm/api/account/create) and [Fanart.tv](https://fanart.tv/get-an-api-key/) (you need a project key, not a personal one). These are free of charge! ## How to install -1) Install Maloja with - - pip3 install malojaserver +1) Download Maloja with the command `pip install malojaserver`. Make sure to use the correct python version (Use `pip3` if necessary). I've provided a simple .sh file to get Maloja going on an Alpine or Ubuntu server / container. 2) Start the server with @@ -51,11 +53,20 @@ Also neat: You can use your **custom artist or track images**. maloja stop maloja restart maloja start - maloja update -3) Various folders have `.info` files with more information on how to use their associated features. +3) Update Maloja with `pip install malojaserver --upgrade --no-cache-dir` -4) If you'd like to implement anything on top of Maloja, visit `/api_explorer`. +4) Various folders have `.info` files with more information on how to use their associated features. + +5) If you'd like to implement anything on top of Maloja, visit `/api_explorer`. + +6) To backup your data, run + + maloja backup + + or, to only backup essential data (no artwork etc) + + maloja backup -l minimal ## How to scrobble diff --git a/cache/.gitignore b/cache/.gitignore deleted file mode 100644 index 306625e..0000000 --- a/cache/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!.gitignore -!*.info diff --git a/clients/.gitignore b/clients/.gitignore deleted file mode 100644 index 77d83b6..0000000 --- a/clients/.gitignore +++ /dev/null @@ -1 +0,0 @@ -!example_file.tsv diff --git a/fixexisting.py b/fixexisting.py deleted file mode 100644 index 5c4538e..0000000 --- a/fixexisting.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -import re -from cleanup import CleanerAgent -from doreah.logging import log -import difflib - -wendigo = CleanerAgent() - -exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)(\t*)([^\t]*)\n" - -for fn in os.listdir("scrobbles/"): - if fn.endswith(".tsv"): - f = open("scrobbles/" + fn) - fnew = open("scrobbles/" + fn + "_new","w") - for l in f: - - a,t = re.sub(exp,r"\3",l), re.sub(exp,r"\5",l) - r1,r2,r3 = re.sub(exp,r"\1\2",l),re.sub(exp,r"\4",l),re.sub(exp,r"\6\7",l) - - a = a.replace("␟",";") - - (al,t) = wendigo.fullclean(a,t) - a = "␟".join(al) - fnew.write(r1 + a + r2 + t + r3 + "\n") - - #print("Artists: " + a) - #print("Title: " + t) - #print("1: " + r1) - #print("2: " + r2) - #print("3: " + r3) - - f.close() - fnew.close() - - #os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn) - with open("scrobbles/" + fn + "_new","r") as newfile: - with open("scrobbles/" + fn,"r") as oldfile: - diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="") - diff = list(diff)[2:] - log("Diff for scrobbles/" + fn + "".join("\n\t" + d for d in diff),module="fixer") - - os.rename("scrobbles/" + fn + "_new","scrobbles/" + fn) - - checkfile = open("scrobbles/" + fn + ".rulestate","w") - checkfile.write(wendigo.checksums) - checkfile.close() diff --git a/images/.gitignore b/images/.gitignore deleted file mode 100644 index 306625e..0000000 --- a/images/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!.gitignore -!*.info diff --git a/info.py b/info.py deleted file mode 100644 index 9603d4f..0000000 --- a/info.py +++ /dev/null @@ -1,10 +0,0 @@ -import os - -author = { - "name":"Johannes Krattenmacher", - "email":"maloja@krateng.dev", - "github": "krateng" -} -version = 1,5,16 -versionstr = ".".join(str(n) for n in version) -dev = os.path.exists("./.dev") diff --git a/install_alpine.sh b/install_alpine.sh new file mode 100644 index 0000000..75c9ac3 --- /dev/null +++ b/install_alpine.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +apk add python3 python3-dev gcc libxml2-dev libxslt-dev py3-pip libc-dev +pip3 install malojaserver diff --git a/install_ubuntu.sh b/install_ubuntu.sh new file mode 100644 index 0000000..ff0bd66 --- /dev/null +++ b/install_ubuntu.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +apt update +apt install python3 python3-pip +pip3 install malojaserver diff --git a/lastfmconverter.py b/lastfmconverter.py deleted file mode 100644 index 162234b..0000000 --- a/lastfmconverter.py +++ /dev/null @@ -1,63 +0,0 @@ -import sys, os, datetime, re, cleanup -from cleanup import * -from utilities import * - - -log = open(sys.argv[1]) -outputlog = open(sys.argv[2],"w") -checksumfile = open(sys.argv[2] + ".rulestate","w") #this file stores an identifier for all rules that were in place when the corresponding file was created - - -c = CleanerAgent() -stamps = [99999999999999] - -for l in log: - l = l.replace("\n","") - data = l.split(",") - - artist = data[0] - album = data[1] - title = data[2] - time = data[3] - - - (artists,title) = c.fullclean(artist,title) - - artistsstr = "␟".join(artists) - - - timeparts = time.split(" ") - (h,m) = timeparts[3].split(":") - - months = {"Jan":1,"Feb":2,"Mar":3,"Apr":4,"May":5,"Jun":6,"Jul":7,"Aug":8,"Sep":9,"Oct":10,"Nov":11,"Dec":12} - - timestamp = int(datetime.datetime(int(timeparts[2]),months[timeparts[1]],int(timeparts[0]),int(h),int(m)).timestamp()) - - - ## We prevent double timestamps in the database creation, so we technically don't need them in the files - ## however since the conversion from lastfm to maloja is a one-time thing, we should take any effort to make the file as good as possible - if (timestamp < stamps[-1]): - pass - elif (timestamp == stamps[-1]): - timestamp -= 1 - else: - while(timestamp in stamps): - timestamp -= 1 - - if (timestamp < stamps[-1]): - stamps.append(timestamp) - else: - stamps.insert(0,timestamp) - - - entry = "\t".join([str(timestamp),artistsstr,title,album]) - entry = entry.replace("#",r"\num") - - outputlog.write(entry) - outputlog.write("\n") - -checksumfile.write(c.checksums) - -log.close() -outputlog.close() -checksumfile.close() diff --git a/maloja b/maloja deleted file mode 100755 index 2686dad..0000000 --- a/maloja +++ /dev/null @@ -1,383 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -import sys -import signal -import os -import stat -import pathlib - - - -neededmodules = [ - "bottle", - "waitress", - "setproctitle", - "doreah", - "nimrodel" -] - -recommendedmodules = [ - "wand" -] - -SOURCE_URL = "https://github.com/krateng/maloja/archive/master.zip" - - - -def blue(txt): return "\033[94m" + txt + "\033[0m" -def green(txt): return "\033[92m" + txt + "\033[0m" -def yellow(txt): return "\033[93m" + txt + "\033[0m" - -## GOTODIR goes to directory that seems to have a maloja install -## SETUP assumes correct directory. sets settings and key -## INSTALL ignores local files, just installs prerequisites -## START INSTALL - GOTODIR - SETUP - starts process -## RESTART STOP - START -## STOP Stops process -## UPDATE GOTODIR - updates from repo -## LOADLASTFM GOTODIR - imports csv data -## INSTALLHERE makes this directory valid - UPDATE - INSTALL - SETUP - - - -def update_version_2(): - - - - if gotodir(): - try: - DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] - assert os.path.exists(DATA_DIR) - except: - DATA_DIR = os.path.join(os.environ["HOME"],".local/share/") - - DATA_DIR = os.path.join(DATA_DIR,"maloja") - os.makedirs(DATA_DIR,exist_ok=True) - - print(yellow("With version 2.0, Maloja has been refactored into a python package. The updater will attempt to make this transition smooth.")) - print("Relocating user data...") - - import shutil - for folder in ["clients","images","logs","rules","scrobbles","settings"]: - shutil.copytree(folder,os.path.join(DATA_DIR,folder)) - - print("Installing pip package...") - - os.system("pip3 install malojaserver --upgrade --no-cache-dir") - - print(yellow("Maloja may now be started from any directory with the command"),blue("maloja start")) - print(yellow("Updates will continue to work with ") + blue("maloja update") + yellow(", but you may also use pip directly")) - print(yellow("Please test your new server installation. If it works correctly with all your scrobbles, rules, settings and custom images, you can delete your old Maloja directory.")) - - if stop(): os.system("maloja start") - -def gotodir(): - if os.path.exists("./server.py"): - return True - elif os.path.exists("/opt/maloja/server.py"): - os.chdir("/opt/maloja/") - return True - - print("Maloja installation could not be found.") - return False - -def setup(): - - from doreah import settings - - # EXTERNAL API KEYS - apikeys = { - "LASTFM_API_KEY":"Last.fm API Key", - "FANARTTV_API_KEY":"Fanart.tv API Key", - "SPOTIFY_API_ID":"Spotify Client ID", - "SPOTIFY_API_SECRET":"Spotify Client Secret" - } - - print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.") - for k in apikeys: - key = settings.get_settings(k) - if key is None: - print("\t" + "Currently not using a " + apikeys[k] + " for image display.") - elif key == "ASK": - print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.") - key = input() - if key == "": key = None - settings.update_settings("settings/settings.ini",{k:key},create_new=True) - else: - print("\t" + apikeys[k] + " found.") - - - # OWN API KEY - if os.path.exists("./clients/authenticated_machines.tsv"): - pass - else: - print("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database. [Y/n]") - answer = input() - if answer.lower() in ["y","yes","yea","1","positive","true",""]: - import random - key = "" - for i in range(64): - key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) - print("Your API Key: " + yellow(key)) - with open("./clients/authenticated_machines.tsv","w") as keyfile: - keyfile.write(key + "\t" + "Default Generated Key") - elif answer.lower() in ["n","no","nay","0","negative","false"]: - pass - -def install(): - toinstall = [] - toinstallr = [] - for m in neededmodules: - try: - exec("import " + m) #I'm sorry - except: - toinstall.append(m) - - for m in recommendedmodules: - try: - exec("import " + m) - except: - toinstallr.append(m) - - if toinstall != []: - print("The following python modules need to be installed:") - for m in toinstall: - print("\t" + yellow(m)) - if toinstallr != []: - print("The following python modules are highly recommended, some features will not work without them:") - for m in toinstallr: - print("\t" + yellow(m)) - - if toinstall != [] or toinstallr != []: - if os.geteuid() != 0: - print("You can install them with",yellow("pip install -r requirements.txt"),"or Maloja can try to install them automatically. For this, you need to run this script as a root user.") - return False - else: - print("You can install them with",yellow("pip install -r requirements.txt"),"or Maloja can try to install them automatically, This might or might not work / bloat your system / cause a nuclear war.") - fail = False - if toinstall != []: - print("Attempt to install required modules? [Y/n]") - answer = input() - - if answer.lower() in ["y","yes","yea","1","positive","true",""]: - for m in toinstall: - try: - print("Installing " + m + " with pip...") - from pip._internal import main as pipmain - #os.system("pip3 install " + m) - pipmain(["install",m]) - print("Success!") - except: - print("Failure!") - fail = True - - elif answer.lower() in ["n","no","nay","0","negative","false"]: - return False #if you dont want to auto install required, you probably dont want to install recommended - else: - print("What?") - return False - if toinstallr != []: - print("Attempt to install recommended modules? [Y/n]") - answer = input() - - if answer.lower() in ["y","yes","yea","1","positive","true",""]: - for m in toinstallr: - try: - print("Installing " + m + " with pip...") - from pip._internal import main as pipmain - #os.system("pip3 install " + m) - pipmain(["install",m]) - print("Success!") - except: - print("Failure!") - fail = True - - elif answer.lower() in ["n","no","nay","0","negative","false"]: - return False - else: - print("What?") - return False - - if fail: return False - print("All modules successfully installed!") - print("Run the script again (without root) to start Maloja.") - return False - - else: - print("All necessary modules seem to be installed.") - return True - -def getInstance(): - try: - output = subprocess.check_output(["pidof","Maloja"]) - pid = int(output) - return pid - except: - return None - -def getInstanceSupervisor(): - try: - output = subprocess.check_output(["pidof","maloja_supervisor"]) - pid = int(output) - return pid - except: - return None - -def start(): - if install(): - - if gotodir(): - setup() - p = subprocess.Popen(["python3","server.py"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - p = subprocess.Popen(["python3","supervisor.py"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - print(green("Maloja started!") + " PID: " + str(p.pid)) - - from doreah import settings - port = settings.get_settings("WEB_PORT") - - print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.") - print("If you're installing this on your local machine, these links should get you there:") - print("\t" + blue("http://localhost:" + str(port))) - print("\t" + blue("http://localhost:" + str(port) + "/setup")) - return True - #else: - # os.chdir("/opt/maloja/") - # p = subprocess.Popen(["python3","server.py"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - # print("Maloja started! PID: " + str(p.pid)) - # return True - - print("Error while starting Maloja.") - return False - -def restart(): - #pid = getInstance() - #if pid == None: - # print("Server is not running.") - #else: - # stop() - #start() - - wasrunning = stop() - start() - return wasrunning - -def stop(): - pid_sv = getInstanceSupervisor() - if pid_sv is not None: - os.kill(pid_sv,signal.SIGTERM) - - pid = getInstance() - if pid is None: - print("Server is not running") - return False - else: - os.kill(pid,signal.SIGTERM) - print("Maloja stopped! PID: " + str(pid)) - return True - -def update(): - - import urllib.request - import shutil - #import tempfile - import zipfile - import distutils.dir_util - - if not gotodir(): return False - - if os.path.exists("./.dev"): - print("Better not overwrite the development server!") - return - - print("Updating Maloja...") - #with urllib.request.urlopen(SOURCE_URL) as response: - # with tempfile.NamedTemporaryFile(delete=True) as tmpfile: - # shutil.copyfileobj(response,tmpfile) - # - # with zipfile.ZipFile(tmpfile.name,"r") as z: - # - # for f in z.namelist(): - # #print("extracting " + f) - # z.extract(f) - - - os.system("wget " + SOURCE_URL) - with zipfile.ZipFile("./master.zip","r") as z: - - # if we ever have a separate directory for the code - # (the calling update script is not the same version as the current - # remote repository, so we better add this check just in case) - if "source/" in z.namelist(): - for f in z.namelist(): - if f.startswith("source/"): - z.extract(f) - for dir,_,files in os.walk("source"): - for f in files: - origfile = os.path.join(dir,f) - newfile = ps.path.join(dir[7:],f) - os.renames(origfile,newfile) #also prunes empty directory - else: - for f in z.namelist(): - z.extract(f) - - os.remove("./master.zip") - - - distutils.dir_util.copy_tree("./maloja-master/","./",verbose=2) - shutil.rmtree("./maloja-master") - print("Done!") - - os.chmod("./maloja",os.stat("./maloja").st_mode | stat.S_IXUSR) - os.chmod("./update_requirements.sh",os.stat("./update_requirements.sh").st_mode | stat.S_IXUSR) - - try: - returnval = os.system("./update_requirements.sh") - assert returnval == 0 - except: - print("Make sure to update required modules! (" + yellow("./update_requirements.sh") + ")") - - if stop(): start() #stop returns whether it was running before, in which case we restart it - -def loadlastfm(): - - try: - filename = sys.argv[2] - filename = os.path.abspath(filename) - except: - print("Please specify a file!") - return - - if gotodir(): - if os.path.exists("./scrobbles/lastfmimport.tsv"): - print("Already imported Last.FM data. Overwrite? [y/N]") - if input().lower() in ["y","yes","yea","1","positive","true"]: - pass - else: - return - print("Please wait...") - os.system("python3 ./lastfmconverter.py " + filename + " ./scrobbles/lastfmimport.tsv") - print("Successfully imported your Last.FM scrobbles!") - -def installhere(): - if len(os.listdir()) > 1: - print("You should install Maloja in an empty directory.") - return False - else: - open("server.py","w").close() - # if it's cheese, but it works, it ain't cheese - update() - install() - setup() - - print("Maloja installed! Start with " + yellow("./maloja start")) - - -if __name__ == "__main__": - if sys.argv[1] == "start": restart() - elif sys.argv[1] == "restart": restart() - elif sys.argv[1] == "stop": stop() - elif sys.argv[1] == "update": update_version_2() - elif sys.argv[1] == "import": loadlastfm() - elif sys.argv[1] == "install": installhere() - else: print("Valid commands: start restart stop update import install") diff --git a/maloja/__init__.py b/maloja/__init__.py new file mode 100644 index 0000000..d1e15a4 --- /dev/null +++ b/maloja/__init__.py @@ -0,0 +1,35 @@ +### PACKAGE DATA + +name = "maloja" +desc = "Self-hosted music scrobble database" +author = { + "name":"Johannes Krattenmacher", + "email":"maloja@krateng.dev", + "github": "krateng" +} +version = 2,2,3 +versionstr = ".".join(str(n) for n in version) + + +requires = [ + "bottle>=0.12.16", + "waitress>=1.3", + "doreah>=1.4.5", + "nimrodel>=0.6.3", + "setproctitle>=1.1.10", + "wand>=0.5.4", + "lesscpy>=0.13" +] +resources = [ + "web/*/*", + "web/*", + "static/*/*", + "data_files/*/*", + "data_files/*/*/*" +] + +commands = { + "maloja":"controller:main" +} + +from . import globalconf diff --git a/maloja/backup.py b/maloja/backup.py new file mode 100644 index 0000000..3a7e9da --- /dev/null +++ b/maloja/backup.py @@ -0,0 +1,35 @@ +import tarfile +from datetime import datetime +import glob +import os +from .globalconf import datadir + + +user_files = { + "minimal":[ + "rules/*.tsv", + "scrobbles" + ], + "full":[ + "clients/authenticated_machines.tsv", + "images/artists", + "images/tracks", + "settings/settings.ini" + ] +} + +def backup(folder,level="full"): + + selected_files = user_files["minimal"] if level == "minimal" else user_files["minimal"] + user_files["full"] + real_files = [] + for g in selected_files: + real_files += glob.glob(datadir(g)) + + now = datetime.utcnow() + timestr = now.strftime("%Y_%m_%d_%H_%M_%S") + filename = "maloja_backup_" + timestr + ".tar.gz" + archivefile = os.path.join(folder,filename) + assert not os.path.exists(archivefile) + with tarfile.open(name=archivefile,mode="x:gz") as archive: + for f in real_files: + archive.add(f) diff --git a/cleanup.py b/maloja/cleanup.py similarity index 83% rename from cleanup.py rename to maloja/cleanup.py index 7961940..6145fdb 100644 --- a/cleanup.py +++ b/maloja/cleanup.py @@ -1,6 +1,8 @@ import re -import utilities +from . import utilities from doreah import tsv, settings +from .globalconf import datadir +import pkg_resources # need to do this as a class so it can retain loaded settings from file # apparently this is not true @@ -11,21 +13,31 @@ class CleanerAgent: self.updateRules() def updateRules(self): - raw = tsv.parse_all("rules","string","string","string","string") + raw = tsv.parse_all(datadir("rules"),"string","string","string","string") self.rules_belongtogether = [b for [a,b,c,d] in raw if a=="belongtogether"] self.rules_notanartist = [b for [a,b,c,d] in raw if a=="notanartist"] self.rules_replacetitle = {b.lower():c for [a,b,c,d] in raw if a=="replacetitle"} self.rules_replaceartist = {b.lower():c for [a,b,c,d] in raw if a=="replaceartist"} self.rules_ignoreartist = [b.lower() for [a,b,c,d] in raw if a=="ignoreartist"] self.rules_addartists = {c.lower():(b.lower(),d) for [a,b,c,d] in raw if a=="addartists"} + self.rules_fixartists = {c.lower():b for [a,b,c,d] in raw if a=="fixartists"} + self.rules_artistintitle = {b.lower():c for [a,b,c,d] in raw if a=="artistintitle"} #self.rules_regexartist = [[b,c] for [a,b,c,d] in raw if a=="regexartist"] #self.rules_regextitle = [[b,c] for [a,b,c,d] in raw if a=="regextitle"] # TODO - # we always need to be able to tell if our current database is made with the current rules - self.checksums = utilities.checksumTSV("rules") + #self.plugin_artistparsers = [] + #self.plugin_titleparsers = [] + #if settings.get_settings("USE_PARSE_PLUGINS"): + # for ep in pkg_resources.iter_entry_points(group='maloja.artistparsers'): + # self.plugin_artistparsers.append(ep.load()) + # for ep in pkg_resources.iter_entry_points(group='maloja.titleparsers'): + # self.plugin_titleparsers.append(ep.load()) + # we always need to be able to tell if our current database is made with the current rules + self.checksums = utilities.checksumTSV(datadir("rules")) + def fullclean(self,artist,title): artists = self.parseArtists(self.removespecial(artist)) @@ -38,6 +50,11 @@ class CleanerAgent: allartists = allartists.split("␟") if set(reqartists).issubset(set(a.lower() for a in artists)): artists += allartists + elif title.lower() in self.rules_fixartists: + allartists = self.rules_fixartists[title.lower()] + allartists = allartists.split("␟") + if len(set(a.lower() for a in allartists) & set(a.lower() for a in artists)) > 0: + artists = allartists artists = list(set(artists)) artists.sort() @@ -120,7 +137,14 @@ class CleanerAgent: t = re.sub(r" \(originally by .*?\)","",t) t = re.sub(r" \(.*?Remaster.*?\)","",t) - return t.strip() + for s in settings.get_settings("REMOVE_FROM_TITLE"): + if s in t: + t = t.replace(s,"") + + t = t.strip() + #for p in self.plugin_titleparsers: + # t = p(t).strip() + return t def parseTitleForArtists(self,t): for d in self.delimiters_feat: @@ -137,7 +161,10 @@ class CleanerAgent: artists += self.parseArtists(re.sub(r"(.*) " + d + " (.*).*",r"\2",t)) return (title,artists) - return (t,[]) + artists = [] + for st in self.rules_artistintitle: + if st in t.lower(): artists += self.rules_artistintitle[st].split("␟") + return (t,artists) @@ -152,7 +179,7 @@ class CollectorAgent: # rules_include dict: credited artist -> all real artists def updateRules(self): - raw = tsv.parse_all("rules","string","string","string") + raw = tsv.parse_all(datadir("rules"),"string","string","string") self.rules_countas = {b:c for [a,b,c] in raw if a=="countas"} self.rules_countas_id = {} self.rules_include = {} #Twice the memory, double the performance! diff --git a/compliant_api.py b/maloja/compliant_api.py similarity index 98% rename from compliant_api.py rename to maloja/compliant_api.py index 5960efa..ae20c26 100644 --- a/compliant_api.py +++ b/maloja/compliant_api.py @@ -1,11 +1,11 @@ from doreah.logging import log import hashlib import random -import database +from . import database import datetime import itertools import sys -from cleanup import CleanerAgent +from .cleanup import CleanerAgent from bottle import response ## GNU-FM-compliant scrobbling @@ -68,7 +68,7 @@ def handle(path,keys): def scrobbletrack(artiststr,titlestr,timestamp): try: - log("Incoming scrobble (compliant API): ARTISTS: " + artiststr + ", TRACK: " + titlestr,module="debug") + log("Incoming scrobble (compliant API): ARTISTS: " + artiststr + ", TRACK: " + titlestr,module="debug") (artists,title) = cla.fullclean(artiststr,titlestr) database.createScrobble(artists,title,timestamp) database.sync() diff --git a/maloja/controller.py b/maloja/controller.py new file mode 100755 index 0000000..8c120df --- /dev/null +++ b/maloja/controller.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 + +import subprocess +import sys +import signal +import os +import shutil +from distutils import dir_util +import stat +import pathlib +import pkg_resources +from doreah.control import mainfunction +from doreah.io import col + +from .globalconf import datadir +from .backup import backup + + + + +def copy_initial_local_files(): + folder = pkg_resources.resource_filename(__name__,"data_files") + #shutil.copy(folder,DATA_DIR) + dir_util.copy_tree(folder,datadir(),update=False) + + +def setup(): + + copy_initial_local_files() + + from doreah import settings + + # EXTERNAL API KEYS + apikeys = { + "LASTFM_API_KEY":"Last.fm API Key", + "FANARTTV_API_KEY":"Fanart.tv API Key", + "SPOTIFY_API_ID":"Spotify Client ID", + "SPOTIFY_API_SECRET":"Spotify Client Secret" + } + + print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.") + for k in apikeys: + key = settings.get_settings(k) + if key is None: + print("\t" + "Currently not using a " + apikeys[k] + " for image display.") + elif key == "ASK": + print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.") + key = input() + if key == "": key = None + settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True) + else: + print("\t" + apikeys[k] + " found.") + + + # OWN API KEY + if os.path.exists(datadir("clients/authenticated_machines.tsv")): + pass + else: + print("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database. [Y/n]") + answer = input() + if answer.lower() in ["y","yes","yea","1","positive","true",""]: + import random + key = "" + for i in range(64): + key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) + print("Your API Key: " + col["yellow"](key)) + with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile: + keyfile.write(key + "\t" + "Default Generated Key") + elif answer.lower() in ["n","no","nay","0","negative","false"]: + pass + + + if settings.get_settings("NAME") is None: + print("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.") + name = input() + if name == "": name = "Generic Maloja User" + settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True) + + if settings.get_settings("SEND_STATS") is None: + print("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)? [Y/n]") + answer = input() + if answer.lower() in ["y","yes","yea","1","positive","true",""]: + settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True) + else: + settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True) + + +def getInstance(): + try: + output = subprocess.check_output(["pidof","Maloja"]) + pid = int(output) + return pid + except: + return None + +def getInstanceSupervisor(): + try: + output = subprocess.check_output(["pidof","maloja_supervisor"]) + pid = int(output) + return pid + except: + return None + +def start(): + setup() + try: + p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + sp = subprocess.Popen(["python3","-m","maloja.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + print(col["green"]("Maloja started!") + " PID: " + str(p.pid)) + + from doreah import settings + port = settings.get_settings("WEB_PORT") + + print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.") + print("If you're installing this on your local machine, these links should get you there:") + print("\t" + col["blue"]("http://localhost:" + str(port))) + print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup")) + return True + except: + print("Error while starting Maloja.") + return False + +def restart(): + wasrunning = stop() + start() + return wasrunning + +def stop(): + pid_sv = getInstanceSupervisor() + if pid_sv is not None: + os.kill(pid_sv,signal.SIGTERM) + + pid = getInstance() + if pid is None: + print("Server is not running") + return False + else: + os.kill(pid,signal.SIGTERM) + print("Maloja stopped! PID: " + str(pid)) + return True + + +def loadlastfm(filename): + + if not os.path.exists(filename): + print("File could not be found.") + return + + if os.path.exists(datadir("scrobbles/lastfmimport.tsv")): + print("Already imported Last.FM data. Overwrite? [y/N]") + if input().lower() in ["y","yes","yea","1","positive","true"]: + pass + else: + return + print("Please wait...") + from .lastfmconverter import convert + convert(filename,datadir("scrobbles/lastfmimport.tsv")) + #os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) + print("Successfully imported your Last.FM scrobbles!") + +def direct(): + from . import server + +def backuphere(): + backup(folder=os.getcwd()) + +def update(): + os.system("pip3 install malojaserver --upgrade --no-cache-dir") + restart() + +def fixdb(): + from .fixexisting import fix + fix() + +@mainfunction({"l":"level"},shield=True) +def main(action,*args,**kwargs): + actions = { + "start":restart, + "restart":restart, + "stop":stop, + "import":loadlastfm, + "debug":direct, + "backup":backuphere, + "update":update, + "fix":fixdb + } + if action in actions: actions[action](*args,**kwargs) + else: print("Valid commands: " + " ".join(a for a in actions)) + + return True + +#if __name__ == "__main__": +# main() diff --git a/logs/dummy b/maloja/data_files/backups/dummy similarity index 100% rename from logs/dummy rename to maloja/data_files/backups/dummy diff --git a/scrobbles/dummy b/maloja/data_files/cache/dummy similarity index 100% rename from scrobbles/dummy rename to maloja/data_files/cache/dummy diff --git a/clients/example_file.tsv b/maloja/data_files/clients/example_file.tsv similarity index 100% rename from clients/example_file.tsv rename to maloja/data_files/clients/example_file.tsv diff --git a/images/images.info b/maloja/data_files/images/images.info similarity index 100% rename from images/images.info rename to maloja/data_files/images/images.info diff --git a/maloja/data_files/logs/dbfix/dummy b/maloja/data_files/logs/dbfix/dummy new file mode 100644 index 0000000..e69de29 diff --git a/maloja/data_files/logs/dummy b/maloja/data_files/logs/dummy new file mode 100644 index 0000000..e69de29 diff --git a/rules/predefined/.gitignore b/maloja/data_files/rules/predefined/.gitignore similarity index 100% rename from rules/predefined/.gitignore rename to maloja/data_files/rules/predefined/.gitignore diff --git a/rules/predefined/krateng_artistsingroups.tsv b/maloja/data_files/rules/predefined/krateng_artistsingroups.tsv similarity index 100% rename from rules/predefined/krateng_artistsingroups.tsv rename to maloja/data_files/rules/predefined/krateng_artistsingroups.tsv diff --git a/rules/predefined/krateng_firefly-soundtrack.tsv b/maloja/data_files/rules/predefined/krateng_firefly-soundtrack.tsv similarity index 100% rename from rules/predefined/krateng_firefly-soundtrack.tsv rename to maloja/data_files/rules/predefined/krateng_firefly-soundtrack.tsv diff --git a/rules/predefined/krateng_jeremysoule.tsv b/maloja/data_files/rules/predefined/krateng_jeremysoule.tsv similarity index 100% rename from rules/predefined/krateng_jeremysoule.tsv rename to maloja/data_files/rules/predefined/krateng_jeremysoule.tsv diff --git a/rules/predefined/krateng_kpopgirlgroups.tsv b/maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv similarity index 97% rename from rules/predefined/krateng_kpopgirlgroups.tsv rename to maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv index a8f414d..28efdb0 100644 --- a/rules/predefined/krateng_kpopgirlgroups.tsv +++ b/maloja/data_files/rules/predefined/krateng_kpopgirlgroups.tsv @@ -108,6 +108,8 @@ replacetitle 여자 대통령 Female President # Mamamoo countas Hwasa Mamamoo +countas Moonbyul Mamamoo +replaceartist Moon Byul Moonbyul replaceartist Hwa Sa Hwasa replaceartist MAMAMOO Mamamoo replacetitle Egotistic(너나 해) Egotistic @@ -154,3 +156,8 @@ replacetitle 벌써 12시 Gotta Go Gotta Go # ITZY replacetitle 달라달라 (DALLA DALLA) Dalla Dalla + + +# Popular Remixes +artistintitle Areia Remix Areia +artistintitle Areia Kpop Areia diff --git a/rules/predefined/krateng_lotr-soundtrack.tsv b/maloja/data_files/rules/predefined/krateng_lotr-soundtrack.tsv similarity index 100% rename from rules/predefined/krateng_lotr-soundtrack.tsv rename to maloja/data_files/rules/predefined/krateng_lotr-soundtrack.tsv diff --git a/maloja/data_files/rules/predefined/krateng_masseffect.tsv b/maloja/data_files/rules/predefined/krateng_masseffect.tsv new file mode 100644 index 0000000..fbcc2e8 --- /dev/null +++ b/maloja/data_files/rules/predefined/krateng_masseffect.tsv @@ -0,0 +1,29 @@ +# NAME: Mass Effect Soundtrack +# DESC: Sets correct artists for the Mass Effect soundtracks + +# 1 +fixartists Jack Wall␟Sam Hulick Mass Effect Theme +fixartists Richard Jacques␟Jack Wall␟Sam Hulick Spectre Induction +fixartists Richard Jacques␟Jack Wall␟Sam Hulick The Citadel +fixartists Richard Jacques␟Jack Wall The Thorian +fixartists Richard Jacques␟Jack Wall␟Sam Hulick The Alien Queen +fixartists Jack Wall␟Sam Hulick Breeding Ground +fixartists Jack Wall␟Sam Hulick In Pursuit of Saren +fixartists David Kates␟Jack Wall␟Sam Hulick Infusion +fixartists David Kates␟Jack Wall␟Sam Hulick Final Assault + +# 2 +fixartists Jack Wall␟David Kates Thane +fixartists Jack Wall␟Sam Hulick The Normandy Attacked +fixartists Jack Wall␟Brian DiDomenico The Collector Base +fixartists Jack Wall␟Sam Hulick New Worlds + +# 3 +fixartists Sascha Dikiciyan␟Cris Velasco The Ardat-Yakshi +fixartists Sascha Dikiciyan␟Cris Velasco Rannoch +fixartists Sascha Dikiciyan␟Cris Velasco I'm Sorry +fixartists Sascha Dikiciyan␟Cris Velasco The Scientists +fixartists Sascha Dikiciyan␟Cris Velasco Aralakh Company +fixartists Sascha Dikiciyan␟Cris Velasco Prothean Beacon +fixartists Sascha Dikiciyan␟Cris Velasco Reaper Chase +fixartists Clint Mansell␟Sam Hulick An End, Once and For All diff --git a/rules/predefined/krateng_monstercat.tsv b/maloja/data_files/rules/predefined/krateng_monstercat.tsv similarity index 100% rename from rules/predefined/krateng_monstercat.tsv rename to maloja/data_files/rules/predefined/krateng_monstercat.tsv diff --git a/rules/predefined/predefined.info b/maloja/data_files/rules/predefined/predefined.info similarity index 100% rename from rules/predefined/predefined.info rename to maloja/data_files/rules/predefined/predefined.info diff --git a/rules/rules.info b/maloja/data_files/rules/rules.info similarity index 86% rename from rules/rules.info rename to maloja/data_files/rules/rules.info index c016d07..46fc7d3 100644 --- a/rules/rules.info +++ b/maloja/data_files/rules/rules.info @@ -20,6 +20,12 @@ The first column defines the type of the rule: Second column is artists that need to be already present for this rule to apply Third column is the song title Fourth column are artists that shoud be added, separated by ␟ + fixartists Similar as above, but simply specifies that if any of the given artists is present, all (and no others) should be present + Second column is correct artists + Third column is the song title + artistintitle Defines title strings that imply the presence of another artist. + Second column is the string + Third column is the artist or artists Rules in non-tsv files are ignored. '#' is used for comments. Additional columns are ignored. To have a '#' in a name, use '\num' Comments are not supported in scrobble lists, but you probably never edit these manually anyway. @@ -35,3 +41,4 @@ replaceartist Dal Shabet Dal★Shabet replaceartist Mr FijiWiji, AgNO3 Mr FijiWiji␟AgNO3 # one artist is replaced by two artists countas Trouble Maker HyunA addartists HyunA Change Jun Hyung +artistintitle Areia Remix Areia diff --git a/maloja/data_files/scrobbles/dummy b/maloja/data_files/scrobbles/dummy new file mode 100644 index 0000000..e69de29 diff --git a/settings/default.ini b/maloja/data_files/settings/default.ini similarity index 84% rename from settings/default.ini rename to maloja/data_files/settings/default.ini index 0087509..1576615 100644 --- a/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -16,6 +16,8 @@ SPOTIFY_API_ID = "ASK" SPOTIFY_API_SECRET = "ASK" CACHE_EXPIRE_NEGATIVE = 30 # after how many days negative results should be tried again CACHE_EXPIRE_POSITIVE = 300 # after how many days positive results should be refreshed +THUMBOR_SERVER = None +THUMBOR_SECRET = "" # Can be 'YouTube', 'YouTube Music', 'Google Play Music', 'Spotify', 'Tidal', 'SoundCloud', 'Deezer', 'Amazon Music' # Omit or set to none to disable @@ -25,6 +27,8 @@ TRACK_SEARCH_PROVIDER = None DB_CACHE_SIZE = 8192 # how many MB on disk each database cache should have available. INVALID_ARTISTS = ["[Unknown Artist]","Unknown Artist","Spotify"] +REMOVE_FROM_TITLE = ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)"] +USE_PARSE_PLUGINS = no [Local Images] @@ -51,9 +55,11 @@ SCROBBLES_GOLD = 250 SCROBBLES_PLATINUM = 500 SCROBBLES_DIAMOND = 1000 # name for comparisons -NAME = "Generic Maloja User" +NAME = None [Misc] EXPERIMENTAL_FEATURES = no -USE_PYHP = no +USE_PYHP = no #not recommended at the moment +FEDERATION = yes #does nothing yet +UPDATE_AFTER_CRASH = no #update when server is automatically restarted diff --git a/database.py b/maloja/database.py similarity index 92% rename from database.py rename to maloja/database.py index 1ae61db..974389a 100644 --- a/database.py +++ b/maloja/database.py @@ -1,12 +1,14 @@ # server from bottle import request, response, FormsDict # rest of the project -from cleanup import CleanerAgent, CollectorAgent -import utilities -from malojatime import register_scrobbletime, time_stamps, ranges -from urihandler import uri_to_internal, internal_to_uri, compose_querystring -import compliant_api -from external import proxy_scrobble +from .cleanup import CleanerAgent, CollectorAgent +from . import utilities +from .malojatime import register_scrobbletime, time_stamps, ranges +from .urihandler import uri_to_internal, internal_to_uri, compose_querystring +from . import compliant_api +from .external import proxy_scrobble +from .__init__ import version +from .globalconf import datadir # doreah toolkit from doreah.logging import log from doreah import tsv @@ -26,6 +28,7 @@ import sys import unicodedata from collections import namedtuple from threading import Lock +import yaml # url handling from importlib.machinery import SourceFileLoader import urllib @@ -46,10 +49,10 @@ Scrobble = namedtuple("Scrobble",["track","timestamp","saved"]) SCROBBLESDICT = {} # timestamps to scrobble mapping STAMPS = [] # sorted #STAMPS_SET = set() # as set for easier check if exists # we use the scrobbles dict for that now -TRACKS_LOWER = [] -ARTISTS_LOWER = [] -ARTIST_SET = set() -TRACK_SET = set() +TRACKS_NORMALIZED = [] +ARTISTS_NORMALIZED = [] +ARTISTS_NORMALIZED_SET = set() +TRACKS_NORMALIZED_SET = set() MEDALS = {} #literally only changes once per year, no need to calculate that on the fly MEDALS_TRACKS = {} @@ -65,14 +68,26 @@ lastsync = 0 # rulestate that the entire current database was built with, or False if the database was built from inconsistent scrobble files db_rulestate = False +try: + with open(datadir("known_servers.yml"),"r") as f: + KNOWN_SERVERS = set(yaml.safe_load(f)) +except: + KNOWN_SERVERS = set() + + +def add_known_server(url): + KNOWN_SERVERS.add(url) + with open(datadir("known_servers.yml"),"w") as f: + f.write(yaml.dump(list(KNOWN_SERVERS))) + ### symmetric keys are fine for now since we hopefully use HTTPS def loadAPIkeys(): global clients - tsv.create("clients/authenticated_machines.tsv") + tsv.create(datadir("clients/authenticated_machines.tsv")) #createTSV("clients/authenticated_machines.tsv") - clients = tsv.parse("clients/authenticated_machines.tsv","string","string") + clients = tsv.parse(datadir("clients/authenticated_machines.tsv"),"string","string") #clients = parseTSV("clients/authenticated_machines.tsv","string","string") log("Authenticated Machines: " + ", ".join([m[1] for m in clients])) @@ -158,16 +173,16 @@ def readScrobble(artists,title,time): def getArtistID(name): obj = name - objlower = name.lower().replace("'","") + obj_normalized = normalize_name(name) - if objlower in ARTIST_SET: - return ARTISTS_LOWER.index(objlower) + if obj_normalized in ARTISTS_NORMALIZED_SET: + return ARTISTS_NORMALIZED.index(obj_normalized) else: i = len(ARTISTS) ARTISTS.append(obj) - ARTIST_SET.add(objlower) - ARTISTS_LOWER.append(objlower) + ARTISTS_NORMALIZED_SET.add(obj_normalized) + ARTISTS_NORMALIZED.append(obj_normalized) # with a new artist added, we might also get new artists that they are credited as cr = coa.getCredited(name) @@ -182,20 +197,24 @@ def getTrackID(artists,title): for a in artists: artistset.add(getArtistID(name=a)) obj = Track(artists=frozenset(artistset),title=title) - objlower = Track(artists=frozenset(artistset),title=title.lower().replace("'","")) + obj_normalized = Track(artists=frozenset(artistset),title=normalize_name(title)) - if objlower in TRACK_SET: - return TRACKS_LOWER.index(objlower) + if obj_normalized in TRACKS_NORMALIZED_SET: + return TRACKS_NORMALIZED.index(obj_normalized) else: i = len(TRACKS) TRACKS.append(obj) - TRACK_SET.add(objlower) - TRACKS_LOWER.append(objlower) + TRACKS_NORMALIZED_SET.add(obj_normalized) + TRACKS_NORMALIZED.append(obj_normalized) return i +import unicodedata - - +# function to turn the name into a representation that can be easily compared, ignoring minor differences +remove_symbols = ["'","`","’"] +def normalize_name(name): + return "".join(char for char in unicodedata.normalize('NFD',name.lower()) + if char not in remove_symbols and unicodedata.category(char) != 'Mn') @@ -232,14 +251,15 @@ def test_server(key=None): @dbserver.get("serverinfo") def server_info(): - import info + response.set_header("Access-Control-Allow-Origin","*") response.set_header("Content-Type","application/json") return { "name":settings.get_settings("NAME"), - "version":info.version + "version":version, + "versionstring":".".join(str(n) for n in version) } ## All database functions are separated - the external wrapper only reads the request keys, converts them into lists and renames them where necessary, and puts the end result in a dict if not already so it can be returned as json @@ -263,6 +283,10 @@ def get_scrobbles(**keys): # info for comparison @dbserver.get("info") def info_external(**keys): + + response.set_header("Access-Control-Allow-Origin","*") + response.set_header("Content-Type","application/json") + result = info() return result @@ -274,7 +298,9 @@ def info(): "name":settings.get_settings("NAME"), "artists":{ chartentry["artist"]:round(chartentry["scrobbles"] * 100 / totalscrobbles,3) - for chartentry in get_charts_artists() if chartentry["scrobbles"]/totalscrobbles >= 0} + for chartentry in get_charts_artists() if chartentry["scrobbles"]/totalscrobbles >= 0 + }, + "known_servers":list(KNOWN_SERVERS) } @@ -826,10 +852,10 @@ def import_rulemodule(**keys): if remove: log("Deactivating predefined rulefile " + filename) - os.remove("rules/" + filename + ".tsv") + os.remove(datadir("rules/" + filename + ".tsv")) else: log("Importing predefined rulefile " + filename) - os.symlink("predefined/" + filename + ".tsv","rules/" + filename + ".tsv") + os.symlink(datadir("rules/predefined/" + filename + ".tsv"),datadir("rules/" + filename + ".tsv")) @@ -841,7 +867,8 @@ def rebuild(**keys): global db_rulestate db_rulestate = False sync() - os.system("python3 fixexisting.py") + from .fixexisting import fix + fix() global cla, coa cla = CleanerAgent() coa = CollectorAgent() @@ -929,7 +956,7 @@ def build_db(): # parse files - db = tsv.parse_all("scrobbles","int","string","string",comments=False) + db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False) #db = parseAllTSV("scrobbles","int","string","string",escape=False) for sc in db: artists = sc[1].split("␟") @@ -960,9 +987,10 @@ def build_db(): #start regular tasks utilities.update_medals() utilities.update_weekly() + utilities.send_stats() global db_rulestate - db_rulestate = utilities.consistentRulestate("scrobbles",cla.checksums) + db_rulestate = utilities.consistentRulestate(datadir("scrobbles"),cla.checksums) log("Database fully built!") @@ -996,9 +1024,9 @@ def sync(): #log("Sorted into months",module="debug") for e in entries: - tsv.add_entries("scrobbles/" + e + ".tsv",entries[e],comments=False) + tsv.add_entries(datadir("scrobbles/" + e + ".tsv"),entries[e],comments=False) #addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False) - utilities.combineChecksums("scrobbles/" + e + ".tsv",cla.checksums) + utilities.combineChecksums(datadir("scrobbles/" + e + ".tsv"),cla.checksums) #log("Written files",module="debug") @@ -1023,7 +1051,7 @@ import copy cache_query = {} if doreah.version >= (0,7,1) and settings.get_settings("EXPERIMENTAL_FEATURES"): - cache_query_permanent = DiskDict(name="dbquery",folder="cache",maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) + cache_query_permanent = DiskDict(name="dbquery",folder=datadir("cache"),maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) else: cache_query_permanent = Cache(maxmemory=1024*1024*500) cacheday = (0,0,0) diff --git a/external.py b/maloja/external.py similarity index 96% rename from external.py rename to maloja/external.py index 5f90197..65efaaf 100644 --- a/external.py +++ b/maloja/external.py @@ -63,7 +63,6 @@ if get_settings("SPOTIFY_API_ID") not in [None,"ASK"] and get_settings("SPOTIFY_ def api_request_artist(artist): for api in apis_artists: if True: - log("API: " + api["name"] + "; Image request: " + artist,module="external") try: artiststring = urllib.parse.quote(artist) var = artiststring @@ -85,7 +84,9 @@ def api_request_artist(artist): for node in step[1]: var = var[node] assert isinstance(var,str) and var != "" - except: + except Exception as e: + log("Error while getting artist image from " + api["name"],module="external") + log(str(e),module="external") continue return var @@ -97,7 +98,6 @@ def api_request_track(track): artists, title = track for api in apis_tracks: if True: - log("API: " + api["name"] + "; Image request: " + "/".join(artists) + " - " + title,module="external") try: artiststring = urllib.parse.quote(", ".join(artists)) titlestring = urllib.parse.quote(title) diff --git a/maloja/fixexisting.py b/maloja/fixexisting.py new file mode 100644 index 0000000..2dba2b6 --- /dev/null +++ b/maloja/fixexisting.py @@ -0,0 +1,64 @@ +import os +from .globalconf import datadir +import re +from .cleanup import CleanerAgent +from doreah.logging import log +import difflib +import datetime +from .backup import backup + +wendigo = CleanerAgent() + +exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)(\t*)([^\t]*)\n" + + + +def fix(): + + backup(level="minimal",folder=datadir("backups")) + + now = datetime.datetime.utcnow() + nowstr = now.strftime("%Y_%m_%d_%H_%M_%S") + datestr = now.strftime("%Y/%m/%d") + timestr = now.strftime("%H:%M:%S") + + with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile: + + logfile.write("Database fix initiated on " + datestr + " " + timestr + " UTC") + logfile.write("\n\n") + + for filename in os.listdir(datadir("scrobbles")): + if filename.endswith(".tsv"): + filename_new = filename + "_new" + + with open(datadir("scrobbles",filename_new),"w") as newfile: + with open(datadir("scrobbles",filename),"r") as oldfile: + + for l in oldfile: + + a,t = re.sub(exp,r"\3",l), re.sub(exp,r"\5",l) + r1,r2,r3 = re.sub(exp,r"\1\2",l),re.sub(exp,r"\4",l),re.sub(exp,r"\6\7",l) + + a = a.replace("␟",";") + + (al,t) = wendigo.fullclean(a,t) + a = "␟".join(al) + newfile.write(r1 + a + r2 + t + r3 + "\n") + + + #os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn) + with open(datadir("scrobbles",filename_new),"r") as newfile: + with open(datadir("scrobbles",filename),"r") as oldfile: + + diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="") + diff = list(diff)[2:] + #log("Diff for scrobbles/" + filename + "".join("\n\t" + d for d in diff),module="fixer") + output = "Diff for scrobbles/" + filename + "".join("\n\t" + d for d in diff) + print(output) + logfile.write(output) + logfile.write("\n") + + os.rename(datadir("scrobbles",filename_new),datadir("scrobbles",filename)) + + with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile: + checkfile.write(wendigo.checksums) diff --git a/maloja/globalconf.py b/maloja/globalconf.py new file mode 100644 index 0000000..6389091 --- /dev/null +++ b/maloja/globalconf.py @@ -0,0 +1,62 @@ +import os + + +# data folder +# must be determined first because getting settings relies on it + +try: + DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] + assert os.path.exists(DATA_DIR) +except: + DATA_DIR = os.path.join(os.environ["HOME"],".local/share/") + +DATA_DIR = os.path.join(DATA_DIR,"maloja") +os.makedirs(DATA_DIR,exist_ok=True) + +def datadir(*args): + return os.path.join(DATA_DIR,*args) + + + +### DOREAH CONFIGURATION + +from doreah import config + +config( + pyhp={ + "version": 2 + }, + logging={ + "logfolder": datadir("logs") + }, + settings={ + "files":[ + datadir("settings/default.ini"), + datadir("settings/settings.ini") + ] + }, + caching={ + "folder": datadir("cache") + }, + regular={ + "autostart": False + } +) + + + +from doreah.settings import get_settings + +# thumbor + +THUMBOR_SERVER, THUMBOR_SECRET = get_settings("THUMBOR_SERVER","THUMBOR_SECRET") +try: + USE_THUMBOR = THUMBOR_SERVER is not None and THUMBOR_SECRET is not None + if USE_THUMBOR: + from libthumbor import CryptoURL + THUMBOR_GENERATOR = CryptoURL(key=THUMBOR_SECRET) + OWNURL = get_settings("PUBLIC_URL") + assert OWNURL is not None +except: + USE_THUMBOR = False + log("Thumbor could not be initialized. Is libthumbor installed?") diff --git a/htmlgenerators.py b/maloja/htmlgenerators.py similarity index 99% rename from htmlgenerators.py rename to maloja/htmlgenerators.py index e443fd7..6c0c16b 100644 --- a/htmlgenerators.py +++ b/maloja/htmlgenerators.py @@ -1,7 +1,7 @@ import urllib from bottle import FormsDict import datetime -from urihandler import compose_querystring +from .urihandler import compose_querystring import urllib.parse from doreah.settings import get_settings diff --git a/htmlmodules.py b/maloja/htmlmodules.py similarity index 98% rename from htmlmodules.py rename to maloja/htmlmodules.py index 6182af2..bf108ae 100644 --- a/htmlmodules.py +++ b/maloja/htmlmodules.py @@ -1,8 +1,8 @@ -from htmlgenerators import * -import database -from utilities import getArtistImage, getTrackImage -from malojatime import * -from urihandler import compose_querystring, internal_to_uri, uri_to_internal +from .htmlgenerators import * +from . import database +from .utilities import getArtistImage, getTrackImage +from .malojatime import * +from .urihandler import compose_querystring, internal_to_uri, uri_to_internal import urllib import datetime import math @@ -568,7 +568,7 @@ def module_paginate(page,pages,perpage,**keys): # THIS FUNCTION USES THE ORIGINAL URI KEYS!!! def module_filterselection(keys,time=True,delimit=False): - from malojatime import today, thisweek, thismonth, thisyear, alltime + from .malojatime import today, thisweek, thismonth, thisyear, alltime filterkeys, timekeys, delimitkeys, extrakeys = uri_to_internal(keys) # drop keys that are not relevant so they don't clutter the URI diff --git a/maloja/lastfmconverter.py b/maloja/lastfmconverter.py new file mode 100644 index 0000000..9f58990 --- /dev/null +++ b/maloja/lastfmconverter.py @@ -0,0 +1,69 @@ +import os, datetime, re +from .cleanup import * +from .utilities import * + + + + +c = CleanerAgent() + + + +def convert(input,output): + + log = open(input,"r") + outputlog = open(output,"w") + checksumfile = open(output + ".rulestate","w") #this file stores an identifier for all rules that were in place when the corresponding file was created + + stamps = [99999999999999] + + for l in log: + l = l.replace("\n","") + data = l.split(",") + + artist = data[0] + album = data[1] + title = data[2] + time = data[3] + + + (artists,title) = c.fullclean(artist,title) + + artistsstr = "␟".join(artists) + + + timeparts = time.split(" ") + (h,m) = timeparts[3].split(":") + + months = {"Jan":1,"Feb":2,"Mar":3,"Apr":4,"May":5,"Jun":6,"Jul":7,"Aug":8,"Sep":9,"Oct":10,"Nov":11,"Dec":12} + + timestamp = int(datetime.datetime(int(timeparts[2]),months[timeparts[1]],int(timeparts[0]),int(h),int(m)).timestamp()) + + + ## We prevent double timestamps in the database creation, so we technically don't need them in the files + ## however since the conversion from lastfm to maloja is a one-time thing, we should take any effort to make the file as good as possible + if (timestamp < stamps[-1]): + pass + elif (timestamp == stamps[-1]): + timestamp -= 1 + else: + while(timestamp in stamps): + timestamp -= 1 + + if (timestamp < stamps[-1]): + stamps.append(timestamp) + else: + stamps.insert(0,timestamp) + + + entry = "\t".join([str(timestamp),artistsstr,title,album]) + entry = entry.replace("#",r"\num") + + outputlog.write(entry) + outputlog.write("\n") + + checksumfile.write(c.checksums) + + log.close() + outputlog.close() + checksumfile.close() diff --git a/malojatime.py b/maloja/malojatime.py similarity index 100% rename from malojatime.py rename to maloja/malojatime.py diff --git a/monkey.py b/maloja/monkey.py similarity index 100% rename from monkey.py rename to maloja/monkey.py diff --git a/server.py b/maloja/server.py similarity index 56% rename from server.py rename to maloja/server.py index 0c558c8..b012bdc 100755 --- a/server.py +++ b/maloja/server.py @@ -1,36 +1,45 @@ #!/usr/bin/env python +import os +from .globalconf import datadir, DATA_DIR + # server stuff from bottle import Bottle, route, get, post, error, run, template, static_file, request, response, FormsDict, redirect, template, HTTPResponse, BaseRequest import waitress # monkey patching -import monkey +from . import monkey # rest of the project -import database -import htmlmodules -import htmlgenerators -import malojatime -import utilities -from utilities import resolveImage -from urihandler import uri_to_internal, remove_identical -import urihandler -import info +from . import database +from . import htmlmodules +from . import htmlgenerators +from . import malojatime +from . import utilities +from .utilities import resolveImage +from .urihandler import uri_to_internal, remove_identical +from . import urihandler +from . import globalconf # doreah toolkit from doreah import settings from doreah.logging import log from doreah.timing import Clock +from doreah.pyhp import file as pyhpfile # technical -from importlib.machinery import SourceFileLoader +#from importlib.machinery import SourceFileLoader +import importlib import _thread import sys import signal import os import setproctitle +import pkg_resources # url handling import urllib + + + #settings.config(files=["settings/default.ini","settings/settings.ini"]) #settings.update("settings/default.ini","settings/settings.ini") MAIN_PORT = settings.get_settings("WEB_PORT") @@ -38,18 +47,30 @@ HOST = settings.get_settings("HOST") THREADS = 12 BaseRequest.MEMFILE_MAX = 15 * 1024 * 1024 +WEBFOLDER = pkg_resources.resource_filename(__name__,"web") +STATICFOLDER = pkg_resources.resource_filename(__name__,"static") +DATAFOLDER = DATA_DIR webserver = Bottle() +pthjoin = os.path.join -import lesscpy -css = "" -for f in os.listdir("website/less"): - css += lesscpy.compile("website/less/" + f) +def generate_css(): + import lesscpy + from io import StringIO + less = "" + for f in os.listdir(pthjoin(STATICFOLDER,"less")): + with open(pthjoin(STATICFOLDER,"less",f),"r") as lessf: + less += lessf.read() -os.makedirs("website/css",exist_ok=True) -with open("website/css/style.css","w") as f: - f.write(css) + css = lesscpy.compile(StringIO(less),minify=True) + return css + +css = generate_css() + +#os.makedirs("web/css",exist_ok=True) +#with open("web/css/style.css","w") as f: +# f.write(css) @webserver.route("") @@ -67,23 +88,11 @@ def mainpage(): @webserver.error(505) def customerror(error): code = int(str(error).split(",")[0][1:]) - log("HTTP Error: " + str(code),module="error") - if os.path.exists("website/errors/" + str(code) + ".html"): - return static_file("website/errors/" + str(code) + ".html",root="") + if os.path.exists(pthjoin(WEBFOLDER,"errors",str(code) + ".pyhp")): + return pyhpfile(pthjoin(WEBFOLDER,"errors",str(code) + ".pyhp"),{"errorcode":code}) else: - with open("website/errors/generic.html") as htmlfile: - html = htmlfile.read() - - # apply global substitutions - with open("website/common/footer.html") as footerfile: - footerhtml = footerfile.read() - with open("website/common/header.html") as headerfile: - headerhtml = headerfile.read() - html = html.replace("",footerhtml + "").replace("",headerhtml + "") - - html = html.replace("ERROR_CODE",str(code)) - return html + return pyhpfile(pthjoin(WEBFOLDER,"errors","generic.pyhp"),{"errorcode":code}) @@ -111,49 +120,63 @@ def dynamic_image(): @webserver.route("/images/") @webserver.route("/images/") def static_image(pth): + if globalconf.USE_THUMBOR: + return static_file(pthjoin("images",pth),root=DATAFOLDER) + + type = pth.split(".")[-1] small_pth = pth + "-small" - if os.path.exists("images/" + small_pth): - response = static_file("images/" + small_pth,root="") + if os.path.exists(datadir("images",small_pth)): + response = static_file(pthjoin("images",small_pth),root=DATAFOLDER) else: try: from wand.image import Image - img = Image(filename="images/" + pth) + img = Image(filename=datadir("images",pth)) x,y = img.size[0], img.size[1] smaller = min(x,y) if smaller > 300: ratio = 300/smaller img.resize(int(ratio*x),int(ratio*y)) - img.save(filename="images/" + small_pth) - response = static_file("images/" + small_pth,root="") + img.save(filename=datadir("images",small_pth)) + response = static_file(pthjoin("images",small_pth),root=DATAFOLDER) else: - response = static_file("images/" + pth,root="") + response = static_file(pthjoin("images",pth),root=DATAFOLDER) except: - response = static_file("images/" + pth,root="") + response = static_file(pthjoin("images",pth),root=DATAFOLDER) #response = static_file("images/" + pth,root="") response.set_header("Cache-Control", "public, max-age=86400") + response.set_header("Content-Type", "image/" + type) return response -#@webserver.route("/") -@webserver.route("/") -@webserver.route("/") -@webserver.route("/") -@webserver.route("/") -@webserver.route("/") -@webserver.route("/") -@webserver.route("/") -def static(name): - response = static_file("website/" + name,root="") + +@webserver.route("/style.css") +def get_css(): + response.content_type = 'text/css' + return css + + +@webserver.route("/.") +def static(name,ext): + assert ext in ["txt","ico","jpeg","jpg","png","less","js"] + response = static_file(ext + "/" + name + "." + ext,root=STATICFOLDER) response.set_header("Cache-Control", "public, max-age=3600") return response +@webserver.route("/media/.") +def static(name,ext): + assert ext in ["ico","jpeg","jpg","png"] + response = static_file(ext + "/" + name + "." + ext,root=STATICFOLDER) + response.set_header("Cache-Control", "public, max-age=3600") + return response + + @webserver.route("/") def static_html(name): - linkheaders = ["; rel=preload; as=style"] + linkheaders = ["; rel=preload; as=style"] keys = remove_identical(FormsDict.decode(request.query)) - pyhp_file = os.path.exists("website/" + name + ".pyhp") - html_file = os.path.exists("website/" + name + ".html") + pyhp_file = os.path.exists(pthjoin(WEBFOLDER,name + ".pyhp")) + html_file = os.path.exists(pthjoin(WEBFOLDER,name + ".html")) pyhp_pref = settings.get_settings("USE_PYHP") adminmode = request.cookies.get("adminmode") == "true" and database.checkAPIkey(request.cookies.get("apikey")) is not False @@ -163,49 +186,51 @@ def static_html(name): # if a pyhp file exists, use this if (pyhp_file and pyhp_pref) or (pyhp_file and not html_file): - from doreah.pyhp import file - environ = {} #things we expose to the pyhp pages - environ["adminmode"] = adminmode - if adminmode: environ["apikey"] = request.cookies.get("apikey") - - # maloja - environ["db"] = database - environ["htmlmodules"] = htmlmodules - environ["htmlgenerators"] = htmlgenerators - environ["malojatime"] = malojatime - environ["utilities"] = utilities - environ["urihandler"] = urihandler - environ["info"] = info - # external - environ["urllib"] = urllib + #things we expose to the pyhp pages + environ = { + "adminmode":adminmode, + "apikey":request.cookies.get("apikey") if adminmode else None, + # maloja + "db": database, + "htmlmodules": htmlmodules, + "htmlgenerators": htmlgenerators, + "malojatime": malojatime, + "utilities": utilities, + "urihandler": urihandler, + "settings": settings.get_settings, + # external + "urllib": urllib + } # request environ["filterkeys"], environ["limitkeys"], environ["delimitkeys"], environ["amountkeys"] = uri_to_internal(keys) + environ["_urikeys"] = keys #temporary! #response.set_header("Content-Type","application/xhtml+xml") - res = file("website/" + name + ".pyhp",environ) - log("Generated page " + name + " in " + str(clock.stop()) + "s (PYHP)",module="debug") + res = pyhpfile(pthjoin(WEBFOLDER,name + ".pyhp"),environ) + log("Generated page {name} in {time}s (PYHP)".format(name=name,time=clock.stop()),module="debug") return res # if not, use the old way else: - with open("website/" + name + ".html") as htmlfile: + with open(pthjoin(WEBFOLDER,name + ".html")) as htmlfile: html = htmlfile.read() # apply global substitutions - with open("website/common/footer.html") as footerfile: + with open(pthjoin(WEBFOLDER,"common/footer.html")) as footerfile: footerhtml = footerfile.read() - with open("website/common/header.html") as headerfile: + with open(pthjoin(WEBFOLDER,"common/header.html")) as headerfile: headerhtml = headerfile.read() html = html.replace("",footerhtml + "").replace("",headerhtml + "") # If a python file exists, it provides the replacement dict for the html file - if os.path.exists("website/" + name + ".py"): - #txt_keys = SourceFileLoader(name,"website/" + name + ".py").load_module().replacedict(keys,DATABASE_PORT) + if os.path.exists(pthjoin(WEBFOLDER,name + ".py")): + #txt_keys = SourceFileLoader(name,"web/" + name + ".py").load_module().replacedict(keys,DATABASE_PORT) try: - txt_keys,resources = SourceFileLoader(name,"website/" + name + ".py").load_module().instructions(keys) + module = importlib.import_module(".web." + name,package="maloja") + txt_keys,resources = module.instructions(keys) except Exception as e: log("Error in website generation: " + str(sys.exc_info()),module="error") raise @@ -229,7 +254,7 @@ def static_html(name): response.set_header("Link",",".join(linkheaders)) log("Generated page " + name + " in " + str(clock.stop()) + "s (Python+HTML)",module="debug") return html - #return static_file("website/" + name + ".html",root="") + #return static_file("web/" + name + ".html",root="") # Shortlinks diff --git a/website/favicon.ico b/maloja/static/ico/favicon.ico similarity index 100% rename from website/favicon.ico rename to maloja/static/ico/favicon.ico diff --git a/website/javascript/cookies.js b/maloja/static/js/cookies.js similarity index 100% rename from website/javascript/cookies.js rename to maloja/static/js/cookies.js diff --git a/website/javascript/datechange.js b/maloja/static/js/datechange.js similarity index 100% rename from website/javascript/datechange.js rename to maloja/static/js/datechange.js diff --git a/website/javascript/neopolitan.js b/maloja/static/js/neopolitan.js similarity index 83% rename from website/javascript/neopolitan.js rename to maloja/static/js/neopolitan.js index 5590597..5293bc7 100644 --- a/website/javascript/neopolitan.js +++ b/maloja/static/js/neopolitan.js @@ -15,7 +15,8 @@ else{for(var key in data){body+=encodeURIComponent(key)+"="+encodeURIComponent(d xhttp.send(body);console.log("Sent XHTTP request to",url)} function xhttprequest(url,data={},method="GET",json=true){var p=new Promise(resolve=>xhttpreq(url,data,method,resolve,json));return p;} function now(){return Math.floor(Date.now()/1000);} -return{getCookie:getCookie,setCookie:setCookie,getCookies:getCookies,saveCookies:saveCookies,xhttpreq:xhttpreq,xhttprequest:xhttprequest,now:now}}();document.addEventListener('DOMContentLoaded',function(){var elements=document.getElementsByClassName("seekable");for(var i=0;i + + +
+ + + +` +const oneresult = html_to_fragment(resulthtml).firstElementChild; + + + + +function searchresult() { + if (this.readyState == 4 && this.status == 200 && document.getElementById("searchinput").value != "") { + // checking if field is empty in case we get an old result coming in that would overwrite our cleared result window + var result = JSON.parse(this.responseText); + var artists = result["artists"].slice(0,5) + var tracks = result["tracks"].slice(0,5) + + while (results_artists.firstChild) { + results_artists.removeChild(results_artists.firstChild); + } + while (results_tracks.firstChild) { + results_tracks.removeChild(results_tracks.firstChild); + } + + for (var i=0;idiv { a:hover { text-decoration:underline; } + + + + +.hide { + display:none; +} diff --git a/website/less/grisonsfont.less b/maloja/static/less/grisonsfont.less similarity index 100% rename from website/less/grisonsfont.less rename to maloja/static/less/grisonsfont.less diff --git a/website/less/maloja.less b/maloja/static/less/maloja.less similarity index 95% rename from website/less/maloja.less rename to maloja/static/less/maloja.less index 992c414..7dc7ffe 100644 --- a/website/less/maloja.less +++ b/maloja/static/less/maloja.less @@ -1,5 +1,3 @@ -@import "website/less/grisons"; - body { padding:15px; padding-bottom:35px; @@ -539,6 +537,8 @@ table.list td.bar div { background-color:@TEXT_COLOR; height:20px; /* can only do this absolute apparently */ position:relative; + display:inline-block; + margin-bottom:-3px; } table.list tr:hover td.bar div { background-color:@FOCUS_COLOR; @@ -698,6 +698,35 @@ table.tiles_3x3 td { width:33.333%; font-size:70% } +table.tiles_4x4 td { + font-size:50% +} +table.tiles_5x5 td { + font-size:40% +} + + +.summary_rank { + background-size:cover; + background-position:center; + display: inline-block; +} + +.summary_rank_1 { + width:300px; + height:300px; + border: 6px solid gold; +} +.summary_rank_2 { + width:250px; + height:250px; + border: 4px solid silver; +} +.summary_rank_3 { + width:240px; + height:240px; + border: 3px solid #cd7f32; +} diff --git a/website/media/chartpos_bronze.png b/maloja/static/png/chartpos_bronze.png similarity index 100% rename from website/media/chartpos_bronze.png rename to maloja/static/png/chartpos_bronze.png diff --git a/website/media/chartpos_gold.png b/maloja/static/png/chartpos_gold.png similarity index 100% rename from website/media/chartpos_gold.png rename to maloja/static/png/chartpos_gold.png diff --git a/website/media/chartpos_normal.png b/maloja/static/png/chartpos_normal.png similarity index 100% rename from website/media/chartpos_normal.png rename to maloja/static/png/chartpos_normal.png diff --git a/website/media/chartpos_silver.png b/maloja/static/png/chartpos_silver.png similarity index 100% rename from website/media/chartpos_silver.png rename to maloja/static/png/chartpos_silver.png diff --git a/website/favicon.png b/maloja/static/png/favicon.png similarity index 100% rename from website/favicon.png rename to maloja/static/png/favicon.png diff --git a/website/media/record_diamond.png b/maloja/static/png/record_diamond.png similarity index 100% rename from website/media/record_diamond.png rename to maloja/static/png/record_diamond.png diff --git a/website/media/record_gold.png b/maloja/static/png/record_gold.png similarity index 100% rename from website/media/record_gold.png rename to maloja/static/png/record_gold.png diff --git a/website/media/record_gold_original.png b/maloja/static/png/record_gold_original.png similarity index 100% rename from website/media/record_gold_original.png rename to maloja/static/png/record_gold_original.png diff --git a/website/media/record_platinum.png b/maloja/static/png/record_platinum.png similarity index 100% rename from website/media/record_platinum.png rename to maloja/static/png/record_platinum.png diff --git a/website/media/star.png b/maloja/static/png/star.png similarity index 100% rename from website/media/star.png rename to maloja/static/png/star.png diff --git a/website/media/star_alt.png b/maloja/static/png/star_alt.png similarity index 100% rename from website/media/star_alt.png rename to maloja/static/png/star_alt.png diff --git a/website/robots.txt b/maloja/static/txt/robots.txt similarity index 100% rename from website/robots.txt rename to maloja/static/txt/robots.txt diff --git a/supervisor.py b/maloja/supervisor.py similarity index 53% rename from supervisor.py rename to maloja/supervisor.py index 824c2a2..a1e5cfe 100644 --- a/supervisor.py +++ b/maloja/supervisor.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 +import os import subprocess import time import setproctitle import signal from doreah.logging import log +from doreah.settings import get_settings setproctitle.setproctitle("maloja_supervisor") @@ -16,10 +18,15 @@ while True: try: output = subprocess.check_output(["pidof","Maloja"]) pid = int(output) - log("Maloja is running, PID " + str(pid),module="supervisor") except: log("Maloja is not running, restarting...",module="supervisor") + if get_settings("UPDATE_AFTER_CRASH"): + log("Updating first...",module="supervisor") + try: + os.system("pip3 install maloja --upgrade --no-cache-dir") + except: + log("Could not update.",module="supervisor") try: - p = subprocess.Popen(["python3","server.py"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) except e: log("Error starting Maloja: " + str(e),module="supervisor") diff --git a/urihandler.py b/maloja/urihandler.py similarity index 98% rename from urihandler.py rename to maloja/urihandler.py index 9ed7943..9a569c1 100644 --- a/urihandler.py +++ b/maloja/urihandler.py @@ -1,6 +1,6 @@ import urllib from bottle import FormsDict -from malojatime import time_fix, time_str, get_range_object +from .malojatime import time_fix, time_str, get_range_object import math # necessary because urllib.parse.urlencode doesnt handle multidicts diff --git a/utilities.py b/maloja/utilities.py similarity index 86% rename from utilities.py rename to maloja/utilities.py index d3850ef..0f95520 100644 --- a/utilities.py +++ b/maloja/utilities.py @@ -14,7 +14,10 @@ from doreah import caching from doreah.logging import log from doreah.regular import yearly, daily -from external import api_request_track, api_request_artist +from .external import api_request_track, api_request_artist +from .__init__ import version +from . import globalconf +from .globalconf import datadir @@ -125,10 +128,20 @@ def consistentRulestate(folder,checksums): ##### +if globalconf.USE_THUMBOR: + def thumborize(url): + if url.startswith("/"): url = globalconf.OWNURL + url + encrypted_url = globalconf.THUMBOR_GENERATOR.generate( + width=300, + height=300, + smart=True, + image_url=url + ) + return globalconf.THUMBOR_SERVER + encrypted_url - - - +else: + def thumborize(url): + return url @@ -217,12 +230,12 @@ def local_files(artist=None,artists=None,title=None): # direct files for ext in ["png","jpg","jpeg","gif"]: #for num in [""] + [str(n) for n in range(0,10)]: - if os.path.exists(purename + "." + ext): + if os.path.exists(datadir(purename + "." + ext)): images.append("/" + purename + "." + ext) # folder try: - for f in os.listdir(purename + "/"): + for f in os.listdir(datadir(purename)): if f.split(".")[-1] in ["png","jpg","jpeg","gif"]: images.append("/" + purename + "/" + f) except: @@ -239,11 +252,6 @@ local_track_cache = caching.Cache(maxage=local_cache_age) def getTrackImage(artists,title,fast=False): -# obj = (frozenset(artists),title) -# filename = "-".join([re.sub("[^a-zA-Z0-9]","",artist) for artist in sorted(artists)]) + "_" + re.sub("[^a-zA-Z0-9]","",title) -# if filename == "": filename = str(hash(obj)) -# filepath = "images/tracks/" + filename - if settings.get_settings("USE_LOCAL_IMAGES"): try: @@ -257,21 +265,6 @@ def getTrackImage(artists,title,fast=False): return urllib.parse.quote(res) - # check if custom image exists -# if os.path.exists(filepath + ".png"): -# imgurl = "/" + filepath + ".png" -# return imgurl -# elif os.path.exists(filepath + ".jpg"): -# imgurl = "/" + filepath + ".jpg" -# return imgurl -# elif os.path.exists(filepath + ".jpeg"): -# imgurl = "/" + filepath + ".jpeg" -# return imgurl -# elif os.path.exists(filepath + ".gif"): -# imgurl = "/" + filepath + ".gif" -# return imgurl - - try: # check our cache # if we have cached the nonexistence of that image, we immediately return the redirect to the artist and let the resolver handle it @@ -313,47 +306,31 @@ def getTrackImage(artists,title,fast=False): def getArtistImage(artist,fast=False): -# obj = artist -# filename = re.sub("[^a-zA-Z0-9]","",artist) -# if filename == "": filename = str(hash(obj)) -# filepath = "images/artists/" + filename -# #filepath_cache = "info/artists_cache/" + filename - if settings.get_settings("USE_LOCAL_IMAGES"): try: - return local_artist_cache.get(artist) + return thumborize(local_artist_cache.get(artist)) + # Local cached image except: + # Get all local images, select one if present images = local_files(artist=artist) if len(images) != 0: #return random.choice(images) res = random.choice(images) local_artist_cache.add(artist,res) - return urllib.parse.quote(res) - - - # check if custom image exists -# if os.path.exists(filepath + ".png"): -# imgurl = "/" + filepath + ".png" -# return imgurl -# elif os.path.exists(filepath + ".jpg"): -# imgurl = "/" + filepath + ".jpg" -# return imgurl -# elif os.path.exists(filepath + ".jpeg"): -# imgurl = "/" + filepath + ".jpeg" -# return imgurl -# elif os.path.exists(filepath + ".gif"): -# imgurl = "/" + filepath + ".gif" -# return imgurl + return thumborize(urllib.parse.quote(res)) + # if no local images (or setting to not use them) try: - #result = cachedArtists[artist] - result = artist_cache.get(artist) #artist_from_cache(artist) - if result is not None: return result + # check cache for foreign image + result = artist_cache.get(artist) + if result is not None: return thumborize(result) else: return "" + # none means non-existence is cached, return empty except: pass + # no cache entry, go on @@ -362,7 +339,6 @@ def getArtistImage(artist,fast=False): # if apikey is None: return "" # DO NOT CACHE THAT - # fast request only retuns cached and local results, generates redirect link for rest if fast: return "/image?artist=" + urllib.parse.quote(artist) @@ -373,7 +349,7 @@ def getArtistImage(artist,fast=False): #cachedArtists[artist] = result artist_cache.add(artist,result) #cache_artist(artist,result) - if result is not None: return result + if result is not None: return thumborize(result) else: return "" def getTrackImages(trackobjectlist,fast=False): @@ -469,7 +445,7 @@ def set_image(b64,**keys): def update_medals(): - from database import MEDALS, MEDALS_TRACKS, STAMPS, get_charts_artists, get_charts_tracks + from .database import MEDALS, MEDALS_TRACKS, STAMPS, get_charts_artists, get_charts_tracks currentyear = datetime.datetime.utcnow().year try: @@ -505,8 +481,8 @@ def update_medals(): @daily def update_weekly(): - from database import WEEKLY_TOPTRACKS, WEEKLY_TOPARTISTS, get_charts_artists, get_charts_tracks - from malojatime import ranges, thisweek + from .database import WEEKLY_TOPTRACKS, WEEKLY_TOPARTISTS, get_charts_artists, get_charts_tracks + from .malojatime import ranges, thisweek WEEKLY_TOPARTISTS.clear() @@ -521,3 +497,29 @@ def update_weekly(): for t in get_charts_tracks(timerange=week): track = (frozenset(t["track"]["artists"]),t["track"]["title"]) if t["rank"] == 1: WEEKLY_TOPTRACKS[track] = WEEKLY_TOPTRACKS.setdefault(track,0) + 1 + + +@daily +def send_stats(): + if settings.get_settings("SEND_STATS"): + + log("Sending daily stats report...") + + from .database import ARTISTS, TRACKS, SCROBBLES + + keys = { + "url":"https://myrcella.krateng.ch/malojastats", + "method":"POST", + "headers":{"Content-Type": "application/json"}, + "data":json.dumps({ + "name":settings.get_settings("NAME"), + "url":settings.get_settings("PUBLIC_URL"), + "version":".".join(str(d) for d in version), + "artists":len(ARTISTS), + "tracks":len(TRACKS), + "scrobbles":len(SCROBBLES) + }).encode("utf-8") + } + req = urllib.request.Request(**keys) + response = urllib.request.urlopen(req) + log("Sent daily report!") diff --git a/website/admin.pyhp b/maloja/web/admin.pyhp similarity index 78% rename from website/admin.pyhp rename to maloja/web/admin.pyhp index 2229e92..098b2a0 100644 --- a/website/admin.pyhp +++ b/maloja/web/admin.pyhp @@ -5,14 +5,25 @@ Maloja - + - + + diff --git a/website/artist.py b/maloja/web/artist.py similarity index 92% rename from website/artist.py rename to maloja/web/artist.py index bf3f2fa..f75c884 100644 --- a/website/artist.py +++ b/maloja/web/artist.py @@ -1,13 +1,13 @@ import urllib -import database -from malojatime import today,thisweek,thismonth,thisyear +from .. import database +from ..malojatime import today,thisweek,thismonth,thisyear def instructions(keys): - from utilities import getArtistImage - from htmlgenerators import artistLink, artistLinks, link_address - from urihandler import compose_querystring, uri_to_internal - from htmlmodules import module_pulse, module_performance, module_trackcharts, module_scrobblelist + from ..utilities import getArtistImage + from ..htmlgenerators import artistLink, artistLinks, link_address + from ..urihandler import compose_querystring, uri_to_internal + from ..htmlmodules import module_pulse, module_performance, module_trackcharts, module_scrobblelist filterkeys, _, _, _ = uri_to_internal(keys,forceArtist=True) artist = filterkeys.get("artist") diff --git a/website/artist.pyhp b/maloja/web/artist.pyhp similarity index 72% rename from website/artist.pyhp rename to maloja/web/artist.pyhp index 6070112..cab590b 100644 --- a/website/artist.pyhp +++ b/maloja/web/artist.pyhp @@ -37,7 +37,7 @@ Maloja - <pyhp echo="artist" /> - + @@ -65,7 +65,7 @@ - + @@ -103,24 +103,19 @@

Pulse

- - - - - - +

- - - - - - + + + + + + @@ -129,24 +124,17 @@

Performance

- - - - - - - +

- - - - - - + + + + + diff --git a/website/charts_artists.html b/maloja/web/charts_artists.html similarity index 91% rename from website/charts_artists.html rename to maloja/web/charts_artists.html index 6b070c9..e203049 100644 --- a/website/charts_artists.html +++ b/maloja/web/charts_artists.html @@ -4,7 +4,7 @@ Maloja - Artist Charts - + diff --git a/website/charts_artists.py b/maloja/web/charts_artists.py similarity index 79% rename from website/charts_artists.py rename to maloja/web/charts_artists.py index b5c951c..04cc582 100644 --- a/website/charts_artists.py +++ b/maloja/web/charts_artists.py @@ -2,10 +2,10 @@ import urllib def instructions(keys): - from utilities import getArtistImage - from urihandler import compose_querystring, uri_to_internal - from htmlmodules import module_artistcharts, module_filterselection, module_artistcharts_tiles - from malojatime import range_desc + from ..utilities import getArtistImage + from ..urihandler import compose_querystring, uri_to_internal + from ..htmlmodules import module_artistcharts, module_filterselection, module_artistcharts_tiles + from ..malojatime import range_desc from doreah.settings import get_settings diff --git a/maloja/web/charts_artists.pyhp b/maloja/web/charts_artists.pyhp new file mode 100644 index 0000000..a2962c3 --- /dev/null +++ b/maloja/web/charts_artists.pyhp @@ -0,0 +1,48 @@ + + + + + + Maloja - Artist Charts + + + + + + + try: + top = db.get_charts_artists(**filterkeys,**limitkeys)[0]["artist"] + img = utilities.getArtistImage(top) + except: + img = "" + + + + + + + + +
+
+ 		+

Artist Charts

View #1 Artists
+ +

+ + +
+ + + + + + + + + + + + + + diff --git a/website/charts_tracks.html b/maloja/web/charts_tracks.html similarity index 100% rename from website/charts_tracks.html rename to maloja/web/charts_tracks.html diff --git a/website/charts_tracks.py b/maloja/web/charts_tracks.py similarity index 81% rename from website/charts_tracks.py rename to maloja/web/charts_tracks.py index 843d0ff..0bfe9a4 100644 --- a/website/charts_tracks.py +++ b/maloja/web/charts_tracks.py @@ -2,11 +2,11 @@ import urllib def instructions(keys): - from utilities import getArtistImage, getTrackImage - from htmlgenerators import artistLink - from urihandler import compose_querystring, uri_to_internal - from htmlmodules import module_trackcharts, module_filterselection, module_trackcharts_tiles - from malojatime import range_desc + from ..utilities import getArtistImage, getTrackImage + from ..htmlgenerators import artistLink + from ..urihandler import compose_querystring, uri_to_internal + from ..htmlmodules import module_trackcharts, module_filterselection, module_trackcharts_tiles + from ..malojatime import range_desc from doreah.settings import get_settings filterkeys, timekeys, _, amountkeys = uri_to_internal(keys) diff --git a/maloja/web/charts_tracks.pyhp b/maloja/web/charts_tracks.pyhp new file mode 100644 index 0000000..9850375 --- /dev/null +++ b/maloja/web/charts_tracks.pyhp @@ -0,0 +1,54 @@ + + + + + + Maloja - Track Charts + + + + + + + try: + try: + img = utilities.getArtistImage(filterkeys['artist']) + except: + top = db.get_charts_tracks(**filterkeys,**limitkeys)[0]["track"] + img = utilities.getTrackImage(**top) + except: + img = "" + + + + + + + + +
+
+ 		+

Track Charts

View #1 Tracks
+ + by + + +

+ + +
+ + + + + + + + + + + + + + diff --git a/website/common/footer.html b/maloja/web/common/footer.html similarity index 86% rename from website/common/footer.html rename to maloja/web/common/footer.html index 733f222..7de3865 100644 --- a/website/common/footer.html +++ b/maloja/web/common/footer.html @@ -9,7 +9,17 @@ - + +
+ Artists + +
+

+ Tracks + +
+
+
diff --git a/website/common/header.html b/maloja/web/common/header.html similarity index 66% rename from website/common/header.html rename to maloja/web/common/header.html index 37d9792..ba3db6d 100644 --- a/website/common/header.html +++ b/maloja/web/common/header.html @@ -5,7 +5,7 @@ --> - - - - + + + + diff --git a/website/compare.html b/maloja/web/compare.html similarity index 100% rename from website/compare.html rename to maloja/web/compare.html diff --git a/website/compare.py b/maloja/web/compare.py similarity index 94% rename from website/compare.py rename to maloja/web/compare.py index febbe64..8a08a70 100644 --- a/website/compare.py +++ b/maloja/web/compare.py @@ -1,8 +1,8 @@ import urllib -import database +from .. import database import json -from htmlgenerators import artistLink -from utilities import getArtistImage +from ..htmlgenerators import artistLink +from ..utilities import getArtistImage def instructions(keys): @@ -15,6 +15,8 @@ def instructions(keys): owninfo = database.info() + database.add_known_server(compareto) + artists = {} for a in owninfo["artists"]: diff --git a/website/errors/generic.html b/maloja/web/errors/generic.pyhp similarity index 68% rename from website/errors/generic.html rename to maloja/web/errors/generic.pyhp index 5ade08e..aafa2e1 100644 --- a/website/errors/generic.html +++ b/maloja/web/errors/generic.pyhp @@ -3,7 +3,8 @@ - Maloja - Error ERROR_CODE + Maloja - Error <pyhp echo="errorcode" /> + @@ -13,7 +14,7 @@
-

Error ERROR_CODE


+

Error


That did not work. Don't ask me why.

diff --git a/website/issues.html b/maloja/web/issues.html similarity index 91% rename from website/issues.html rename to maloja/web/issues.html index 31e10f0..1a8bb9e 100644 --- a/website/issues.html +++ b/maloja/web/issues.html @@ -4,7 +4,7 @@ Maloja - Issues - + @@ -43,7 +43,7 @@ var xhttp = new XMLHttpRequest(); - xhttp.open("POST","/db/newrule?", true); + xhttp.open("POST","/api/newrule?", true); xhttp.send(keys); e = arguments[0] line = e.parentNode @@ -56,7 +56,7 @@ apikey = document.getElementById("apikey").value var xhttp = new XMLHttpRequest(); - xhttp.open("POST","/db/rebuild", true); + xhttp.open("POST","/api/rebuild", true); xhttp.send("key=" + encodeURIComponent(apikey)) window.location = "/wait"; } diff --git a/website/issues.py b/maloja/web/issues.py similarity index 96% rename from website/issues.py rename to maloja/web/issues.py index d389ad0..eac9cdc 100644 --- a/website/issues.py +++ b/maloja/web/issues.py @@ -1,6 +1,6 @@ import urllib -import database -from htmlgenerators import artistLink +from .. import database +from ..htmlgenerators import artistLink def instructions(keys): diff --git a/website/manual.html b/maloja/web/manual.html similarity index 99% rename from website/manual.html rename to maloja/web/manual.html index b642a8d..e880d5c 100644 --- a/website/manual.html +++ b/maloja/web/manual.html @@ -4,7 +4,7 @@ Maloja - + + + - - + + diff --git a/website/start.py b/maloja/web/start.py similarity index 94% rename from website/start.py rename to maloja/web/start.py index 246e8b6..11c6076 100644 --- a/website/start.py +++ b/maloja/web/start.py @@ -1,10 +1,10 @@ import urllib from datetime import datetime, timedelta -import database +from .. import database from doreah.timing import clock, clockp from doreah.settings import get_settings -from htmlmodules import module_scrobblelist, module_pulse, module_artistcharts_tiles, module_trackcharts_tiles +from ..htmlmodules import module_scrobblelist, module_pulse, module_artistcharts_tiles, module_trackcharts_tiles def instructions(keys): @@ -17,7 +17,7 @@ def instructions(keys): #clock() - from malojatime import today,thisweek,thismonth,thisyear + from ..malojatime import today,thisweek,thismonth,thisyear # artists diff --git a/maloja/web/summary.pyhp b/maloja/web/summary.pyhp new file mode 100644 index 0000000..e195336 --- /dev/null +++ b/maloja/web/summary.pyhp @@ -0,0 +1,85 @@ + + + + + + Maloja - Summary + + + + + + + + + + + + +
+
+ 		+

Summary


+ + + +

+ + +
+ + + + + + + + + + + + + + + + + +
+ +

Artists

+ + +

Tracks

+ + + + entry = type[r] + isartist = type is artists + entity = entry["artist"] if isartist else entry["track"] + image = utilities.getArtistImage(entity,fast=True) if isartist else utilities.getTrackImage(entity['artists'],entity['title'],fast=True) + rank = entry["rank"] + + + + +
+ +
+ + + +
+ + + + + + diff --git a/website/top_artists.html b/maloja/web/top_artists.html similarity index 100% rename from website/top_artists.html rename to maloja/web/top_artists.html diff --git a/website/top_artists.py b/maloja/web/top_artists.py similarity index 75% rename from website/top_artists.py rename to maloja/web/top_artists.py index 0d3f523..82b864d 100644 --- a/website/top_artists.py +++ b/maloja/web/top_artists.py @@ -2,11 +2,11 @@ import urllib def instructions(keys): - from utilities import getArtistImage, getTrackImage - from htmlgenerators import artistLink - from urihandler import compose_querystring, uri_to_internal - from htmlmodules import module_topartists, module_filterselection - from malojatime import range_desc + from ..utilities import getArtistImage, getTrackImage + from ..htmlgenerators import artistLink + from ..urihandler import compose_querystring, uri_to_internal + from ..htmlmodules import module_topartists, module_filterselection + from ..malojatime import range_desc _, timekeys, delimitkeys, _ = uri_to_internal(keys) diff --git a/website/top_tracks.html b/maloja/web/top_tracks.html similarity index 100% rename from website/top_tracks.html rename to maloja/web/top_tracks.html diff --git a/website/top_tracks.py b/maloja/web/top_tracks.py similarity index 76% rename from website/top_tracks.py rename to maloja/web/top_tracks.py index aa7e547..2ffd80a 100644 --- a/website/top_tracks.py +++ b/maloja/web/top_tracks.py @@ -2,11 +2,11 @@ import urllib def instructions(keys): - from utilities import getArtistImage, getTrackImage - from htmlgenerators import artistLink - from urihandler import compose_querystring, uri_to_internal - from htmlmodules import module_toptracks, module_filterselection - from malojatime import range_desc + from ..utilities import getArtistImage, getTrackImage + from ..htmlgenerators import artistLink + from ..urihandler import compose_querystring, uri_to_internal + from ..htmlmodules import module_toptracks, module_filterselection + from ..malojatime import range_desc filterkeys, timekeys, delimitkeys, _ = uri_to_internal(keys) diff --git a/website/track.html b/maloja/web/track.html similarity index 96% rename from website/track.html rename to maloja/web/track.html index c67b824..1fdd265 100644 --- a/website/track.html +++ b/maloja/web/track.html @@ -4,8 +4,8 @@ Maloja - KEY_TRACKTITLE - - + + diff --git a/website/track.py b/maloja/web/track.py similarity index 91% rename from website/track.py rename to maloja/web/track.py index 8f108e7..0110576 100644 --- a/website/track.py +++ b/maloja/web/track.py @@ -1,13 +1,13 @@ import urllib -import database -from malojatime import today,thisweek,thismonth,thisyear +from .. import database +from ..malojatime import today,thisweek,thismonth,thisyear def instructions(keys): - from utilities import getArtistImage, getTrackImage - from htmlgenerators import artistLinks - from urihandler import compose_querystring, uri_to_internal - from htmlmodules import module_scrobblelist, module_pulse, module_performance + from ..utilities import getArtistImage, getTrackImage + from ..htmlgenerators import artistLinks + from ..urihandler import compose_querystring, uri_to_internal + from ..htmlmodules import module_scrobblelist, module_pulse, module_performance filterkeys, _, _, _ = uri_to_internal(keys,forceTrack=True) diff --git a/website/track.pyhp b/maloja/web/track.pyhp similarity index 60% rename from website/track.pyhp rename to maloja/web/track.pyhp index 589aba1..811f527 100644 --- a/website/track.pyhp +++ b/maloja/web/track.pyhp @@ -18,7 +18,7 @@ Maloja - <pyhp echo="track['title']" /> - +