Updated database fix

maloja/controller.py

@@ -198,6 +198,10 @@ def backup(level="full"):
 def update():
 	os.system("pip3 install malojaserver --upgrade --no-cache-dir")
 
+def fixdb():
+	from .fixexisting import fix
+	fix()
+
 @mainfunction({"l":"level"},shield=True)
 def main(action,*args,**kwargs):
 	actions = {
@@ -207,7 +211,8 @@ def main(action,*args,**kwargs):
 		"import":loadlastfm,
 		"debug":direct,
 		"backup":backup,
-		"update":update
+		"update":update,
+		"fix":fixdb
 	}
 	if action in actions: actions[action](*args,**kwargs)
 	else: print("Valid commands: " + " ".join(a for a in actions))

maloja/fixexisting.py

@@ -1,18 +1,40 @@
 import os
+from .__init__ import DATA_DIR
+os.chdir(DATA_DIR)
+
 import re
 from .cleanup import CleanerAgent
 from doreah.logging import log
 import difflib
+import datetime
 
 wendigo = CleanerAgent()
 
 exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)(\t*)([^\t]*)\n"
 
-for fn in os.listdir("scrobbles/"):
+pthj = os.path.join
-	if fn.endswith(".tsv"):
+
-		f = open("scrobbles/" + fn)
+
-		fnew = open("scrobbles/" + fn + "_new","w")
+def fix():
-		for l in f:
+
+	now = datetime.datetime.utcnow()
+	nowstr = now.strftime("%Y_%m_%d_%H_%M_%S")
+	datestr = now.strftime("%Y/%m/%d")
+	timestr = now.strftime("%H:%M:%S")
+
+	with open(pthj(DATA_DIR,"logs","dbfix",nowstr + ".log"),"a") as logfile:
+
+		logfile.write("Database fix initiated on " + datestr + " " + timestr + " UTC")
+		logfile.write("\n\n")
+
+		for filename in os.listdir(pthj(DATA_DIR,"scrobbles")):
+			if filename.endswith(".tsv"):
+				filename_new = filename + "_new"
+
+				with open(pthj(DATA_DIR,"scrobbles",filename_new),"w") as newfile:
+					with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile:
+
+						for l in oldfile:
 
 							a,t = re.sub(exp,r"\3",l), re.sub(exp,r"\5",l)
 							r1,r2,r3 = re.sub(exp,r"\1\2",l),re.sub(exp,r"\4",l),re.sub(exp,r"\6\7",l)
@@ -21,26 +43,22 @@ for fn in os.listdir("scrobbles/"):
 
 							(al,t) = wendigo.fullclean(a,t)
 							a = "␟".join(al)
-			fnew.write(r1 + a + r2 + t + r3 + "\n")
+							newfile.write(r1 + a + r2 + t + r3 + "\n")
 
-			#print("Artists: " + a)
-			#print("Title: " + t)
-			#print("1: " + r1)
-			#print("2: " + r2)
-			#print("3: " + r3)
-
-		f.close()
-		fnew.close()
 
 				#os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn)
-		with open("scrobbles/" + fn + "_new","r") as newfile:
+				with open(pthj(DATA_DIR,"scrobbles",filename_new),"r") as newfile:
-			with open("scrobbles/" + fn,"r") as oldfile:
+					with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile:
+
 						diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="")
 						diff = list(diff)[2:]
-				log("Diff for scrobbles/" + fn + "".join("\n\t" + d for d in diff),module="fixer")
+						#log("Diff for scrobbles/" + filename + "".join("\n\t" + d for d in diff),module="fixer")
+						output = "Diff for scrobbles/" + filename + "".join("\n\t" + d for d in diff)
+						print(output)
+						logfile.write(output)
+						logfile.write("\n")
 
-		os.rename("scrobbles/" + fn + "_new","scrobbles/" + fn)
+				os.rename(pthj(DATA_DIR,"scrobbles",filename_new),pthj(DATA_DIR,"scrobbles",filename))
 
-		checkfile = open("scrobbles/" + fn + ".rulestate","w")
+				with open(pthj(DATA_DIR,"scrobbles",filename + ".rulestate"),"w") as checkfile:
 					checkfile.write(wendigo.checksums)
-		checkfile.close()