Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Change the source for recent user agent data

Browse Source 
The old source no longer works
This commit is contained in:
Kovid Goyal 2019-04-01 18:08:29 +05:30
parent b78e24417a
commit ccb0391844
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
setup/browser_data.py
View File

@ -5,7 +5,9 @@
from __future__ import absolute_import, division, print_function, unicode_literals from __future__ import absolute_import, division, print_function, unicode_literals
import os import os
import re import json
import gzip
import io
from datetime import datetime from datetime import datetime
from setup import download_securely from setup import download_securely
@ -39,10 +41,14 @@ def common_user_agents():
] ]
print('Getting recent UAs...') print('Getting recent UAs...')
raw = download_securely( raw = download_securely(
'https://techblog.willshouse.com/2012/01/03/most-common-user-agents/').decode('utf-8') 'https://raw.githubusercontent.com/intoli/user-agents/master/src/user-agents.json.gz')
lines = re.search( data = json.loads(gzip.GzipFile(fileobj=io.BytesIO(raw)).read())
r'<textarea.+"get-the-list".+>([^<]+)</textarea>', raw).group(1).splitlines() uas = []
ans = filter_ans(lines) for item in data:
ua = item['userAgent']
if not ua.startswith('Opera'):
uas.append(ua)
ans = filter_ans(uas)[:256]
if not ans: if not ans:
raise ValueError('Failed to download list of common UAs') raise ValueError('Failed to download list of common UAs')
return ans return ans

2
setup/resources.py
View File

@ -270,7 +270,7 @@ class RecentUAs(Command): # {{{
from setup.browser_data import get_data from setup.browser_data import get_data
data = get_data() data = get_data()
with open(self.UA_PATH, 'wb') as f: with open(self.UA_PATH, 'wb') as f:
f.write(json.dumps(data, indent=2, ensure_ascii=False).encode('utf-8')) f.write(json.dumps(data, indent=2, ensure_ascii=False, sort_keys=True).encode('utf-8'))
# }}} # }}}