Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Remove list of chrome/firefox versions from user agent data

Browse Source 
The upstream data source (Wikipedia) was deleted and these arent currently used
anyway
This commit is contained in:
Kovid Goyal 2023-04-26 08:51:57 +05:30
parent 809ce3dc44
commit f5b35bb97e
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 0 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
setup/browser_data.py
View File

@ -8,8 +8,6 @@ import sys
from datetime import datetime from datetime import datetime
from urllib.request import urlopen from urllib.request import urlopen
from setup import download_securely
def download_from_calibre_server(url): def download_from_calibre_server(url):
ca = os.path.join(sys.resources_location, 'calibre-ebook-root-CA.crt') ca = os.path.join(sys.resources_location, 'calibre-ebook-root-CA.crt')
@ -34,49 +32,6 @@ def common_user_agents():
return ans, list(sorted(ans, reverse=True, key=ans.__getitem__)) return ans, list(sorted(ans, reverse=True, key=ans.__getitem__))
def firefox_versions():
print('Getting firefox versions...')
import html5lib
raw = download_securely(
'https://www.mozilla.org/en-US/firefox/releases/').decode('utf-8')
root = html5lib.parse(raw, treebuilder='lxml', namespaceHTMLElements=False)
ol = root.xpath('//main[@id="main-content"]/ol')[0]
ol.xpath('descendant::li/strong/a[@href]')
ans = filter_ans(ol.xpath('descendant::li/strong/a[@href]/text()'))
if not ans:
raise ValueError('Failed to download list of firefox versions')
return ans
def chrome_versions():
print('Getting chrome versions...')
import html5lib
raw = download_securely(
'https://en.wikipedia.org/wiki/Google_Chrome_version_history').decode('utf-8')
root = html5lib.parse(raw, treebuilder='lxml', namespaceHTMLElements=False)
table = root.xpath('//*[@id="mw-content-text"]//tbody')[-1]
ans = []
for tr in table.iterchildren('tr'):
cells = tuple(tr.iterchildren('td'))
if not cells:
continue
if not cells[2].text or not cells[2].text.strip():
continue
s = cells[0].get('style')
if '#a0e75a' not in s and 'salmon' not in s:
break
chrome_version = cells[0].text.strip()
ts = datetime.strptime(cells[1].text.strip().split()[
0], '%Y-%m-%d').date().strftime('%Y-%m-%d')
try:
webkit_version = cells[2].text.strip().split()[1]
except IndexError:
continue
ans.append({'date': ts, 'chrome_version': chrome_version,
'webkit_version': webkit_version})
return list(reversed(ans))
def all_desktop_platforms(user_agents): def all_desktop_platforms(user_agents):
ans = set() ans = set()
for ua in user_agents: for ua in user_agents:
@ -92,8 +47,6 @@ def all_desktop_platforms(user_agents):
def get_data(): def get_data():
ua_freq_map, common = common_user_agents() ua_freq_map, common = common_user_agents()
ans = { ans = {
'chrome_versions': chrome_versions(),
'firefox_versions': firefox_versions(),
'common_user_agents': common, 'common_user_agents': common,
'user_agents_popularity': ua_freq_map, 'user_agents_popularity': ua_freq_map,
'timestamp': datetime.utcnow().isoformat() + '+00:00', 'timestamp': datetime.utcnow().isoformat() + '+00:00',

8
src/calibre/utils/random_ua.py
View File

@ -48,18 +48,10 @@ def user_agents_popularity_map():
return user_agent_data().get('user_agents_popularity', {}) return user_agent_data().get('user_agents_popularity', {})
def all_firefox_versions(limit=10):
return user_agent_data()['firefox_versions'][:limit]
def random_desktop_platform(): def random_desktop_platform():
return random.choice(user_agent_data()['desktop_platforms']) return random.choice(user_agent_data()['desktop_platforms'])
def all_chrome_versions(limit=10):
return user_agent_data()['chrome_versions'][:limit]
def accept_header_for_ua(ua): def accept_header_for_ua(ua):
# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Content_negotiation/List_of_default_Accept_values # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Content_negotiation/List_of_default_Accept_values
if 'Firefox/' in ua: if 'Firefox/' in ua: