Update login mechanism for Times Online

Fixes #1025 (Update login mechanism)
Fixes #1026 (Fix login mechanism)

recipes/sunday_times_magazine.recipe

@@ -1,13 +1,13 @@
 __license__ = 'GPL v3'
-__copyright__ = '2010-2013, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2010-2019'
 '''
 www.thetimes.co.uk/magazine/the-sunday-times-magazine/
 '''
+
+from mechanize import Request
+
+from calibre import random_user_agent
 from calibre.web.feeds.news import BasicNewsRecipe
-try:
-    from urllib.parse import urlencode
-except ImportError:
-    from urllib import urlencode
 
 
 def classes(classes):
@@ -31,8 +31,9 @@ class TimesOnline(BasicNewsRecipe):
     delay = 1
     needs_subscription = True
     publication_type = 'newspaper'
-    INDEX = 'http://www.thetimes.co.uk/'
+    INDEX = 'https://www.thetimes.co.uk'
-    PREFIX = u'http://www.thetimes.co.uk/'
+    LOGIN = 'https://login.thetimes.co.uk/'
+    PREFIX = u'https://www.thetimes.co.uk'
     extra_css = """
         .author-name,.authorName{font-style: italic}
         .published-date,.multi-position-photo-text{
@@ -48,16 +49,30 @@ class TimesOnline(BasicNewsRecipe):
         'publisher': publisher,
         'language': language}
 
-    def get_browser(self):
+    def get_browser(self, *a, **kw):
-        br = BasicNewsRecipe.get_browser(self)
+        start_url = self.INDEX
-        br.open('http://www.thetimes.co.uk/')
+        kw['user_agent'] = random_user_agent(allow_ie=False)
-        if self.username is not None and self.password is not None:
+        br = BasicNewsRecipe.get_browser(self, *a, **kw)
-            data = urlencode({
+        self.log('Starting login process...')
-                'gotoUrl': self.INDEX,
+        res = br.open(start_url)
-                'username': self.username,
+        sso_url = res.geturl()
-                'password': self.password})
+        self.log(sso_url)
-            br.open('https://login.thetimes.co.uk/', data)
+        request_query = {
+            'username': self.username,
+            'password': self.password,
+            's': 1,
+            'gotoUrl': self.INDEX,
+        }
+        rq = Request(self.LOGIN, headers={
+            'Accept': 'text/html',
+            'Accept-Language': 'en-US,en;q=0.8',
+            'X-HTTP-Method-Override': 'POST',
+            'X-Requested-With': 'XMLHttpRequest',
+        }, data=request_query)
+        self.log('Sending login request...')
+        res = br.open(rq)
         return br
+    # }}}
 
     def get_cover_url(self):
         from datetime import date

recipes/times_online.recipe

@@ -1,16 +1,15 @@
 __license__ = 'GPL v3'
-__copyright__ = '2010-2017, Bobby Steel <bob at xdca.com>, Darko Miletic'
+__copyright__ = '2010-2019, Bobby Steel <bob at xdca.com>, Darko Miletic'
 '''
 www.thetimes.co.uk
 '''
-import html5lib
+from mechanize import Request
-try:
+from calibre import random_user_agent
-    from urllib.parse import urlencode
-except ImportError:
-    from urllib import urlencode
-from lxml import html
 from calibre.web.feeds.news import BasicNewsRecipe
 
+import html5lib
+from lxml import html
+
 
 def classes(classes):
     q = frozenset(classes.split(' '))
@@ -35,6 +34,7 @@ class TimesOnline(BasicNewsRecipe):
     needs_subscription = True
     publication_type = 'newspaper'
     INDEX = 'http://www.thetimes.co.uk/'
+    LOGIN = 'https://login.thetimes.co.uk/'
     PREFIX = u'http://www.thetimes.co.uk'
     extra_css = """
         .author-name,.authorName{font-style: italic}
@@ -78,15 +78,28 @@ class TimesOnline(BasicNewsRecipe):
             br.open(cover)
         return cover
 
-    def get_browser(self):
+    def get_browser(self, *a, **kw):
-        br = BasicNewsRecipe.get_browser(self)
+        start_url = self.INDEX
-        br.open('http://www.thetimes.co.uk/')
+        kw['user_agent'] = random_user_agent(allow_ie=False)
-        if self.username is not None and self.password is not None:
+        br = BasicNewsRecipe.get_browser(self, *a, **kw)
-            data = urlencode({
+        self.log('Starting login process...')
-                'gotoUrl': self.INDEX,
+        res = br.open(start_url)
-                'username': self.username,
+        sso_url = res.geturl()
-                'password': self.password})
+        self.log(sso_url)
-            br.open('https://login.thetimes.co.uk/', data)
+        request_query = {
+            'username': self.username,
+            'password': self.password,
+            's': 1,
+            'gotoUrl': self.INDEX,
+        }
+        rq = Request(self.LOGIN, headers={
+            'Accept': 'text/html',
+            'Accept-Language': 'en-US,en;q=0.8',
+            'X-HTTP-Method-Override': 'POST',
+            'X-Requested-With': 'XMLHttpRequest',
+        }, data=request_query)
+        self.log('Sending login request...')
+        res = br.open(rq)
         return br
 
     remove_tags = [