Replace use of obsolete Qt WebKit in the google images metadata source plugin

src/calibre/ebooks/metadata/sources/google_images.py

@@ -9,9 +9,10 @@ __docformat__ = 'restructuredtext en'
 
 from collections import OrderedDict
 
-from calibre import as_unicode
 from calibre.ebooks.metadata.sources.base import Source, Option
 
+USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0'
+
 class GoogleImages(Source):
 
     name = 'Google Images'
@@ -19,6 +20,7 @@ class GoogleImages(Source):
     capabilities = frozenset(['cover'])
     config_help_message = _('Configure the Google Image Search plugin')
     can_get_multiple_covers = True
+    supports_gzip_transfer_encoding = True
     options = (Option('max_covers', 'number', 5, _('Maximum number of covers to get'),
                       _('The maximum number of covers to process from the google search result')),
                Option('size', 'choices', 'svga', _('Cover size'),
@@ -45,66 +47,38 @@ class GoogleImages(Source):
         urls = self.get_image_urls(title, author, log, abort, timeout)
         self.download_multiple_covers(title, authors, urls, get_best_cover, timeout, result_queue, abort, log)
 
+    @property
+    def user_agent(self):
+        return USER_AGENT
+
     def get_image_urls(self, title, author, log, abort, timeout):
-        from calibre.utils.ipc.simple_worker import fork_job, WorkerError
+        from calibre.utils.cleantext import clean_ascii_chars
-        try:
+        from urllib import quote_plus
-            return fork_job('calibre.ebooks.metadata.sources.google_images',
+        import html5lib
-                    'search', args=(title, author, self.prefs['size'], timeout), no_output=True, abort=abort, timeout=timeout)['result']
+        import json
-        except WorkerError as e:
+        from collections import OrderedDict
-            if e.orig_tb:
+        ans = OrderedDict()
-                log.error(e.orig_tb)
+        br = self.browser
-            log.exception('Searching google failed:' + as_unicode(e))
+        q = quote_plus('%s %s'%(title, author))
-        except Exception as e:
+        sz = self.prefs['size']
-            log.exception('Searching google failed:' + as_unicode(e))
+        if sz == 'any':
-
+            sz = ''
-        return []
+        elif sz == 'l':
-
+            sz = 'isz:l,'
-USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0'
+        else:
-
+            sz = 'isz:lt,islt:%s,' % sz
-def find_image_urls(br, ans):
+        url = 'https://www.google.com/search?as_st=y&tbm=isch&as_q={}&as_epq=&as_oq=&as_eq=&cr=&as_sitesearch=&safe=images&tbs={}iar:t,ift:jpg'.format(q, sz)
-    import urlparse
+        log('Search URL: ' + url)
-    for w in br.page.mainFrame().documentElement().findAll('div#ires a.rg_l[href]'):
+        raw = br.open(url).read()
-        try:
+        root = html5lib.parse(clean_ascii_chars(raw), treebuilder='lxml', namespaceHTMLElements=False)
-            imgurl = urlparse.parse_qs(urlparse.urlparse(unicode(w.attribute('href'))).query)['imgurl'][0]
+        for div in root.xpath('//div[@class="rg_meta"]'):
-        except:
+            try:
-            # import traceback
+                data = json.loads(div.text)
-            # traceback.print_exc()
+            except Exception:
-            continue
+                continue
-        if imgurl not in ans:
+            if 'ou' in data:
-            ans.append(imgurl)
+                ans[data['ou']] = True
-
+        return list(ans.iterkeys())
-def search(title, author, size, timeout, debug=False):
-    import time
-    from calibre.web.jsbrowser.browser import Browser, Timeout
-    ans = []
-    start_time = time.time()
-    br = Browser(user_agent=USER_AGENT, enable_developer_tools=debug, headless=not debug)
-    br.visit('https://www.google.com/advanced_image_search')
-    f = br.select_form('form[action="/search"]')
-    f['as_q'] = '%s %s'%(title, author)
-    if size != 'any':
-        f['imgsz'] = size
-    f['imgar'] = 't|xt'
-    f['as_filetype'] = 'jpg'
-    br.submit(wait_for_load=False)
-
-    # Loop until the page finishes loading or at least five image urls are
-    # found
-    while len(ans) < 5:
-        br.run_for_a_time(0.2)
-        find_image_urls(br, ans)
-        if time.time() - start_time > timeout:
-            raise Timeout('Timed out trying to load google image search page')
-    find_image_urls(br, ans)
-    if debug:
-        br.show_browser()
-    br.close()
-    del br  # Needed to prevent PyQt from segfaulting
-    return ans
-
-def test_google():
-    import pprint
-    pprint.pprint(search('heroes', 'abercrombie', 'svga', 60, debug=True))
 
 def test():
     from Queue import Queue
@@ -119,4 +93,3 @@ def test():
 
 if __name__ == '__main__':
     test()
-