Recover from yet another Amazon EU website change. This time they introduced a new table form with a very different structure. They switch between the form types on a seemingly random basis.

2025-07-09 03:04:10 -04:00 · 2013-07-12 10:01:42 +02:00 · 2013-07-12 10:01:42 +02:00 · 89f712dc53
commit 89f712dc53
parent 433dcea35a
5 changed files with 251 additions and 71 deletions
--- a/src/calibre/gui2/store/stores/amazon_de_plugin.py
+++ b/src/calibre/gui2/store/stores/amazon_de_plugin.py
@ -52,25 +52,61 @@ class AmazonDEKindleStore(StorePlugin):

    def search(self, query, max_results=10, timeout=60):
        url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+')
+        #print(url)
        br = browser()

        counter = max_results
        with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())#.decode('latin-1', 'replace'))
+            allText = f.read()
+            doc = html.fromstring(allText)#.decode('latin-1', 'replace'))
+
+            if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'):
+                #print('grid form')
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'):
+                #print('ilo form')
+                data_xpath = '//li[(@class="ilo")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                # Results can be in a grid (table) or a column
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'):
+                #print('list form')
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            else:
+                # URK -- whats this?
+                print('unknown result table form for Amazon EU search')
+                #with open("c:/amazon_search_results.html", "w") as out:
+                #    out.write(allText)
+                return

-            data_xpath = '//div[contains(@class, "prod")]'
-            # Results can be in a grid (table) or a column
-            format_xpath = (
-                    './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
-                    '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
-            asin_xpath = '@name'
-            cover_xpath = './/img[@class="productImage"]/@src'
-            title_xpath = './/h3[@class="newaps"]/a//text()'
-            author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
-            # Results can be in a grid (table) or a column
-            price_xpath = (
-                    './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
-                    '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')

            for data in doc.xpath(data_xpath):
                if counter <= 0:
@ -120,4 +156,3 @@ class AmazonDEKindleStore(StorePlugin):

    def get_details(self, search_result, timeout):
        pass
-
--- a/src/calibre/gui2/store/stores/amazon_es_plugin.py
+++ b/src/calibre/gui2/store/stores/amazon_es_plugin.py
@ -51,25 +51,61 @@ class AmazonESKindleStore(StorePlugin):

    def search(self, query, max_results=10, timeout=60):
        url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+')
+        #print(url)
        br = browser()

        counter = max_results
        with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())#.decode('latin-1', 'replace'))
+            allText = f.read()
+            doc = html.fromstring(allText)#.decode('latin-1', 'replace'))
+
+            if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'):
+                #print('grid form')
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'):
+                #print('ilo form')
+                data_xpath = '//li[(@class="ilo")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                # Results can be in a grid (table) or a column
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'):
+                #print('list form')
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            else:
+                # URK -- whats this?
+                print('unknown result table form for Amazon EU search')
+                #with open("c:/amazon_search_results.html", "w") as out:
+                #    out.write(allText)
+                return

-            data_xpath = '//div[contains(@class, "prod")]'
-            # Results can be in a grid (table) or a column
-            format_xpath = (
-                    './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
-                    '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
-            asin_xpath = '@name'
-            cover_xpath = './/img[@class="productImage"]/@src'
-            title_xpath = './/h3[@class="newaps"]/a//text()'
-            author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
-            # Results can be in a grid (table) or a column
-            price_xpath = (
-                    './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
-                    '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')

            for data in doc.xpath(data_xpath):
                if counter <= 0:
--- a/src/calibre/gui2/store/stores/amazon_fr_plugin.py
+++ b/src/calibre/gui2/store/stores/amazon_fr_plugin.py
@ -48,25 +48,61 @@ class AmazonFRKindleStore(StorePlugin):

    def search(self, query, max_results=10, timeout=60):
        url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+')
+        #print(url)
        br = browser()

        counter = max_results
        with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())#.decode('latin-1', 'replace'))
+            allText = f.read()
+            doc = html.fromstring(allText)#.decode('latin-1', 'replace'))
+
+            if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'):
+                #print('grid form')
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'):
+                #print('ilo form')
+                data_xpath = '//li[(@class="ilo")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                # Results can be in a grid (table) or a column
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'):
+                #print('list form')
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            else:
+                # URK -- whats this?
+                print('unknown result table form for Amazon EU search')
+                #with open("c:/amazon_search_results.html", "w") as out:
+                #    out.write(allText)
+                return

-            data_xpath = '//div[contains(@class, "prod")]'
-            # Results can be in a grid (table) or a column
-            format_xpath = (
-                    './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
-                    '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
-            asin_xpath = '@name'
-            cover_xpath = './/img[@class="productImage"]/@src'
-            title_xpath = './/h3[@class="newaps"]/a//text()'
-            author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
-            # Results can be in a grid (table) or a column
-            price_xpath = (
-                    './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
-                    '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')

            for data in doc.xpath(data_xpath):
                if counter <= 0:
--- a/src/calibre/gui2/store/stores/amazon_it_plugin.py
+++ b/src/calibre/gui2/store/stores/amazon_it_plugin.py
@ -51,25 +51,61 @@ class AmazonITKindleStore(StorePlugin):

    def search(self, query, max_results=10, timeout=60):
        url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+')
+        #print(url)
        br = browser()

        counter = max_results
        with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())#.decode('latin-1', 'replace'))
+            allText = f.read()
+            doc = html.fromstring(allText)#.decode('latin-1', 'replace'))
+
+            if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'):
+                #print('grid form')
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'):
+                #print('ilo form')
+                data_xpath = '//li[(@class="ilo")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                # Results can be in a grid (table) or a column
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'):
+                #print('list form')
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            else:
+                # URK -- whats this?
+                print('unknown result table form for Amazon EU search')
+                #with open("c:/amazon_search_results.html", "w") as out:
+                #    out.write(allText)
+                return

-            data_xpath = '//div[contains(@class, "prod")]'
-            # Results can be in a grid (table) or a column
-            format_xpath = (
-                    './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
-                    '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
-            asin_xpath = '@name'
-            cover_xpath = './/img[@class="productImage"]/@src'
-            title_xpath = './/h3[@class="newaps"]/a//text()'
-            author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
-            # Results can be in a grid (table) or a column
-            price_xpath = (
-                    './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
-                    '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')

            for data in doc.xpath(data_xpath):
                if counter <= 0:
@ -119,3 +155,4 @@ class AmazonITKindleStore(StorePlugin):

    def get_details(self, search_result, timeout):
        pass
+
--- a/src/calibre/gui2/store/stores/amazon_uk_plugin.py
+++ b/src/calibre/gui2/store/stores/amazon_uk_plugin.py
@ -55,25 +55,61 @@ class AmazonUKKindleStore(StorePlugin):

    def search(self, query, max_results=10, timeout=60):
        url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+')
+        #print(url)
        br = browser()

        counter = max_results
        with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())#.decode('latin-1', 'replace'))
+            allText = f.read()
+            doc = html.fromstring(allText)#.decode('latin-1', 'replace'))
+
+            if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'):
+                #print('grid form')
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'):
+                #print('ilo form')
+                data_xpath = '//li[(@class="ilo")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                # Results can be in a grid (table) or a column
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'):
+                #print('list form')
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            else:
+                # URK -- whats this?
+                print('unknown result table form for Amazon EU search')
+                #with open("c:/amazon_search_results.html", "w") as out:
+                #    out.write(allText)
+                return

-            data_xpath = '//div[contains(@class, "prod")]'
-            # Results can be in a grid (table) or a column
-            format_xpath = (
-                    './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
-                    '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
-            asin_xpath = '@name'
-            cover_xpath = './/img[@class="productImage"]/@src'
-            title_xpath = './/h3[@class="newaps"]/a//text()'
-            author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
-            # Results can be in a grid (table) or a column
-            price_xpath = (
-                    './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
-                    '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')

            for data in doc.xpath(data_xpath):
                if counter <= 0: