mirror of
				https://github.com/searxng/searxng.git
				synced 2025-11-04 03:27:06 -05:00 
			
		
		
		
	update generalfile engine and add comments
This commit is contained in:
		
							parent
							
								
									80b9312e42
								
							
						
					
					
						commit
						c5d83059d5
					
				@ -1,35 +1,60 @@
 | 
				
			|||||||
 | 
					## General Files (Files)
 | 
				
			||||||
 | 
					# 
 | 
				
			||||||
 | 
					# @website     http://www.general-files.org
 | 
				
			||||||
 | 
					# @provide-api no (nothing found)
 | 
				
			||||||
 | 
					# 
 | 
				
			||||||
 | 
					# @using-api   no (because nothing found)
 | 
				
			||||||
 | 
					# @results     HTML (using search portal)
 | 
				
			||||||
 | 
					# @stable      no (HTML can change)
 | 
				
			||||||
 | 
					# @parse       url, title, content
 | 
				
			||||||
 | 
					#
 | 
				
			||||||
 | 
					# @todo        detect torrents?
 | 
				
			||||||
 | 
					
 | 
				
			||||||
from lxml import html
 | 
					from lxml import html
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# engine dependent config
 | 
				
			||||||
 | 
					categories = ['files']
 | 
				
			||||||
 | 
					paging = True
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# search-url
 | 
				
			||||||
base_url = 'http://www.general-file.com'
 | 
					base_url = 'http://www.general-file.com'
 | 
				
			||||||
search_url = base_url + '/files-{letter}/{query}/{pageno}'
 | 
					search_url = base_url + '/files-{letter}/{query}/{pageno}'
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# specific xpath variables
 | 
				
			||||||
result_xpath = '//table[@class="block-file"]'
 | 
					result_xpath = '//table[@class="block-file"]'
 | 
				
			||||||
title_xpath = './/h2/a//text()'
 | 
					title_xpath = './/h2/a//text()'
 | 
				
			||||||
url_xpath = './/h2/a/@href'
 | 
					url_xpath = './/h2/a/@href'
 | 
				
			||||||
content_xpath = './/p//text()'
 | 
					content_xpath = './/p//text()'
 | 
				
			||||||
 | 
					
 | 
				
			||||||
paging = True
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# do search-request
 | 
				
			||||||
def request(query, params):
 | 
					def request(query, params):
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    params['url'] = search_url.format(query=query,
 | 
					    params['url'] = search_url.format(query=query,
 | 
				
			||||||
                                      letter=query[0],
 | 
					                                      letter=query[0],
 | 
				
			||||||
                                      pageno=params['pageno'])
 | 
					                                      pageno=params['pageno'])
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    return params
 | 
					    return params
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# get response from search-request
 | 
				
			||||||
def response(resp):
 | 
					def response(resp):
 | 
				
			||||||
 | 
					 | 
				
			||||||
    results = []
 | 
					    results = []
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    dom = html.fromstring(resp.text)
 | 
					    dom = html.fromstring(resp.text)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    # parse results
 | 
				
			||||||
    for result in dom.xpath(result_xpath):
 | 
					    for result in dom.xpath(result_xpath):
 | 
				
			||||||
        url = result.xpath(url_xpath)[0]
 | 
					        url = result.xpath(url_xpath)[0]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        # skip fast download links
 | 
					        # skip fast download links
 | 
				
			||||||
        if not url.startswith('/'):
 | 
					        if not url.startswith('/'):
 | 
				
			||||||
            continue
 | 
					            continue
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        # append result
 | 
				
			||||||
        results.append({'url': base_url + url,
 | 
					        results.append({'url': base_url + url,
 | 
				
			||||||
                        'title': ''.join(result.xpath(title_xpath)),
 | 
					                        'title': ''.join(result.xpath(title_xpath)),
 | 
				
			||||||
                        'content': ''.join(result.xpath(content_xpath))})
 | 
					                        'content': ''.join(result.xpath(content_xpath))})
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    # return results
 | 
				
			||||||
    return results
 | 
					    return results
 | 
				
			||||||
 | 
				
			|||||||
@ -62,7 +62,6 @@ engines:
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
  - name : general-file
 | 
					  - name : general-file
 | 
				
			||||||
    engine : generalfile
 | 
					    engine : generalfile
 | 
				
			||||||
    categories : files
 | 
					 | 
				
			||||||
    shortcut : gf
 | 
					    shortcut : gf
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  - name : github
 | 
					  - name : github
 | 
				
			||||||
 | 
				
			|||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user