diff --git a/.gitignore b/.gitignore index accd508..01afa93 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ xgoogle.egg-info .project .pydevproject .settings +*.output +output.* diff --git a/README.md b/README.md index 31e4ac8..745c721 100755 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ +xgoogle +======= + Python wrapper to Google Search service. +This is an command line search tool, which designed to fetch search results +from some famous search engine include Google, Baidu, Weibo, Qihoo, etc. + Provide a wrapper for the following services: * Google Search * Google Translate diff --git a/autosearch.conf b/autosearch.conf new file mode 100644 index 0000000..30aaa91 --- /dev/null +++ b/autosearch.conf @@ -0,0 +1,108 @@ +[google] +PAGE_MODE=1 +SEARCH_URL=http://www.google.com.hk/search?q=%(query)s&start=%(num)d + +SEARCH_URL_0=http://www.google.%(tld)s/search?hl=%(lang)s&q=%(query)s&btnG=Google+Search +NEXT_PAGE_0=http://www.google.%(tld)s/search?hl=%(lang)s&q=%(query)s&start=%(start)d +SEARCH_URL_1=http://www.google.%(tld)s/search?hl=%(lang)s&q=%(query)s&num=%(num)d&btnG=Google+Search +NEXT_PAGE_1=http://www.google.%(tld)s/search?hl=%(lang)s&q=%(query)s&num=%(num)d&start=%(start)d + +page_nums=10 + +total_tag=div +total_tag_filter={'id': 'resultStats'} + +result_tag=li +result_tag_filter={'class': 'g'} +title_tag=a +title_tag_filter={} +desc_tag=span +desc_tag_filter={'class': 'st'} + +[baidu] +PAGE_MODE=1 +SEARCH_URL=http://www.baidu.com/s?wd=%(query)s&pn=%(num)d&ie=utf-8 + +page_nums=10 + +total_tag=span +total_tag_filter={'class': 'nums'} + +result_tag=table +result_tag_filter={'class': 'result'} +title_tag=a +title_tag_filter={} +desc_tag=font +desc_tag_filter={} + +[qihoo] +PAGE_MODE=2 +SEARCH_URL=http://qihoo.com/wenda.php?kw=%(query)s&page=%(num)d + +page_nums=10 + +total_tag=em +total_tag_filter={'id': 'search-result'} + +result_tag=dl +result_tag_filter={} +title_tag=a +title_tag_filter={} +desc_tag=dd +desc_tag_filter={'class': 'content'} + +encoding=gb2312 + +[maopu] +PAGE_MODE=1 +SEARCH_URL=http://www.baidu.com/s?wd=site:mop.com+%(query)s&pn=%(num)d&ie=utf-8 + +page_nums=10 + +total_tag=span +total_tag_filter={'class': 'nums'} + +result_tag=table +result_tag_filter={'class': 'result'} +title_tag=a +title_tag_filter={} +desc_tag=font +desc_tag_filter={} + +[tianya] +PAGE_MODE=2 +SEARCH_URL=http://www.tianya.cn/search/bbs?q=%(query)s&pn=%(num)d + +page_nums=10 + +total_tag=em +total_tag_filter={} + +result_container_tag=div +result_container_tag_filter={'class': 'searchListOne'} +result_tag=li +result_tag_filter={} +title_tag=a +title_tag_filter={} +desc_tag=p +desc_tag_filter={} + +[weibo] + +[tq] +PAGE_MODE=2 +SEARCH_URL=http://search.t.qq.com/index.php?k=%(query)s&p=%(num)d + +page_nums=15 + +total_tag=span +total_tag_filter={'class': 'tabNum left'} + +result_container_tag=ul +result_container_tag_filter={'id': 'talkList'} +result_tag=li +result_tag_filter={} +title_tag=a +title_tag_filter={'ctype': '2'} +desc_tag=div +desc_tag_filter={'class': 'msgCnt'} diff --git a/autosearch.py b/autosearch.py new file mode 100644 index 0000000..d454344 --- /dev/null +++ b/autosearch.py @@ -0,0 +1,198 @@ +#!/bin/python +## coding=utf-8 ## +#-*- encoding: utf8 -*- + +### handle options +import sys, getopt, re, os + +KEYWORD="" +ENGINE=[] +INTERNAL=0 +NUM=10 +#PREFERENCE=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'autosearch.conf') +PREFERENCE='autosearch.conf' +FORMAT=['title', 'url', 'desc'] +WRITE="autosearch.output" +FILTER="" +CHART="" +SORT="" +VERBOSE=False + +ENGINES=['google', 'baidu', 'qihoo', 'maopu', 'tianya', 'weibo', 'tq'] +FORMATS=['title', 'url', 'desc'] +DATA=[] + +ENCODING = sys.getfilesystemencoding() + +def opt(): + global KEYWORD, ENGINE, INTERNAL, NUM, PREFERENCE, FORMAT, WRITE, FILTER, CHART, SORT, VERBOSE + global ENGINES, FORMATS + + blank = re.compile('\s') + opts, args = getopt.getopt(sys.argv[1:], "k:e:i:n:p:o:w:f:c:s:vh", + ["keyword=", "engine=", "internal=", "num=", "preference=", "format=", "write=", "filter=", "chart=", "sort=", "verbose", "help"]) + for op, value in opts: + if op == "-k" or op == "--keyword": + KEYWORD = value.strip() + elif op == "-e" or op == "--engine": + ENGINE = blank.sub('', value).split(',') + for e in ENGINE: + if not e in ENGINES: + error( "warning: %s is not supported, ignored" % e ) + ENGINE.remove(e) + else: + DATA.append([e, [[0,0]]]) + elif op == "-i" or op == "--internal": + INTERNAL = float(value) + elif op == "-n" or op == "--num": + NUM = int(value) + elif op == "-p" or op == "--preference": + PREFERENCE = value + elif op == "-o" or op == "--format": + FORMAT = blank.sub('', value).split(',') + for f in FORMAT: + if not f in FORMATS: + error( "warning: format %s is not supported, ignored" % f ) + FORMAT.remove(f) + elif op == "-w" or op == "--write": + WRITE = value + elif op == "-f" or op == "--filter": + FILTER = value + elif op == "-c" or op == "--chart": + CHART = value + elif op == "-s" or op == "--sort": + SORT = value + elif op == "-v" or op == "--verbose": + VERBOSE = True + elif op == "-h" or op == "--help": + usage() + sys.exit() + else: + error( "warning: option %s not recognized, ignored" % op ) + +def usage(): + print """autosearch is an automatically tools used in command line, the usage:""" + print """autosearch + -k,--keyword=keyword ---if have blank,put the keyword in "",like this "cars tree" + -e,--engine=google,baidu,qihoo,maopu,tianya,weibo,tq ---at least one of these search engines + -i,--internal=NUMBER ---seconds, default is 0 + -n,--num=NUMBER ---topmost search results, default is 10 + -p,--preference=FILE ---set preference file name, default is autosearch.conf + -o,--format=title,url,desc ---set output formate, default is title,url,desc + -w,--write=FILE ---set output file, default is autosearch.output + -f,--filter=STRING ---set filter string + -c,--chart ---got output chart + -s,--sort ---set sort type + -v,--verbose ---if set, print the output to screen also + -h,--help ---this help information""" + +def error(msg): + print 50*'*' + print "*** %s" % msg + print 50*'*' + usage() + sys.exit() + +def try_output(content): + #s = str.format( content ) + if VERBOSE: + print content + if WRITE: + with open(WRITE, 'a+') as f: + f.write(content.encode('utf-8')) + f.write("\n") + +import cairo +import pycha +import pycha.bar +def try_chart(): + global DATA + #print DATA + + if CHART=='': + return + + width, height = (500,400) + surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, width, height) + chart = pycha.bar.VerticalBarChart(surface) + chart.addDataset( DATA ) + chart.render() + surface.write_to_png('output.png') + +try: + opt() +except Exception, e: + error( "error: %s" % e ) + usage() + sys.exit() +else: + pass +finally: + pass + +if KEYWORD=='': + error( "error: no keyword to query, exit!!!" ) + +if len(ENGINE)==0: + error( "error: must assign at least 1 search engine!!!" ) + +if not os.path.exists(PREFERENCE): + error( "error: preference file %s not exists!!!" % PREFERENCE ) + +if WRITE: + with open(WRITE, 'w', ) as f: + pass + +### main logic +from xgoogle.GeneralSearch import GeneralSearch +import time +import string +import time + +start_time = 1 +while True: + for e in ENGINE: + gs=GeneralSearch(KEYWORD.decode(ENCODING), e, PREFERENCE) + results = gs.get_results() + print gs.num_results + #print gs._last_search_url + + #s1 = str.format( "%s: %d results of \"" % ( e.upper(), gs.num_results ) ) + #s2 = str.format( "\" --- %s" % ( time.strftime("%Y-%m-%d %X", time.localtime())) ) + #s = s1 + KEYWORD.decode(ENCODING) + s2 + #try_output( s ) + try_output( "%s: %d results of \"%s\" --- %s" % ( e.upper(), gs.num_results, KEYWORD.decode(ENCODING), time.strftime("%Y-%m-%d %X", time.localtime())) ) + try_output( 80*'-' ) + + index = ENGINE.index(e) + if DATA[index][0]==e: + #DATA[index][1].append([time.strftime("%Y-%m-%d %X", time.localtime()), gs.num_results]) + DATA[index][1].append([start_time, gs.num_results]) + + count=1 + while True: + for r in results: + if count>NUM: + break + try_output( "results[%d]: " % count ) + for k in FORMAT: + s = "r.%s" % k + c = eval( s ) + + try_output( "%s" % c ) + try_output( 80*'+' ) + count=count+1 + if count>NUM: + break + results = gs.get_results() + + del gs + + try_chart() + + if INTERNAL: + print "info: loop search every %d seconds, press CTRL+C to exit." % INTERNAL + time.sleep(string.atof(INTERNAL)) + start_time += INTERNAL + else: + sys.exit() diff --git a/autoweibo.py b/autoweibo.py new file mode 100644 index 0000000..671f3b1 --- /dev/null +++ b/autoweibo.py @@ -0,0 +1,89 @@ + # -*- coding: UTF-8 -*- + from weibopy.auth import OAuthHandler + from weibopy.api import API + import urllib, urllib2, cookielib, hashlib,threading,BeautifulSoup,webbrowser,getpass + import re, os, time, random，sys + + reload(sys) + sys.setdefaultencoding('utf-8') + + print "Python for weibo.com sender\n" + + + id=raw_input("input your weibo.com username:") + passwd=getpass.getpass("input your weibo.com password:") + + def log(s): + s = "\n" + str(s) + f = open('log.txt', 'a+') + f.write(s) + f.close() + sys.stdout.write(s) + sys.stdout.flush() + + + class WeiboCn: + all = ('python weibo.com sender') + url = 'http://www.weibo.com' + header = { + 'User-Agent' : 'Mozilla/5.0 (Windows NT 5.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', + } + def __init__(self, username, password, keyword = None, *args): + self.user = username + self.keyword = keyword + self.all = args or self.all + self.cj = cookielib.LWPCookieJar() + self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) + urllib2.install_opener(self.opener) + self.tryLogin(username, password) + + def tryLogin(self, username, password): + bodies = dict(_=int(time.time()),callback='sinaSSOController.preloginCallBack',client='ssologin.js(v1.3.12)',entry='miniblog',user=id) + print "pre-login,get servertime&nonce arg(secret password)" + preloadurl = 'http://login.sina.com.cn/sso/prelogin.php?' + urllib.urlencode(bodies) + content = self._request(preloadurl)[1].read() + bodies = eval(re.findall('\{.*?\}',content)[0]) + password = hashlib.sha1(hashlib.sha1(hashlib.sha1(password).hexdigest()).hexdigest() + str(bodies['servertime']) + bodies['nonce']).hexdigest() + print "Hash Password<%s>" % password + bodies.update(dict(client='ssologin.js(v1.3.12)',encoding='utf-8',entry='miniblog',gateway='1',password=password,pwencode='wsse',returntype='META',savestate='7',service='miniblog',ssosimplelogin='1',url='http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',username=username,useticket=1)) + response = self._request('http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.3.12)', bodies)[1] + content = response.read() + moreurl = re.findall('replace\([\'|"](.*?)[\'|"]\)', content) + if len(moreurl) == 0: print "Login Failed!" + content = self._request(moreurl[0], dict(Referer='http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.3.12)',Host='weibo.com'))[1].read() + if username in content: + print "Login Successed" + self.afterLogin() + + def afterLogin(self): + content = self._request('http://weibo.com/')[1].read() + #self.uid = re.findall('\$uid.*?"(\d+)"', content)[0] + #self.uid=[1265020392] + + + def _request(self, url, bodies = {}, headers = {}): + request = urllib2.Request(url, urllib.urlencode(bodies), headers = headers) + return (request, self.opener.open(request)) + + def _readMainPage(self): + return self._request(self.url)[1].read() + + def sinaapp(self,app_key,app_secret): + auth = OAuthHandler(app_key, app_secret) ##认证 + auth_url = auth.get_authorization_url() ##返回授权页面链接，用浏览器打开 + #webbrowser.open(auth_url) + content = self._request(auth_url)[1].read() + soup=BeautifulSoup.BeautifulSoup(''.join(content)) + #print content + pin=soup.span.string #自动获取pin码 + #print 'Please authorize: ' + auth_url ##输入获得的Pin码 + #verifier = raw_input('输入您在浏览器页面显示的PIN码: ').strip() + auth.get_access_token(pin) + api = API(auth) #整合函数 + connent=raw_input('What are you want to say?') + status = api.update_status(status=connent)#发布微博 + print "Send Successed" + raw_input('Press enter to exit ......') + + login=WeiboCn(id,passwd) + login.sinaapp('3837289606','14416dc6caadc1b14c4b0b09f58cc459') \ No newline at end of file diff --git a/autoweibo2.py b/autoweibo2.py new file mode 100644 index 0000000..63441ac --- /dev/null +++ b/autoweibo2.py @@ -0,0 +1,94 @@ +#! /usr/bin/env python +#coding=utf8 +import urllib +import urllib2 +import cookielib +import base64 +import re +import json +import hashlib + +cj = cookielib.LWPCookieJar() +cookie_support = urllib2.HTTPCookieProcessor(cj) +opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler) +urllib2.install_opener(opener) +postdata = { + 'entry': 'weibo', + 'gateway': '1', + 'from': '', + 'savestate': '7', + 'userticket': '1', + 'ssosimplelogin': '1', + 'vsnf': '1', + 'vsnval': '', + 'su': '', + 'service': 'miniblog', + 'servertime': '', + 'nonce': '', + 'pwencode': 'wsse', + 'sp': '', + 'encoding': 'UTF-8', + 'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack', + 'returntype': 'META' +} + +def get_servertime(): + url = 'http://login.sina.com.cn/sso/prelogin.php?entry=weibo&callback=sinaSSOController.preloginCallBack&su=dW5kZWZpbmVk&client=ssologin.js(v1.3.18)&_=1329806375939' + data = urllib2.urlopen(url).read() + p = re.compile('\((.*)\)') + try: + json_data = p.search(data).group(1) + data = json.loads(json_data) + servertime = str(data['servertime']) + nonce = data['nonce'] + return servertime, nonce + except: + print 'Get severtime error!' + return None + +def get_pwd(pwd, servertime, nonce): + pwd1 = hashlib.sha1(pwd).hexdigest() + pwd2 = hashlib.sha1(pwd1).hexdigest() + pwd3_ = pwd2 + servertime + nonce + pwd3 = hashlib.sha1(pwd3_).hexdigest() + return pwd3 + +def get_user(username): + username_ = urllib.quote(username) + username = base64.encodestring(username_)[:-1] + return username + + +def login(): + username = 'caozhzh_qf@126.com' + pwd = 'da42busay' + url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.3.18)' + try: + servertime, nonce = get_servertime() + except: + return + global postdata + postdata['servertime'] = servertime + postdata['nonce'] = nonce + postdata['su'] = get_user(username) + postdata['sp'] = get_pwd(pwd, servertime, nonce) + postdata = urllib.urlencode(postdata) + headers = {'User-Agent':'Mozilla/5.0 (X11; Linux i686; rv:8.0) Gecko/20100101 Firefox/8.0'} + req = urllib2.Request( + url = url, + data = postdata, + headers = headers + ) + result = urllib2.urlopen(req) + text = result.read() + print text + p = re.compile('location\.replace\(\'(.*?)\'\)') + try: + login_url = p.search(text).group(1) + #print login_url + urllib2.urlopen(login_url) + print "登录成功!" + except: + print 'Login error!' + +login() \ No newline at end of file diff --git a/autoweibo3.py b/autoweibo3.py new file mode 100644 index 0000000..9a38dda --- /dev/null +++ b/autoweibo3.py @@ -0,0 +1,63 @@ +import re +import json +import urllib +import base64 +import hashlib +import requests + + +WBCLIENT = 'ssologin.js(v.1.3.18)' +sha1 = lambda x: hashlib.sha1(x).hexdigest() + + +def wblogin(username, password): + session = requests.session( + headers={ + 'User-Agent': 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHT' + 'ML, like Gecko) Chrome/21.0.1180.89 Safari/537.1' + } + ) + resp = session.get( + 'http://login.sina.com.cn/sso/prelogin.php?entry=weibo&callback=sina' + 'SSOController.preloginCallBack&su=%s&client=%s' % + (base64.b64encode(username), WBCLIENT) + ) + pre_login_str = re.match(r'[^{]+({.+?})', resp.content).group(1) + pre_login_json = json.loads(pre_login_str) + data = { + 'entry': 'weibo', + 'gateway': 1, + 'from': '', + 'savestate': 7, + 'useticket': 1, + 'ssosimplelogin': 1, + 'su': base64.b64encode(urllib.quote(username)), + 'service': 'miniblog', + 'servertime': pre_login_json['servertime'], + 'nonce': pre_login_json['nonce'], + 'pcid': pre_login_json['pcid'], + 'vsnf': 1, + 'vsnval': '', + 'pwencode': 'wsse', + 'sp': sha1(sha1(sha1(password)) + + str(pre_login_json['servertime']) + + pre_login_json['nonce']), + 'encoding': 'UTF-8', + 'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.si' + 'naSSOController.feedBackUrlCallBack', + 'returntype': 'META' + } + resp = session.post( + 'http://login.sina.com.cn/sso/login.php?client=%s' % WBCLIENT, + data=data + ) + login_url = re.search(r'replace\([\"\']([^\'\"]+)[\"\']', + resp.content).group(1) + resp = session.get(login_url) + login_str = re.match(r'[^{]+({.+?}})', resp.content).group(1) + return json.loads(login_str) + + +if __name__ == '__main__': + from pprint import pprint + pprint(wblogin('XXXX@gmail.com', 'XXXX')) \ No newline at end of file diff --git a/png.py b/png.py new file mode 100644 index 0000000..90be2de --- /dev/null +++ b/png.py @@ -0,0 +1,26 @@ +import cairo +width, height = (500,400) +surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, width, height) +dataSet1=[ + ['dataSet 1', [[0,1], [1,3], [2,2.5]]], + ['dataSet 2', [[0,2], [1,4], [2,3]]], + ['dataSet 3', [[0,5], [1,1], [2,0.5]]], +] +dataSet2=[ + ['dataSet 1', [[0,1], [1,3], [2,2.5]]], + ['dataSet 2', [[0,2], [1,4], [2,3]]], + ['dataSet 3', [[0,5], [1,1], [2,0.5]]], +] +dataSet=[ + ['dataSet 1', [[0,1], [1,3], [2,2.5]]], + ['dataSet 2', [[0,2], [1,4], [2,3]]], + ['dataSet 3', [[0,5], [1,1], [2,0.5]]], +] + +import pycha.bar +chart = pycha.bar.VerticalBarChart(surface) +chart.addDataset( dataSet ) +chart.render() + +surface.write_to_png('output.png') + diff --git a/setup.py b/setup.py index 3bb276f..5c1bd45 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages import sys -__version__ = '1.4' +__version__ = '1.3.0.1000.2' import os def _read(fname): @@ -10,8 +10,8 @@ def _read(fname): setup( name='xgoogle', version=__version__, - description="Python library to Google services (Google Search, Google Images, Google Videos, Google Translate, Google Real-Time)", - long_description=_read('README.md'), + description="Python library to Google services (google search, google sets, google translate, sponsored links, google realtime)", + long_description=_read('readme.txt'), classifiers=[], keywords='google search', author='Peteris Krumins', @@ -25,7 +25,6 @@ def _read(fname): include_package_data=True, zip_safe=False, install_requires=[ - # -*- Extra requirements: -*- - 'nltk==2.0.4' + # -*- Extra requirements: -*- ], ) diff --git a/xgoogle/GeneralSearch.py b/xgoogle/GeneralSearch.py new file mode 100755 index 0000000..17d356b --- /dev/null +++ b/xgoogle/GeneralSearch.py @@ -0,0 +1,385 @@ +#!/usr/bin/python +# encoding: utf-8 +# +# Peteris Krumins (peter@catonmat.net) +# http://www.catonmat.net -- good coders code, great reuse +# +# http://www.catonmat.net/blog/python-library-for-google-search/ +# +# Code is licensed under MIT license. +# +# caozhzh@gmail.com +# try make general search for multi search engine +# + +import re +import urllib +from htmlentitydefs import name2codepoint +from BeautifulSoup import BeautifulSoup + +### ini handler +import ConfigParser +import string, sys + +import ast + +from browser import Browser, BrowserError + +class GeneralSearchError(Exception): + """ + Base class for General Search exceptions. + """ + pass + +class GeneralParseError(GeneralSearchError): + """ + Parse error in General results. + self.msg attribute contains explanation why parsing failed + self.tag attribute contains BeautifulSoup object with the most relevant tag that failed to parse + Thrown only in debug mode + """ + + def __init__(self, msg, tag): + self.msg = msg + self.tag = tag + + def __str__(self): + return self.msg + + def html(self): + return self.tag.prettify() + +class GeneralSearchResult: + def __init__(self, title, url, desc): + self.title = title + self.url = url + self.desc = desc + + def __str__(self): + return 'General Search Result: "%s"' % self.title + +class GeneralSearch(object): + def __init__(self, query, engine="google", conf="", random_agent=True, debug=False, lang="en", tld="com.hk", re_search_strings=None): + + # read ini + self.cf = ConfigParser.RawConfigParser() + self.conf = conf + self.cf.read( self.conf ) + + self.query = query + self.debug = debug + self.engine = engine + self.browser = Browser(debug=debug) + self.results_info = None + self.eor = False # end of results + self._page = 0 + self._first_indexed_in_previous = None + self._filetype = None + self._last_search_url = None + self._results_per_page = self.cf.getint(self.engine, "page_nums") + self._last_from = 0 + self._lang = lang + self._tld = tld + self._encoding = 'utf8' + + try: + self._encoding = self.cf.get(self.engine, "encoding") + except Exception, e: + pass + else: + pass + finally: + pass + + #print self._encoding + self.query = self.query.encode(self._encoding, 'ignore') + + if re_search_strings: + self._re_search_strings = re_search_strings + elif lang == "de": + self._re_search_strings = ("Ergebnisse", "von", u"ungefähr") + elif lang == "es": + self._re_search_strings = ("Resultados", "de", "aproximadamente") + elif lang == "fr": + self._re_search_strings = ("résultats", "de", "Environ") + # add more localised versions here + else: + self._re_search_strings = ("Results", "of", "about") + + if random_agent: + self.browser.set_random_user_agent() + + @property + def num_results(self): + if not self.results_info: + page = self._general_get_results_page() + total = self._general_extract_total(page) + if total == 0: + self.eor = True + return self.results_info['total'] + + @property + def last_search_url(self): + return self._last_search_url + + def _get_page(self): + return self._page + + def _set_page(self, page): + self._page = page + + page = property(_get_page, _set_page) + + def _get_first_indexed_in_previous(self): + return self._first_indexed_in_previous + + def _set_first_indexed_in_previous(self, interval): + if interval == "day": + self._first_indexed_in_previous = 'd' + elif interval == "week": + self._first_indexed_in_previous = 'w' + elif interval == "month": + self._first_indexed_in_previous = 'm' + elif interval == "year": + self._first_indexed_in_previous = 'y' + else: + # a floating point value is a number of months + try: + num = float(interval) + except ValueError: + raise GeneralSearchError, "Wrong parameter to first_indexed_in_previous: %s" % (str(interval)) + self._first_indexed_in_previous = 'm' + str(interval) + + first_indexed_in_previous = property(_get_first_indexed_in_previous, _set_first_indexed_in_previous, doc="possible values: day, week, month, year, or a float value of months") + + def _get_filetype(self): + return self._filetype + + def _set_filetype(self, filetype): + self._filetype = filetype + + filetype = property(_get_filetype, _set_filetype, doc="file extension to search for") + + def _get_results_per_page(self): + return self._results_per_page + + def _set_results_par_page(self, rpp): + self._results_per_page = rpp + + results_per_page = property(_get_results_per_page, _set_results_par_page) + + def get_results(self): + """ Gets a page of results """ + if self.eor: + return [] + MAX_VALUE = 1000000 + page = self._general_get_results_page() + + total = self._general_extract_total(page) + results = self._general_extract_results(page) + + page.decompose() + + search_info = {'from': self.results_per_page*self._page+1, + 'to': self.results_per_page*self._page + len(results), + 'total': total} + if not self.results_info: + self.results_info = search_info + if total == 0: + self.eor = True + return [] + if not results: + self.eor = True + return [] + if self._page > 0 and search_info['from'] == self._last_from: + self.eor = True + return [] + if search_info['to'] == search_info['total']: + self.eor = True + self._page += 1 + self._last_from = search_info['from'] + return results + + def _maybe_raise(self, cls, *arg): + if self.debug: + raise cls(*arg) + + # not used, but some code should be referenced in future + def _get_results_page_google(self): + if self._page == 0: + if self._results_per_page == 10: + url = self.cf.get(self.engine, "SEARCH_URL_0") + else: + url = self.cf.get(self.engine, "SEARCH_URL_1") + else: + if self._results_per_page == 10: + url = self.cf.get(self.engine, "NEXT_PAGE_0") + else: + url = self.cf.get(self.engine, "NEXT_PAGE_1") + + safe_url = [url % { 'query': urllib.quote_plus(self.query), + 'start': self._page * self._results_per_page, + 'num': self._results_per_page, + 'tld' : self._tld, + 'lang' : self._lang }] + + # possibly extend url with optional properties + if self._first_indexed_in_previous: + safe_url.extend(["&as_qdr=", self._first_indexed_in_previous]) + if self._filetype: + safe_url.extend(["&as_filetype=", self._filetype]) + + safe_url = "".join(safe_url) + self._last_search_url = safe_url + + try: + page = self.browser.get_page(safe_url) + except BrowserError, e: + raise GeneralSearchError, "Failed getting %s: %s" % (e.url, e.error) + return BeautifulSoup(page) + + def _general_get_results_page(self): + url = self.cf.get(self.engine, "SEARCH_URL") + page_mode = self.cf.getint(self.engine, "PAGE_MODE") + + if page_mode==1: + safe_url = [url % { 'query': urllib.quote_plus(self.query), + 'num': self._page * self._results_per_page }] + elif page_mode==2: + safe_url = [url % { 'query': urllib.quote_plus(self.query), + 'num': self._page + 1 }] + else: + safe_url = [url % { 'query': urllib.quote_plus(self.query), + 'num': self._page + 1 }] + + safe_url = "".join(safe_url) + self._last_search_url = safe_url + + print safe_url + + try: + page = self.browser.get_page(safe_url).decode(self._encoding, 'ignore') + except BrowserError, e: + raise GeneralSearchError, "Failed getting %s: %s" % (e.url, e.error) + return BeautifulSoup(page) + + def _general_extract_total(self, soup): + total_tag = self.cf.get(self.engine, "total_tag") + total_tag_filter = self.cf.get(self.engine, "total_tag_filter") + total_tag_filter = ast.literal_eval(total_tag_filter) + + div_ssb = soup.find(total_tag, total_tag_filter) + if not div_ssb: + self._maybe_raise(GeneralParseError, "Span with class:num of results was not found on Baidu search page", soup) + return 0 + p = div_ssb + txt = ''.join(p.findAll(text=True)) + txt = txt.replace(',', '') + txt = txt.replace(' ', '') + #matches = re.search(r'(\d+) - (\d+) %s (?:%s )?(\d+)' % self._re_search_strings, txt, re.U) + #matches = re.search(r'(\d+) %s' % self._re_search_strings[0], txt, re.U|re.I) + matches = re.search(r'(\d+)', txt, re.U) + + if not matches: + print self._re_search_strings[0] + print txt + return 0 + return int(matches.group(1)) + + def _general_extract_results(self, soup): + result_container_tag = '' + result_container_tag_filter = {} + try: + result_container_tag = self.cf.get(self.engine, "result_container_tag") + result_container_tag_filter = self.cf.get(self.engine, "result_container_tag_filter") + result_container_tag_filter = ast.literal_eval(result_container_tag_filter) + except Exception, e: + pass + else: + pass + finally: + pass + + result_tag = self.cf.get(self.engine, "result_tag") + result_tag_filter = self.cf.get(self.engine, "result_tag_filter") + result_tag_filter = ast.literal_eval(result_tag_filter) + + title_tag = self.cf.get(self.engine, "title_tag") + title_tag_filter = self.cf.get(self.engine, "title_tag_filter") + title_tag_filter = ast.literal_eval(title_tag_filter) + + desc_tag = self.cf.get(self.engine, "desc_tag") + desc_tag_filter = self.cf.get(self.engine, "desc_tag_filter") + desc_tag_filter = ast.literal_eval(desc_tag_filter) + + if result_container_tag!='': + soup = soup.find(result_container_tag, result_container_tag_filter) + + results = soup.findAll(result_tag, result_tag_filter) + ret_res = [] + for result in results: + title_a = result.find(title_tag, title_tag_filter) + if not title_a: + self._maybe_raise(GeneralParseError, "Title tag in Google search result was not found", result) + continue + + title = ''.join(title_a.findAll(text=True)) + title = self._html_unescape(title) + url = title_a['href'] + match = re.match(r'/url\?q=(http[^&]+)&', url) + if match: + url = urllib.unquote(match.group(1)) + + desc_div = result.find(desc_tag, desc_tag_filter) + if not desc_div: + self._maybe_raise(GeneralParseError, "Description tag in Google search result was not found", result) + continue + + desc_strs = [] + def looper(tag): + if not tag: return + for t in tag: + try: + if t.name == 'br': break + except AttributeError: + pass + + try: + desc_strs.append(t.string) + except AttributeError: + desc_strs.append(t) + + looper(desc_div) + looper(desc_div.find('wbr')) # BeautifulSoup does not self-close + + desc = ''.join(s for s in desc_strs if s) + desc = self._html_unescape(desc) + + if not title or not url or not desc: + eres = None + else: + eres = GeneralSearchResult(title, url, desc) + + if eres: + ret_res.append(eres) + return ret_res + + def _html_unescape(self, str): + def entity_replacer(m): + entity = m.group(1) + if entity in name2codepoint: + return unichr(name2codepoint[entity]) + else: + return m.group(0) + + def ascii_replacer(m): + cp = int(m.group(1)) + if cp <= 255: + return unichr(cp) + else: + return m.group(0) + + s = re.sub(r'&#(\d+);', ascii_replacer, str, re.U) + return re.sub(r'&([^;]+);', entity_replacer, s, re.U) + +#class GoogleSearch(GeneralSearch): \ No newline at end of file diff --git a/xgoogle/browser.py b/xgoogle/browser.py index 290343d..650d6e4 100755 --- a/xgoogle/browser.py +++ b/xgoogle/browser.py @@ -1,5 +1,4 @@ #!/usr/bin/python -# -*- coding: utf8 -*- # # Peteris Krumins (peter@catonmat.net) # http://www.catonmat.net -- good coders code, great reuse @@ -22,7 +21,6 @@ # sort -rn | # head -20 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0) Gecko/20100101 Firefox/4.0', - 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6 (.NET CLR 3.5.30729)', @@ -35,6 +33,7 @@ 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.5) Gecko/2008121621 Ubuntu/8.04 (hardy) Firefox/3.0.5', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)', + 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)' ) @@ -74,8 +73,6 @@ def http_open(self, req): return self.do_open(PoolHTTPConnection, req) class Browser(object): - """Provide a simulated browser object. - """ def __init__(self, user_agent=BROWSERS[0], debug=False, use_pool=False): self.headers = { 'User-Agent': user_agent, @@ -106,3 +103,4 @@ def get_page(self, url, data=None): def set_random_user_agent(self): self.headers['User-Agent'] = random.choice(BROWSERS) return self.headers['User-Agent'] + diff --git a/xgoogle/search.py b/xgoogle/search.py index 9ad3839..0193590 100755 --- a/xgoogle/search.py +++ b/xgoogle/search.py @@ -644,6 +644,8 @@ def __init__(self, query, random_agent=False, debug=False, lang="en", tld="com", self._re_search_strings = ("Ergebnisse", "von", u"ungefähr") elif lang == "es": self._re_search_strings = ("Resultados", "de", "aproximadamente") + elif lang == "fr": + self._re_search_strings = ("résultats", "de", "Environ") # add more localised versions here else: self._re_search_strings = ("Results", "of", "about") @@ -772,25 +774,31 @@ def _get_results_page(self): page = self.browser.get_page(safe_url) except BrowserError, e: raise SearchError, "Failed getting %s: %s" % (e.url, e.error) - return BeautifulSoup(page) def _extract_info(self, soup): empty_info = {'from': 0, 'to': 0, 'total': 0} - div_ssb = soup.find('div', id='ssb') + div_ssb = soup.find('div', id='resultStats') if not div_ssb: self._maybe_raise(ParseError, "Div with number of results was not found on Google search page", soup) return empty_info - p = div_ssb.find('p') + #p = div_ssb.find('p') + p = div_ssb if not p: self._maybe_raise(ParseError, """

tag within

was not found on Google search page""", soup) return empty_info txt = ''.join(p.findAll(text=True)) txt = txt.replace(',', '') - matches = re.search(r'%s (\d+) - (\d+) %s (?:%s )?(\d+)' % self._re_search_strings, txt, re.U) + txt = txt.replace(' ', '') + #matches = re.search(r'(\d+) - (\d+) %s (?:%s )?(\d+)' % self._re_search_strings, txt, re.U) + #matches = re.search(r'(\d+) %s' % self._re_search_strings[0], txt, re.U|re.I) + matches = re.search(r'(\d+)', txt, re.U) + if not matches: + print self._re_search_strings[0] + print txt return empty_info - return {'from': int(matches.group(1)), 'to': int(matches.group(2)), 'total': int(matches.group(3))} + return {'from': 0, 'to': 0, 'total': int(matches.group(1))} def _extract_results(self, soup): # Should extract