python/googlequery.py at master · dillonko/python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
import urllib
import mechanize
from bs4 import BeautifulSoup
import re


def googlesearch(links):
    br = mechanize.Browser()
    br.set_handle_robots(False)
    br.addheaders = [('user-agent','chrome')]

    term = links.replace(" ", "+")

    query = "http://www.google.com/search?num=100&q="+term

    htmltext = br.open(query).read()

    #print htmltext


    soup = BeautifulSoup(htmltext)
    search = soup.findAll('div',attrs={'id':'search'})
    soup1 = BeautifulSoup(str(search[0]))
    listitems =  soup1.findAll('li')

    #print listitems[0]

    regex = "q(?!.*q).*?&amp"
    pattern = re.compile(regex)

    result_list = []

    for li in listitems:
        souplinks = BeautifulSoup(str(li)).findAll('a')
        firstlink = souplinks[0]
        soupurl = re.findall(pattern, str(firstlink))
        if len(soupurl) >0:
            result_list.append(soupurl[0].replace('q=',"").replace("&amp",""))
    return result_list

print googlesearch("stock market")