Web_Crawler爬top100电影信息

 1 import requests 
 2 import pymongo
 3 import re
 4 from requests.exceptions import RequestException
 5 from multiprocessing import Pool

 1 
 2 def get_one_page(url):
 3    try:
 4        res = requests.get(url,headers = headers)
 5        if res.status_code == 200:
 6            return res.text
 7        return None
 8    except RequestException:
 9        return None
10 
11 # Building the page compiler
12 def parse_one_page(html):
13         pattern = re.compile('<dd>.*?board-index.*?>(d+)</i>.*?data-src="(.*?)".*?name"><a'
14                              + '.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
15                              + '.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>', re.S)
16         items = re.findall(pattern, html)
17         for item in items:
18             yield {
19                 'index': item[0],
20                 'image': item[1],
21                 'title': item[2],
22                 'actor': item[3].strip()[3:],
23                 'time': item[4].strip()[5:],
24                 'score': item[5] + item[6]
25             }
26 
27 # Stoing the data from pages (Into mongoDB)
28 def write_to_mongoDB(item):
29     #content.insert_one({'index': item[0], 'image': item[1], 'title': item[2], 'actor': item[3].strip()[3:], 'time':item[4].strip()[5:],'score':item[5] + item[6]})
30     content.insert_one(item)
31 
32 # Main function
33 def main(offset):
34     url = 'http://maoyan.com/board/4?offset=' + str(offset)
35     html = get_one_page(url)
36     for item in parse_one_page(html):
37         print(item)
38         write_to_mongoDB(item)
39 
40 
41 if __name__ == '__main__':
42     p = Pool()
43     p.map(main,[i*10 for i in range(10)])
44