爬取猫眼电影排行榜

1.爬取首页源代码

• 爬取首页的基本网页源代码
• 发现不同页码的网址规律

import requests

def get_url(url):
    header = {
        'User-Agent': 'XXXX' #换取自己的
    }
    response = requests.get(url, headers=header)
    content = response.text
    if response.status_code == 200:
        return content
    return None

def main():
    # 1.爬取首页
    for i in range(0, 1):
        url = "https://maoyan.com/board/4?offset={}".format(i * 10)
        html = get_url(url)
        print(html)


if __name__ == "__main__":
    main()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

2.用正则筛选网页信息

2.1通过单个正则表达式的筛选，输出，判断正则表达式的正确性

• 电影排名

ranks = re.findall('<dd>.*?(\d+)</i>',content,re.S)
print(ranks)
1
2

• 电影名字

file_names = re.findall('<p class="name">.*?<a href.*?title="(.*?)"',content,re.S)
print(file_names)
1
2

• 电影主演明星

file_stars = re.findall('<p class="star">(.*?)</p>',content,re.S)
print(file_stars)  
1
2

• 电影上映时间

file_releasetimes = re.findall('releasetime.*?(\d+.*?)</p>',content,re.S)
print(file_releasetimes)
1
2

• 电影评分

file_scores_integer = re.findall('<p.*?score.*?integer.*?(\d.)</i>',content,re.S)
    file_scores_fraction = re.findall('<i.*?fraction.*?(\d)</i>',content,re.S)
#print(type(file_scores_fraction[0]))
print(file_scores_integer)
print(file_scores_fraction)
1
2
3
4
5

2.2一起整合，编写函数

• items 返回由元组组成的列表
• 元组里面是每个电影的信息

def parse_html(html):
    pattern = re.compile('<dd>.*?(\d+)</i>.*?<p class="name">.*?<a href.*?title="(.*?)".*?<p class="star">(.*?)</p>.*?releasetime.*?(\d+.*?)</p>.*?<p.*?score.*?integer.*?(\d.)</i>.*?<i.*?fraction.*?(\d)</i>',re.S)
    items = re.findall(pattern,html)
    # for item in items:
    #     print(type(item[1]))
    return items
1
2
3
4
5
6

3.写进文本中

def write_cvs(items):
    for item in items:
        with open('mao.csv','a+',encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow([item[0],item[1],item[2].strip(),item[3],str(item[4])+str(item[5])])
1
2
3
4
5

4.全部代码

4.1 单一代码

import requests
import re
import csv
# 获取 HTML 源码
header = {
    'User-Agent' : 'xxx'
}
for i in range (0,100):
    url = "https://maoyan.com/board/4?offset={}".format(i*10)
    response = requests.get(url,headers=header)
    content = response.text
# pattern = re.compile('<dd>.*?(\d)</i>')
# findall() 方法用compile 对象会出现错误
    ranks = re.findall('<dd>.*?(\d+)</i>',content,re.S)
# print(ranks)
    file_names = re.findall('<p class="name">.*?<a href.*?title="(.*?)"',content,re.S)
# print(file_names)
    file_stars = re.findall('<p class="star">(.*?)</p>',content,re.S)

# print(file_stars)

    file_releasetimes = re.findall('releasetime.*?(\d+.*?)</p>',content,re.S)
# print(file_releasetimes)
    file_scores_integer = re.findall('<p.*?score.*?integer.*?(\d.)</i>',content,re.S)
    file_scores_fraction = re.findall('<i.*?fraction.*?(\d)</i>',content,re.S)
#print(type(file_scores_fraction[0]))

# print(file_scores_integer)
# print(file_scores_fraction)
    i = 0
    for rank in ranks:

        file_score = file_scores_integer[i] + file_scores_fraction[i]
       # print(ranks[i],file_names[i],file_score,file_stars[i].strip(),file_releasetimes[i])

        with open('maoyan.cvs','a+') as csvfile:
            writer = csv.writer(csvfile)
        
            writer.writerow([str(ranks[i]),str(file_names[i]),str(file_score),str(file_stars[i].strip()),str(file_releasetimes[i])])
            i += 1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

4.2 整合代码

import requests
import re
import csv


def get_url(url):
    header = {
        'User-Agent': 'XXXX'
    }
    response = requests.get(url, headers=header)
    content = response.text
    if response.status_code == 200:
        return content
    return None

def parse_html(html):
    pattern = re.compile('<dd>.*?(\d+)</i>.*?<p class="name">.*?<a href.*?title="(.*?)".*?<p class="star">(.*?)</p>.*?releasetime.*?(\d+.*?)</p>.*?<p.*?score.*?integer.*?(\d.)</i>.*?<i.*?fraction.*?(\d)</i>',re.S)
    items = re.findall(pattern,html)
    # for item in items:
    #     print(type(item[1]))
    return items

    #print(items)


def write_cvs(items):
    for item in items:
        with open('mao.csv','a+',encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow([item[0],item[1],item[2].strip(),item[3],str(item[4])+str(item[5])])



def main():
    # 1.爬取首页
    for i in range(0, 10):
        url = "https://maoyan.com/board/4?offset={}".format(i * 10)
        html = get_url(url)
        #parse_html(html)
        items = parse_html(html)
        write_cvs(items)

        # print(items)

        # print(html)


if __name__ == "__main__":
    main()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49