发布于2019-08-30 14:44 阅读(1012) 评论(0) 点赞(15) 收藏(3)
使用python抓取网页图片的示例(python网络爬虫)。
代码:
#-*- encoding: utf-8 -*-'''Created on 2014-4-24
@author: Leon Wong'''import urllib2import urllibimport reimport timeimport osimport uuid#获取二级页面urldef findUrl2(html):
re1 = r'http://tuchong.com/\d+/\d+/|http://\w+(?<!photos).tuchong.com/\d+/'url2list = re.findall(re1,html)
url2lstfltr = list(set(url2list))
url2lstfltr.sort(key=url2list.index)#print url2lstfltrreturn url2lstfltr#获取html文本def getHtml(url):
html = urllib2.urlopen(url).read().decode('utf-8')#解码为utf-8return html#下载图片到本地def download(html_page , pageNo):
#定义文件夹的名字x = time.localtime(time.time())
foldername = str(x.__getattribute__("tm_year"))+"-"+str(x.__getattribute__("tm_mon"))+"-"+str(x.__getattribute__("tm_mday"))
re2=r'http://photos.tuchong.com/.+/f/.+\.jpg'imglist=re.findall(re2,html_page)print imglist
download_img=Nonefor imgurl in imglist:
picpath = 'D:\\TuChong\\%s\\%s' % (foldername,str(pageNo))
filename = str(uuid.uuid1())if not os.path.exists(picpath):
os.makedirs(picpath)
target = picpath+"\\%s.jpg" % filenameprint "The photos location is:"+target
download_img = urllib.urlretrieve(imgurl, target)#将图片下载到指定路径中time.sleep(1)print(imgurl)return download_img# def callback(blocknum, blocksize, totalsize):# '''回调函数 www.jbxue.com# @blocknum: 已经下载的数据块# @blocksize: 数据块的大小# @totalsize: 远程文件的大小# '''# print str(blocknum),str(blocksize),str(totalsize)# if blocknum * blocksize >= totalsize:# print '下载完成'def quitit():print "Bye!"exit(0)if __name__ == '__main__':print ''' *****************************************
** Welcome to Spider for TUCHONG **
** Created on 2014-4-24 **
** @author: Leon Wong **
*****************************************'''pageNo = raw_input("Input the page number you want to scratch (1-100),please input 'quit' if you want to quit>")while not pageNo.isdigit() or int(pageNo) > 100 :if pageNo == 'quit':quitit()print "Param is invalid , please try again."pageNo = raw_input("Input the page number you want to scratch >")#针对图虫人像模块来爬取html = getHtml("http://tuchong.com/tags/%E4%BA%BA%E5%83%8F/?page="+str(pageNo))
detllst = findUrl2(html)for detail in detllst:
html2 = getHtml(detail)
download(html2,pageNo)print "Finished."
作者:放羊人
链接:https://www.pythonheidong.com/blog/article/71294/0e9b85574b913b0d8517/
来源:python黑洞网
任何形式的转载都请注明出处,如有侵权 一经发现 必将追究其法律责任
昵称:
评论内容:(最多支持255个字符)
---无人问津也好,技不如人也罢,你都要试着安静下来,去做自己该做的事,而不是让内心的烦躁、焦虑,坏掉你本来就不多的热情和定力
Copyright © 2018-2021 python黑洞网 All Rights Reserved 版权所有,并保留所有权利。 京ICP备18063182号-1
投诉与举报,广告合作请联系vgs_info@163.com或QQ3083709327
免责声明:网站文章均由用户上传,仅供读者学习交流使用,禁止用做商业用途。若文章涉及色情,反动,侵权等违法信息,请向我们举报,一经核实我们会立即删除!