下面是小编为大家整理的Python实现多线程抓取妹子图（共含6篇），欢迎阅读与收藏。同时，但愿您也能像本文投稿人“majortom”一样，积极向本站投稿分享好文章。
本文给大家汇总了3款由Python制作的多线程批量抓取美图的代码，主要是将获取图片链接任务和下载图片任务用线程分开来处理了，而且这次的爬虫不仅仅可以爬第一页的图片链接的，有类似需求的小伙伴可以参考下，
心血来潮写了个多线程抓妹子图，虽然代码还是有一些瑕疵，但是还是记录下来，分享给大家。
Pic_downloader.py
# -*- coding: utf-8 -*-“”“Created on Fri Aug 07 17:30:58 @author: Dreace”“”import urllib2import sysimport timeimport osimport randomfrom multiprocessing.dummy import Pool as ThreadPool type_ = sys.getfilesystemencodingdef rename(): return time.strftime(“%Y%m%d%H%M%S”)def rename_2(name): if len(name) == 2: name = ‘0‘ + name + ‘.jpg‘ elif len(name) == 1: name = ‘00‘ + name + ‘.jpg‘ else: name = name + ‘.jpg‘ return namedef download_pic(i): global count global time_out if Filter(i): try: content = urllib2.urlopen(i,timeout = time_out)url_content = content.read()f = open(repr(random.randint(10000,999999999)) + “_” + rename_2(repr(count)),“wb”)f.write(url_content)f.close()count += 1 except Exception, e:print i + “下载超时，跳过！”.decode(“utf-8”).encode(type_)def Filter(content): for line in Filter_list: line=line.strip(‘\n‘) if content.find(line) == -1:return Truedef get_pic(url_address): global pic_list try: str_ = urllib2.urlopen(url_address, timeout = time_out).read() url_content = str_.split(“\”“) for i in url_content:if i.find(”.jpg“) != -1: pic_list.append(i) except Exception, e: print ”获取图片超时，跳过！“.decode(”utf-8“).encode(type_)MAX = 2count = 0time_out = 60thread_num = 30pic_list = []page_list = []Filter_list = [”imgsize.ph.126.net“,”img.ph.126.net“,”img2.ph.126.net“]dir_name = ”C:\Photos\\“+rename()os.makedirs(dir_name)os.chdir(dir_name)start_time = time.time()url_address = ”sexy.faceks.com/?page=“for i in range(1,MAX + 1): page_list.append(url_address + repr(i))page_pool = ThreadPool(thread_num)page_pool.map(get_pic,page_list)print ”获取到“.decode(”utf-8“).encode(type_),len(pic_list),”张图片，开始下载！“.decode(”utf-8“).encode(type_)pool = ThreadPool(thread_num) pool.map(download_pic,pic_list)pool.close() pool.join()print count,”张图片保存在“.decode(”utf-8“).encode(type_) + dir_nameprint ”共耗时“.decode(”utf-8“).encode(type_),time.time() - start_time,”s“
我们来看下一个网友的作品
#coding: utf-8 ############################################################## File Name: main.py# Author: mylonly# mail: mylonly@gmail.com# Created Time: Wed 11 Jun 08:22:12 PM CST##########################################################################!/usr/bin/pythonimport re,urllib2,HTMLParser,threading,Queue,time#各图集入口链接htmlDoorList = []#包含图片的Hmtl链接htmlUrlList = []#图片Url链接QueueimageUrlList = Queue.Queue(0)#捕获图片数量imageGetCount = 0#已下载图片数量imageDownloadCount = 0#每个图集的起始地址，用于判断终止nextHtmlUrl = ‘‘#本地保存路径localSavePath = ‘/data/1920x1080/‘#如果你想下你需要的分辨率的，请修改replace_str,有如下分辨率可供选择1920x1200，1980x1920,1680x1050,1600x900,1440x900,1366x768,1280x1024,1024x768,1280x800replace_str = ‘1920x1080‘replaced_str = ‘960x600‘#内页分析处理类class ImageHtmlParser(HTMLParser.HTMLParser):def __init__(self):self.nextUrl = ‘‘HTMLParser.HTMLParser.__init__(self)def handle_starttag(self,tag,attrs):global imageUrlListif(tag == ‘img‘ and len(attrs) >2 ):if(attrs[0] == (‘id‘,‘bigImg‘)):url = attrs[1][1]url = url.replace(replaced_str,replace_str)imageUrlList.put(url)global imageGetCountimageGetCount = imageGetCount + 1print urlelif(tag == ‘a‘ and len(attrs) == 4):if(attrs[0] == (‘id‘,‘pageNext‘) and attrs[1] == (‘class‘,‘next‘)):global nextHtmlUrlnextHtmlUrl = attrs[2][1];#首页分析类class IndexHtmlParser(HTMLParser.HTMLParser):def __init__(self):self.urlList = []self.index = 0self.nextUrl = ‘‘self.tagList = [‘li‘,‘a‘]self.classList = [‘photo-list-padding‘,‘pic‘]HTMLParser.HTMLParser.__init__(self)def handle_starttag(self,tag,attrs):if(tag == self.tagList[self.index]):for attr in attrs:if (attr[1] == self.classList[self.index]):if(self.index == 0):#第一层找到了self.index = 1else:#第二层找到了self.index = 0print attrs[1][1]self.urlList.append(attrs[1][1])breakelif(tag == ‘a‘):for attr in attrs:if (attr[0] == ‘id‘ and attr[1] == ‘pageNext‘):self.nextUrl = attrs[1][1]print ‘nextUrl:‘,self.nextUrlbreak#首页Hmtl解析器indexParser = IndexHtmlParser()#内页Html解析器imageParser = ImageHtmlParser()#根据首页得到所有入口链接print ‘开始扫描首页...‘host = ‘desk.zol.com.cn‘indexUrl = ‘/meinv/‘while (indexUrl != ‘‘):print ‘正在抓取网页:‘,host+indexUrlrequest = urllib2.Request(host+indexUrl)try:m = urllib2.urlopen(request)con = m.read()indexParser.feed(con)if (indexUrl == indexParser.nextUrl):breakelse:indexUrl = indexParser.nextUrlexcept urllib2.URLError,e:print e.reasonprint ‘首页扫描完成，所有图集链接已获得：‘htmlDoorList = indexParser.urlList#根据入口链接得到所有图片的urlclass getImageUrl(threading.Thread):def __init__(self):threading.Thread.__init__(self)def run(self):for door in htmlDoorList:print ‘开始获取图片地址,入口地址为:‘,doorglobal nextHtmlUrlnextHtmlUrl = ‘‘while(door != ‘‘):print ‘开始从网页%s获取图片...‘% (host+door)if(nextHtmlUrl != ‘‘):request = urllib2.Request(host+nextHtmlUrl)else:request = urllib2.Request(host+door)try:m = urllib2.urlopen(request)con = m.read()imageParser.feed(con)print ‘下一个页面地址为:‘,nextHtmlUrlif(door == nextHtmlUrl):breakexcept urllib2.URLError,e:print e.reasonprint ‘所有图片地址均已获得:‘,imageUrlListclass getImage(threading.Thread):def __init__(self):threading.Thread.__init__(self)def run(self):global imageUrlListprint ‘开始下载图片...‘while(True):print ‘目前捕获图片数量:‘,imageGetCountprint ‘已下载图片数量:‘,imageDownloadCountimage = imageUrlList.get()print ‘下载文件路径:‘,imagetry:cont = urllib2.urlopen(image).read()patter = ‘[0-9]*\.jpg‘;match = re.search(patter,image);if match:print ‘正在下载文件：‘,match.group()filename = localSavePath+match.group()f = open(filename,‘wb‘)f.write(cont)f.close()global imageDownloadCountimageDownloadCount = imageDownloadCount + 1else:print ‘no match‘if(imageUrlList.empty()):breakexcept urllib2.URLError,e:print e.reasonprint ‘文件全部下载完成...‘get = getImageUrl()get.start()print ‘获取图片链接线程启动:‘time.sleep(2)download = getImage()download.start()print ‘下载图片链接线程启动:‘
批量抓取指定网页上的所有图片
# -*- coding:utf-8 -*-# coding=UTF-8 import os,urllib,urllib2,re url = u”image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=26592&cl=2&lm=-1&st=-1&fm=index&fr=&sf=1&fmq=&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word=python&oq=python&rsp=-1“outpath = ”t:\\“ def getHtml(url): webfile = urllib.urlopen(url) uthtml = webfile.read() print outhtml return outhtml def getImageList(html): restr=ur‘(‘ restr+=ur‘http:\/\/[^\s,”]*\.jpg‘ restr+=ur‘|http:\/\/[^\s,“]*\.jpeg‘ restr+=ur‘|http:\/\/[^\s,”]*\.png‘ restr+=ur‘|http:\/\/[^\s,“]*\.gif‘ restr+=ur‘|http:\/\/[^\s,”]*\.bmp‘ restr+=ur‘|https:\/\/[^\s,“]*\.jpeg‘ restr+=ur‘|https:\/\/[^\s,”]*\.jpeg‘ restr+=ur‘|https:\/\/[^\s,“]*\.png‘ restr+=ur‘|https:\/\/[^\s,”]*\.gif‘ restr+=ur‘|https:\/\/[^\s,“]*\.bmp‘ restr+=ur‘)‘ htmlurl = re.compile(restr) imgList = re.findall(htmlurl,html) print imgList return imgList def download(imgList, page): x = 1 for imgurl in imgList: filepathname=str(outpath+‘pic_%09d_%010d‘%(page,x)+str(os.path.splitext(urllib2.unquote(imgurl).decode(‘utf8‘).split(‘/‘)[-1])[1])).lower() print ‘[Debug] Download file :‘+ imgurl+‘ >>‘+filepathname urllib.urlretrieve(imgurl,filepathname) x+=1 def downImageNum(pagenum): page = 1 pageNumber = pagenum while(page <= pageNumber): html = getHtml(url)#获得url指向的html内容 imageList = getImageList(html)#获得所有图片的地址，返回列表 download(imageList,page)#下载所有的图片 page = page+1 if __name__ == ‘__main__‘: downImageNum(1)
以上就是给大家汇总的3款Python实现的批量抓取妹纸图片的代码了，希望对大家学习Python爬虫能够有所帮助，
前面我们给大家介绍了使用nodejs来爬取妹纸图片的方法，下面我们来看下使用Python是如何实现的呢，有需要的小伙伴参考下吧，
Python Scrapy爬虫，听说妹子图挺火，我整站爬取了，上周一共搞了大概8000多张图片。和大家分享一下。
核心爬虫代码
# -*- coding: utf-8 -*-from scrapy.selector import Selectorimport scrapyfrom scrapy.contrib.loader import ItemLoader, Identityfrom fun.items import MeizituItem class MeizituSpider(scrapy.Spider): name = ”meizitu“ allowed_domains = [”meizitu.com“] start_urls = ( ‘www.meizitu.com/‘, ) def parse(self, response): sel = Selector(response) for link in sel.xpath(‘//h2/a/@href‘).extract():request = scrapy.Request(link, callback=self.parse_item)yield request pages = sel.xpath(”//div[@class=‘navigation‘]/div[@id=‘wp_page_numbers‘]/ul/li/a/@href“).extract() print(‘pages: %s‘ % pages) if len(pages) >2:page_link = pages[-2]page_link = page_link.replace(‘/a/‘, ‘‘) request = scrapy.Request(‘www.meizitu.com/a/%s‘ % page_link, callback=self.parse)yield request def parse_item(self, response): l = ItemLoader(item=MeizituItem(), response=response) l.add_xpath(‘name‘, ‘//h2/a/text()‘) l.add_xpath(‘tags‘, ”//div[@id=‘maincontent‘]/div[@class=‘postmeta. clearfix‘]/div[@class=‘metaRight‘]/p“) l.add_xpath(‘image_urls‘, ”//div[@id=‘picture‘]/p/img/@src“, Identity()) l.add_value(‘url‘, response.url) return l.load_item()
项目地址：github.com/ZhangBohan/fun_crawler
以上所述就是本文的全部内容了，希望大家能够喜欢，
本文给大家介绍的是一则使用Python实现抓取城市的PM2.5数据和排名，
主机环境：（Python2.7.9 / Win8_64 / bs4）
利用BeautifulSoup4来抓取 www.pm25.com 上的PM2.5数据，之所以抓取这个网站，是因为上面有城市PM2.5浓度排名（其实真正的原因是，它是百度搜PM2.5出来的第一个网站！）
程序里只对比了两个城市，所以多线程的速度提升并不是很明显，大家可以弄10个城市并开10个线程试试，
最后吐槽一下：上海的空气质量怎么这么差！！！
PM25.py
代码如下:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# by ustcwq
import urllib2
import threading
from time import ctime
from bs4 import BeautifulSoup
def getPM25(cityname):
site = ‘www.pm25.com/‘ + cityname + ‘.html‘
html = urllib2.urlopen(site)
soup = BeautifulSoup(html)
city = soup.find(class_ = ‘bi_loaction_city‘) # 城市名称
aqi = soup.find(”a“,{”class“,”bi_aqiarea_num“}) # AQI指数
quality = soup.select(”.bi_aqiarea_right span“) # 空气质量等级
result = soup.find(”div“,class_ =‘bi_aqiarea_bottom‘) # 空气质量描述
print city.text + u‘AQI指数：‘ + aqi.text + u‘\n空气质量：‘ + quality[0].text + result.text
print ‘*‘*20 + ctime + ‘*‘*20
def one_thread(): # 单线程
print ‘One_thread Start: ‘ + ctime() + ‘\n‘
getPM25(‘hefei‘)
getPM25(‘shanghai‘)
def two_thread(): # 多线程
print ‘Two_thread Start: ‘ + ctime() + ‘\n‘
threads = []
t1 = threading.Thread(target=getPM25,args=(‘hefei‘,))
threads.append(t1)
t2 = threading.Thread(target=getPM25,args=(‘shanghai‘,))
threads.append(t2)
for t in threads:
# t.setDaemon(True)
t.start()
if __name__ == ‘__main__‘:
one_thread()
print ‘\n‘ * 2
two_thread()
以上就是本文所述的全部内容了，希望大家能够喜欢，
作者：断鸿 字体：[增加 减小] 类型：
这篇文章主要介绍了Python实现登录人人网并抓取新鲜事的方法,可实现Python模拟登陆并抓取新鲜事的功能,需要的朋友可以参考下
本文实例讲述了Python实现登录人人网并抓取新鲜事的方法，分享给大家供大家参考。具体如下：
这里演示了Python登录人人网并抓取新鲜事的方法（抓取后的排版不太美观~~）
from sgmllib import SGMLParserimport sys,urllib2,urllib,cookielibclass spider(SGMLParser): def __init__(self,email,password): SGMLParser.__init__(self) self.h3=False self.h3_is_ready=False self.div=False self.h3_and_div=False self.a=False self.depth=0 self.names=”“ self.dic={}self.email=email self.password=password self.domain=‘renren.com‘ try:cookie=cookielib.CookieJar()cookieProc=urllib2.HTTPCookieProcessor(cookie) except:raise else:pener=urllib2.build_opener(cookieProc)urllib2.install_opener(opener)def login(self): url=‘www.renren.com/PLogin.do‘ postdata={ ‘email‘:self.email, ‘password‘:self.password, ‘domain‘:self.domain } req=urllib2.Request( url, urllib.urlencode(postdata) ) self.file=urllib2.urlopen(req).read() #print self.file def start_h3(self,attrs): self.h3 = True def end_h3(self): self.h3=False self.h3_is_ready=True def start_a(self,attrs): if self.h3 or self.div:self.a=True def end_a(self): self.a=False def start_div(self,attrs): if self.h3_is_ready == False:return if self.div==True:self.depth += 1 for k,v in attrs:if k == ‘class‘ and v == ‘content‘: self.div=True; self.h3_and_div=True #h3 and div is connected def end_div(self): if self.depth == 0:self.div=Falseself.h3_and_div=Falseself.h3_is_ready=Falseself.names=”“ if self.div == True:self.depth-=1 def handle_data(self,text): #record the name if self.h3 and self.a:self.names+=text #record says if self.h3 and (self.a==False):if not text:passelse: self.dic.setdefault(self.names,[]).append(text)return if self.h3_and_div:self.dic.setdefault(self.names,[]).append(text) def show(self): type = sys.getfilesystemencoding() for key in self.dic:print ( (‘‘.join(key)).replace(‘ ‘,‘‘)).decode(‘utf-8‘).encode(type), \ ( (‘‘.join(self.dic[key])).replace(‘ ‘,‘‘)).decode(‘utf-8‘).encode(type)renrenspider=spider(‘your email‘,‘your password‘)renrenspider.login()renrenspider.feed(renrenspider.file)renrenspider.show()
希望本文所述对大家的Python程序设计有所帮助，
这篇文章主要介绍了Python多线程编程（七）：使用Condition实现复杂同步,本文讲解通过很著名的“生产者-消费者”模型来来演示在Python中使用Condition实现复杂同步,需要的朋友可以参考下
目前我们已经会使用Lock去对公共资源进行互斥访问了，也探讨了同一线程可以使用RLock去重入锁，但是尽管如此我们只不过才处理了一些程序中简单的同步现象，我们甚至还不能很合理的去解决使用Lock锁带来的死锁问题，所以我们得学会使用更深层的解决同步问题。
Python提供的Condition对象提供了对复杂线程同步问题的支持。Condition被称为条件变量，除了提供与Lock类似的acquire和release方法外，还提供了wait和notify方法。
使用Condition的主要方式为：线程首先acquire一个条件变量，然后判断一些条件。如果条件不满足则wait；如果条件满足，进行一些处理改变条件后，通过notify方法通知其他线程，其他处于wait状态的线程接到通知后会重新判断条件。不断的重复这一过程，从而解决复杂的同步问题。
下面我们通过很著名的“生产者-消费者”模型来来演示下，在Python中使用Condition实现复杂同步。
代码如下:
‘‘‘
Created on -9-8
@author: walfred
@module: thread.TreadTest7
‘‘‘
import threading
import time
condition = threading.Condition
products = 0
class Producer(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
global condition, products
while True:
if condition.acquire():
if products < 10:
products += 1;
print ”Producer(%s):deliver one, now products:%s“ %(self.name, products)
condition.notify()
else:
print ”Producer(%s):already 10, stop deliver, now products:%s“ %(self.name, products)
condition.wait();
condition.release()
time.sleep(2)
class Consumer(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
global condition, products
while True:
if condition.acquire():
if products >1:
products -= 1
print ”Consumer(%s):consume one, now products:%s“ %(self.name, products)
condition.notify()
else:
print ”Consumer(%s):only 1, stop consume, products:%s“ %(self.name, products)
condition.wait();
condition.release()
time.sleep(2)
if __name__ == ”__main__“:
for p in range(0, 2):
p = Producer()
p.start()
for c in range(0, 10):
c = Consumer()
c.start()
代码中主要实现了生产者和消费者线程，双方将会围绕products来产生同步问题，首先是2个生成者生产products ，而接下来的10个消费者将会消耗products，代码运行如下：
代码如下:
Producer(Thread-1):deliver one, now products:1
Producer(Thread-2):deliver one, now products:2
Consumer(Thread-3):consume one, now products:1
Consumer(Thread-4):only 1, stop consume, products:1
Consumer(Thread-5):only 1, stop consume, products:1
Consumer(Thread-6):only 1, stop consume, products:1
Consumer(Thread-7):only 1, stop consume, products:1
Consumer(Thread-8):only 1, stop consume, products:1
Consumer(Thread-10):only 1, stop consume, products:1
Consumer(Thread-9):only 1, stop consume, products:1
Consumer(Thread-12):only 1, stop consume, products:1
Consumer(Thread-11):only 1, stop consume, products:1
另外：Condition对象的构造函数可以接受一个Lock/RLock对象作为参数，如果没有指定，则Condition对象会在内部自行创建一个RLock；除了notify方法外，Condition对象还提供了notifyAll方法，可以通知waiting池中的所有线程尝试acquire内部锁，
由于上述机制，处于waiting状态的线程只能通过notify方法唤醒，所以notifyAll的作用在于防止有线程永远处于沉默状态。
这篇文章主要介绍了Python多线程编程（八）：使用Event实现线程间通信,,需要的朋友可以参考下
使用threading.Event可以实现线程间相互通信，之前的Python：使用threading模块实现多线程编程七[使用Condition实现复杂同步]我们已经初步实现了线程间通信的基本功能，但是更为通用的一种做法是使用threading.Event对象，使用threading.Event可以使一个线程等待其他线程的通知，我们把这个Event传递到线程对象中，Event默认内置了一个标志，初始值为False。一旦该线程通过wait()方法进入等待状态，直到另一个线程调用该Event的set()方法将内置标志设置为True时，该Event会通知所有等待状态的线程恢复运行。
代码如下:
‘‘‘
Created on 2012-9-9
@author: walfred
@module: thread.TreadTest8
‘‘‘
import threading
import time
class MyThread(threading.Thread):
def __init__(self, signal):
threading.Thread.__init__(self)
self.singal = signal
def run(self):
print ”I am %s,I will sleep ...“%self.name
self.singal.wait()
print ”I am %s, I awake...“ %self.name
if __name__ == ”__main__“:
singal = threading.Event()
for t in range(0, 3):
thread = MyThread(singal)
thread.start()
print ”main thread sleep 3 seconds... "
time.sleep(3)
singal.set()
运行效果如下：
代码如下:
I am Thread-1,I will sleep ...
I am Thread-2,I will sleep ...
I am Thread-3,I will sleep ...
main thread sleep 3 seconds...
I am Thread-1, I awake...I am Thread-2, I awake...
I am Thread-3, I awake...
