WooYun.org

#! /usr/bin/env python
#coding=utf-8
'''
Author:	kevin
Date:	2015/7/13
thread:	mul
'''
import time
import sys
import requests
import BeautifulSoup
import tablib
import threading
import Queue
info_list   = []
    
def codeSet():
    if(sys.getdefaultencoding()=='ascii'):
        reload(sys)
        sys.setdefaultencoding('utf-8')
        
def crawl(queue):
    global info_list
    while True:
        if queue.empty() == True:
            break
        url = queue.get()
        headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0"}
        try:
            res     = requests.get(url,headers=headers,timeout=20)       
            res.raise_for_status()
            html    = res.text
            soup    = BeautifulSoup.BeautifulSoup(html)                                        #默认编码就是utf-8
            crawlList = []
            dataInfo = soup.findAll(attrs={"target":"_blank"})
            for i in range(len(dataInfo)):
                idstr   = dataInfo[i].attrs[0][1]
                info_list.append(idstr+'\r\n')
            
            sys.stdout.write(threading.current_thread().getName()+"\r\n")
            sys.stdout.flush()                     
        except Exception,ex:
            print Exception,ex
        queue.task_done()  
        #time.sleep(1)       #########~~~~~~
def dataSave(info_list):
    try:
        with open('res.txt','w') as f:
            f.writelines(info_list)
    except Exception,err:
        print err
def main():
    print threading.current_thread().getName()
    codeSet()                                       #Default code is ascii not unicode
    threads =[]
    raw_url = 'http://main.jiajiaozaixian.com/searchGaojiUserinfo.action?pagelimit=no&renyuanSearchType=01&sortContent=&isGoldAsc=&filterContent=&turnPage=Y&direction=&page='
    for i in xrange(3000):
        print i
        url = raw_url+str(i)
        queue.put(url,block=True,timeout=None)
        
    for i in range(10):
        print str(i)
        t   = threading.Thread(target=crawl,args=(queue,))
        threads.append(t)
        
    for i in range(len(threads)):
        
        #threads[i].join()
        #threads[i].setDaemon(True)
        threads[i].start()
        #threads[i].join()
    for i in range(len(threads)):
        threads[i].join()
        
    dataSave(info_list)
    
    end_time    = time.time()
    print "Elapse time is %s"%(end_time-start_time)    
if __name__ == '__main__':
    
    queue = Queue.Queue(3000)
    start_time  = time.time()
    main()