多线程批量判断网站响应状态

  • 内容
  • 相关
# -*- coding:utf-8 -*-

# @Time    : 2017/09/19 14:38

# @Author  : zlmfslx

# @File    : quick_urls.py

# @Purpose : 判断网站响应类型



import requests

import threading

import time

import json

from Queue import Queue



class jiandanSpider(threading.Thread):

    def __init__(self,queue):

        threading.Thread.__init__(self)

        self._queue = queue



    def run(self):

        while not self._queue.empty():

            page_url = self._queue.get_nowait()

            self.spider(page_url)

    def spider(self,url):

        #t1 = datetime.datetime.now()



        try:

            headers = {

            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'

            }

            url = url.strip()

            time_start = time.time()

            r =requests.get(url,headers=headers,allow_redirects = False)

            times = time.time()-time_start

            sTring = url+'---'+str(r.status_code)+'---'+str(times)

            print url,r.status_code,times,r.history,str(r)

            #t2 =datetime.datetime.now()

            #if r.status_code !=200:



            f=open('f.txt','a')

            f.write(sTring+'\n')

            f.close()

        #print r.status_code,time.ctime()

        except requests.exceptions.SSLError:

            print url+'---ssl---'

            f=open('f.txt','a')

            f.write(str(url)+'---ssl---')

            f.close()

            pass

        except requests.exceptions.ConnectTimeout:

            print url+'---404---'

            f=open('f.txt','a')

            f.write(str(url)+'---404---')

            f.close()

            pass

        except:

            print url,r.history,str(r)

            f=open('f.txt','a')

            f.write(str(url)+'------')

            f.close()

            pass

        time.sleep(2)







def main():

    queue = Queue()

    thread_counts = 100

    for i in open("f1.txt"):

        queue.put(str(i))

    threads =[]

    for i in range(thread_counts):

        threads.append(jiandanSpider(queue))

    for t in threads:

        t.start()

    for i in range(thread_counts):

        t.join()





if __name__ == '__main__':

    main()

    f=open('f1.txt','w')

    f.close()

本文标签:

版权声明:若无特殊注明,本文皆为《颓废》原创,转载请保留文章出处。

收录状态:[百度已收录] | [360已收录] | [搜狗已收录]

本文链接:多线程批量判断网站响应状态 - https://www.0dayhack.com/post-712.html

严重声明:本站内容来自于互联网,仅适于网络安全技术爱好者学习研究使用,学习中请遵循国家相关法律法规,黑客不是骇客,黑客维护网络安全

发表评论

电子邮件地址不会被公开。 必填项已用*标注