python-自定义异步非阻塞爬虫框架

api

import socket
import select


class MySock:
    def __init__(self, sock, data):
        self.sock = sock
        self.data = data

    def __getattr__(self, item):
        return getattr(self.sock, item)


class YinBing:
    def __init__(self):
        self.r_list = []
        self.w_list = []

    def add(self, req_info):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.setblocking(0)
        try:
            sock.connect((req_info.get("host"), req_info.get("port")))
        except BlockingIOError:
            pass
        s = MySock(sock, req_info)
        self.r_list.append(s)
        self.w_list.append(s)

    def run(self):
        while True:
            rl, wl, el = select.select(self.r_list, self.w_list, [], 0.5)
            for sock in wl:
                sock.send(("GET %s Http/1.1\r\nHost:%s\r\n\r\n" % (sock.data["path"], sock.data["host"])).encode("utf-8"))
            for sock in rl:
                response = sock.recv(1024)
                callback = sock.data.get("callback")
                if callback: callback(response)
                self.r_list.remove(sock)

            if not self.r_list:
                breakimport socket
import select


class MySoc