文章詳情頁

網頁爬蟲 - python3.4.1 request模塊報錯 ’list’ object has no attribute ’get’

瀏覽：200日期：2022-06-29 11:59:14

問題描述

用python 寫了一個爬取ip地址的爬蟲，由于該網站是反爬蟲的，所以寫了代理使用線程池開啟10個線程來爬取ip地址然而直接報錯’list’ object has no attribute ’get’不知道如何解決，貼上本人代碼。from bs4 import BeautifulSoupimport requestsimport reimport timefrom multiprocessing import Poolimport pymysqlimport randomfrom threadpool import *

隨機請求頭

def randHeader():

head_connection = [’Keep-Alive’, ’close’]head_accept = [’text/html, application/xhtml+xml, */*’]head_accept_language = [’zh-CN,fr-FR;q=0.5’, ’en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3’]head_user_agent = [’Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko’, ’Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36’, ’Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; rv:11.0) like Gecko)’, ’Mozilla/5.0 (Windows; U; Windows NT 5.2) Gecko/2008070208 Firefox/3.0.1’, ’Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070309 Firefox/2.0.0.3’, ’Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070803 Firefox/1.5.0.12’, ’Opera/9.27 (Windows NT 5.2; U; zh-cn)’, ’Mozilla/5.0 (Macintosh; PPC Mac OS X; U; en) Opera 8.0’, ’Opera/8.0 (Macintosh; PPC Mac OS X; U; en)’, ’Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080219 Firefox/2.0.0.12 Navigator/9.0.0.6’, ’Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Win64; x64; Trident/4.0)’, ’Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)’, ’Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)’, ’Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Maxthon/4.0.6.2000 Chrome/26.0.1410.43 Safari/537.1 ’, ’Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E; QQBrowser/7.3.9825.400)’, ’Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0 ’, ’Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.92 Safari/537.1 LBBROWSER’, ’Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; BIDUBrowser 2.x)’, ’Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/3.0 Safari/536.11’]header = { ’Connection’: head_connection[0], ’Accept’: head_accept[0], ’Accept-Language’: head_accept_language[1], ’User-Agent’: head_user_agent[random.randrange(0, len(head_user_agent))]}return header

def randproxy():

config = { ’host’: ’127.0.0.1’, ’port’: 3306, ’user’: ’root’, ’password’: ’’, ’db’: ’autohome’, ’charset’: ’utf8’, # ’cursorclass’: pymysql.cursors.DictCursor,}# 創建連接list_ip = []connection = pymysql.connect(**config)cursor = connection.cursor()sql = ’select ip,port from can_use’try: cursor.execute(sql) results = cursor.fetchall() for row in results:data = { ’ip’: row[0], ’port’: row[1]}list_ip.append(data)except: print('error')# time.sleep(1)finally: connection.close()return random.choice(list_ip)

def download(url):

proxy = randproxy()proxy_host = 'http://' + proxy[’ip’] + ':' + proxy[’port’]proxy_temp = {'http': proxy_host}parse_url = requests.get(url[0],headers=randHeader(),timeout=12,proxies=proxy_temp)soup = BeautifulSoup(parse_url.text,’lxml’)pre_proxys = soup.find(’table’, id=’ip_list’).find_all(’tr’)for i in pre_proxys[1:]: try:td = i.find_all(’td’)id = td[1].get_text()port = td[2].get_text()# 執行sql語句config = { ’host’: ’127.0.0.1’, ’port’: 3306, ’user’: ’root’, ’password’: ’’, ’db’: ’autohome’, ’charset’: ’utf8’, ’cursorclass’: pymysql.cursors.DictCursor,}# 創建連接connection = pymysql.connect(**config)data = { ’ip’:id, ’port’:port,}with connection.cursor() as cursor: # 執行sql語句，插入記錄 sql = ’INSERT INTO proxyip (ip,port) VALUES (%s,%s)’ cursor.execute(sql, (data[’ip’],data[’port’])) # 沒有設置默認自動提交，需要主動提交，以保存所執行的語句connection.commit() except:print('error') # time.sleep(1) finally:connection.close()time.sleep(2)

def proxy_url_list():

url = 'http://www.xicidaili.com/wt/{}'url_list = []for i in range(1,1387): new_url = url.format(i) url_list.append(new_url)return url_list

if name =='__main__':

pool = ThreadPool(2)requests = makeRequests(download,proxy_url_list())[pool.putRequest(req) for req in requests]pool.wait()# url = 'http://www.baidu.com'# proxy = randproxy()# proxy_host = 'http://' + proxy[’ip’] + ':' + proxy[’port’]# proxy_temp = {'http': proxy_host}# test = requests.get(url,headers=randHeader(),timeout=10,proxies=proxy_temp)# soup = BeautifulSoup(test.text,’lxml’)# print(soup)圖片發布不了，現在只能貼上錯誤提示了 File 'C:Pythonlibsite-packagesthreadpool.py', line 158, in runresult = request.callable(*request.args, **request.kwds)