注意:由于网站升级等原因,各关卡的访问链接会有所变化,请注意修改代码中的url变量。
# coding=utf-8

import requests
from lxml import etree
import re

base_url = 'http://spiderbuf.cn/e03'
# http://spiderbuf.cn/e03/5f685274073b

myheaders = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36'}

# 取页数
html = requests.get(base_url, headers=myheaders).text
root = etree.HTML(html)
print(html)
# 
  • 共5页
  • #
  • 1
  • #
  • 2
  • #
  • 3
  • #
  • 4
  • #
  • 5
  • lis = root.xpath('//ul[@class="pagination"]/li/a/@href') print(lis) # ['./2fe6286a4e5f', './5f685274073b', './279fcd874c72', './8a3d4d640516', './fbd076c39d28'] i = 1 for item in lis: print(item) s = item.replace('.','') print(base_url + s) url = base_url + s # print(url) html = requests.get(url, headers=myheaders).text # print(html) # f = open('./data/e03/e03_%d.html' % i, 'w', encoding='utf-8') f.write(html) f.close() # root = etree.HTML(html) trs = root.xpath('//tr') f = open('./data/e03/e03_%d.txt' % i, 'w', encoding='utf-8') for tr in trs: tds = tr.xpath('./td') s = '' for td in tds: s = s + str(td.xpath('string(.)')) + '|' # s = s + str(td.text) + '|' print(s) if s != '': f.write(s + '\n') f.close() i += 1

    粤ICP备2021093459. All rights reserved.