爬取起点中文网某本小说实例：
1 # -*-coding:utf8-*- 2 import requests 3 import urllib 4 import urllib2 5 from bs4 import BeautifulSoup 6 import sys 7 reload(sys) 8 sys.setdefaultencoding('utf-8') #读写中文内容 9 10 class QDZW:11def __init__(self, url):12 self.url = url #网址13 self.user_agent = 'Mozilla/5.0 (Windows NT 6.1; (Windows NT 6.1; WOW64; rv:62.0) Gecko/0101 Firefox/61.0'#请求头属性（模拟浏览器访问）14 self.headers = {'User-Agent' : self.user_agent}15 16#获取网址静态HTML（如果未访问成功，则持续访问，直到访问成功为止）17def gethtml(self):18 19 try:20 #return requests.get(self.url，headers = self.headers,time_out = 2)21 request = urllib2.Request(self.url,headers = self.headers)22 response = urllib2.urlopen(request,time_out = 2) #设置访问时长限制23 temp = False24 return response25 except Exception as e:26 print e27#解析HTML内容，获取需要的内容 28def novel(self):29 while True:30 r = self.gethtml()31 while r == 'None': #r内容为空表示访问超时，则继续访问32 r = self.gethtml()33 #print r.encoding #输出编码类型34 soup = BeautifulSoup(r ,"html.parser")#利用BeautifulSoup解析网址内容，class前加 . id前加 #35 #select输出内容为数组，select参数从body中一级结点开始即可，可逐级叠加定位36 #XXX.text为读取标签中的内容 XXX.attrs['xx']为读取 XXX中属性为xx的值 ，XXX为HTML格式的文本37 section_name = soup.select(' .wrap #j_readMainWrap #j_chapterBox .text-wrap .main-text-wrap .text-head .j_chapterName')[0].text 38 print section_name39 section_text = soup.select('.wrap #j_readMainWrap #j_chapterBox .text-wrap .main-text-wrap .read-content')[0].text40 section_text = self.spliting(section_text)41 #print section_text42 self.writing(section_name + "\r\n" + section_text)43 44 next_url = soup.select('.wrap #j_readMainWrap .chapter-control #j_chapterNext')[0]45 #print next_url.text 46 if next_url.text == "下一章":47 self.url = "https:" + next_url.attrs['href']48 else:49 break50#在去掉<p></p>的位置添加换行符号“\r\n”51def spliting(self, string):52 str_split = string.split() #以split()中参数为标准，切割字符串53 string = ""54 for text in str_split:55 if text != "":56 string = string + " " + text + "\r\n"57 return string58#输出读取的novel内容59def writing(self, novel):60 outputs = open(unicode('至尊剑皇.txt','utf8'),'a')#a为追加型写入61 outputs.write(novel)62 outputs.close()63 64 qdzw = QDZW('/chapter/JDViC81SWM41/pskSqRrpbXDgn4SMoDUcDQ2')#参数为初始网址 65 qdzw.novel()
