由于内涵段子被封了,换了一个网站,利用python爬取内涵吧段子标题和内容,使用python2实现
finditer 方法
finditer 方法的行为跟 findall 的行为类似，也是搜索整个字符串，获得所有匹配的结果。但它返回一个顺序访问每一个匹配结果（Match 对象）的迭代器。
#!/usr/bin/env python# encoding:utf-8import reimport requestsclass NeihanSpider(object):"""内涵吧段子爬取:"""def __init__(self):self.page = 1self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.5478.400 QQBrowser/10.1.1550.400 Name"}self.pattern = r'\s+<h3><a href="/article/\d+.html" class="title" title=".*?">(.*?)</a></h3>\s+<div class="desc">(.*?)</div>'def send_request(self, url, params={}):"""发送请求"""print("[INFO]:正在请求<{}>".format(url))html = requests.get(url, params=params, headers=self.headers).contentreturn htmldef save_file(self, iter):"""保存段子"""with open("./段子.txt", "ab") as f:while True:try:ret = next(iter)title = ret.group(1)content = ret.group(2)# 处理掉空格content = re.sub(r'', "", content)f.write(title + "\n")f.write(content + "\n\n")except StopIteration:breakdef parse_page(self, html):"""正则处理文本"""# re.finditer()返回一个顺序访问每一个结果的match对象迭代器# 这里用finditer的原因是想要同时匹配到标题和内容result_iter = re.finditer(self.pattern, html)return result_iterdef main(self):while True:url = "/article/index_" + str(self.page) + ".html"if self.page == 1:url = "/article/index.html"html = self.send_request(url)result_iter = self.parse_page(html)self.save_file(result_iter)print("[INFO]:第<{}>页处理完成".format(self.page))if raw_input("回车键继续,按q退出...") == "q":breakself.page += 1if __name__ == '__main__':spider = NeihanSpider()spider.main()
运行结果如下:
结果:
