之所以选择selenium实现登录主要是为了处理验证码，招聘狗网站的验证码图片是拼接出来的，所以我的方法是通过webdriver截图来实现，然后通过打码兔平台获取验证码坐标实现自动自动登录。列表页和详情页用requests库实现。具体实现过程如下：
招聘狗的验证码如下：
首先你得注册一个账号，可以跳过企业验证，招聘狗网站是给企业HR使用的，所以一般要求企业验证，这里我们直接跳过企业验证，下面是实现过程，有详细注释：
import jsonimport osimport randomimport reimport sysimport tracebackimport timefrom PIL import Imagefrom lxml import html as lxml_htmlimport seleniumfrom selenium import webdriverfrom mon.exceptions import NoSuchElementExceptionfrom selenium.webdriver import ActionChainsimport requestsimport base64from requests.exceptions import ConnectionErrorimport http.cookiejarimport loggingfrom dama2_API import Dama2API#随机获取useragent的第三方库from fake_useragent import UserAgentua = UserAgent()class RTC_zhaopingou(object):def __init__(self, account: dict, debug=False, visible=-1, last_try=False):assert account['user_id']assert account['password']logging.info('Change webdriver to FireFox')#创建seeion对象，爬取列表页和详情页使用self.session = requests.Session()self.session.headers = {'Host': "","Origin":"","Referer":"","User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36",}#需要注册打码兔账号，从打码兔平台下载代码self.dama2 = Dama2API()def login(self):l = loggingl.info("Processing Login...")self.driver = webdriver.Firefox()self.driver.set_window_size(1920, 1080)self.driver.implicitly_wait(10)driver = self.driver# login_url = '/zhaopingou_interface/security_login?timestamp='+str(int(time.time()*1000))login_url = '/'driver.get(login_url)#打开页面后出现的需要选择城市driver.find_element_by_xpath('//div[@class="city-now citys"]').click()#找到用户名和密码元素，模仿人手动输入for i in self.account['username']:driver.find_element_by_xpath('//input[@placeholder="请输入手机号/邮箱/狗狗号"]').send_keys(i)time.sleep(random.uniform(0.2,0.8))for j in self.account['password']:driver.find_element_by_xpath('//input[@placeholder="请输入密码"]').send_keys(j)time.sleep(random.uniform(0.2, 0.8))# 获取弹出验证码的按钮元素，这里有一个坑，按钮元素在iframe节点中，不能直接获取，需要通过driver.find_element_by_tag_name("iframe")切入到第一个iframe中，然后在通过xpath获取按钮元素# iframe = driver.find_element_by_id('captcha_widget_aiwaylekc')driver.switch_to.frame(driver.find_element_by_tag_name("iframe"))# driver.switch_to.frame('captcha_widget_aiwaylekc')driver.find_element_by_xpath('//span[@class="captcha-widget-text"]').click()#等待5秒，避免出现有时候还未加载出来的情况，通过driver.switch_to.default_content()从iframe切换到主html页面time.sleep(5)driver.switch_to.default_content()#点击弹出验证码按钮后出现一个新的iframe,此时有两个iframe，并列的，从这页面切入到第二个iframedriver.switch_to.frame(driver.find_elements_by_tag_name("iframe")[1])# 验证码区域captcha_xpath = '//div[@class="lc-panel"]'# captcha_xpath = '#l-captcha-float_aiwaylekc're = self._login_process_captcha(captcha_xpath)#登录成功if re:driver.switch_to.default_content()driver.find_element_by_id('form_login').click()time.sleep(3)current_url = driver.current_url#判断登录后的url是否是期望值expect_url = '/'if current_url==expect_url:l.info('login sucess!!!')#获取cookie，并将cookie保存到session中，以便爬虫列表页和详情页使用cookie = dict()print(driver.get_cookies())for item in driver.get_cookies():# cookie += "; {}={}".format(item['name'], item["value"])cookie[item['name']] = item['value']if item['name'] == 'hrkeepToken':self.token = item['value']# 存储cookieself.session.cookies = requests.utils.cookiejar_from_dict(cookie, self.cookiejar)l.info("get cookie: {}".format(cookie))#登录成功，退出driver，后面不使用了self.driver.quit()return Trueelse:l.info('login failed due to CAPTCHA, submit_count')return Falsedef _login_process_captcha(self,captcha_xpath):l = loggingdriver = self.drivercaptcha_element = driver.find_element_by_xpath(captcha_xpath)#验证码坐标和大小offset = captcha_element.locationprint('offset:',offset)size = captcha_element.size# 验证码接口dama2 = self.dama2#保存验证码图片shm_dir = r'/tmp/zhaopingou/'if os.path.exists(shm_dir) is False:os.makedirs(shm_dir)captcha_img_path = os.path.join(shm_dir, 'captcha_img_{user_id}.png'.format(user_id=self.account['user_id']))maximum = 20attempt = 0while attempt<=maximum:l.info(f'Trying to decode CAPTCHA: {attempt}/{maximum}')#验证码元素captcha_element = driver.find_element_by_xpath(captcha_xpath)#截取验证码图片保存到captcha_img_pathcaptcha_element.screenshot(captcha_img_path)try:#调用打码兔接口，传入验证码类型，验证码图片文件，返回坐标值coordinate_listcaptcha_id, coordinate_list = dama2.decode_captcha(captcha_type=6137, file_path=captcha_img_path)l.info(f'coordinate_list:{coordinate_list}')except Exception as err:err_str = str(err)tb = traceback.format_exc()msg = f'Exception occurred when decode CAPTCHA, err: {err_str}, tb:\n{tb}'l.warning(msg)attempt+=1# 发生异常时先返回主页面continue#将鼠标移动到返回的坐标位置并点击for xy in coordinate_list:action = ActionChains(driver)action.move_to_element_with_offset(captcha_element, xy[0], xy[1]).click()action.perform()time.sleep(random.uniform(0.5,2))#先切回到主html，再切到第一个iframe，获取之前的弹出验证按钮，判断内容是否是验证成功driver.switch_to.default_content()driver.switch_to.frame(driver.find_elements_by_tag_name("iframe")[0])text = driver.find_element_by_xpath('//span[@class="captcha-widget-text"]').textif text.find('验证成功')!=-1:l.info('验证码验证成功！')time.sleep(random.uniform(1,2))return Trueelse: #失败则再切回到第二个iframe，从新获取验证码driver.switch_to.default_content()driver.switch_to.frame(driver.find_elements_by_tag_name("iframe")[1])l.info('fail,and try it again')attempt+=1time.sleep(2)continuereturn False#通过搜索关键字获取列表页面，并定位到某一页def search(self, keyword, page_to_go):'''搜索简历，得到列表页面，数据为json格式'''l = loggingassert keywordself.keyword = keyword# 使用firefox浏览器抓取post请求参数params = {"pageSize":page_to_go,"pageNo":"25","keyStr":keyword,"companyName":"","schoolName":"","keyStrPostion":"","postionStr":"","startDegrees":"-1","endDegress":"-1","startAge":"0","endAge":"0","gender":"-1","region":"","timeType":"-1","startWorkYear":"-1","endWorkYear":"-1","beginTime":"","endTime":"","isMember":"-1","hopeAdressStr":"","cityId":"-1","updateTime":"","tradeId":"","clientNo":"","userToken":self.token,"clientType":"2"}retry = 0while True:#抓包获取请求的真实URL，后面是随机的数字字符串search_url = "/zhaopingou_interface/find_warehouse_by_position_new?timestamp=" + str(int(time.time() * 1000))l.info('search_url:{}'.format(search_url))self.current_url = search_urll.debug(f'Open search page. url,params,keyword,userToken: {search_url},{params},{keyword},{self.token}')retry += 1if retry == 11:return ''try:#使用session请求res = self.session.post(search_url, data=params)except ConnectionError:l.info("ConnectionError! Sleep 5 minutes and retry...")time.sleep(300)self.current_url = search_urlcontinueelse:l.info('current url is:{}'.format(res.url))if res.url != search_url:login_result = self.login(load=False)if login_result:continueelse:l.warning("Login failed!")sys.exit('login failed')elif not res.text:l.info("Service is busy. Wait 5 minutes and retry...")time.sleep(300)l.info('Continue Searching...')continue#返回的数据异常，内容很少elif len(str(res.text))<2000:#若返回‘请您登录后查看简历’，则重新登录后在爬取if '请您登录后查看简历' in str(res.text):self.login(load=False)continueresult = str(res.text)#更换useragentself.session.headers['User-Agent'] = ua.firefoxl.info(f'errorcode msg:{result}')l.info('Too frequent operation, please try again in a minute')time.sleep(random.randint(61,100))continueelse:try:#返回的正常数据，通过json.dumps()获取json数据resume_list = json.loads(res.text)resume_list["current_page"]=page_to_go# 在列表页面加入搜索页面res = json.dumps(resume_list,ensure_ascii=False)l.info(f'search_resume_list_info:{res}')return resexcept:l.warning(res.text)l.warning("something wrong!sleep 5 minutes and retry...")time.sleep(300)continuedef open_resume(self, url):'''打开简历，得到详情页面url可通过base64加密的用户id构造'''l = loggingl.debug(f'Open a resume: request_url: {url}')resumeHtmlId=(url.split("="))[1]# 设置前链#self.session.headers['Referer'] = "/resume?key="+self.keyword# 抓包获取简历详情页的请求参数open_resume_data={"resumeHtmlId": resumeHtmlId,"keyStr":"","keyPositionName":"","tradeId":"","postionStr":"","jobId":"0","companyName":"","schoolName":"","clientNo":"","userToken":self.token,"clientType":"2"}retry = 0while True:#抓包获取详情页真实urlopenresumeurl = "/zhaopingou_interface/zpg_find_resume_html_details?timestamp=" + str(int(time.time() * 1000))l.info('resume_url:{}'.format(openresumeurl))retry += 1if retry == 11:return ''try:res = self.session.post(url=openresumeurl,data=open_resume_data)except ConnectionError:l.info("ConnectionError! Sleep 5 minutes and retry...")time.sleep(300)continueelse:# 返回的html页面l.info('current url is:{}'.format(res.url))if res.url != openresumeurl:l.info("cookie is invalid. Login with webdriver")login_result = self.login(load=False)if login_result:continueelse:l.warning("Login failed!")sys.exit('login failed')if not res.text:l.info("Service is busy. Wait 5 minutes and retry...")time.sleep(300)continueelif len(str(res.text))<2000:print('errorcode:',res.text)result = str(res.text)l.info(f'errorcode msg:{result}')l.info('Too frequent operation, please try again in a minute')time.sleep(random.randint(61, 100))continueelse:try:page_len = len(res.text)self.current_url = openresumeurll.info(f'Downloaded a resume, len: {page_len:,d}, current_url: {url}')resp_json=json.loads(res.text)res_utf=json.dumps(resp_json,ensure_ascii=False)return res_utfexcept:l.warning(res.text)l.warning("something wrong! sleep 5 minutes and retry...")time.sleep(300)continueif __name__ == '__main__':#账号密码是假的，大家填写自己的账号密码rtc_zhaopingou = RTC_zhaopingou(account={'user_id': '-701', 'username': '13419696888', 'password': '123'},debug=False,visible=1, last_try=False)rtc_zhaopingou.login()keyword_list = ['python','大数据','人工智能','java']for kw in keyword_list:for i in range(1,200):search_result = rtc_zhaopingou.search(kw, i)print('****************************************************************')res = rtc_zhaopingou.open_resume(' /resume/detail?resumeId=5761920')print(res)
打码兔平台的代码需要自己下载，放在同级目录后可以跑一下
