100字范文 > 爬虫 spider05——使用httpclient发送get请求 post请求

爬虫 spider05——使用httpclient发送get请求 post请求

时间：2020-06-05 03:10:52

百度解释

HttpClient 是Apache Jakarta Common 下的子项目，可以用来提供高效的、最新的、功能丰富的支持 HTTP 协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本和建议。

httpClient专为java发送http请求而生的, 如果要httpClient ,需要先进行导包

<dependency><groupId>org.apache.httpcomponents</groupId><artifactId>httpclient</artifactId><version>4.5.4</version></dependency>

Get

import org.apache.http.Header;import org.apache.http.HttpEntity;import org.apache.http.StatusLine;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.util.EntityUtils;import java.io.IOException;public class HttpClientGet {public static void main(String[] args) throws IOException {//请求地址String toUrl="";//封装地址到get请求中HttpGet httpGet=new HttpGet(toUrl);//设置请求头信息：User-AgenthttpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36");//创建执行请求对象CloseableHttpClient closeableHttpClient = HttpClients.createDefault();//执行请求,获取响应对象CloseableHttpResponse closeableHttpResponse = closeableHttpClient.execute(httpGet);//获取http请求响应信息StatusLine statusLine = closeableHttpResponse.getStatusLine();//获取状态码int respCode=statusLine.getStatusCode();System.out.println("respCode:"+respCode);//获取响应的Content typeHeader[] headersType = closeableHttpResponse.getHeaders("Content-Type");System.out.println(headersType.length);System.out.println(headersType[0].getName()+"----"+headersType[0].getValue());//获取响应数据,并封装到实体对象HttpEntity httpEntity = closeableHttpResponse.getEntity();//从实体类中取出爬取的数据String pageMessage = EntityUtils.toString(httpEntity, "utf-8");//System.out.println(pageMessage);}}

结果展示：

Post

import org.apache.http.Consts;import org.apache.http.Header;import org.apache.http.HttpEntity;import org.apache.http.NameValuePair;import org.apache.http.client.entity.UrlEncodedFormEntity;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.client.methods.HttpPost;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.message.BasicNameValuePair;import org.apache.http.util.EntityUtils;import org.jsoup.Jsoup;import org.jsoup.nodes.Document;import org.jsoup.select.Elements;import java.io.IOException;import java.io.UnsupportedEncodingException;import java.util.ArrayList;import java.util.List;/*** post请求登录慢慢买网站*/public class HttpClientPost {public static void main(String[] args) throws IOException {//登录请求地址String toLogin="/login.aspx";//封装请求HttpPost httpPost=new HttpPost(toLogin);//设置请求头信息，模拟浏览器登录//httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36");httpPost.setHeader("Referer","/login.aspx?tourl=http%3a%2f%%2fdefault.aspx");//httpPost.setHeader("Content-Type","application/x-www-form-urlencoded");httpPost.setHeader("Cookie","_ga=GA1.2.1458535014.1567059656; _gid=GA1.2.1328820533.1570633933; Hm_lvt_85f48cee3e51cd48eaba80781b243db3=1570634150,1570709948,1570717507,1570756038; _SessionId=rjnpykzbfs1yyvmdrtclz455; Hm_lpvt_85f48cee3e51cd48eaba80781b243db3=1570763535; _gat_gtag_UA_145348783_1=1");httpPost.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");//封装表单数据BasicNameValuePair txtName= new BasicNameValuePair("txtUser", "13699268827");BasicNameValuePair txtPass= new BasicNameValuePair("txtPass", "manmanmai");BasicNameValuePair autoLogin= new BasicNameValuePair("autoLogin", "on");BasicNameValuePair btnLogin= new BasicNameValuePair("btnLogin", "登陆");BasicNameValuePair viewState= new BasicNameValuePair("__VIEWSTATE", "/wEPDwULLTIwNjQ3Mzk2NDFkGAEFHl9fQ29udHJvbHNSZXF1aXJlUG9zdEJhY2tLZXlfXxYBBQlhdXRvTG9naW4voj01ABewCkGpFHsMsZvOn9mEZg==");BasicNameValuePair eventValidation= new BasicNameValuePair("__EVENTVALIDATION", "/wEWBQLW+t7HAwLB2tiHDgLKw6LdBQKWuuO2AgKC3IeGDJ4BlQgowBQGYQvtxzS54yrOdnbC");// 表单数据封装到集合中List<NameValuePair> formParams = new ArrayList<NameValuePair>();formParams.add(txtName);formParams.add(txtPass);formParams.add(autoLogin);formParams.add(btnLogin);formParams.add(viewState);formParams.add(eventValidation);//把表单集合对象封装到表单实体中UrlEncodedFormEntity urlEncodedFormEntity=new UrlEncodedFormEntity(formParams, Consts.UTF_8);//把表单实体封装到post请求中httpPost.setEntity(urlEncodedFormEntity);//*******************************************************//创建请求对象CloseableHttpClient closeableHttpClient = HttpClients.createDefault();//用户登录成功之后的响应对象CloseableHttpResponse closeableHttpResponse = closeableHttpClient.execute(httpPost);int code=closeableHttpResponse.getStatusLine().getStatusCode();if(code==302){//状态码为302意味着登录成功，并且服务器端做了重定向跳转//通过响应对象获取locationHeader[] locations = closeableHttpResponse.getHeaders("Location");String respUrl=locations[0].getValue();//爬取用户中心页面String toUserCenterPage=""+respUrl;//用户中心get请求HttpGet httpGet=new HttpGet(toUserCenterPage);//--httpGet.setHeader("Referer","/default.aspx");//--//获取第一次登录成功后的cookieHeader[] headerCookies = closeableHttpResponse.getHeaders("Set-Cookie");System.out.println("headerCookies:"+headerCookies.length);//System.out.println(headerCookies[0].getName()+"----"+headerCookies[0].getValue());//获取第一请求响应后的cookie，并把其cookie值赋予第二个请求httpGet.setHeader("Cookie",headerCookies[0].getValue());//Httpclient对象closeableHttpClient = HttpClients.createDefault();//执行请求 user center pageCloseableHttpResponse closeableHttpResponseUserCenter = closeableHttpClient.execute(httpGet);//获取页面信息 user center pageHttpEntity userCenterEntity = closeableHttpResponseUserCenter.getEntity();//String userCenterPage = EntityUtils.toString(userCenterEntity, "gb2312");//把用户中心的页面转换成document对象Document userCenterDocument = Jsoup.parse(userCenterPage);Elements integrals = userCenterDocument.select("#aspnetForm > div.udivright > div:nth-child(2) > table > tbody > tr > td:nth-child(1) > table:nth-child(2) > tbody > tr > td:nth-child(2) > div:nth-child(1) > font");System.out.println(integrals.text());}}}

结果展示：

总结

总结: httpClient使用步骤:

导包确定首页URL创建httpClient对象: HttpClients.createDefault();设置请求方式: HttpGet HttpPost

设置请求参数和请求头

post设置参数:

```java List list = new ArrayList();

list.add(new BasicNameValuePair("username","rose"));

list.add(new BasicNameValuePair("password","123"));

HttpEntity entity = new UrlEncodedFormEntity(list);

httpPost.setEntity(entity);

```