From da89100b4cd43071cb9b24dfc47fec59149f73ac Mon Sep 17 00:00:00 2001 From: chenrui <1950717904@qq.com> Date: Mon, 11 Nov 2024 19:38:49 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E7=88=AC=E8=99=AB=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E9=A1=B5=E9=9D=A2=E5=B7=A5=E5=85=B7=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/EmergencyService/pom.xml | 10 ++++ .../java/com/zdxt/code/utils/CrawlerUtil.java | 58 +++++++++++++++++++ .../java/com/zdxt/code/utils/HttpUtil.java | 58 +++++++++---------- 3 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 lib/EmergencyService/zdxtLawEnforcementCodeService/src/main/java/com/zdxt/code/utils/CrawlerUtil.java diff --git a/lib/EmergencyService/pom.xml b/lib/EmergencyService/pom.xml index 64552e3c..05fb62ef 100644 --- a/lib/EmergencyService/pom.xml +++ b/lib/EmergencyService/pom.xml @@ -488,6 +488,16 @@ jasypt-spring-boot-starter 2.1.2 + + net.sourceforge.htmlunit + htmlunit + 2.41.0 + + + xml-apis + xml-apis + 1.4.01 + diff --git a/lib/EmergencyService/zdxtLawEnforcementCodeService/src/main/java/com/zdxt/code/utils/CrawlerUtil.java b/lib/EmergencyService/zdxtLawEnforcementCodeService/src/main/java/com/zdxt/code/utils/CrawlerUtil.java new file mode 100644 index 00000000..54710119 --- /dev/null +++ b/lib/EmergencyService/zdxtLawEnforcementCodeService/src/main/java/com/zdxt/code/utils/CrawlerUtil.java @@ -0,0 +1,58 @@ +package com.zdxt.code.utils; + +import com.gargoylesoftware.htmlunit.BrowserVersion; +import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController; +import com.gargoylesoftware.htmlunit.WebClient; +import com.gargoylesoftware.htmlunit.WebClientOptions; +import com.gargoylesoftware.htmlunit.html.DomElement; +import com.gargoylesoftware.htmlunit.html.HtmlPage; +import net.sourceforge.htmlunit.corejs.javascript.EcmaError; + +import java.util.Objects; +import java.util.logging.LogManager; + +/** + * 爬虫 读取页面工具 + * @author 陈瑞 + * @date 2024-11-11 18:33 + */ +public class CrawlerUtil { + + public static WebClient webClient; + + static { + webClient = new WebClient(BrowserVersion.CHROME); + WebClientOptions options = webClient.getOptions(); + options.setUseInsecureSSL(true); + options.setCssEnabled(false); + options.setThrowExceptionOnScriptError(false); + options.setThrowExceptionOnFailingStatusCode(false); + options.setJavaScriptEnabled(true); + options.setTimeout(10000); + webClient.setAjaxController(new NicelyResynchronizingAjaxController()); + LogManager.getLogManager().reset(); + } + + + /** + * 爬取页面固定数据 + * @param url + * @return + * @throws Exception + */ + public static String getCreditCode(String url) throws Exception { + HtmlPage htmlPage = null; + try { + htmlPage = webClient.getPage(url); + } catch (EcmaError e) { + } + if (Objects.nonNull(htmlPage)) { + DomElement codeElement = htmlPage.getElementById("lblEntSCCode"); + if (codeElement != null) { + String code = codeElement.asText(); + return code; + } + } + return null; + } +} diff --git a/lib/EmergencyService/zdxtLawEnforcementCodeService/src/main/java/com/zdxt/code/utils/HttpUtil.java b/lib/EmergencyService/zdxtLawEnforcementCodeService/src/main/java/com/zdxt/code/utils/HttpUtil.java index 77e242da..1e8865de 100644 --- a/lib/EmergencyService/zdxtLawEnforcementCodeService/src/main/java/com/zdxt/code/utils/HttpUtil.java +++ b/lib/EmergencyService/zdxtLawEnforcementCodeService/src/main/java/com/zdxt/code/utils/HttpUtil.java @@ -3,6 +3,12 @@ package com.zdxt.code.utils; import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.IdUtil; import com.alibaba.fastjson.JSONObject; +import com.gargoylesoftware.htmlunit.BrowserVersion; +import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController; +import com.gargoylesoftware.htmlunit.WebClient; +import com.gargoylesoftware.htmlunit.WebClientOptions; +import com.gargoylesoftware.htmlunit.html.DomElement; +import com.gargoylesoftware.htmlunit.html.HtmlPage; import com.zdxt.auth.dto.DictDataBean; import com.zdxt.code.service.impl.MssqApiServiceImpl; import com.zdxt.common.util.StringUtils; @@ -19,16 +25,27 @@ import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.StringEntity; +import org.apache.http.impl.DefaultConnectionReuseStrategy; +import org.apache.http.impl.DefaultHttpClientConnection; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeader; +import org.apache.http.message.BasicHttpRequest; import org.apache.http.message.BasicNameValuePair; +import org.apache.http.params.BasicHttpParams; +import org.apache.http.params.HttpParams; +import org.apache.http.params.HttpProtocolParams; +import org.apache.http.protocol.*; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,17 +54,12 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import java.io.BufferedReader; import java.io.DataOutputStream; +import java.io.IOException; import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLConnection; +import java.net.*; import java.nio.charset.StandardCharsets; import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; public class HttpUtil { private static final Logger log = LoggerFactory.getLogger(HttpUtil.class); @@ -147,9 +159,11 @@ public class HttpUtil { public static String HttpSendForm(String url, Map param,String accessToken) throws Exception{ HttpPost post = new HttpPost(url); List paramList = new ArrayList<>(); - for(String key : param.keySet()){ - NameValuePair pair = new BasicNameValuePair(key,param.get(key)); - paramList.add(pair); + if(MapUtil.isNotEmpty(param)){ + for(String key : param.keySet()){ + NameValuePair pair = new BasicNameValuePair(key,param.get(key)); + paramList.add(pair); + } } CloseableHttpResponse response=null; SSLContext sc = SSLContext.getInstance("SSL"); @@ -265,25 +279,11 @@ public class HttpUtil { return null; } + static String url = "http://jdm.szssfj.cn/sjWeb/QyxyDetail.aspx?rid=536993e1c2504e7eb55b7234c90d597e&cid=91440300769156389U"; - public static void main(String[] args) { - try { -// String s = HttpSendGet("https://qr.szcredit.org.cn/GJQYCredit/GSZJGSPT/QYGS_Jump.aspx?rid=836E3C11540954A7A32077D8E724981E995A0B0521B91F8ECB1984EF3C175A05189FF39173BB0049", ""); - //// System.out.println(s); - // // 选择 id 为 "main" 的元素 - HttpClient httpClient = HttpClients.createDefault(); - HttpGet pmethod = new HttpGet("https://qr.szcredit.org.cn/GJQYCredit/GSZJGSPT/QYGS_Jump.aspx?rid=836E3C11540954A7A32077D8E724981E995A0B0521B91F8ECB1984EF3C175A05189FF39173BB0049"); - pmethod.addHeader("Connection", "keep-alive"); - pmethod.addHeader("Cache-Control", "max-age=0"); - pmethod.addHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0) "); - pmethod.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/;q=0.8"); - HttpResponse response = httpClient.execute(pmethod); - String s = EntityUtils.toString(response.getEntity(), "utf-8"); - Document doc = Jsoup.parse(s); - String mainContent = doc.select("#lblEntSCCode").text(); - } catch (Exception e) { - e.printStackTrace(); - } + public static void main(String[] args) throws Exception { + String creditCode = CrawlerUtil.getCreditCode(url); + System.out.println(creditCode); } }