提交爬虫解析页面工具类

lib/EmergencyService/pom.xml

@@ -488,6 +488,16 @@
             <artifactId>jasypt-spring-boot-starter</artifactId>
             <version>2.1.2</version>
         </dependency>
+        <dependency>
+            <groupId>net.sourceforge.htmlunit</groupId>
+            <artifactId>htmlunit</artifactId>
+            <version>2.41.0</version>
+        </dependency>
+        <dependency>
+            <groupId>xml-apis</groupId>
+            <artifactId>xml-apis</artifactId>
+            <version>1.4.01</version>
+        </dependency>
 <!--         <dependency>-->
 <!--            <groupId>EmergencyXxlJob</groupId>-->
 <!--            <artifactId>xxl-job-core</artifactId>-->

lib/EmergencyService/zdxtLawEnforcementCodeService/src/main/java/com/zdxt/code/utils/CrawlerUtil.java

@@ -0,0 +1,58 @@
+package com.zdxt.code.utils;
+
+import com.gargoylesoftware.htmlunit.BrowserVersion;
+import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gargoylesoftware.htmlunit.WebClientOptions;
+import com.gargoylesoftware.htmlunit.html.DomElement;
+import com.gargoylesoftware.htmlunit.html.HtmlPage;
+import net.sourceforge.htmlunit.corejs.javascript.EcmaError;
+
+import java.util.Objects;
+import java.util.logging.LogManager;
+
+/**
+ * 爬虫 读取页面工具
+ * @author  陈瑞
+ * @date 2024-11-11 18:33
+ */
+public class CrawlerUtil {
+
+    public static WebClient webClient;
+
+    static {
+        webClient = new WebClient(BrowserVersion.CHROME);
+        WebClientOptions options = webClient.getOptions();
+        options.setUseInsecureSSL(true);
+        options.setCssEnabled(false);
+        options.setThrowExceptionOnScriptError(false);
+        options.setThrowExceptionOnFailingStatusCode(false);
+        options.setJavaScriptEnabled(true);
+        options.setTimeout(10000);
+        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
+        LogManager.getLogManager().reset();
+    }
+
+
+    /**
+     * 爬取页面固定数据
+     * @param url
+     * @return
+     * @throws Exception
+     */
+    public static String getCreditCode(String url) throws Exception {
+        HtmlPage htmlPage = null;
+        try {
+            htmlPage = webClient.getPage(url);
+        } catch (EcmaError e) {
+        }
+        if (Objects.nonNull(htmlPage)) {
+            DomElement codeElement = htmlPage.getElementById("lblEntSCCode");
+            if (codeElement != null) {
+                String code = codeElement.asText();
+                 return code;
+            }
+        }
+        return null;
+    }
+}

lib/EmergencyService/zdxtLawEnforcementCodeService/src/main/java/com/zdxt/code/utils/HttpUtil.java

@@ -3,6 +3,12 @@ package com.zdxt.code.utils;
 import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.util.IdUtil;
 import com.alibaba.fastjson.JSONObject;
+import com.gargoylesoftware.htmlunit.BrowserVersion;
+import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gargoylesoftware.htmlunit.WebClientOptions;
+import com.gargoylesoftware.htmlunit.html.DomElement;
+import com.gargoylesoftware.htmlunit.html.HtmlPage;
 import com.zdxt.auth.dto.DictDataBean;
 import com.zdxt.code.service.impl.MssqApiServiceImpl;
 import com.zdxt.common.util.StringUtils;
@@ -19,16 +25,27 @@ import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
 import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.DefaultConnectionReuseStrategy;
+import org.apache.http.impl.DefaultHttpClientConnection;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.message.BasicHttpRequest;
 import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.HttpParams;
+import org.apache.http.params.HttpProtocolParams;
+import org.apache.http.protocol.*;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -37,17 +54,12 @@ import javax.net.ssl.TrustManager;
 import javax.net.ssl.X509TrustManager;
 import java.io.BufferedReader;
 import java.io.DataOutputStream;
+import java.io.IOException;
 import java.io.InputStreamReader;
-import java.net.HttpURLConnection;
+import java.net.*;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLConnection;
 import java.nio.charset.StandardCharsets;
 import java.security.cert.X509Certificate;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
 
 public class HttpUtil {
     private static final Logger log = LoggerFactory.getLogger(HttpUtil.class);
@@ -147,9 +159,11 @@ public class HttpUtil {
     public static String HttpSendForm(String url, Map<String,String> param,String accessToken) throws Exception{
         HttpPost post = new HttpPost(url);
         List<NameValuePair> paramList = new ArrayList<>();
-        for(String key : param.keySet()){
+        if(MapUtil.isNotEmpty(param)){
-            NameValuePair pair = new BasicNameValuePair(key,param.get(key));
+            for(String key : param.keySet()){
-            paramList.add(pair);
+                NameValuePair pair = new BasicNameValuePair(key,param.get(key));
+                paramList.add(pair);
+            }
         }
         CloseableHttpResponse response=null;
         SSLContext sc = SSLContext.getInstance("SSL");
@@ -265,25 +279,11 @@ public class HttpUtil {
         return null;
     }
 
+    static String url = "http://jdm.szssfj.cn/sjWeb/QyxyDetail.aspx?rid=536993e1c2504e7eb55b7234c90d597e&cid=91440300769156389U";
 
-    public static void main(String[] args) {
+    public static void main(String[] args) throws Exception {
-        try {
+        String creditCode = CrawlerUtil.getCreditCode(url);
-//            String s = HttpSendGet("https://qr.szcredit.org.cn/GJQYCredit/GSZJGSPT/QYGS_Jump.aspx?rid=836E3C11540954A7A32077D8E724981E995A0B0521B91F8ECB1984EF3C175A05189FF39173BB0049", "");
+        System.out.println(creditCode);
-            ////            System.out.println(s);
-            //            // 选择 id 为 "main" 的元素
-            HttpClient httpClient = HttpClients.createDefault();
-            HttpGet pmethod = new HttpGet("https://qr.szcredit.org.cn/GJQYCredit/GSZJGSPT/QYGS_Jump.aspx?rid=836E3C11540954A7A32077D8E724981E995A0B0521B91F8ECB1984EF3C175A05189FF39173BB0049");
-            pmethod.addHeader("Connection", "keep-alive");
-            pmethod.addHeader("Cache-Control", "max-age=0");
-            pmethod.addHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0) ");
-            pmethod.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/;q=0.8");
-            HttpResponse response = httpClient.execute(pmethod);
-            String s = EntityUtils.toString(response.getEntity(), "utf-8");
-            Document doc = Jsoup.parse(s);
-            String mainContent = doc.select("#lblEntSCCode").text();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
     }
 
 }