福特抓取网页

modules/i3plus-ext-mes-pcn-api/src/main/java/cn/estsh/i3plus/ext/mes/pcn/api/busi/ford/IFordFetchWebService.java

@@ -1,6 +1,7 @@
 package cn.estsh.i3plus.ext.mes.pcn.api.busi.ford;
 
 import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordFetchParameter;
+import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordJsaSortInfoWithFetchWeb;
 
 import java.util.List;
 
@@ -18,5 +19,5 @@ public interface IFordFetchWebService {
 
     List<MesFordFetchParameter> queryParams(String organizeCode);
 
-    void doFetchData(Long id,String organizeCode);
+    List<MesFordJsaSortInfoWithFetchWeb> doFetchData(Long id, String organizeCode);
 }

modules/i3plus-ext-mes-pcn-apiservice/pom.xml

@@ -14,6 +14,10 @@
     <packaging>jar</packaging>
 
     <dependencies>
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+        </dependency>
         <!-- CXF依赖 -->
         <dependency>
             <groupId>org.apache.cxf</groupId>

modules/i3plus-ext-mes-pcn-apiservice/src/main/java/cn/estsh/i3plus/ext/mes/pcn/apiservice/controller/busi/ford/MesFordFetchWebController.java

@@ -3,6 +3,7 @@ package cn.estsh.i3plus.ext.mes.pcn.apiservice.controller.busi.ford;
 import cn.estsh.i3plus.ext.mes.pcn.api.busi.ford.IFordFetchWebService;
 import cn.estsh.i3plus.ext.mes.pcn.pojo.constant.MesCommonConstant;
 import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordFetchParameter;
+import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordJsaSortInfoWithFetchWeb;
 import cn.estsh.impp.framework.boot.util.ResultBean;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
@@ -10,6 +11,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 
+import java.util.ArrayList;
 import java.util.List;
 
 @RestController
@@ -82,12 +84,13 @@ public class MesFordFetchWebController {
     @GetMapping(value = "/param/data/{id}/{organizeCode}")
     @ApiOperation(value = "定时抓单")
     public ResultBean queryData(@PathVariable Long id,@PathVariable String organizeCode) {
-
+        List<MesFordJsaSortInfoWithFetchWeb> mesFordJsaSortInfoWithFetchWebs;
         try {
+            mesFordJsaSortInfoWithFetchWebs = fordFetchWebService.doFetchData(id, organizeCode);
         } catch (Exception e) {
             return ResultBean.fail(e.getMessage());
         }
-        return null;
+        return ResultBean.success("查询成功").setResultList(mesFordJsaSortInfoWithFetchWebs);
     }
 
 

modules/i3plus-ext-mes-pcn-apiservice/src/main/java/cn/estsh/i3plus/ext/mes/pcn/apiservice/serviceimpl/ford/MesFordFetchWebServiceImpl.java

@@ -5,19 +5,22 @@ import cn.estsh.i3plus.platform.common.convert.ConvertBean;
 import cn.estsh.i3plus.pojo.base.bean.DdlPackBean;
 import cn.estsh.i3plus.pojo.base.tool.DdlPreparedPack;
 import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordFetchParameter;
+import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordJsaSortInfoWithFetchWeb;
 import cn.estsh.i3plus.pojo.mes.repository.ford.MesFordFetchParameterRepository;
+import cn.estsh.i3plus.pojo.mes.repository.ford.MesFordJsaSortInfoWithFetchWebRepository;
 import cn.estsh.impp.framework.boot.exception.ImppExceptionBuilder;
 import cn.hutool.http.HttpRequest;
 import cn.hutool.http.HttpResponse;
 import lombok.extern.slf4j.Slf4j;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.net.HttpCookie;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
+import java.util.*;
 
 @Service
 @Slf4j
@@ -25,6 +28,9 @@ public class MesFordFetchWebServiceImpl implements IFordFetchWebService {
     @Autowired
     private MesFordFetchParameterRepository paramRao;
 
+    @Autowired
+    private MesFordJsaSortInfoWithFetchWebRepository fetchWebRao;
+
     @Override
     public void insertParams(MesFordFetchParameter parameter) {
        ConvertBean.saveOrUpdate(parameter, parameter.getCreateUser());
@@ -57,7 +63,8 @@ public class MesFordFetchWebServiceImpl implements IFordFetchWebService {
     }
 
     @Override
-    public void doFetchData(Long id, String organizeCode) {
+    public List<MesFordJsaSortInfoWithFetchWeb> doFetchData(Long id, String organizeCode) {
+        List<MesFordJsaSortInfoWithFetchWeb> infoList = new ArrayList<>();
         DdlPackBean ddlPackBean = DdlPackBean.getDdlPackBean(organizeCode);
         DdlPreparedPack.getNumEqualPack(id,"id",ddlPackBean);
         List<MesFordFetchParameter> list = paramRao.findByHqlWhere(ddlPackBean);
@@ -68,12 +75,15 @@ public class MesFordFetchWebServiceImpl implements IFordFetchWebService {
         //尝试登录
         Map<String, Object> paramMap = new HashMap<>();
         paramMap.put("username", parameter.getUid());
-        paramMap.put("passwork", parameter.getPwd());
-        Map<String,Object> header = new HashMap<>();
-        header.put("content-type","application/x-www-form-urlencoded");
-        HttpResponse loginResponse = HttpRequest.post(parameter.getLoginUrl()).form(paramMap).execute();
+        paramMap.put("password", parameter.getPwd());
+        paramMap.put("t", String.valueOf(System.currentTimeMillis()));
+        String body = "username="+parameter.getUid()+"&password="+parameter.getPwd()+"&t="+ System.currentTimeMillis();
+        String loginUrl = parameter.getLoginUrl();
+        HttpResponse loginResponse = HttpRequest.post(loginUrl).body(body)
+                .header("Content-Type","application/x-www-form-urlencoded").timeout(60000).execute();
+        log.info("doFetchData-打印的返回值是:{}",loginResponse.body());
         if (loginResponse.getStatus() != 200){
-            log.info("登录失败,需要重新登录!返回内容为:{}",loginResponse.body());
+            log.info("doFetchData-登录失败,需要重新登录!返回内容为:{}",loginResponse.body());
         }
         List<HttpCookie> cookies = loginResponse.getCookies();
         //拼接请求参数
@@ -86,6 +96,42 @@ public class MesFordFetchWebServiceImpl implements IFordFetchWebService {
         map.put("catacode","0");
         map.put("itemcode","0");
         HttpResponse response = HttpRequest.get(parameter.getFetchUrl()).cookie(cookies).timeout(60000).form(map).execute();
-        log.info("查询出来的内容为:{}",response.body());
+        log.info("doFetchData-查询出来的内容为:{}",response.body());
+        Document doc = Jsoup.parse(response.body());
+        Elements trList = doc.select("tr");
+        for (Element tr : trList) {
+            Elements tdList = tr.select("td");
+            MesFordJsaSortInfoWithFetchWeb webInfo = new MesFordJsaSortInfoWithFetchWeb();
+            webInfo.setSeq(Long.parseLong(tdList.get(0).text()));
+            webInfo.setFetchDate(tdList.get(1).text());
+            webInfo.setFetchTime(tdList.get(2).text());
+            webInfo.setVinCode(tdList.get(3).text());
+            webInfo.setPartNo(tdList.get(4).text());
+            webInfo.setPartName(tdList.get(5).text());
+            webInfo.setColorCode(tdList.get(6).text());
+            webInfo.setInfo0008(tdList.get(7).text());
+            webInfo.setSeqContext(tdList.get(8).text());
+            webInfo.setRemark(tdList.get(9).text());
+            webInfo.setInfoPointDesc(tdList.get(10).text());
+            webInfo.setFetchType(1); //抓取类型
+            webInfo.setDataStatus(200);
+            webInfo.setOrganizeCode(organizeCode);
+            webInfo.setInfoPointCode(parameter.getInfoPointCode());
+            webInfo.setCustOrganizeCode(parameter.getCustOrganizeCode());
+            ConvertBean.saveOrUpdate(webInfo,"doFetchData");
+            //校验是否已经查询过
+            DdlPackBean webInfoPackBean = DdlPackBean.getDdlPackBean(organizeCode);
+            DdlPreparedPack.getNumEqualPack(webInfo.getSeq(),"seq",webInfoPackBean);
+            DdlPreparedPack.getNumEqualPack(webInfo.getFetchDate(),"fetchDate",webInfoPackBean);
+            DdlPreparedPack.getNumEqualPack(webInfo.getFetchTime(),"fetchTime",webInfoPackBean);
+            int count = fetchWebRao.findByHqlWhereCount(webInfoPackBean);
+            if (count > 0){
+                continue;
+            }
+            fetchWebRao.insert(webInfo);
+            infoList.add(webInfo);
+        }
+
+        return infoList;
     }
 }

pom.xml

@@ -41,6 +41,11 @@
 
     <dependencyManagement>
         <dependencies>
+            <dependency>
+                <groupId>org.jsoup</groupId>
+                <artifactId>jsoup</artifactId>
+                <version>1.11.3</version>
+            </dependency>
             <!-- CXF依赖 -->
             <dependency>
                 <groupId>org.apache.cxf</groupId>