package org.jeecg.modules.demo.openapi;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.swagger.annotations.Api;
import kong.unirest.HttpResponse;
import kong.unirest.JsonNode;
import kong.unirest.Unirest;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.config.TenantContext;
import org.jeecg.modules.demo.externalinternalsite.entity.ExternalInternalSite;
import org.jeecg.modules.demo.externalinternalsite.service.IExternalInternalSiteService;
import org.jeecg.modules.demo.linkcaptureerrorlog.service.ILinkCaptureErrorLogService;
import org.jeecg.modules.demo.linkcaptureerrormessage.service.ILinkCaptureErrorMessageService;
import org.jeecg.modules.demo.linkchainerrordetection.entity.LinkChainErrorDetection;
import org.jeecg.modules.demo.linkchainerrordetection.service.ILinkChainErrorDetectionService;
import org.jeecg.modules.demo.linkchainerrorlog.entity.LinkChainErrorLog;
import org.jeecg.modules.demo.linkchainerrorlog.service.ILinkChainErrorLogService;
import org.jeecg.modules.demo.linkerrorstatistics.entity.LinkErrorStatistics;
import org.jeecg.modules.demo.linkerrorstatistics.service.ILinkErrorStatisticsService;
import org.jeecg.modules.demo.linkerrorwordstatistics.service.ILinkErrorWordStatisticsService;
import org.jeecg.modules.demo.linkouterchaindetection.entity.LinkOuterChainDetection;
import org.jeecg.modules.demo.linkouterchaindetection.service.ILinkOuterChainDetectionService;
import org.jeecg.modules.demo.linkouterchainlog.service.ILinkOuterChainLogService;
import org.jeecg.modules.demo.linkouterouterstatistics.entity.LinkOuterouterStatistics;
import org.jeecg.modules.demo.linkouterouterstatistics.service.ILinkOuterouterStatisticsService;
import org.jeecg.modules.demo.linkpersonalbankcarddetection.entity.LinkPersonalBankCardDetection;
import org.jeecg.modules.demo.linkpersonalbankcarddetection.service.ILinkPersonalBankCardDetectionService;
import org.jeecg.modules.demo.linkpersonalidnumberdetection.entity.LinkPersonalIdNumberDetection;
import org.jeecg.modules.demo.linkpersonalidnumberdetection.service.ILinkPersonalIdNumberDetectionService;
import org.jeecg.modules.demo.linkpersonalphonedetection.entity.LinkPersonalPhoneDetection;
import org.jeecg.modules.demo.linkpersonalphonedetection.service.ILinkPersonalPhoneDetectionService;
import org.jeecg.modules.demo.linkpublishingtimegrasping.entity.LinkPublishingTimeGrasping;
import org.jeecg.modules.demo.linkpublishingtimegrasping.service.ILinkPublishingTimeGraspingService;
import org.jeecg.modules.demo.linkpublishingtitle.entity.LinkPublishingTitle;
import org.jeecg.modules.demo.linkpublishingtitle.service.ILinkPublishingTitleService;
import org.jeecg.modules.demo.linkwechatbankcarddetection.entity.LinkWechatBankCardDetection;
import org.jeecg.modules.demo.linkwechatbankcarddetection.service.ILinkWechatBankCardDetectionService;
import org.jeecg.modules.demo.linkwechatidnumberdetection.entity.LinkWechatIdNumberDetection;
import org.jeecg.modules.demo.linkwechatidnumberdetection.service.ILinkWechatIdNumberDetectionService;
import org.jeecg.modules.demo.linkwechatphonedetection.entity.LinkWechatPhoneDetection;
import org.jeecg.modules.demo.linkwechatphonedetection.service.ILinkWechatPhoneDetectionService;
import org.jeecg.modules.demo.linkweiboidnumberdetection.entity.LinkWeiboIdNumberDetection;
import org.jeecg.modules.demo.linkweiboidnumberdetection.service.ILinkWeiboIdNumberDetectionService;
import org.jeecg.modules.demo.linkwhitelist.service.ILinkWhiteListService;
import org.jeecg.modules.demo.linkwhiteword.entity.LinkWhiteWord;
import org.jeecg.modules.demo.linkwhiteword.service.ILinkWhiteWordService;
import org.jeecg.modules.demo.mainsite.entity.MainSite;
import org.jeecg.modules.demo.mainsite.service.IMainSiteService;
import org.jeecg.modules.demo.pagecontentcapture.entity.JsResult;
import org.jeecg.modules.demo.pagecontentcapture.entity.PageContentCapture;
import org.jeecg.modules.demo.pagecontentcapture.service.IPageContentCaptureService;
import org.jeecg.modules.demo.wechatpostsword.entity.WechatPostsWord;
import org.jeecg.modules.demo.wechatpostsword.service.IWechatPostsWordService;
import org.jeecg.modules.spyderUtils.SpyderUtils;
import org.jeecg.modules.spyderUtils.spider.*;
import org.jeecg.modules.system.entity.SysDict;
import org.jeecg.modules.system.entity.SysDictItem;
import org.jeecg.modules.system.service.ISysDictItemService;
import org.jeecg.modules.system.service.ISysDictService;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.DigestUtils;
import org.springframework.util.ObjectUtils;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.net.URL;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Slf4j
@Api(tags = "单表DEMO")
@RestController
@RequestMapping("/openai")
public class OpenApiCrawling {
    @Autowired
    private IPageContentCaptureService iPageContentCaptureService;
    @Autowired
    private ILinkPublishingTitleService linkPublishingTitleService;

    @Autowired
    private ILinkPublishingTimeGraspingService iLinkPublishingTimeGraspingService;

    @Autowired
    private IMainSiteService iMainSiteService;

    @Autowired
    private ISysDictService sysDictService;

    @Autowired
    private ILinkWhiteWordService iLinkWhiteWordService;

    @Autowired
    private ISysDictItemService iSysDictItemService;


    @Autowired
    private IExternalInternalSiteService externalInternalSiteService;

    @Autowired
    private IPageContentCaptureService pageContentCaptureService;


    @Autowired
    private ILinkCaptureErrorLogService iLinkCaptureErrorLogService;

    @Autowired
    private ILinkCaptureErrorMessageService iLinkCaptureErrorMessageService;

    @Autowired
    private ILinkErrorWordStatisticsService iLinkErrorWordStatisticsService;

    @Autowired
    private TrendsSpyder trendsSpyder;

    @Autowired
    private ILinkOuterChainDetectionService linkOuterChainDetectionService;

    @Autowired
    private ILinkChainErrorLogService iLinkChainErrorLogService;

    @Autowired
    private ILinkOuterChainLogService linkOuterChainLogService;

    @Autowired
    private ILinkOuterouterStatisticsService iLinkOuterouterStatisticsService;

    @Autowired
    private ILinkWhiteListService linkWhiteListService;

    @Autowired
    private ILinkChainErrorDetectionService linkChainErrorDetectionService;

    @Autowired
    private ILinkErrorStatisticsService iLinkErrorStatisticsService;
    @Autowired
    private FangCunUtils fangCunUtils;

    @Autowired
    private LinkOrWordSaveUtils linkOrWordSaveUtils;

    @Autowired
    private JsoupUtils jsoupUtils;

    @Autowired
    private SpyderUtils spyderUtils;

    @Autowired
    private CrawlingUtils crawlingUtils;

    @Autowired
    private ILinkPersonalPhoneDetectionService iLinkPersonalPhoneDetectionService;

    @Autowired
    private ILinkPersonalIdNumberDetectionService linkPersonalIdNumberDetectionService;

    @Autowired
    private ILinkPersonalBankCardDetectionService iLinkPersonalBankCardDetectionService;

    @Autowired
    private IWechatPostsWordService iWechatPostsWordService;

    @Autowired
    private ILinkWechatPhoneDetectionService iLinkWechatPhoneDetectionService;

    @Autowired
    private ILinkWeiboIdNumberDetectionService iLinkWeiboIdNumberDetectionService;

    @Autowired
    private ILinkWechatIdNumberDetectionService iLinkWechatIdNumberDetectionService;

    @Autowired
    private ILinkWechatBankCardDetectionService linkWechatBankCardDetectionService;
    @Data
    class ReturnResult{
        PageContentCapture byId;
        List<String> errorMsg;
        List<Integer> index;
    }

    @GetMapping(value = "/listMyBank")
    public Result<LinkPersonalBankCardDetection> listMyBank(
            @RequestParam(name = "id") String id
    ) throws IOException {
        LinkPersonalBankCardDetection linkPersonalBankCardDetection = iLinkPersonalBankCardDetectionService.getById(id);
        String ahtmlList = linkPersonalBankCardDetection.getAhtmlList();
        String[] split = ahtmlList.split(",");
        List<String> aHrefHtmlList = new ArrayList<String>();
        for (String strA : split) {
            aHrefHtmlList.add(strA);
        }
        linkPersonalBankCardDetection.setAHrefHtmlList(aHrefHtmlList);
        if(ObjectUtils.isEmpty(linkPersonalBankCardDetection)){
            return Result.error("没有找到对应数据");
        }
        return Result.ok(linkPersonalBankCardDetection);
    }

    @GetMapping(value = "/listIdNumber")
    public Result<LinkPersonalIdNumberDetection> listIdNumber(
            @RequestParam(name = "id") String id
    ) throws IOException {
        LinkPersonalIdNumberDetection linkPersonalIdNumberDetection = linkPersonalIdNumberDetectionService.getById(id);
        String ahtmlList = linkPersonalIdNumberDetection.getAhtmlList();
        String[] split = ahtmlList.split(",");
        List<String> aHrefHtmlList = new ArrayList<String>();
        for (String strA : split) {
            aHrefHtmlList.add(strA);
        }
        linkPersonalIdNumberDetection.setAHrefHtmlList(aHrefHtmlList);
        if(ObjectUtils.isEmpty(linkPersonalIdNumberDetection)){
            return Result.error("没有找到对应数据");
        }
        return Result.ok(linkPersonalIdNumberDetection);
    }

    @GetMapping(value = "/listWXIdNumber")
    public Result<LinkWechatIdNumberDetection> listWXIdNumber(
            @RequestParam(name = "id") String id
    ) throws IOException {

        LinkWechatIdNumberDetection linkPersonalIdNumberDetection = iLinkWechatIdNumberDetectionService.getById(id);
        String ahtmlList = linkPersonalIdNumberDetection.getAhtmlList();
        String[] split = ahtmlList.split(",");
        List<String> aHrefHtmlList = new ArrayList<String>();
        for (String strA : split) {
            aHrefHtmlList.add(strA);
        }
        linkPersonalIdNumberDetection.setAHrefHtmlList(aHrefHtmlList);
        if(ObjectUtils.isEmpty(linkPersonalIdNumberDetection)){
            return Result.error("没有找到对应数据");
        }
        return Result.ok(linkPersonalIdNumberDetection);
    }



    @GetMapping(value = "/listWXBankCard")
    public Result<LinkWechatBankCardDetection> listWXBankCard(
            @RequestParam(name = "id") String id
    ) throws IOException {

        LinkWechatBankCardDetection linkPersonalIdNumberDetection = linkWechatBankCardDetectionService.getById(id);
        String ahtmlList = linkPersonalIdNumberDetection.getAhtmlList();
        String[] split = ahtmlList.split(",");
        List<String> aHrefHtmlList = new ArrayList<String>();
        for (String strA : split) {
            aHrefHtmlList.add(strA);
        }
        linkPersonalIdNumberDetection.setAHrefHtmlList(aHrefHtmlList);
        if(ObjectUtils.isEmpty(linkPersonalIdNumberDetection)){
            return Result.error("没有找到对应数据");
        }
        return Result.ok(linkPersonalIdNumberDetection);
    }



    @GetMapping(value = "/listPhone")
    public Result<LinkPersonalPhoneDetection> listPhone(
            @RequestParam(name = "id") String id
    ) throws IOException {
        LinkPersonalPhoneDetection linkPersonalPhoneDetection = iLinkPersonalPhoneDetectionService.getById(id);
        String ahtmlList = linkPersonalPhoneDetection.getAhtmlList();
        String[] split = ahtmlList.split(",");
        List<String> aHrefHtmlList = new ArrayList<String>();
        for (String strA : split) {
            aHrefHtmlList.add(strA);
        }
        linkPersonalPhoneDetection.setAHrefHtmlList(aHrefHtmlList);
        if(ObjectUtils.isEmpty(linkPersonalPhoneDetection)){
            return Result.error("没有找到对应数据");
        }
        return Result.ok(linkPersonalPhoneDetection);
    }

    @GetMapping(value = "/listWXPhone")
    public Result<LinkWechatPhoneDetection> listWXPhone(
            @RequestParam(name = "id") String id
    ) throws IOException {

        LinkWechatPhoneDetection byId = iLinkWechatPhoneDetectionService.getById(id);
        String ahtmlList = byId.getAhtmlList();
        String[] split = ahtmlList.split(",");
        List<String> aHrefHtmlList = new ArrayList<String>();
        for (String strA : split) {
            aHrefHtmlList.add(strA);
        }
        byId.setAHrefHtmlList(aHrefHtmlList);
        if(ObjectUtils.isEmpty(byId)){
            return Result.error("没有找到对应数据");
        }
        return Result.ok(byId);
    }



    /**
     * 分页列表查询
     * @return
     */
    @GetMapping(value = "/list")
    public Result<ReturnResult> list(
            @RequestParam(name = "id") String id
    ) throws IOException {
        ReturnResult returnResult = new ReturnResult();
        List<String> errorMsg = new ArrayList<String>();
        PageContentCapture pageContentCapture = iPageContentCaptureService.getById(id);
        //爬取页面
        Document document = Jsoup.parse(new URL(pageContentCapture.getSource()), 10000);
        String baseUri = document.baseUri();
        document = jsoupUtils.setUrl(document, baseUri);
        String stringHTml = document.toString();
        stringHTml = stringHTml.replaceAll("amp;","");
        pageContentCapture.setContent(stringHTml);
        //获取文章发布的标题
        QueryWrapper<LinkPublishingTitle> linkPublishingTitleQueryWrapper =new QueryWrapper<LinkPublishingTitle>();
        linkPublishingTitleQueryWrapper.eq("main_site_url",pageContentCapture.getMainUrl());
        List<LinkPublishingTitle> linkPublishingTitles = linkPublishingTitleService.list(linkPublishingTitleQueryWrapper);
        String title = null;
        for (LinkPublishingTitle linkPublishingTitle : linkPublishingTitles) {
            String typeCssXpath = linkPublishingTitle.getTypeCssXpath();
            try {
                if(typeCssXpath.equals("selector")){
                    Elements select = document.select(linkPublishingTitle.getElementPositioningType());
                    String string = select.toString();
                    String text = Jsoup.parse(string).text();
                    if(text != null && text != ""){
                        title = text;
                    }
                }
                if(typeCssXpath.equals("xpath")){
                    // 5.将document转换成xpath能解析的文档树结构
                    JXDocument jdoc = new JXDocument(document.getAllElements()); // 此处可以使用xpath语法
                    // 6.使用xpath语法来解析html文档来完成数据清洗
                    List<JXNode> elNodes = jdoc.selN(linkPublishingTitle.getElementPositioningType());
                    for (JXNode elNode : elNodes) {
                        String string = elNode.toString();
                        String text = Jsoup.parse(string).text();
                        if(text != null && text != ""){
                            title = text;
                        }
                    }
                }
            }catch (Exception e){
            }
        }
        if(title != null && title != "" && title.length() != 0){
            pageContentCapture.setTitle(title);
        }else{
            //抓取网页title的内容
            Elements eles = document.getElementsByTag("title");
            String string = eles.text();
            pageContentCapture.setTitle(string);
        }
        QueryWrapper<LinkPublishingTimeGrasping> linkPublishingTimeGraspingQueryWrapper =new QueryWrapper<LinkPublishingTimeGrasping>();
        linkPublishingTimeGraspingQueryWrapper.eq("main_site_url",pageContentCapture.getMainUrl());
        //获取文章发布的时间
        List<LinkPublishingTimeGrasping> linkPublishingTimeGraspings = iLinkPublishingTimeGraspingService.list(linkPublishingTimeGraspingQueryWrapper);
        String time = null;
        for (LinkPublishingTimeGrasping linkPublishingTimeGrasping : linkPublishingTimeGraspings) {
            String typeCssXpathTime = linkPublishingTimeGrasping.getTypeCssXpath();
            try {
                if(typeCssXpathTime.equals("selector")){
                    Elements select = document.select(linkPublishingTimeGrasping.getElementPositioningType());
                    String string = select.toString();
                    String text = Jsoup.parse(string).text();
                    if(text != null && text != ""){
                        time = text;
                    }
                }
                if(typeCssXpathTime.equals("xpath")){
                    // 5.将document转换成xpath能解析的文档树结构
                    JXDocument jdoc = new JXDocument(document.getAllElements()); // 此处可以使用xpath语法
                    // 6.使用xpath语法来解析html文档来完成数据清洗
                    List<JXNode> elNodes = jdoc.selN(linkPublishingTimeGrasping.getElementPositioningType());
                    for (JXNode elNode : elNodes) {
                        String string = elNode.toString();
                        String text = Jsoup.parse(string).text();
                        if(text != null && text != ""){
                            time = text;
                        }
                    }
                }
            }catch (Exception e){
            }
        }
        //格式化时间
        Date parse = null;
        if(time != null && time != "" && time.length() != 0){
            pageContentCapture.setTitle(title);
            for (LinkPublishingTimeGrasping linkPublishingTimeGrasping : linkPublishingTimeGraspings) {
                String timeFormat = linkPublishingTimeGrasping.getTimeFormat();
                QueryWrapper<SysDict> sysDictQueryWrapper = new QueryWrapper<SysDict>();
                sysDictQueryWrapper.eq("dict_code","time_type");
                List<SysDict> listSysDict = sysDictService.list(sysDictQueryWrapper);
                SysDict sysDict = null;
                if(listSysDict.size() >= 1){
                    sysDict = listSysDict.get(0);
                }
                QueryWrapper<SysDictItem> sysDictItemQueryWrapper = new QueryWrapper<SysDictItem>();
                sysDictItemQueryWrapper.eq("dict_id",sysDict.getId());
                sysDictItemQueryWrapper.eq("item_value",timeFormat);
                List<SysDictItem> sysDictItems = iSysDictItemService.list(sysDictItemQueryWrapper);
                for (SysDictItem sysDictItem : sysDictItems) {
                    try {
                        String itemText = sysDictItem.getItemText();
                        //正则表达式
                        String description = sysDictItem.getDescription();
                        Pattern r = Pattern.compile(description);
                        Matcher m = r.matcher(time);
                        String dateStr = null;
                        if (m.find()) {
                            dateStr = m.group(0);
                        }
                        if(dateStr != null && dateStr != ""){
                            SimpleDateFormat simpleDateFormat = new SimpleDateFormat(itemText);
                            parse = simpleDateFormat.parse(dateStr);
                        }
                    }catch (Exception e){
                    }
                }
            }
        }
        if(!ObjectUtils.isEmpty(parse)){
            pageContentCapture.setArticleReleaseTime(parse);
        }
        //错敏词校验-----开始
        Integer[] IntStr = {1000,1001,1002,1004,2000,2001,3000,3001,4000,5000,7001,7002};
        Object oIntStr = JSONArray.toJSON(IntStr);
        Map<String, Object> paramMap = new HashMap<>();
        paramMap.put("userid", "quyanhong");
        paramMap.put("ukey", "K103fhnBhXak");
        paramMap.put("text", URLEncoder.encode( pageContentCapture.getContent()));
        paramMap.put("error_types",oIntStr);
        paramMap.put("threshold",0.2);
        paramMap.put("strict_degree","m");
        paramMap.put("domains","test");
        HttpResponse<JsonNode> response = Unirest.post("http://api.ifuncun.cn:8081/v1/proofreadHtml")
                .header("Content-Type", "application/json")
                .body( JSONObject.toJSONString(paramMap)
                )
                .asJson();
        String string = response.getBody().toString();
        JsResult jsonObject = JSONObject.parseObject(string, JsResult.class);
        String ret_msg = jsonObject.getRet_msg();
        pageContentCapture.setErrorCount(jsonObject.getErrorCount());
        pageContentCapture.setRetMsg(jsonObject.getRet_msg());
        pageContentCapture.setRetCode(jsonObject.getRet_code());
        if(!ObjectUtils.isEmpty(ret_msg)){
            //跳过白名单词汇表
            QueryWrapper<LinkWhiteWord> linkWhiteWordQueryWrapper = new QueryWrapper<LinkWhiteWord>();
            linkWhiteWordQueryWrapper.eq("main_site_url",pageContentCapture.getMainUrl());
            List<LinkWhiteWord> linkWhiteWordList = iLinkWhiteWordService.list(linkWhiteWordQueryWrapper);
            for (LinkWhiteWord linkWhiteWord : linkWhiteWordList) {
                int index = ret_msg.indexOf(linkWhiteWord.getWord());
                if(index > 0){
                    Document parseHtml = Jsoup.parse(ret_msg);
                    Elements elements = parseHtml.select("span.proofreadSignClass.errorWordMark");
                    for (Element element : elements) {
                        String text = element.text();
                        if(text.equals(linkWhiteWord.getWord())){
                            String string1 = parseHtml.toString();
                            String replace = string1.replace(element.toString(), linkWhiteWord.getWord());
                            pageContentCapture.setRetMsg(replace);
                            pageContentCapture.setErrorCount(0);
                        }
                    }
                }
            }
        }
        pageContentCapture.setRetCode(jsonObject.getRet_code());
        pageContentCapture.setHeavyCount(jsonObject.getHeavyCount());
        //错敏词校验-----结束
        returnResult.setById(pageContentCapture);
        String retMsg = pageContentCapture.getRetMsg();
        Document parseRetMsg = Jsoup.parse(retMsg);
        String parseRetMsgHTMl = parseRetMsg.toString();
        Elements elements = parseRetMsg.select("span.proofreadSignClass.errorWordMark");
        Integer indexIn = 0;
        for (Element element : elements) {
            String reHtml = "<span id='a"+indexIn+"'>"+element.toString()+"</span>";
            parseRetMsgHTMl = parseRetMsgHTMl.replace(element.toString(),reHtml);
            String myClass = element.attr("title");
            errorMsg.add("<a href='#a"+indexIn+"'>"+myClass+"</a>");
            indexIn++;
        }
        pageContentCapture.setRetMsg(parseRetMsgHTMl);
        returnResult.setErrorMsg(errorMsg);
        return Result.OK("成功",returnResult);
    }


    /**
     * 分页列表查询
     * @return
     */
    @GetMapping(value = "/listOld")
    public Result<ReturnResult> listOld(
            @RequestParam(name = "id") String id
    ) throws IOException {
        ReturnResult returnResult = new ReturnResult();
        List<String> errorMsg = new ArrayList<String>();
        PageContentCapture byId = iPageContentCaptureService.getById(id);
        returnResult.setById(byId);
        String retMsg = byId.getRetMsg();
        Document parse = Jsoup.parse(retMsg);
        Elements elements = parse.select("span.proofreadSignClass.errorWordMark");
        Integer indexIn = 0;
        for (Element element : elements) {
            String reHtml = "<span id='a"+indexIn+"'>"+element.toString()+"</span>";
            retMsg = retMsg.replace(element.toString(),reHtml);
            String myClass = element.attr("title");
            errorMsg.add("<a href='#a"+indexIn+"'>"+myClass+"</a>");
            indexIn++;
        }
        byId.setRetMsg(retMsg);
        returnResult.setById(byId);
        returnResult.setErrorMsg(errorMsg);
        return Result.OK("成功",returnResult);
    }


    @GetMapping(value = "/listLink")
    public Result<ExternalInternalSite> listLink(
            @RequestParam(name = "id") String id
    ) throws IOException {
        ExternalInternalSite externalInternalSite = externalInternalSiteService.getById(id);
        String source = externalInternalSite.getSource();
        Document document = Jsoup.parse(new URL(source), 10000);
        String baseUri = document.baseUri();
        document = jsoupUtils.setUrl(document, baseUri);
        String string = document.toString();
        string = string.replaceAll("amp;","");
        externalInternalSite.setDoc(string);
        return Result.OK(externalInternalSite);
    }

    @GetMapping(value = "/listLinkSpyder")
    public  Result<String> listLinkSpyder(
            @RequestParam(name = "url") String url,
            @RequestParam(name = "tenantId") String tenantId
    ) throws Exception {
        QueryWrapper<MainSite> queryWrapper = new QueryWrapper<MainSite>();
        queryWrapper.eq("url",url);
        Page<MainSite> page = new Page<MainSite>(1, 1);
        TenantContext.setTenant(tenantId);
        IPage<MainSite> pageList = iMainSiteService.page(page, queryWrapper);
        List<MainSite> records = pageList.getRecords();
        MainSite mainSite = null;
        if(records.size() >= 1){
            mainSite = records.get(0);
        }
        if(!ObjectUtils.isEmpty(mainSite)){
            System.out.println("抓取页面数据");
            crawlingContentCapture(mainSite,tenantId);
            System.out.println("抓取外链数据");
            crawlingLinkOuter(mainSite,tenantId);
            System.out.println("抓取错链数据");
            crawlingErrorLink(mainSite,tenantId);
        }
        return Result.ok("成功");
    }
    //抓取页面信息
    public void crawlingContentCapture(MainSite mainSite,String  tenantId){
        TenantContext.setTenant(tenantId);
        Integer count =  externalInternalSiteService.countSource(mainSite.getMainSiteUrl());
        System.out.println(count);
        Integer pageSize = 100;
        Integer pageNum = (int)Math.ceil((double)count/pageSize);
        for(int i = 1; i <= pageNum;i+=1){
            Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
            TenantContext.setTenant(tenantId);
            IPage<ExternalInternalSite> pageList = externalInternalSiteService.pageSourceMainUrl(page,mainSite.getMainSiteUrl());
            for (ExternalInternalSite record : pageList.getRecords()) {
                String LinkCaptureErrorLogId = IdWorker.getIdStr();
                PageContentCapture pageContentCapture = new PageContentCapture();
                pageContentCapture.setId(DigestUtils.md5DigestAsHex(record.getSource().getBytes()));
                //查询当前id有没有对应的内容
                try {
                    TenantContext.setTenant(tenantId);
                    PageContentCapture byId = iPageContentCaptureService.getById(DigestUtils.md5DigestAsHex(record.getSource().getBytes()));
                    if(!ObjectUtils.isEmpty(byId)){
                        pageContentCapture = byId;
                    }
                }catch (Exception e){
                    e.printStackTrace();
                }
                pageContentCapture.setCreateTime(new Date());
                pageContentCapture.setUpdateTime(new Date());
                pageContentCapture.setSource(record.getSource());
                if(!ObjectUtils.isEmpty(record.getSource())){
                    //爬取页面
                    Document document = null;
                    String baseUri = null;
                    String grabPage = mainSite.getGrabPage();
                    try {
                        SpyderReturn spyderReturn = trendsSpyder.companyTrendsSpyderMessage(record.getSource(), grabPage);
                        document = spyderReturn.getDocument();
                        baseUri = spyderReturn.getBaseUri();
                        pageContentCapture.setIsInvalid("2");
                    }catch (Exception e){
                        pageContentCapture.setIsInvalid("1");
                        document = null;
                    }
                    if(!ObjectUtils.isEmpty(document)){
                        document= jsoupUtils.setUrl(document, baseUri);
                        String sHtml = document.toString();
                        sHtml = sHtml.replaceAll("amp;","");
                        pageContentCapture.setContent(sHtml);

                        //获取文章发布的标题
                        QueryWrapper<LinkPublishingTitle> linkPublishingTitleQueryWrapper =new QueryWrapper<LinkPublishingTitle>();
                        linkPublishingTitleQueryWrapper.eq("main_site_url",record.getMainUrl());
                        String title = null;
                        title = crawlingUtils.getPublishingTitle(record.getMainUrl(), document,tenantId);
                        if(title != null && title != "" && title.length() != 0){
                            pageContentCapture.setTitle(title);
                        }else{
                            //抓取网页title的内容
                            Elements eles = document.getElementsByTag("title");
                            String string = eles.text();
                            pageContentCapture.setTitle(string);
                        }
                        QueryWrapper<LinkPublishingTimeGrasping> linkPublishingTimeGraspingQueryWrapper =new QueryWrapper<LinkPublishingTimeGrasping>();
                        linkPublishingTimeGraspingQueryWrapper.eq("main_site_url",record.getMainUrl());
                        //获取文章发布的时间
                        Date parse = crawlingUtils.getPublishingTimeGrasping(record.getMainUrl(), document,tenantId);
                        if(!ObjectUtils.isEmpty(parse)){
                            pageContentCapture.setArticleReleaseTime(parse);
                        }
                        //父地址类型  typeSource
                        pageContentCapture.setTypeSource(record.getTypeSource());
                        pageContentCapture.setStateSource(record.getStateSource());
                        pageContentCapture.setMainUrl(record.getMainUrl());
                        //错敏词校验-----开始
                        JsResult jsonObject = fangCunUtils.getJsResult(pageContentCapture.getContent());
                        String ret_msg = jsonObject.getRet_msg();
                        pageContentCapture.setErrorCount(jsonObject.getErrorCount());
                        pageContentCapture.setRetMsg(jsonObject.getRet_msg());
                        pageContentCapture.setRetCode(jsonObject.getRet_code());
                        if(!ObjectUtils.isEmpty(ret_msg)){
                            //跳过白名单词汇
                            pageContentCapture =  crawlingUtils.JumpLinkWhiteWord(ret_msg,record.getMainUrl(),pageContentCapture,tenantId);
                            //保存错词信息
                            linkOrWordSaveUtils.saveLinkCaptureErrorMessage(LinkCaptureErrorLogId,pageContentCapture);
                            Document parse1 = Jsoup.parse(pageContentCapture.getRetMsg());
                            Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
                            pageContentCapture.setSensitiveList(elements1.toString());
                            pageContentCapture.setSensitiveCount(0);
                            pageContentCapture.setSensitiveCount(elements1.size());
                            if(elements1.size() > 0){
                                pageContentCapture.setCheckErrorCount(elements1.size());
                                pageContentCapture.setErrorCount(1);
                                pageContentCapture.setModifyState("1");
                            }
                        }
                        pageContentCapture.setRetCode(jsonObject.getRet_code());
                        pageContentCapture.setHeavyCount(jsonObject.getHeavyCount());
                    }
                    //错敏词校验-----结束
                }
                pageContentCapture.setIsIgnore("2");
                String id = pageContentCapture.getId();
                TenantContext.setTenant(tenantId);
                PageContentCapture byId = iPageContentCaptureService.getById(id);
                if(!ObjectUtils.isEmpty(byId)){
                    Integer numberScans = byId.getNumberScans();
                    if(!ObjectUtils.isEmpty(numberScans)){
                        numberScans = numberScans + 1;
                        pageContentCapture.setNumberScans(numberScans);
                        pageContentCapture.setCreateTime(byId.getCreateTime());
                    }
                    pageContentCapture.setIsIgnore(byId.getIsIgnore());
                }else{
                    pageContentCapture.setNumberScans(1);
                }
                pageContentCapture.setCheckState("2");
                if(!ObjectUtils.isEmpty(pageContentCapture))
                {
                    pageContentCapture.setTenantId(record.getTenantId());
                    iPageContentCaptureService.saveOrUpdate(pageContentCapture);
                    String sensitiveList = pageContentCapture.getSensitiveList();
                    if(!ObjectUtils.isEmpty(sensitiveList)){
                        linkOrWordSaveUtils.iLinkCaptureErrorLogServicesave(pageContentCapture,LinkCaptureErrorLogId);
                    }
                }
            }
        }
        try {
            //扫描结束对信息进行统计
            linkOrWordSaveUtils.LinkErrorWordStatisticsSave(mainSite,"1",tenantId);
        }catch (Exception e){
            e.printStackTrace();
        }
    }

    private void crawlingLinkOuter(MainSite mainSite,String tenantId) {
        //只统计外链
        TenantContext.setTenant(tenantId);
        Integer count =  externalInternalSiteService.countSourceLinkOuter(mainSite.getMainSiteUrl());
        Integer pageSize = 100;
        Integer pageNum = (int)Math.ceil((double)count/pageSize);
        for(int i = 1; i <= pageNum;i+=1){
            Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
            //只统计外链
            TenantContext.setTenant(tenantId);
            IPage<ExternalInternalSite> pageList = externalInternalSiteService.pageSourceMainUrlLinkOuter(page,mainSite.getMainSiteUrl());
            for (ExternalInternalSite externalInternalSite : pageList.getRecords()) {
                LinkOuterChainDetection outerChainDetection = new LinkOuterChainDetection();
                outerChainDetection.setId(externalInternalSite.getId());
                outerChainDetection.setCreateBy(externalInternalSite.getMainUrl());
                outerChainDetection.setCreateTime(new Date());
                outerChainDetection.setUpdateBy(externalInternalSite.getMainUrl());
                outerChainDetection.setUpdateTime(new Date());
                QueryWrapper<MainSite> mainSiteQueryWrapper = new QueryWrapper<MainSite>();
                String mainUrl1 = externalInternalSite.getMainUrl();
                mainSiteQueryWrapper.eq("main_site_url",mainUrl1);
                TenantContext.setTenant(tenantId);
                List<MainSite> mainSiteList = iMainSiteService.list(mainSiteQueryWrapper);
                if(mainSiteList.size() >= 1){
                    MainSite mainSite1 = mainSiteList.get(0);
                    outerChainDetection.setMainSiteId(mainSite1.getId());
                    outerChainDetection.setMainSiteName(mainSite1.getMainSiteName());
                    outerChainDetection.setMainSiteUrl(mainSite1.getMainSiteUrl());
                }
                outerChainDetection.setOuterUrl(externalInternalSite.getUrl());
                outerChainDetection.setOuterUrlType(externalInternalSite.getTypeUrl());
                outerChainDetection.setOuterUrlState(externalInternalSite.getStateUrl());
                //设置引用页数
                QueryWrapper<ExternalInternalSite> siteQueryWrapper = new QueryWrapper<ExternalInternalSite>();
                siteQueryWrapper.eq("url",externalInternalSite.getUrl());
                TenantContext.setTenant(tenantId);
                long countSiteQuery = externalInternalSiteService.count(siteQueryWrapper);
                outerChainDetection.setNumberApplicationPages(Integer.parseInt(String.valueOf(countSiteQuery)));
                QueryWrapper<LinkOuterChainDetection> linkOuterChainDetectionQueryWrapper = new QueryWrapper<LinkOuterChainDetection>();
                linkOuterChainDetectionQueryWrapper.eq("id",externalInternalSite.getId());
                Page<LinkOuterChainDetection> pageListS = new Page<LinkOuterChainDetection>(1, 1);
                TenantContext.setTenant(tenantId);
                Page<LinkOuterChainDetection> page1 = linkOuterChainDetectionService.page(pageListS,linkOuterChainDetectionQueryWrapper);
                List<LinkOuterChainDetection> outerChainDetectionList = page1.getRecords();
                LinkOuterChainDetection chainDetection = null;
                if(outerChainDetectionList.size() >=1){
                    chainDetection = outerChainDetectionList.get(0);
                }
                if(!ObjectUtils.isEmpty(chainDetection)){
                    outerChainDetection.setUpdateTime(chainDetection.getUpdateTime());
                    Integer numberScans = chainDetection.getNumberScans();
                    numberScans = numberScans +1;
                    outerChainDetection.setNumberScans(numberScans);
                    outerChainDetection.setIsIgnore(chainDetection.getIsIgnore());
                    outerChainDetection.setCheckState(chainDetection.getCheckState());
                    outerChainDetection.setIsRevise(chainDetection.getIsRevise());
                }else{
                    outerChainDetection.setNumberScans(1);
                    outerChainDetection.setIsIgnore("2");
                    outerChainDetection.setCheckState("2");
                    outerChainDetection.setIsRevise("2");
                }
                outerChainDetection.setTenantId(externalInternalSite.getTenantId());
                linkOuterChainDetectionService.saveOrUpdate(outerChainDetection);
                //记录日志信息
                linkOrWordSaveUtils.savelinkOuterChainLog(outerChainDetection,externalInternalSite.getTenantId());
            }
        }
        try {
            TenantContext.setTenant(tenantId);
            LinkOuterouterStatistics linkOuterouterStatistics = iLinkOuterouterStatisticsService.listCountByMainUrl(mainSite.getMainSiteUrl());
            if(!ObjectUtils.isEmpty(linkOuterouterStatistics)){
                linkOuterouterStatistics.setId(IdWorker.getIdStr());
                linkOuterouterStatistics.setScanTime(linkOuterouterStatistics.getUpdateTime());
                linkOuterouterStatistics.setTestType("1");
                linkOuterouterStatistics.setTenantId(Integer.parseInt(tenantId));
                iLinkOuterouterStatisticsService.save(linkOuterouterStatistics);
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }

    private void crawlingErrorLink(MainSite mainSite,String  tenantId) {
        String mainUrl = mainSite.getMainSiteUrl();
        //只统计错链
        TenantContext.setTenant(tenantId);
        Integer count =  externalInternalSiteService.countSourceLinkError(mainUrl);
        System.out.println(count);
        Integer pageSize = 100;
        Integer pageNum = (int)Math.ceil((double)count/pageSize);
        for(int i = 1; i <= pageNum;i+=1){
            Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
            //只统计错链
            TenantContext.setTenant(tenantId);
            IPage<ExternalInternalSite> pageList = externalInternalSiteService.pageSourceErrorMainUrl(page,mainUrl);
            for (ExternalInternalSite internalSite : pageList.getRecords()) {
                //爬取页面，判断当前链接是否是错链
                Document document = null;
                try {
                    document = Jsoup.parse(new URL(internalSite.getUrl()), 10000);
                }catch (Exception e){
                }
                if(!org.apache.commons.lang3.ObjectUtils.isEmpty(document)){
                    continue;
                }
                LinkChainErrorDetection errorDetection = new LinkChainErrorDetection();
                errorDetection.setId(internalSite.getId());
                errorDetection.setCreateBy(internalSite.getMainUrl());
                errorDetection.setCreateTime(new Date());
                errorDetection.setUpdateBy(internalSite.getMainUrl());
                errorDetection.setUpdateTime(new Date());
                QueryWrapper<MainSite> mainSiteQueryWrapper = new QueryWrapper<MainSite>();
                String mainUrl1 = internalSite.getMainUrl();
                mainSiteQueryWrapper.eq("main_site_url",mainUrl1);
                TenantContext.setTenant(tenantId);
                List<MainSite> mainSiteList = iMainSiteService.list(mainSiteQueryWrapper);
                if(mainSiteList.size() >= 1){
                    MainSite mainSiteIn = mainSiteList.get(0);
                    errorDetection.setMainSiteId(mainSiteIn.getId());
                    errorDetection.setMainSiteName(mainSiteIn.getMainSiteName());
                    errorDetection.setMainSiteUrl(mainSiteIn.getMainSiteUrl());
                }
                errorDetection.setErrorUrl(internalSite.getUrl());
                errorDetection.setErrorUrlType(internalSite.getTypeUrl());
                errorDetection.setErrorUrlState(internalSite.getStateUrl());
                //设置引用页数
                QueryWrapper<ExternalInternalSite> siteQueryWrapper = new QueryWrapper<ExternalInternalSite>();
                siteQueryWrapper.eq("url",internalSite.getUrl());
                TenantContext.setTenant(tenantId);
                long countSiteQuery = externalInternalSiteService.count(siteQueryWrapper);
                errorDetection.setNumberApplicationPages(Integer.parseInt(String.valueOf(countSiteQuery)));

                QueryWrapper<LinkChainErrorDetection> linkChainErrorDetectionQueryWrapper = new QueryWrapper<LinkChainErrorDetection>();
                linkChainErrorDetectionQueryWrapper.eq("id",internalSite.getId());
                Page<LinkChainErrorDetection> pageListS = new Page<LinkChainErrorDetection>(1, 1);
                TenantContext.setTenant(tenantId);
                Page<LinkChainErrorDetection> page1 = linkChainErrorDetectionService.page(pageListS,linkChainErrorDetectionQueryWrapper);
                List<LinkChainErrorDetection> chainErrorDetections = page1.getRecords();
                LinkChainErrorDetection chainDetection = null;
                if(chainErrorDetections.size() >=1){
                    chainDetection = chainErrorDetections.get(0);
                }
                if(!ObjectUtils.isEmpty(chainDetection)){
                    errorDetection.setUpdateTime(chainDetection.getUpdateTime());
                    Integer numberScans = chainDetection.getNumberScans();
                    numberScans = numberScans + 1;
                    errorDetection.setNumberScans(numberScans);
                    errorDetection.setIsIgnore(chainDetection.getIsIgnore());
                    errorDetection.setCheckState(chainDetection.getCheckState());
                }else{
                    errorDetection.setNumberScans(1);
                    errorDetection.setIsIgnore("2");
                    errorDetection.setCheckState("2");
                }
                errorDetection.setIsRevise("2");
                errorDetection.setTenantId(internalSite.getTenantId());
                linkChainErrorDetectionService.saveOrUpdate(errorDetection);
                saveLinkChainErrorLog(errorDetection);
            }
        }
        try {
            //统计数据
            LinkErrorStatistics linkErrorStatistics = iLinkErrorStatisticsService.listCountByMainUrl(mainUrl);
            if(!ObjectUtils.isEmpty(linkErrorStatistics)){
                linkErrorStatistics.setId(IdWorker.getIdStr());
                linkErrorStatistics.setTestType("1");
                linkErrorStatistics.setScanTime(linkErrorStatistics.getUpdateTime());
                linkErrorStatistics.setTenantId(Integer.parseInt(tenantId));
                iLinkErrorStatisticsService.save(linkErrorStatistics);
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }

    public void saveLinkChainErrorLog(LinkChainErrorDetection errorDetection){
        //记录日志信息
        LinkChainErrorLog linkChainErrorLog = new LinkChainErrorLog();
        linkChainErrorLog.setId(IdWorker.getIdStr());
        linkChainErrorLog.setCreateBy(errorDetection.getCreateBy());
        linkChainErrorLog.setCreateTime(errorDetection.getCreateTime());
        linkChainErrorLog.setUpdateBy(errorDetection.getUpdateBy());
        linkChainErrorLog.setUpdateTime(errorDetection.getUpdateTime());
        linkChainErrorLog.setErrorDetectionId(errorDetection.getId());
        linkChainErrorLog.setMainSiteId(errorDetection.getMainSiteId());
        linkChainErrorLog.setMainSiteName(errorDetection.getMainSiteName());
        linkChainErrorLog.setMainSiteUrl(errorDetection.getMainSiteUrl());
        linkChainErrorLog.setErrorUrl(errorDetection.getErrorUrl());
        linkChainErrorLog.setErrorUrlType(errorDetection.getErrorUrlType());
        linkChainErrorLog.setErrorUrlState(errorDetection.getErrorUrlState());
        linkChainErrorLog.setNumberApplicationPages(errorDetection.getNumberApplicationPages());
        linkChainErrorLog.setNumberScans(errorDetection.getNumberScans());
        linkChainErrorLog.setNumberScans(errorDetection.getNumberScans());
        linkChainErrorLog.setIsRevise(errorDetection.getIsRevise());
        linkChainErrorLog.setIsIgnore(errorDetection.getIsIgnore());
        linkChainErrorLog.setCheckTime(errorDetection.getCheckTime());
        linkChainErrorLog.setCheckState(errorDetection.getCheckState());
        linkChainErrorLog.setCheckMessage(errorDetection.getCheckMessage());
        linkChainErrorLog.setTenantId(errorDetection.getTenantId());
        boolean save = iLinkChainErrorLogService.save(linkChainErrorLog);
    }

    @Data
    class ReturnResultWx{
        WechatPostsWord byId;
        List<String> errorMsg;
        List<Integer> index;
    }

    @GetMapping(value = "/listWx")
    public Result<ReturnResultWx> listWx(
            @RequestParam(name = "id") String id
    ) throws IOException {
        ReturnResultWx returnResult = new ReturnResultWx();
        WechatPostsWord wechatPostsWord = iWechatPostsWordService.getById(id);
        List<String> errorMsg = new ArrayList<String>();
        if(!ObjectUtils.isEmpty(wechatPostsWord)){
            String link = wechatPostsWord.getLink();
            String spyderGetHTMLByUrl = spyderUtils.getSpyderGetHTMLByUrl(link, null, "2");
            spyderGetHTMLByUrl = spyderGetHTMLByUrl.replaceAll("amp;","");
            Document parse = Jsoup.parse(spyderGetHTMLByUrl);
            Element js_article = parse.getElementById("js_article");
            //将其中内容进行校验
            String sjs_article = js_article.toString();
            JsResult jsResult = fangCunUtils.getJsResult(sjs_article);
            String ret_msg = jsResult.getRet_msg();
            returnResult.setErrorMsg(errorMsg);
            Document parseRetMsg = Jsoup.parse(ret_msg);
            Elements elements = parseRetMsg.select("span.proofreadSignClass.errorWordMark");
            Integer indexIn = 0;
            for (Element element : elements) {
                String reHtml = "<span id='a"+indexIn+"'>"+element.toString()+"</span>";
                ret_msg = ret_msg.replace(element.toString(),reHtml);
                String myClass = element.attr("title");
                errorMsg.add("<a href='#a"+indexIn+"'>"+myClass+"</a>");
                indexIn++;
            }
            wechatPostsWord.setRetMsg(ret_msg);
        }
        returnResult.setById(wechatPostsWord);
        return Result.OK("成功",returnResult);
    }
}
