package org.jeecg.modules.demo.job;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import kong.unirest.HttpResponse;
import kong.unirest.JsonNode;
import kong.unirest.Unirest;
import lombok.extern.slf4j.Slf4j;
import org.jeecg.common.config.TenantContext;
import org.jeecg.common.util.DateUtils;
import org.jeecg.modules.demo.externalinternalsite.entity.ExternalInternalSite;
import org.jeecg.modules.demo.externalinternalsite.service.IExternalInternalSiteService;
import org.jeecg.modules.demo.linkcaptureerrorlog.entity.LinkCaptureErrorLog;
import org.jeecg.modules.demo.linkcaptureerrorlog.service.ILinkCaptureErrorLogService;
import org.jeecg.modules.demo.linkcaptureerrormessage.entity.LinkCaptureErrorMessage;
import org.jeecg.modules.demo.linkcaptureerrormessage.service.ILinkCaptureErrorMessageService;
import org.jeecg.modules.demo.linkchainerrordetection.entity.LinkChainErrorDetection;
import org.jeecg.modules.demo.linkchainerrordetection.service.ILinkChainErrorDetectionService;
import org.jeecg.modules.demo.linkchainerrorlog.entity.LinkChainErrorLog;
import org.jeecg.modules.demo.linkchainerrorlog.service.ILinkChainErrorLogService;
import org.jeecg.modules.demo.linkerrorstatistics.entity.LinkErrorStatistics;
import org.jeecg.modules.demo.linkerrorstatistics.service.ILinkErrorStatisticsService;
import org.jeecg.modules.demo.linkerrorwordstatistics.entity.LinkErrorWordStatistics;
import org.jeecg.modules.demo.linkerrorwordstatistics.service.ILinkErrorWordStatisticsService;
import org.jeecg.modules.demo.linkouterchaindetection.entity.LinkOuterChainDetection;
import org.jeecg.modules.demo.linkouterchaindetection.service.ILinkOuterChainDetectionService;
import org.jeecg.modules.demo.linkouterchainlog.entity.LinkOuterChainLog;
import org.jeecg.modules.demo.linkouterchainlog.service.ILinkOuterChainLogService;
import org.jeecg.modules.demo.linkouterouterstatistics.entity.LinkOuterouterStatistics;
import org.jeecg.modules.demo.linkouterouterstatistics.service.ILinkOuterouterStatisticsService;
import org.jeecg.modules.demo.linkpublishingtimegrasping.entity.LinkPublishingTimeGrasping;
import org.jeecg.modules.demo.linkpublishingtimegrasping.service.ILinkPublishingTimeGraspingService;
import org.jeecg.modules.demo.linkpublishingtitle.entity.LinkPublishingTitle;
import org.jeecg.modules.demo.linkpublishingtitle.service.ILinkPublishingTitleService;
import org.jeecg.modules.demo.linkwhitelist.service.ILinkWhiteListService;
import org.jeecg.modules.demo.linkwhiteword.entity.LinkWhiteWord;
import org.jeecg.modules.demo.linkwhiteword.service.ILinkWhiteWordService;
import org.jeecg.modules.demo.mainsite.entity.MainSite;
import org.jeecg.modules.demo.mainsite.service.IMainSiteService;
import org.jeecg.modules.demo.pagecontentcapture.entity.JsResult;
import org.jeecg.modules.demo.pagecontentcapture.entity.PageContentCapture;
import org.jeecg.modules.demo.pagecontentcapture.service.IPageContentCaptureService;
import org.jeecg.modules.spyderUtils.spider.SpyderReturn;
import org.jeecg.modules.spyderUtils.spider.TrendsSpyder;
import org.jeecg.modules.system.entity.SysDict;
import org.jeecg.modules.system.entity.SysDictItem;
import org.jeecg.modules.system.service.ISysDictItemService;
import org.jeecg.modules.system.service.ISysDictService;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.DigestUtils;
import org.springframework.util.ObjectUtils;

import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Slf4j
@Component
public class JobCheck implements Job {

    @Autowired
    private IMainSiteService mainSiteService;
    @Autowired
    private IPageContentCaptureService iPageContentCaptureService;
    @Autowired
    private ILinkPublishingTitleService linkPublishingTitleService;

    @Autowired
    private ILinkPublishingTimeGraspingService iLinkPublishingTimeGraspingService;

    @Autowired
    private IMainSiteService iMainSiteService;

    @Autowired
    private ISysDictService sysDictService;

    @Autowired
    private ILinkWhiteWordService iLinkWhiteWordService;

    @Autowired
    private ISysDictItemService iSysDictItemService;


    @Autowired
    private IExternalInternalSiteService externalInternalSiteService;

    @Autowired
    private IPageContentCaptureService pageContentCaptureService;


    @Autowired
    private ILinkCaptureErrorLogService iLinkCaptureErrorLogService;

    @Autowired
    private ILinkCaptureErrorMessageService iLinkCaptureErrorMessageService;

    @Autowired
    private ILinkErrorWordStatisticsService iLinkErrorWordStatisticsService;

    @Autowired
    private TrendsSpyder trendsSpyder;

    @Autowired
    private ILinkOuterChainDetectionService linkOuterChainDetectionService;

    @Autowired
    private ILinkChainErrorLogService iLinkChainErrorLogService;

    @Autowired
    private ILinkOuterChainLogService linkOuterChainLogService;

    @Autowired
    private ILinkOuterouterStatisticsService iLinkOuterouterStatisticsService;

    @Autowired
    private ILinkWhiteListService linkWhiteListService;

    @Autowired
    private ILinkChainErrorDetectionService linkChainErrorDetectionService;

    @Autowired
    private ILinkErrorStatisticsService iLinkErrorStatisticsService;

    /**
     * 若参数变量名修改 QuartzJobController中也需对应修改
     */
    private String parameter;

    public void setParameter(String parameter) {
        this.parameter = parameter;
    }

    @Override
    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
        String TenantId = parameter.split("&&&")[1];
        parameter =  parameter.split("&&&")[0];
        log.info(" Job Execution key："+jobExecutionContext.getJobDetail().getKey());
        log.info( String.format("welcome %s! Jeecg-Boot 带参数定时任务 SampleParamJob !   时间:" + DateUtils.now(),parameter));
        QueryWrapper<MainSite> queryWrapper = new QueryWrapper<MainSite>();
        Page<MainSite> page = new Page<MainSite>(1, 1);
        queryWrapper.eq("main_site_url",parameter);
        TenantContext.setTenant(TenantId);
        IPage<MainSite> pageList = mainSiteService.page(page, queryWrapper);
        List<MainSite> records = pageList.getRecords();
        MainSite mainSite = null;
        if(records.size() >= 1){
            mainSite = records.get(0);
        }
        if(!ObjectUtils.isEmpty(mainSite)){
            //错敏词复核
            checkPageContent(mainSite.getMainSiteUrl());
            //外链复核
            checkLinkOuterByMain(mainSite);
            //错链复核
            checkErrorLinkByMain(mainSite);
        }
    }

    public void checkPageContent(String mainUrl){
        if(ObjectUtils.isEmpty(mainUrl) && mainUrl != "" && mainUrl != null){
            QueryWrapper<PageContentCapture> queryWrapper = new QueryWrapper<PageContentCapture>();
            queryWrapper.isNotNull("sensitive_list");
            queryWrapper.ne("sensitive_list","");
            queryWrapper.gt("error_count",0);
            queryWrapper.eq("main_url",mainUrl);
            long count = pageContentCaptureService.count(queryWrapper);
            Integer pageSize = 100;
            Integer pageNum = (int)Math.ceil((double)count/pageSize);
            Integer tenantId = null;
            for(int i = 1; i <= pageNum;i+=1){
                Page<PageContentCapture> page = new Page<PageContentCapture>(i, pageSize);
                IPage<PageContentCapture> pageList = pageContentCaptureService.page(page, queryWrapper);
                for (PageContentCapture contentCapture : pageList.getRecords()) {
                    tenantId = contentCapture.getTenantId();
                    String source = contentCapture.getSource();
                    contentCapture.setUpdateTime(new Date());
                    String LinkCaptureErrorLogId = IdWorker.getIdStr();
                    if(!ObjectUtils.isEmpty(source)){
                        //爬取页面
                        Document document = null;
                        try {
                            document = Jsoup.parse(new URL(source), 10000);
                            contentCapture.setIsInvalid("2");
                        }catch (Exception e){
                            contentCapture.setIsInvalid("1");
                        }
                        if(!ObjectUtils.isEmpty(document)){
                            String baseUri = document.baseUri();
                            Elements tags = document.select("link[href],img[src],*[style*=background-image],script[src],td[background],a[href]");
                            try {
                                for (Element tag : tags) {
                                    String attr = tag.attr("href");
                                    if (attr == null || attr.isEmpty()) continue;
                                    if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                        String absUrl = new URI(baseUri).resolve(attr).toString();
                                        absUrl = new URI(absUrl).normalize().toString();
                                        tag.attr("href", absUrl);
                                    }
                                }
                            }catch (Exception e){
                                e.printStackTrace();
                            }
                            try {
                                for (Element tag : tags) {
                                    String attr = tag.attr("src");
                                    if (attr == null || attr.isEmpty()) continue;
                                    if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                        String absUrl = new URI(baseUri).resolve(attr).toString();
                                        absUrl = new URI(absUrl).normalize().toString();
                                        tag.attr("src", absUrl);
                                    }
                                }
                            }catch (Exception e){
                                e.printStackTrace();
                            }
                            try {
                                for (Element tag : tags) {
                                    String attr = tag.attr("background");
                                    if (attr == null || attr.isEmpty()) continue;
                                    if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                        String absUrl = new URI(baseUri).resolve(attr).toString();
                                        absUrl = new URI(absUrl).normalize().toString();
                                        tag.attr("background", absUrl);
                                    }
                                }
                            }catch (Exception e){
                                e.printStackTrace();
                            }
                            String sHtml = document.toString();
                            sHtml = sHtml.replaceAll("amp;","");
                            contentCapture.setContent(sHtml);
                            //获取文章发布的标题
                            QueryWrapper<LinkPublishingTitle> linkPublishingTitleQueryWrapper =new QueryWrapper<LinkPublishingTitle>();
                            linkPublishingTitleQueryWrapper.eq("main_site_url",contentCapture.getMainUrl());
                            List<LinkPublishingTitle> linkPublishingTitles = linkPublishingTitleService.list(linkPublishingTitleQueryWrapper);
                            String title = null;
                            for (LinkPublishingTitle linkPublishingTitle : linkPublishingTitles) {
                                String typeCssXpath = linkPublishingTitle.getTypeCssXpath();
                                try {
                                    if(typeCssXpath.equals("selector")){
                                        Elements select = document.select(linkPublishingTitle.getElementPositioningType());
                                        String string = select.toString();
                                        String text = Jsoup.parse(string).text();
                                        if(text != null && text != ""){
                                            title = text;
                                        }
                                    }
                                    if(typeCssXpath.equals("xpath")){
                                        // 5.将document转换成xpath能解析的文档树结构
                                        JXDocument jdoc = new JXDocument(document.getAllElements()); // 此处可以使用xpath语法
                                        // 6.使用xpath语法来解析html文档来完成数据清洗
                                        List<JXNode> elNodes = jdoc.selN(linkPublishingTitle.getElementPositioningType());
                                        for (JXNode elNode : elNodes) {
                                            String string = elNode.toString();
                                            String text = Jsoup.parse(string).text();
                                            if(text != null && text != ""){
                                                title = text;
                                            }
                                        }
                                    }
                                }catch (Exception e){
                                }
                            }
                            if(title != null && title != "" && title.length() != 0){
                                contentCapture.setTitle(title);
                            }else{
                                //抓取网页title的内容
                                Elements eles = document.getElementsByTag("title");
                                String string = eles.text();
                                contentCapture.setTitle(string);
                            }
                            QueryWrapper<LinkPublishingTimeGrasping> linkPublishingTimeGraspingQueryWrapper =new QueryWrapper<LinkPublishingTimeGrasping>();
                            linkPublishingTimeGraspingQueryWrapper.eq("main_site_url",contentCapture.getMainUrl());
                            //获取文章发布的时间
                            List<LinkPublishingTimeGrasping> linkPublishingTimeGraspings = iLinkPublishingTimeGraspingService.list(linkPublishingTimeGraspingQueryWrapper);
                            String time = null;
                            for (LinkPublishingTimeGrasping linkPublishingTimeGrasping : linkPublishingTimeGraspings) {
                                String typeCssXpathTime = linkPublishingTimeGrasping.getTypeCssXpath();
                                try {
                                    if(typeCssXpathTime.equals("selector")){
                                        Elements select = document.select(linkPublishingTimeGrasping.getElementPositioningType());
                                        String string = select.toString();
                                        String text = Jsoup.parse(string).text();
                                        if(text != null && text != ""){
                                            time = text;
                                        }
                                    }
                                    if(typeCssXpathTime.equals("xpath")){
                                        // 5.将document转换成xpath能解析的文档树结构
                                        JXDocument jdoc = new JXDocument(document.getAllElements()); // 此处可以使用xpath语法
                                        // 6.使用xpath语法来解析html文档来完成数据清洗
                                        List<JXNode> elNodes = jdoc.selN(linkPublishingTimeGrasping.getElementPositioningType());
                                        for (JXNode elNode : elNodes) {
                                            String string = elNode.toString();
                                            String text = Jsoup.parse(string).text();
                                            if(text != null && text != ""){
                                                time = text;
                                            }
                                        }
                                    }
                                }catch (Exception e){
                                }
                            }
                            //格式化时间
                            Date parse = null;
                            if(time != null && time != "" && time.length() != 0){
                                contentCapture.setTitle(title);
                                for (LinkPublishingTimeGrasping linkPublishingTimeGrasping : linkPublishingTimeGraspings) {
                                    String timeFormat = linkPublishingTimeGrasping.getTimeFormat();
                                    QueryWrapper<SysDict> sysDictQueryWrapper = new QueryWrapper<SysDict>();
                                    sysDictQueryWrapper.eq("dict_code","time_type");
                                    List<SysDict> listSysDict = sysDictService.list(sysDictQueryWrapper);
                                    SysDict sysDict = null;
                                    if(listSysDict.size() >= 1){
                                        sysDict = listSysDict.get(0);
                                    }
                                    QueryWrapper<SysDictItem> sysDictItemQueryWrapper = new QueryWrapper<SysDictItem>();
                                    sysDictItemQueryWrapper.eq("dict_id",sysDict.getId());
                                    sysDictItemQueryWrapper.eq("item_value",timeFormat);
                                    List<SysDictItem> sysDictItems = iSysDictItemService.list(sysDictItemQueryWrapper);
                                    for (SysDictItem sysDictItem : sysDictItems) {
                                        try {
                                            String itemText = sysDictItem.getItemText();
                                            //正则表达式
                                            String description = sysDictItem.getDescription();
                                            Pattern r = Pattern.compile(description);
                                            Matcher m = r.matcher(time);
                                            String dateStr = null;
                                            if (m.find()) {
                                                dateStr = m.group(0);
                                            }
                                            if(dateStr != null && dateStr != ""){
                                                SimpleDateFormat simpleDateFormat = new SimpleDateFormat(itemText);
                                                parse = simpleDateFormat.parse(dateStr);
                                            }
                                        }catch (Exception e){
                                        }
                                    }
                                }
                            }
                            if(!ObjectUtils.isEmpty(parse)){
                                contentCapture.setArticleReleaseTime(parse);
                            }
                            //父地址类型  typeSource
                            contentCapture.setTypeSource(contentCapture.getTypeSource());
                            contentCapture.setStateSource(contentCapture.getStateSource());
                            contentCapture.setMainUrl(contentCapture.getMainUrl());
                            //错敏词校验-----开始
                            Integer[] IntStr = {1000,1001,1002,1004,2000,2001,3000,3001,4000,5000,7001,7002};
                            Object oIntStr = JSONArray.toJSON(IntStr);
                            Map<String, Object> paramMap = new HashMap<>();
                            paramMap.put("userid", "quyanhong");
                            paramMap.put("ukey", "K103fhnBhXak");
                            paramMap.put("text", URLEncoder.encode( contentCapture.getContent()));
                            paramMap.put("error_types",oIntStr);
                            paramMap.put("threshold",0.2);
                            paramMap.put("strict_degree","m");
                            paramMap.put("domains","test");
                            HttpResponse<JsonNode> response = Unirest.post("http://api.ifuncun.cn:8081/v1/proofreadHtml")
                                    .header("Content-Type", "application/json")
                                    .body( JSONObject.toJSONString(paramMap))
                                    .asJson();
                            String string = response.getBody().toString();
                            JsResult jsonObject = JSONObject.parseObject(string, JsResult.class);
                            String ret_msg = jsonObject.getRet_msg();
                            contentCapture.setErrorCount(jsonObject.getErrorCount());
                            contentCapture.setRetMsg(jsonObject.getRet_msg());
                            contentCapture.setRetCode(jsonObject.getRet_code());
                            if(!ObjectUtils.isEmpty(ret_msg)){
                                //跳过白名单词汇表
                                QueryWrapper<LinkWhiteWord> linkWhiteWordQueryWrapper = new QueryWrapper<LinkWhiteWord>();
                                linkWhiteWordQueryWrapper.eq("main_site_url",contentCapture.getMainUrl());
                                List<LinkWhiteWord> linkWhiteWordList = iLinkWhiteWordService.list(linkWhiteWordQueryWrapper);
                                for (LinkWhiteWord linkWhiteWord : linkWhiteWordList) {
                                    int index = ret_msg.indexOf(linkWhiteWord.getWord());
                                    if(index > 0){
                                        Document parseHtml = Jsoup.parse(ret_msg);
                                        Elements elements = parseHtml.select("span.proofreadSignClass.errorWordMark");
                                        for (Element element : elements) {
                                            String text = element.text();
                                            if(text.equals(linkWhiteWord.getWord())){
                                                String string1 = parseHtml.toString();
                                                String replace = string1.replace(element.toString(), linkWhiteWord.getWord());
                                                contentCapture.setRetMsg(replace);
                                                contentCapture.setErrorCount(0);
                                            }
                                        }
                                    }
                                }
                                Document parse1 = Jsoup.parse(contentCapture.getRetMsg());
                                Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
                                for(Element element : elements1){
                                    LinkCaptureErrorMessage linkCaptureErrorMessage = new LinkCaptureErrorMessage();
                                    linkCaptureErrorMessage.setErrorMessage(element.toString());
                                    linkCaptureErrorMessage.setErrerWord(element.text());
                                    linkCaptureErrorMessage.setLinkCaptureErrorLogId(LinkCaptureErrorLogId);
                                    linkCaptureErrorMessage.setId(DigestUtils.md5DigestAsHex((linkCaptureErrorMessage.getErrorMessage()+contentCapture.getId()).getBytes()));
                                    linkCaptureErrorMessage.setLinkPageId(contentCapture.getId());
                                    linkCaptureErrorMessage.setTenantId(contentCapture.getTenantId());
                                    iLinkCaptureErrorMessageService.saveOrUpdate(linkCaptureErrorMessage);
                                }
                                if(elements1.size() > 0){


                                    contentCapture.setCheckErrorCount(elements1.size());
                                    contentCapture.setErrorCount(1);
                                }
                            }
                            contentCapture.setRetCode(jsonObject.getRet_code());
                            contentCapture.setHeavyCount(jsonObject.getHeavyCount());
                        }
                        //错敏词校验-----结束
                        //判断是否修改完成
                        String id = contentCapture.getId();
                        PageContentCapture byId = iPageContentCaptureService.getById(id);
                        if(!ObjectUtils.isEmpty(byId)){
                            Integer numberScans = byId.getNumberScans();
                            if(!ObjectUtils.isEmpty(numberScans)){
                                numberScans = numberScans + 1;
                                contentCapture.setNumberScans(numberScans);
                                contentCapture.setCreateTime(byId.getCreateTime());
                            }
                        }else{
                            contentCapture.setNumberScans(1);
                        }
                        contentCapture.setIsIgnore("2");
                        contentCapture.setCheckState("1");
                        contentCapture.setCheckTime(new Date());
                        contentCapture.setTenantId(contentCapture.getTenantId());
                        iPageContentCaptureService.saveOrUpdate(contentCapture);
                        iLinkCaptureErrorLogServicesave(contentCapture,LinkCaptureErrorLogId);
                    }
                }
                try {
                    //扫描结束对信息进行统计
                    PageContentCapture pageContentCapture = iPageContentCaptureService.listCountByMainUrl(mainUrl);
                    //聚合错敏感词信息，将其结果保存到
                    if(!ObjectUtils.isEmpty(pageContentCapture)){
                        LinkErrorWordStatistics linkErrorWordStatistics = new LinkErrorWordStatistics();
                        linkErrorWordStatistics.setId(IdWorker.getIdStr());
                        linkErrorWordStatistics.setCreateBy(pageContentCapture.getCreateBy());
                        linkErrorWordStatistics.setCreateTime(pageContentCapture.getCreateTime());
                        linkErrorWordStatistics.setUpdateTime(pageContentCapture.getUpdateTime());
                        linkErrorWordStatistics.setUpdateBy(pageContentCapture.getUpdateBy());
                        String mainUrl1 = pageContentCapture.getMainUrl();
                        //通过站点id查询对应的站点信息
                        linkErrorWordStatistics.setMainSiteUrl(mainUrl1);
                        QueryWrapper<MainSite> mainSiteQueryWrapper = new QueryWrapper<MainSite>();
                        mainSiteQueryWrapper.eq("main_site_url",mainUrl1);
                        List<MainSite> mainSiteList = iMainSiteService.list(mainSiteQueryWrapper);
                        if(mainSiteList.size() >= 1){
                            MainSite mainSite = mainSiteList.get(0);
                            linkErrorWordStatistics.setMainSiteId(mainSite.getId());
                            linkErrorWordStatistics.setMainSiteName(mainSite.getMainSiteName());
                        }
                        linkErrorWordStatistics.setTestType("2");
                        linkErrorWordStatistics.setScanTime(pageContentCapture.getUpdateTime());
                        linkErrorWordStatistics.setErrorWordCount(pageContentCapture.getHeavyCount());
                        linkErrorWordStatistics.setTenantId(tenantId);
                        if(!ObjectUtils.isEmpty(linkErrorWordStatistics.getMainSiteUrl())){
                            String mainSiteId = linkErrorWordStatistics.getMainSiteId();
                            if(!ObjectUtils.isEmpty(mainSiteId)){
                                iLinkErrorWordStatisticsService.save(linkErrorWordStatistics);
                            }
                        }
                    }
                }catch (Exception e){
                    e.printStackTrace();
                }
            }
        }
    }
    public boolean iLinkCaptureErrorLogServicesave(PageContentCapture pageContentCapture, String LinkCaptureErrorLogId ){
        LinkCaptureErrorLog linkCaptureErrorLog = new LinkCaptureErrorLog();
        linkCaptureErrorLog.setId(LinkCaptureErrorLogId);
        linkCaptureErrorLog.setCheckTime(new Date());
        linkCaptureErrorLog.setUpdateTime(new Date());
        try {
            linkCaptureErrorLog.setLinkPageId(pageContentCapture.getId());
        }catch (Exception e){
        }
        linkCaptureErrorLog.setSource(pageContentCapture.getSource());
        linkCaptureErrorLog.setTitle(pageContentCapture.getTitle());
        linkCaptureErrorLog.setTypeSource(pageContentCapture.getTypeSource());
        linkCaptureErrorLog.setStateSource(pageContentCapture.getStateSource());
        linkCaptureErrorLog.setMainUrl(pageContentCapture.getMainUrl());
        linkCaptureErrorLog.setSensitiveList(pageContentCapture.getSensitiveList());
        linkCaptureErrorLog.setSensitiveCount(pageContentCapture.getSensitiveCount());
        linkCaptureErrorLog.setRetCode(pageContentCapture.getRetCode());
        linkCaptureErrorLog.setErrorCount(pageContentCapture.getErrorCount());
        linkCaptureErrorLog.setHeavyCount(pageContentCapture.getHeavyCount());
        linkCaptureErrorLog.setModifyState(pageContentCapture.getModifyState());
        linkCaptureErrorLog.setArticleReleaseTime(pageContentCapture.getArticleReleaseTime());
        linkCaptureErrorLog.setNumberScans(pageContentCapture.getNumberScans());
        linkCaptureErrorLog.setIsIgnore(pageContentCapture.getIsIgnore());
        linkCaptureErrorLog.setCheckTime(pageContentCapture.getCheckTime());
        linkCaptureErrorLog.setCheckState(pageContentCapture.getCheckState());
        linkCaptureErrorLog.setCheckErrorCount(pageContentCapture.getCheckErrorCount());
        linkCaptureErrorLog.setTenantId(pageContentCapture.getTenantId());
        //扫描日志记录
        boolean save = iLinkCaptureErrorLogService.saveOrUpdate(linkCaptureErrorLog);
        return save;
    }

    private void checkErrorLinkByMain(MainSite mainSite){
        QueryWrapper<LinkChainErrorDetection> queryWrapper = new QueryWrapper<LinkChainErrorDetection>();
        queryWrapper.eq("main_site_url",mainSite.getMainSiteUrl());
        //只统计错链
        Long count =  linkChainErrorDetectionService.count(queryWrapper);
        Integer pageSize = 100;
        Integer pageNum = (int)Math.ceil((double)count/pageSize);
        for(int i = 1; i <= pageNum;i+=1){
            Page<LinkChainErrorDetection> page = new Page<LinkChainErrorDetection>(i, pageSize);
            IPage<LinkChainErrorDetection> pageList = linkChainErrorDetectionService.page(page, queryWrapper);
            //并发执行
            //List<List<LinkChainErrorDetection>> lists = groupList(pageList.getRecords());
            checkErrorLink(pageList.getRecords());
        }
        //统计数据
        LinkErrorStatistics linkErrorStatistics = iLinkErrorStatisticsService.listCountByMainUrl(mainSite.getMainSiteUrl());
        if(!ObjectUtils.isEmpty(linkErrorStatistics)){
            linkErrorStatistics.setId(IdWorker.getIdStr());
            linkErrorStatistics.setTestType("2");
            linkErrorStatistics.setScanTime(linkErrorStatistics.getUpdateTime());
            linkErrorStatistics.setTenantId(mainSite.getTenantId());
            iLinkErrorStatisticsService.save(linkErrorStatistics);
        }
    }

    private void checkErrorLink(List<LinkChainErrorDetection> exportList){
        for (LinkChainErrorDetection errorDetection : exportList) {
            String errorUrl = errorDetection.getErrorUrl();
            String mainSiteId = errorDetection.getMainSiteId();
            MainSite mainSite = iMainSiteService.getById(mainSiteId);
            QueryWrapper<ExternalInternalSite> internalSiteQueryWrapper = new QueryWrapper<ExternalInternalSite>();
            internalSiteQueryWrapper.eq("url",errorUrl);
            long count = externalInternalSiteService.count(internalSiteQueryWrapper);
            Integer pageSize = 100;
            Integer pageNum = (int)Math.ceil((double)count/pageSize);
            int countError = 0;
            for(int i = 1; i <= pageNum;i+=1){
                Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
                IPage<ExternalInternalSite> pageList = externalInternalSiteService.page(page,internalSiteQueryWrapper);
                List<ExternalInternalSite> records = pageList.getRecords();
                for (ExternalInternalSite record : records) {
                    //爬取页面
                    Document document = null;
                    String grabPage = mainSite.getGrabPage();
                    try {
                        SpyderReturn spyderReturn = trendsSpyder.companyTrendsSpyderMessage(record.getSource(), grabPage);
                        document = spyderReturn.getDocument();
                        String baseUri = spyderReturn.getBaseUri();
                        Elements tags = document.select("link[href],img[src],*[style*=background-image],script[src],td[background],a[href],iframe[src]");
                        try {
                            for (Element tag : tags) {
                                String attr = tag.attr("href");
                                if (attr == null || attr.isEmpty()) continue;
                                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                    String absUrl = new URI(baseUri).resolve(attr).toString();
                                    absUrl = new URI(absUrl).normalize().toString();
                                    tag.attr("href", absUrl);
                                }
                            }
                        }catch (Exception e){
                            e.printStackTrace();
                        }
                        try {
                            for (Element tag : tags) {
                                String attr = tag.attr("src");
                                if (attr == null || attr.isEmpty()) continue;
                                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                    String absUrl = new URI(baseUri).resolve(attr).toString();
                                    absUrl = new URI(absUrl).normalize().toString();
                                    tag.attr("src", absUrl);
                                }
                            }
                        }catch (Exception e){
                            e.printStackTrace();
                        }
                        try {
                            for (Element tag : tags) {
                                String attr = tag.attr("background");
                                if (attr == null || attr.isEmpty()) continue;
                                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                    String absUrl = new URI(baseUri).resolve(attr).toString();
                                    absUrl = new URI(absUrl).normalize().toString();
                                    tag.attr("background", absUrl);
                                }
                            }
                        }catch (Exception e){
                            e.printStackTrace();
                        }
                        try {
                            for (Element tag : tags) {
                                String attr = tag.attr("iframe");
                                if (attr == null || attr.isEmpty()) continue;
                                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                    String absUrl = new URI(baseUri).resolve(attr).toString();
                                    absUrl = new URI(absUrl).normalize().toString();
                                    tag.attr("iframe", absUrl);
                                }
                            }
                        }catch (Exception e){
                            e.printStackTrace();
                        }
                    }catch (Exception e){
                        document = null;
                    }
                    if(!ObjectUtils.isEmpty(document)){
                        String sHtml = document.toString();
                        sHtml = sHtml.replaceAll("amp;","");
                        int i1 = sHtml.indexOf(errorUrl);
                        if(i1 > 0){
                            countError++;
                        }

                    }
                }
            }
            if(countError > 0){
                errorDetection.setIsRevise("2");
                errorDetection.setCheckNumberApplicationPages(countError);
            }
            if(countError == 0){
                errorDetection.setIsRevise("1");
            }
            errorDetection.setCreateTime(new Date());
            errorDetection.setUpdateTime(new Date());
            errorDetection.setCheckTime(new Date());
            errorDetection.setCheckState("1");
            boolean b = linkChainErrorDetectionService.updateById(errorDetection);
            saveLinkChainErrorLog(errorDetection);
        }
    }

    public void checkLinkOuterByMain(MainSite mainSite){
        QueryWrapper<LinkOuterChainDetection> queryWrapper = new QueryWrapper<LinkOuterChainDetection>();
        queryWrapper.eq("main_site_url",mainSite.getMainSiteUrl());
        //只统计错链
        Long count =  linkOuterChainDetectionService.count(queryWrapper);
        Integer pageSize = 100;
        Integer pageNum = (int)Math.ceil((double)count/pageSize);
        for(int i = 1; i <= pageNum;i+=1){
            Page<LinkOuterChainDetection> page = new Page<LinkOuterChainDetection>(i, pageSize);
            IPage<LinkOuterChainDetection> pageList = linkOuterChainDetectionService.page(page, queryWrapper);
            checkLinkOuter(pageList.getRecords());
        }
        //统计信息
        LinkOuterouterStatistics linkOuterouterStatistics = iLinkOuterouterStatisticsService.listCountByMainUrl(mainSite.getMainSiteUrl());
        if(!ObjectUtils.isEmpty(linkOuterouterStatistics)){
            linkOuterouterStatistics.setId(IdWorker.getIdStr());
            linkOuterouterStatistics.setScanTime(linkOuterouterStatistics.getUpdateTime());
            linkOuterouterStatistics.setTestType("2");
            linkOuterouterStatistics.setTenantId(mainSite.getTenantId());
            iLinkOuterouterStatisticsService.save(linkOuterouterStatistics);
        }
    }

    private void checkLinkOuter(List<LinkOuterChainDetection> exportList){
        for (LinkOuterChainDetection linkOuterChainDetection : exportList) {
            System.out.println(linkOuterChainDetection);
            String outerUrl = linkOuterChainDetection.getOuterUrl();
            String mainSiteId = linkOuterChainDetection.getMainSiteId();
            MainSite mainSite = iMainSiteService.getById(mainSiteId);
            QueryWrapper<ExternalInternalSite> internalSiteQueryWrapper = new QueryWrapper<ExternalInternalSite>();
            internalSiteQueryWrapper.eq("url",outerUrl);
            long count = externalInternalSiteService.count(internalSiteQueryWrapper);
            Integer pageSize = 100;
            Integer pageNum = (int)Math.ceil((double)count/pageSize);
            int countError = 0;
            for(int i = 1; i <= pageNum;i+=1){
                Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
                IPage<ExternalInternalSite> pageList = externalInternalSiteService.page(page,internalSiteQueryWrapper);
                List<ExternalInternalSite> records = pageList.getRecords();
                for (ExternalInternalSite record : records) {
                    //爬取页面
                    Document document = null;
                    String grabPage = mainSite.getGrabPage();
                    try {
                        SpyderReturn spyderReturn = trendsSpyder.companyTrendsSpyderMessage(record.getSource(), grabPage);
                        document = spyderReturn.getDocument();
                        String baseUri = spyderReturn.getBaseUri();
                        Elements tags = document.select("link[href],img[src],*[style*=background-image],script[src],td[background],a[href],iframe[src]");
                        try {
                            for (Element tag : tags) {
                                String attr = tag.attr("href");
                                if (attr == null || attr.isEmpty()) continue;
                                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                    String absUrl = new URI(baseUri).resolve(attr).toString();
                                    absUrl = new URI(absUrl).normalize().toString();
                                    tag.attr("href", absUrl);
                                }
                            }
                        }catch (Exception e){
                            e.printStackTrace();
                        }
                        try {
                            for (Element tag : tags) {
                                String attr = tag.attr("src");
                                if (attr == null || attr.isEmpty()) continue;
                                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                    String absUrl = new URI(baseUri).resolve(attr).toString();
                                    absUrl = new URI(absUrl).normalize().toString();
                                    tag.attr("src", absUrl);
                                }
                            }
                        }catch (Exception e){
                            e.printStackTrace();
                        }
                        try {
                            for (Element tag : tags) {
                                String attr = tag.attr("background");
                                if (attr == null || attr.isEmpty()) continue;
                                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                    String absUrl = new URI(baseUri).resolve(attr).toString();
                                    absUrl = new URI(absUrl).normalize().toString();
                                    tag.attr("background", absUrl);
                                }
                            }
                        }catch (Exception e){
                            e.printStackTrace();
                        }
                        try {
                            for (Element tag : tags) {
                                String attr = tag.attr("iframe");
                                if (attr == null || attr.isEmpty()) continue;
                                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                                    String absUrl = new URI(baseUri).resolve(attr).toString();
                                    absUrl = new URI(absUrl).normalize().toString();
                                    tag.attr("iframe", absUrl);
                                }
                            }
                        }catch (Exception e){
                            e.printStackTrace();
                        }
                    }catch (Exception e){
                        document = null;
                    }
                    if(!ObjectUtils.isEmpty(document)){
                        String sHtml = document.toString();
                        sHtml = sHtml.replaceAll("amp;","");
                        int i1 = sHtml.indexOf(outerUrl);
                        if(i1 > 0){
                            countError++;
                        }
                    }
                }
            }
            if(countError > 0){
                linkOuterChainDetection.setIsRevise("2");
                linkOuterChainDetection.setCheckNumberApplicationPagesv(countError);
            }
            if(countError == 0){
                linkOuterChainDetection.setIsRevise("1");
            }
            linkOuterChainDetection.setCheckTime(new Date());
            linkOuterChainDetection.setCheckState("1");
            System.out.println("=========================");
            System.out.println(linkOuterChainDetection);
            linkOuterChainDetectionService.updateById(linkOuterChainDetection);
            LinkOuterChainLog(linkOuterChainDetection);
        }
    }

    public void saveLinkChainErrorLog(LinkChainErrorDetection errorDetection){
        //记录日志信息
        LinkChainErrorLog linkChainErrorLog = new LinkChainErrorLog();
        linkChainErrorLog.setId(IdWorker.getIdStr());
        linkChainErrorLog.setCreateBy(errorDetection.getCreateBy());
        linkChainErrorLog.setCreateTime(errorDetection.getCreateTime());
        linkChainErrorLog.setUpdateBy(errorDetection.getUpdateBy());
        linkChainErrorLog.setUpdateTime(errorDetection.getUpdateTime());
        linkChainErrorLog.setErrorDetectionId(errorDetection.getId());
        linkChainErrorLog.setMainSiteId(errorDetection.getMainSiteId());
        linkChainErrorLog.setMainSiteName(errorDetection.getMainSiteName());
        linkChainErrorLog.setMainSiteUrl(errorDetection.getMainSiteUrl());
        linkChainErrorLog.setErrorUrl(errorDetection.getErrorUrl());
        linkChainErrorLog.setErrorUrlType(errorDetection.getErrorUrlType());
        linkChainErrorLog.setErrorUrlState(errorDetection.getErrorUrlState());
        linkChainErrorLog.setNumberApplicationPages(errorDetection.getNumberApplicationPages());
        linkChainErrorLog.setNumberScans(errorDetection.getNumberScans());
        linkChainErrorLog.setNumberScans(errorDetection.getNumberScans());
        linkChainErrorLog.setIsRevise(errorDetection.getIsRevise());
        linkChainErrorLog.setIsIgnore(errorDetection.getIsIgnore());
        linkChainErrorLog.setCheckTime(errorDetection.getCheckTime());
        linkChainErrorLog.setCheckState(errorDetection.getCheckState());
        linkChainErrorLog.setCheckMessage(errorDetection.getCheckMessage());
        linkChainErrorLog.setTenantId(errorDetection.getTenantId());
        boolean save = iLinkChainErrorLogService.save(linkChainErrorLog);
    }


    public void LinkOuterChainLog(LinkOuterChainDetection linkOuterChainDetection){
        //记录日志信息
        LinkOuterChainLog linkOuterChainLog = new LinkOuterChainLog();
        linkOuterChainLog.setId(IdWorker.getIdStr());
        linkOuterChainLog.setOuterChainDetectionId(linkOuterChainDetection.getId());
        linkOuterChainLog.setCreateBy(linkOuterChainDetection.getCreateBy());
        linkOuterChainLog.setCreateTime(linkOuterChainDetection.getCreateTime());
        linkOuterChainLog.setUpdateBy(linkOuterChainDetection.getUpdateBy());
        linkOuterChainLog.setUpdateTime(linkOuterChainDetection.getUpdateTime());
        linkOuterChainLog.setMainSiteId(linkOuterChainDetection.getMainSiteId());
        linkOuterChainLog.setMainSiteName(linkOuterChainDetection.getMainSiteName());
        linkOuterChainLog.setMainSiteUrl(linkOuterChainDetection.getMainSiteUrl());
        linkOuterChainLog.setOuterUrl(linkOuterChainDetection.getOuterUrl());
        linkOuterChainLog.setOuterUrlType(linkOuterChainDetection.getOuterUrlType());
        linkOuterChainLog.setOuterUrlState(linkOuterChainDetection.getOuterUrlState());
        linkOuterChainLog.setNumberApplicationPages(linkOuterChainDetection.getNumberApplicationPages());
        linkOuterChainLog.setNumberScans(linkOuterChainDetection.getNumberScans());
        linkOuterChainLog.setIsRevise(linkOuterChainDetection.getIsRevise());
        linkOuterChainLog.setIsIgnore(linkOuterChainDetection.getIsIgnore());
        linkOuterChainLog.setCheckTime(linkOuterChainDetection.getCheckTime());
        linkOuterChainLog.setCheckState(linkOuterChainDetection.getCheckState());
        linkOuterChainLog.setCheckNumberApplicationPagesv(linkOuterChainDetection.getCheckNumberApplicationPagesv());
        linkOuterChainLog.setCheckMessage(linkOuterChainDetection.getCheckMessage());
        linkOuterChainLog.setTenantId(linkOuterChainDetection.getTenantId());
        linkOuterChainLogService.save(linkOuterChainLog);
    }
}
