package org.jeecg.modules.demo.externalinternalsite.controller;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.swagger.models.auth.In;
import kong.unirest.HttpResponse;
import kong.unirest.JsonNode;
import kong.unirest.Unirest;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.shiro.SecurityUtils;
import org.apache.shiro.subject.Subject;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.config.TenantContext;
import org.jeecg.common.system.query.QueryGenerator;
import org.jeecg.common.system.vo.LoginUser;
import org.jeecg.common.util.DateUtils;
import org.jeecg.modules.demo.externalinternalsite.entity.ExternalInternalSite;
import org.jeecg.modules.demo.externalinternalsite.service.IExternalInternalSiteService;
import org.jeecg.modules.demo.linkcaptureerrorlog.entity.LinkCaptureErrorLog;
import org.jeecg.modules.demo.linkcaptureerrorlog.service.ILinkCaptureErrorLogService;
import org.jeecg.modules.demo.linkcaptureerrormessage.entity.LinkCaptureErrorMessage;
import org.jeecg.modules.demo.linkcaptureerrormessage.service.ILinkCaptureErrorMessageService;
import org.jeecg.modules.demo.linkerrorwordstatistics.entity.LinkErrorWordStatistics;
import org.jeecg.modules.demo.linkerrorwordstatistics.service.ILinkErrorWordStatisticsService;
import org.jeecg.modules.demo.linkpublishingtimegrasping.entity.LinkPublishingTimeGrasping;
import org.jeecg.modules.demo.linkpublishingtimegrasping.service.ILinkPublishingTimeGraspingService;
import org.jeecg.modules.demo.linkpublishingtitle.entity.LinkPublishingTitle;
import org.jeecg.modules.demo.linkpublishingtitle.service.ILinkPublishingTitleService;
import org.jeecg.modules.demo.linkwhiteword.entity.LinkWhiteWord;
import org.jeecg.modules.demo.linkwhiteword.service.ILinkWhiteWordService;
import org.jeecg.modules.demo.mainsite.entity.MainSite;
import org.jeecg.modules.demo.mainsite.service.IMainSiteService;
import org.jeecg.modules.demo.pagecontentcapture.entity.JsResult;
import org.jeecg.modules.demo.pagecontentcapture.entity.PageContentCapture;
import org.jeecg.modules.demo.pagecontentcapture.service.IPageContentCaptureService;
import org.jeecg.modules.system.entity.SysDict;
import org.jeecg.modules.system.entity.SysDictItem;
import org.jeecg.modules.system.entity.SysUser;
import org.jeecg.modules.system.service.ISysDictItemService;
import org.jeecg.modules.system.service.ISysDictService;
import org.jeecg.modules.system.service.ISysUserService;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.DigestUtils;

import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 示例带参定时任务
 * 
 * @Author Scott
 */
@Slf4j
@Component
public class ExternalInternalSiteParamJob{

	private String parameter;

	@Autowired
	private IExternalInternalSiteService externalInternalSiteService;

	@Autowired
	private IPageContentCaptureService iPageContentCaptureService;

	@Autowired
	private ILinkPublishingTitleService linkPublishingTitleService;

	@Autowired
	private ILinkPublishingTimeGraspingService iLinkPublishingTimeGraspingService;

	@Autowired
	private ISysDictService sysDictService;

	@Autowired
	private ISysDictItemService iSysDictItemService;

	@Autowired
	private ILinkWhiteWordService iLinkWhiteWordService;

	@Autowired
	private ILinkCaptureErrorLogService iLinkCaptureErrorLogService;

	@Autowired
	private ILinkCaptureErrorMessageService iLinkCaptureErrorMessageService;

	@Autowired
	private ILinkErrorWordStatisticsService iLinkErrorWordStatisticsService;

	@Autowired
	private IMainSiteService iMainSiteService;

	@Autowired
	private ISysUserService userService;

	private Integer ThreadpageNum = 0;

	public synchronized boolean execute(String parameter) throws JobExecutionException {
		this.parameter = parameter;
		try {
			String TenantId = this.parameter.split("&&&")[1];
			this.parameter =  this.parameter.split("&&&")[0];
			log.info( String.format("welcome %s! Jeecg-Boot 带参数定时任务 SampleParamJob !   时间:" + DateUtils.now(), this.parameter));
			QueryWrapper<MainSite> mainSiteQueryWrapper = new QueryWrapper<MainSite>();
			Page<MainSite> page = new Page<MainSite>(1, 1);
			System.out.println(this.parameter);
			System.out.println(iMainSiteService.list());
			mainSiteQueryWrapper.eq("main_site_url",this.parameter);
			TenantContext.setTenant(TenantId);
			Page<MainSite> page1 = iMainSiteService.page(page, mainSiteQueryWrapper);
			log.info(page1.getRecords()+"");
			MainSite mainSite = null;
			if(page1.getRecords().size() >= 1){
				mainSite = page1.getRecords().get(0);
			}
			log.info(mainSite+"");
			Map<String,ExternalInternalSite> stringExternalInternalSiteMap = new HashMap<String,ExternalInternalSite>();
			if(!ObjectUtils.isEmpty(mainSite)){
				Integer threadNumber = mainSite.getThreadNumber();
				//将对应的扫描列表的起始页分为上述的
				if(threadNumber > 1){
					multithreading(threadNumber,mainSite,TenantId);
				}
				if(threadNumber == 1 || threadNumber == 0){
					onethreading(mainSite,TenantId);
				}
				try {
					//扫描结束对信息进行统计
					TenantContext.setTenant(TenantId);
					PageContentCapture pageContentCapture = iPageContentCaptureService.listCountByMainUrl(mainSite.getMainSiteUrl());
					//聚合错敏感词信息，将其结果保存到
					if(!org.springframework.util.ObjectUtils.isEmpty(pageContentCapture)){
						LinkErrorWordStatistics linkErrorWordStatistics = new LinkErrorWordStatistics();
						linkErrorWordStatistics.setId(IdWorker.getIdStr());
						linkErrorWordStatistics.setCreateBy(pageContentCapture.getCreateBy());
						linkErrorWordStatistics.setCreateTime(pageContentCapture.getCreateTime());
						linkErrorWordStatistics.setUpdateTime(pageContentCapture.getUpdateTime());
						linkErrorWordStatistics.setUpdateBy(pageContentCapture.getUpdateBy());
						String mainUrl1 = pageContentCapture.getMainUrl();
						//通过站点id查询对应的站点信息
						linkErrorWordStatistics.setMainSiteUrl(mainUrl1);
						QueryWrapper<MainSite> mainSiteQueryWrapperQQ = new QueryWrapper<MainSite>();
						mainSiteQueryWrapper.eq("main_site_url",mainUrl1);
						TenantContext.setTenant(TenantId);
						List<MainSite> mainSiteList = iMainSiteService.list(mainSiteQueryWrapperQQ);
						if(mainSiteList.size() >= 1){
							MainSite mainSiteQQ = mainSiteList.get(0);
							linkErrorWordStatistics.setMainSiteId(mainSiteQQ.getId());
							linkErrorWordStatistics.setMainSiteName(mainSiteQQ.getMainSiteName());
						}
						linkErrorWordStatistics.setTestType("1");
						linkErrorWordStatistics.setScanTime(pageContentCapture.getUpdateTime());
						linkErrorWordStatistics.setErrorWordCount(pageContentCapture.getHeavyCount());
						linkErrorWordStatistics.setTenantId(pageContentCapture.getTenantId());
						TenantContext.setTenant(TenantId);
						iLinkErrorWordStatisticsService.save(linkErrorWordStatistics);
					}
				}catch (Exception e){
					e.printStackTrace();
				}
			}
		}catch (Exception e){
			e.printStackTrace();
		}
		return true;
	}
	public void onethreading(MainSite mainSiteMM,String TenantId ){
		String mainUrl = mainSiteMM.getMainSiteUrl();
		if(org.springframework.util.ObjectUtils.isEmpty(mainUrl) || mainUrl == "" || mainUrl == null){
			mainUrl = "http://www.hetaodaxue.com/hyywxx/";
		}
		TenantContext.setTenant(TenantId);
		Integer count =  externalInternalSiteService.countSource(mainUrl);
		Integer pageSize = 100;
		Integer pageNum = (int)Math.ceil((double)count/pageSize);
		for(int i = 1; i <= pageNum;i+=1){
			Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
			TenantContext.setTenant(TenantId);
			IPage<ExternalInternalSite> pageList = externalInternalSiteService.pageSourceMainUrl(page,mainUrl);
			for (ExternalInternalSite record : pageList.getRecords()) {
				String LinkCaptureErrorLogId = IdWorker.getIdStr();
				PageContentCapture pageContentCapture = new PageContentCapture();
				pageContentCapture.setId(DigestUtils.md5DigestAsHex(record.getSource().getBytes()));
				Integer tenantId = record.getTenantId();
				//查询当前id有没有对应的内容
				try {
					TenantContext.setTenant(TenantId);
					PageContentCapture byId = iPageContentCaptureService.getById(DigestUtils.md5DigestAsHex(record.getSource().getBytes()));
					if(!org.springframework.util.ObjectUtils.isEmpty(byId)){
						pageContentCapture = byId;
					}
				}catch (Exception e){
					e.printStackTrace();
				}
				pageContentCapture.setCreateTime(new Date());
				pageContentCapture.setUpdateTime(new Date());
				pageContentCapture.setSource(record.getSource());
				if(!org.springframework.util.ObjectUtils.isEmpty(record.getSource())){
					//爬取页面
					Document document = null;
					try {
						document = Jsoup.parse(new URL(record.getSource()), 10000);
						pageContentCapture.setIsInvalid("2");
					}catch (Exception e){
						pageContentCapture.setIsInvalid("1");
					}
					if(!org.springframework.util.ObjectUtils.isEmpty(document)){
						String baseUri = document.baseUri();
						Elements tags = document.select("link[href],img[src],*[style*=background-image],script[src],td[background],a[href]");
						try {
							for (Element tag : tags) {
								String attr = tag.attr("href");
								if (attr == null || attr.isEmpty()) continue;
								if (!attr.startsWith("http") && !attr.startsWith("data")) {
									String absUrl = new URI(baseUri).resolve(attr).toString();
									absUrl = new URI(absUrl).normalize().toString();
									tag.attr("href", absUrl);
								}
							}
						}catch (Exception e){
							e.printStackTrace();
						}
						try {
							for (Element tag : tags) {
								String attr = tag.attr("src");
								if (attr == null || attr.isEmpty()) continue;
								if (!attr.startsWith("http") && !attr.startsWith("data")) {
									String absUrl = new URI(baseUri).resolve(attr).toString();
									absUrl = new URI(absUrl).normalize().toString();
									tag.attr("src", absUrl);
								}
							}
						}catch (Exception e){
							e.printStackTrace();
						}
						try {
							for (Element tag : tags) {
								String attr = tag.attr("background");
								if (attr == null || attr.isEmpty()) continue;
								if (!attr.startsWith("http") && !attr.startsWith("data")) {
									String absUrl = new URI(baseUri).resolve(attr).toString();
									absUrl = new URI(absUrl).normalize().toString();
									tag.attr("background", absUrl);
								}
							}
						}catch (Exception e){
							e.printStackTrace();
						}
						try {
							for (Element tag : tags) {
								try {
									String attr = tag.attr("iframe");
									if (attr == null || attr.isEmpty()) continue;
									if (!attr.startsWith("http") && !attr.startsWith("data")) {
										String absUrl = new URI(baseUri).resolve(attr).toString();
										absUrl = new URI(absUrl).normalize().toString();
										tag.attr("iframe", absUrl);
									}
								}catch (Exception e){
									e.printStackTrace();
								}
							}
						}catch (Exception e){
							e.printStackTrace();
						}
						String sHtml = document.toString();
						sHtml = sHtml.replaceAll("amp;","");
						pageContentCapture.setContent(sHtml);
						//获取文章发布的标题
						QueryWrapper<LinkPublishingTitle> linkPublishingTitleQueryWrapper =new QueryWrapper<LinkPublishingTitle>();
						linkPublishingTitleQueryWrapper.eq("main_site_url",record.getMainUrl());
						TenantContext.setTenant(TenantId);
						List<LinkPublishingTitle> linkPublishingTitles = linkPublishingTitleService.list(linkPublishingTitleQueryWrapper);
						String title = null;
						for (LinkPublishingTitle linkPublishingTitle : linkPublishingTitles) {
							String typeCssXpath = linkPublishingTitle.getTypeCssXpath();
							try {
								if(typeCssXpath.equals("selector")){
									Elements select = document.select(linkPublishingTitle.getElementPositioningType());
									String string = select.toString();
									String text = Jsoup.parse(string).text();
									if(text != null && text != ""){
										title = text;
									}
								}
								if(typeCssXpath.equals("xpath")){
									// 5.将document转换成xpath能解析的文档树结构
									JXDocument jdoc = new JXDocument(document.getAllElements()); // 此处可以使用xpath语法
									// 6.使用xpath语法来解析html文档来完成数据清洗
									List<JXNode> elNodes = jdoc.selN(linkPublishingTitle.getElementPositioningType());
									for (JXNode elNode : elNodes) {
										String string = elNode.toString();
										String text = Jsoup.parse(string).text();
										if(text != null && text != ""){
											title = text;
										}
									}
								}
							}catch (Exception e){
							}
						}
						if(title != null && title != "" && title.length() != 0){
							pageContentCapture.setTitle(title);
						}else{
							//抓取网页title的内容
							Elements eles = document.getElementsByTag("title");
							String string = eles.text();
							pageContentCapture.setTitle(string);
						}
						QueryWrapper<LinkPublishingTimeGrasping> linkPublishingTimeGraspingQueryWrapper =new QueryWrapper<LinkPublishingTimeGrasping>();
						linkPublishingTimeGraspingQueryWrapper.eq("main_site_url",record.getMainUrl());
						//获取文章发布的时间
						TenantContext.setTenant(TenantId);
						List<LinkPublishingTimeGrasping> linkPublishingTimeGraspings = iLinkPublishingTimeGraspingService.list(linkPublishingTimeGraspingQueryWrapper);
						String time = null;
						for (LinkPublishingTimeGrasping linkPublishingTimeGrasping : linkPublishingTimeGraspings) {
							String typeCssXpathTime = linkPublishingTimeGrasping.getTypeCssXpath();
							try {
								if(typeCssXpathTime.equals("selector")){
									Elements select = document.select(linkPublishingTimeGrasping.getElementPositioningType());
									String string = select.toString();
									String text = Jsoup.parse(string).text();
									if(text != null && text != ""){
										time = text;
									}
								}
								if(typeCssXpathTime.equals("xpath")){
									// 5.将document转换成xpath能解析的文档树结构
									JXDocument jdoc = new JXDocument(document.getAllElements()); // 此处可以使用xpath语法
									// 6.使用xpath语法来解析html文档来完成数据清洗
									List<JXNode> elNodes = jdoc.selN(linkPublishingTimeGrasping.getElementPositioningType());
									for (JXNode elNode : elNodes) {
										String string = elNode.toString();
										String text = Jsoup.parse(string).text();
										if(text != null && text != ""){
											time = text;
										}
									}
								}
							}catch (Exception e){
							}
						}
						System.out.println(time);
						//格式化时间
						Date parse = null;
						if(time != null && time != "" && time.length() != 0){
							for (LinkPublishingTimeGrasping linkPublishingTimeGrasping : linkPublishingTimeGraspings) {
								String timeFormat = linkPublishingTimeGrasping.getTimeFormat();
								QueryWrapper<SysDict> sysDictQueryWrapper = new QueryWrapper<SysDict>();
								sysDictQueryWrapper.eq("dict_code","time_type");
								TenantContext.setTenant(TenantId);
								List<SysDict> listSysDict = sysDictService.list(sysDictQueryWrapper);
								SysDict sysDict = null;
								if(listSysDict.size() >= 1){
									sysDict = listSysDict.get(0);
								}

								QueryWrapper<SysDictItem> sysDictItemQueryWrapper = new QueryWrapper<SysDictItem>();
								sysDictItemQueryWrapper.eq("dict_id",sysDict.getId());
								sysDictItemQueryWrapper.eq("item_value",timeFormat);
								TenantContext.setTenant(TenantId);
								List<SysDictItem> sysDictItems = iSysDictItemService.list(sysDictItemQueryWrapper);
								for (SysDictItem sysDictItem : sysDictItems) {
									try {
										String itemText = sysDictItem.getItemText();
										//正则表达式
										String description = sysDictItem.getDescription();
										Pattern r = Pattern.compile(description);
										Matcher m = r.matcher(time);
										String dateStr = null;
										if (m.find()) {
											dateStr = m.group(0);
										}
										if(dateStr != null && dateStr != ""){
											SimpleDateFormat simpleDateFormat = new SimpleDateFormat(itemText);
											parse = simpleDateFormat.parse(dateStr);
										}
									}catch (Exception e){
									}
								}
							}
						}
						if(!org.springframework.util.ObjectUtils.isEmpty(parse)){
							pageContentCapture.setArticleReleaseTime(parse);
						}
						//父地址类型  typeSource
						pageContentCapture.setTypeSource(record.getTypeSource());
						pageContentCapture.setStateSource(record.getStateSource());
						pageContentCapture.setMainUrl(record.getMainUrl());
						//错敏词校验-----开始
						Integer[] IntStr = {1000,1001,1002,1004,2000,2001,3000,3001,4000,5000,7001,7002};
						Object oIntStr = JSONArray.toJSON(IntStr);
						Map<String, Object> paramMap = new HashMap<>();
						paramMap.put("userid", "quyanhong");
						paramMap.put("ukey", "K103fhnBhXak");
						paramMap.put("text", URLEncoder.encode( pageContentCapture.getContent()));
						paramMap.put("error_types",oIntStr);
						paramMap.put("threshold",0.2);
						paramMap.put("strict_degree","m");
						paramMap.put("domains","test");
						HttpResponse<JsonNode> response = Unirest.post("http://api.ifuncun.cn:8081/v1/proofreadHtml")
								.header("Content-Type", "application/json")
								.body( JSONObject.toJSONString(paramMap)
								)
								.asJson();
						String string = response.getBody().toString();
						JsResult jsonObject = JSONObject.parseObject(string, JsResult.class);
						String ret_msg = jsonObject.getRet_msg();
						pageContentCapture.setErrorCount(jsonObject.getErrorCount());
						pageContentCapture.setRetMsg(jsonObject.getRet_msg());
						pageContentCapture.setRetCode(jsonObject.getRet_code());
						if(!org.springframework.util.ObjectUtils.isEmpty(ret_msg)){
							//跳过白名单词汇表
							QueryWrapper<LinkWhiteWord> linkWhiteWordQueryWrapper = new QueryWrapper<LinkWhiteWord>();
							linkWhiteWordQueryWrapper.eq("main_site_url",record.getMainUrl());
							TenantContext.setTenant(TenantId);
							List<LinkWhiteWord> linkWhiteWordList = iLinkWhiteWordService.list(linkWhiteWordQueryWrapper);
							for (LinkWhiteWord linkWhiteWord : linkWhiteWordList) {
								int index = ret_msg.indexOf(linkWhiteWord.getWord());
								if(index > 0){
									Document parseHtml = Jsoup.parse(ret_msg);
									Elements elements = parseHtml.select("span.proofreadSignClass.errorWordMark");
									for (Element element : elements) {
										String text = element.text();
										if(text.equals(linkWhiteWord.getWord())){
											String string1 = parseHtml.toString();
											String replace = string1.replace(element.toString(), linkWhiteWord.getWord());
											pageContentCapture.setRetMsg(replace);
											pageContentCapture.setErrorCount(0);
										}
									}
								}
							}
							Document parse1 = Jsoup.parse(pageContentCapture.getRetMsg());
							Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
							System.out.println("==============="+elements1.size());
							for (Element element : elements1) {
								LinkCaptureErrorMessage linkCaptureErrorMessage = new LinkCaptureErrorMessage();
								linkCaptureErrorMessage.setErrorMessage(element.toString());
								linkCaptureErrorMessage.setErrerWord(element.text());
								linkCaptureErrorMessage.setLinkCaptureErrorLogId(LinkCaptureErrorLogId);
								linkCaptureErrorMessage.setId(DigestUtils.md5DigestAsHex((linkCaptureErrorMessage.getErrorMessage()+pageContentCapture.getId()).getBytes()));
								linkCaptureErrorMessage.setLinkPageId(pageContentCapture.getId());
								linkCaptureErrorMessage.setTenantId(pageContentCapture.getTenantId());
								iLinkCaptureErrorMessageService.saveOrUpdate(linkCaptureErrorMessage);
							}
							pageContentCapture.setSensitiveList(elements1.toString());
							pageContentCapture.setSensitiveCount(0);
							pageContentCapture.setSensitiveCount(elements1.size());
							if(elements1.size() > 0){
								pageContentCapture.setErrorCount(1);
							}
						}
						pageContentCapture.setRetCode(jsonObject.getRet_code());
						pageContentCapture.setHeavyCount(jsonObject.getHeavyCount());
					}
					//错敏词校验-----结束
				}

				pageContentCapture.setModifyState("1");
				String id = pageContentCapture.getId();
				TenantContext.setTenant(TenantId);
				PageContentCapture byId = iPageContentCaptureService.getById(id);

				if(!org.springframework.util.ObjectUtils.isEmpty(byId)){
					Integer numberScans = byId.getNumberScans();
					if(!org.springframework.util.ObjectUtils.isEmpty(numberScans)){
						numberScans = numberScans + 1;
						pageContentCapture.setNumberScans(numberScans);
						pageContentCapture.setCreateTime(byId.getCreateTime());
					}
				}else{
					pageContentCapture.setNumberScans(1);
				}
				pageContentCapture.setIsIgnore("2");
				pageContentCapture.setCheckState("2");
				if(
						!org.springframework.util.ObjectUtils.isEmpty(pageContentCapture)
								&&
								!org.springframework.util.ObjectUtils.isEmpty(pageContentCapture.getContent())
								&&
								pageContentCapture.getContent() != ""
								&&
								pageContentCapture.getContent() != null
				)
				{
					pageContentCapture.setTenantId(record.getTenantId());
					iPageContentCaptureService.saveOrUpdate(pageContentCapture);
					String sensitiveList = pageContentCapture.getSensitiveList();
					if(!org.springframework.util.ObjectUtils.isEmpty(sensitiveList)){
						iLinkCaptureErrorLogServicesave(pageContentCapture,LinkCaptureErrorLogId);
					}
				}
			}
		}
		try {
			//扫描结束对信息进行统计
			TenantContext.setTenant(TenantId);
			PageContentCapture pageContentCapture = iPageContentCaptureService.listCountByMainUrl(mainUrl);
			//聚合错敏感词信息，将其结果保存到
			if(!org.springframework.util.ObjectUtils.isEmpty(pageContentCapture)){
				LinkErrorWordStatistics linkErrorWordStatistics = new LinkErrorWordStatistics();
				linkErrorWordStatistics.setId(IdWorker.getIdStr());
				linkErrorWordStatistics.setCreateBy(pageContentCapture.getCreateBy());
				linkErrorWordStatistics.setCreateTime(pageContentCapture.getCreateTime());
				linkErrorWordStatistics.setUpdateTime(pageContentCapture.getUpdateTime());
				linkErrorWordStatistics.setUpdateBy(pageContentCapture.getUpdateBy());
				String mainUrl1 = pageContentCapture.getMainUrl();
				//通过站点id查询对应的站点信息
				linkErrorWordStatistics.setMainSiteUrl(mainUrl1);
				QueryWrapper<MainSite> mainSiteQueryWrapper = new QueryWrapper<MainSite>();
				mainSiteQueryWrapper.eq("main_site_url",mainUrl1);
				TenantContext.setTenant(TenantId);
				List<MainSite> mainSiteList = iMainSiteService.list(mainSiteQueryWrapper);
				if(mainSiteList.size() >= 1){
					MainSite mainSite = mainSiteList.get(0);
					linkErrorWordStatistics.setMainSiteId(mainSite.getId());
					linkErrorWordStatistics.setMainSiteName(mainSite.getMainSiteName());
				}
				linkErrorWordStatistics.setTestType("1");
				linkErrorWordStatistics.setScanTime(pageContentCapture.getUpdateTime());
				linkErrorWordStatistics.setErrorWordCount(pageContentCapture.getHeavyCount());
				linkErrorWordStatistics.setTenantId(pageContentCapture.getTenantId());
				iLinkErrorWordStatisticsService.save(linkErrorWordStatistics);
			}
		}catch (Exception e){
			e.printStackTrace();
		}
	}
	public void multithreading(Integer threadNumber,MainSite mainSite,String TenantId ) throws InterruptedException {

		QueryWrapper<ExternalInternalSite> siteQueryWrapper = new QueryWrapper<ExternalInternalSite>();
		siteQueryWrapper.eq("main_url",mainSite.getMainSiteUrl());
		TenantContext.setTenant(TenantId);
		long countExternalInternalSite = externalInternalSiteService.count(siteQueryWrapper);
		this.ThreadpageNum = (int)Math.ceil((double)countExternalInternalSite/threadNumber);
		log.info(countExternalInternalSite+"");
		log.info("ThreadpageNum========"+ThreadpageNum+"");
		List<Map<Integer,Integer>> integerList = new ArrayList<Map<Integer,Integer>>();
		for (Integer i = 0; i < threadNumber; i++) {
			Map<Integer,Integer> hashMap = new HashMap<Integer,Integer>();
			Integer index =  ThreadpageNum*i;
			if(index == 0){
				index =1 ;
			}
			hashMap.put(index,ThreadpageNum*(i+1));
			integerList.add(hashMap);
		}
		for (Map<Integer, Integer> integerIntegerMap : integerList) {
			Iterator<Map.Entry<Integer, Integer>> iterator = integerIntegerMap.entrySet().iterator();
			Integer threadIndex = 1;
			Vector<Thread> threadVector = new Vector<Thread>();
			while(iterator.hasNext()){
				Map.Entry<Integer,Integer> entry = iterator.next();
				Integer keyStart = entry.getKey();
				Integer valueEnd = entry.getValue();
				Integer finalThreadIndex = threadIndex;
				Thread thread = new Thread() {
					@Override
					public void run() {
						Integer count = valueEnd - keyStart;
						Integer pageSize = 100;
						Integer pageNum = (int) Math.ceil((double) count / pageSize);
						for (Integer i = keyStart; i <= valueEnd; i += pageSize) {
							System.out.println(i);
							System.out.println(keyStart);
							System.out.println(valueEnd);
							TenantContext.setTenant(TenantId);
							List<ExternalInternalSite> externalInternalSiteList = externalInternalSiteService.listPage(i, pageSize);
							System.out.println("externalInternalSiteList==" + externalInternalSiteList);
							for (ExternalInternalSite record : externalInternalSiteList) {
								System.out.println(record);
								String LinkCaptureErrorLogId = IdWorker.getIdStr();
								PageContentCapture pageContentCapture = new PageContentCapture();
								pageContentCapture.setId(DigestUtils.md5DigestAsHex(record.getSource().getBytes()));
								Integer tenantId = record.getTenantId();

								//查询当前id有没有对应的内容
								try {
									TenantContext.setTenant(TenantId);
									PageContentCapture byId = iPageContentCaptureService.getById(DigestUtils.md5DigestAsHex(record.getSource().getBytes()));
									if (!org.springframework.util.ObjectUtils.isEmpty(byId)) {
										pageContentCapture = byId;
									}
								} catch (Exception e) {
									e.printStackTrace();
								}
								pageContentCapture.setCreateTime(new Date());
								pageContentCapture.setUpdateTime(new Date());
								pageContentCapture.setSource(record.getSource());
								if (!org.springframework.util.ObjectUtils.isEmpty(record.getSource())) {
									//爬取页面
									Document document = null;
									try {
										document = Jsoup.parse(new URL(record.getSource()), 10000);
										pageContentCapture.setIsInvalid("2");
									} catch (Exception e) {
										pageContentCapture.setIsInvalid("1");
									}
									if (!org.springframework.util.ObjectUtils.isEmpty(document)) {
										String baseUri = document.baseUri();
										Elements tags = document.select("link[href],img[src],*[style*=background-image],script[src],td[background],a[href]");
										try {
											for (Element tag : tags) {
												String attr = tag.attr("href");
												if (attr == null || attr.isEmpty()) continue;
												if (!attr.startsWith("http") && !attr.startsWith("data")) {
													String absUrl = new URI(baseUri).resolve(attr).toString();
													absUrl = new URI(absUrl).normalize().toString();
													tag.attr("href", absUrl);
												}
											}
										} catch (Exception e) {
											e.printStackTrace();
										}
										try {
											for (Element tag : tags) {
												String attr = tag.attr("src");
												if (attr == null || attr.isEmpty()) continue;
												if (!attr.startsWith("http") && !attr.startsWith("data")) {
													String absUrl = new URI(baseUri).resolve(attr).toString();
													absUrl = new URI(absUrl).normalize().toString();
													tag.attr("src", absUrl);
												}
											}
										} catch (Exception e) {
											e.printStackTrace();
										}
										try {
											for (Element tag : tags) {
												String attr = tag.attr("background");
												if (attr == null || attr.isEmpty()) continue;
												if (!attr.startsWith("http") && !attr.startsWith("data")) {
													String absUrl = new URI(baseUri).resolve(attr).toString();
													absUrl = new URI(absUrl).normalize().toString();
													tag.attr("background", absUrl);
												}
											}
										} catch (Exception e) {
											e.printStackTrace();
										}
										try {
											for (Element tag : tags) {
												try {
													String attr = tag.attr("iframe");
													if (attr == null || attr.isEmpty()) continue;
													if (!attr.startsWith("http") && !attr.startsWith("data")) {
														String absUrl = new URI(baseUri).resolve(attr).toString();
														absUrl = new URI(absUrl).normalize().toString();
														tag.attr("iframe", absUrl);
													}
												}catch (Exception e){
													e.printStackTrace();
												}
											}
										}catch (Exception e){
											e.printStackTrace();
										}
										String sHtml = document.toString();
										sHtml = sHtml.replaceAll("amp;","");
										pageContentCapture.setContent(sHtml);
										pageContentCapture.setContent(document.toString());
										//获取文章发布的标题
										QueryWrapper<LinkPublishingTitle> linkPublishingTitleQueryWrapper = new QueryWrapper<LinkPublishingTitle>();
										linkPublishingTitleQueryWrapper.eq("main_site_url", record.getMainUrl());
										TenantContext.setTenant(TenantId);
										List<LinkPublishingTitle> linkPublishingTitles = linkPublishingTitleService.list(linkPublishingTitleQueryWrapper);
										String title = null;
										for (LinkPublishingTitle linkPublishingTitle : linkPublishingTitles) {
											String typeCssXpath = linkPublishingTitle.getTypeCssXpath();
											try {
												if (typeCssXpath.equals("selector")) {
													Elements select = document.select(linkPublishingTitle.getElementPositioningType());
													String string = select.toString();
													String text = Jsoup.parse(string).text();
													if (text != null && text != "") {
														title = text;
													}
												}
												if (typeCssXpath.equals("xpath")) {
													// 5.将document转换成xpath能解析的文档树结构
													JXDocument jdoc = new JXDocument(document.getAllElements()); // 此处可以使用xpath语法
													// 6.使用xpath语法来解析html文档来完成数据清洗
													List<JXNode> elNodes = jdoc.selN(linkPublishingTitle.getElementPositioningType());
													for (JXNode elNode : elNodes) {
														String string = elNode.toString();
														String text = Jsoup.parse(string).text();
														if (text != null && text != "") {
															title = text;
														}
													}
												}
											} catch (Exception e) {
											}
										}
										if (title != null && title != "" && title.length() != 0) {
											pageContentCapture.setTitle(title);
										} else {
											//抓取网页title的内容
											Elements eles = document.getElementsByTag("title");
											String string = eles.text();
											pageContentCapture.setTitle(string);
										}
										QueryWrapper<LinkPublishingTimeGrasping> linkPublishingTimeGraspingQueryWrapper = new QueryWrapper<LinkPublishingTimeGrasping>();
										linkPublishingTimeGraspingQueryWrapper.eq("main_site_url", record.getMainUrl());
										//获取文章发布的时间
										TenantContext.setTenant(TenantId);
										List<LinkPublishingTimeGrasping> linkPublishingTimeGraspings = iLinkPublishingTimeGraspingService.list(linkPublishingTimeGraspingQueryWrapper);
										String time = null;
										for (LinkPublishingTimeGrasping linkPublishingTimeGrasping : linkPublishingTimeGraspings) {
											String typeCssXpathTime = linkPublishingTimeGrasping.getTypeCssXpath();
											try {
												if (typeCssXpathTime.equals("selector")) {
													Elements select = document.select(linkPublishingTimeGrasping.getElementPositioningType());
													String string = select.toString();
													String text = Jsoup.parse(string).text();
													if (text != null && text != "") {
														time = text;
													}
												}
												if (typeCssXpathTime.equals("xpath")) {
													// 5.将document转换成xpath能解析的文档树结构
													JXDocument jdoc = new JXDocument(document.getAllElements()); // 此处可以使用xpath语法
													// 6.使用xpath语法来解析html文档来完成数据清洗
													List<JXNode> elNodes = jdoc.selN(linkPublishingTimeGrasping.getElementPositioningType());
													for (JXNode elNode : elNodes) {
														String string = elNode.toString();
														String text = Jsoup.parse(string).text();
														if (text != null && text != "") {
															time = text;
														}
													}
												}
											} catch (Exception e) {
											}
										}
										System.out.println(time);
										//格式化时间
										Date parse = null;
										if (time != null && time != "" && time.length() != 0) {
											for (LinkPublishingTimeGrasping linkPublishingTimeGrasping : linkPublishingTimeGraspings) {
												String timeFormat = linkPublishingTimeGrasping.getTimeFormat();
												QueryWrapper<SysDict> sysDictQueryWrapper = new QueryWrapper<SysDict>();
												sysDictQueryWrapper.eq("dict_code", "time_type");
												TenantContext.setTenant(TenantId);
												List<SysDict> listSysDict = sysDictService.list(sysDictQueryWrapper);
												SysDict sysDict = null;
												if (listSysDict.size() >= 1) {
													sysDict = listSysDict.get(0);
												}
												QueryWrapper<SysDictItem> sysDictItemQueryWrapper = new QueryWrapper<SysDictItem>();
												sysDictItemQueryWrapper.eq("dict_id", sysDict.getId());
												sysDictItemQueryWrapper.eq("item_value", timeFormat);
												TenantContext.setTenant(TenantId);
												List<SysDictItem> sysDictItems = iSysDictItemService.list(sysDictItemQueryWrapper);
												for (SysDictItem sysDictItem : sysDictItems) {
													try {
														String itemText = sysDictItem.getItemText();
														//正则表达式
														String description = sysDictItem.getDescription();
														Pattern r = Pattern.compile(description);
														Matcher m = r.matcher(time);
														String dateStr = null;
														if (m.find()) {
															dateStr = m.group(0);
														}
														if (dateStr != null && dateStr != "") {
															SimpleDateFormat simpleDateFormat = new SimpleDateFormat(itemText);
															parse = simpleDateFormat.parse(dateStr);
														}
													} catch (Exception e) {
													}
												}
											}
										}
										if (!org.springframework.util.ObjectUtils.isEmpty(parse)) {
											pageContentCapture.setArticleReleaseTime(parse);
										}
										//父地址类型  typeSource
										pageContentCapture.setTypeSource(record.getTypeSource());
										pageContentCapture.setStateSource(record.getStateSource());
										pageContentCapture.setMainUrl(record.getMainUrl());
										//错敏词校验-----开始
										Integer[] IntStr = {1000, 1001, 1002, 1004, 2000, 2001, 3000, 3001, 4000, 5000, 7001, 7002};
										Object oIntStr = JSONArray.toJSON(IntStr);
										Map<String, Object> paramMap = new HashMap<>();
										paramMap.put("userid", "quyanhong");
										paramMap.put("ukey", "K103fhnBhXak");
										paramMap.put("text", URLEncoder.encode(pageContentCapture.getContent()));
										paramMap.put("error_types", oIntStr);
										paramMap.put("threshold", 0.2);
										paramMap.put("strict_degree", "m");
										paramMap.put("domains", "test");
										HttpResponse<JsonNode> response = Unirest.post("http://api.ifuncun.cn:8081/v1/proofreadHtml")
												.header("Content-Type", "application/json")
												.body(JSONObject.toJSONString(paramMap)
												)
												.asJson();
										String string = response.getBody().toString();
										JsResult jsonObject = JSONObject.parseObject(string, JsResult.class);
										String ret_msg = jsonObject.getRet_msg();
										pageContentCapture.setErrorCount(jsonObject.getErrorCount());
										pageContentCapture.setRetMsg(jsonObject.getRet_msg());
										pageContentCapture.setRetCode(jsonObject.getRet_code());
										if (!org.springframework.util.ObjectUtils.isEmpty(ret_msg)) {
											//跳过白名单词汇表
											QueryWrapper<LinkWhiteWord> linkWhiteWordQueryWrapper = new QueryWrapper<LinkWhiteWord>();
											linkWhiteWordQueryWrapper.eq("main_site_url", record.getMainUrl());
											TenantContext.setTenant(TenantId);
											List<LinkWhiteWord> linkWhiteWordList = iLinkWhiteWordService.list(linkWhiteWordQueryWrapper);
											for (LinkWhiteWord linkWhiteWord : linkWhiteWordList) {
												int index = ret_msg.indexOf(linkWhiteWord.getWord());
												if (index > 0) {
													Document parseHtml = Jsoup.parse(ret_msg);
													Elements elements = parseHtml.select("span.proofreadSignClass.errorWordMark");
													for (Element element : elements) {
														String text = element.text();
														if (text.equals(linkWhiteWord.getWord())) {
															String string1 = parseHtml.toString();
															String replace = string1.replace(element.toString(), linkWhiteWord.getWord());
															pageContentCapture.setRetMsg(replace);
															pageContentCapture.setErrorCount(0);
														}
													}
												}
											}
											Document parse1 = Jsoup.parse(pageContentCapture.getRetMsg());
											Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
											System.out.println("===============" + elements1.size());
											for (Element element : elements1) {
												LinkCaptureErrorMessage linkCaptureErrorMessage = new LinkCaptureErrorMessage();
												linkCaptureErrorMessage.setErrorMessage(element.toString());
												linkCaptureErrorMessage.setErrerWord(element.text());
												linkCaptureErrorMessage.setLinkCaptureErrorLogId(LinkCaptureErrorLogId);
												linkCaptureErrorMessage.setId(DigestUtils.md5DigestAsHex((linkCaptureErrorMessage.getErrorMessage() + pageContentCapture.getId()).getBytes()));
												linkCaptureErrorMessage.setLinkPageId(pageContentCapture.getId());
												linkCaptureErrorMessage.setTenantId(pageContentCapture.getTenantId());
												iLinkCaptureErrorMessageService.saveOrUpdate(linkCaptureErrorMessage);
											}
											pageContentCapture.setSensitiveList(elements1.toString());
											pageContentCapture.setSensitiveCount(0);
											pageContentCapture.setSensitiveCount(elements1.size());
											if (elements1.size() > 0) {
												pageContentCapture.setErrorCount(1);
											}
										}
										pageContentCapture.setRetCode(jsonObject.getRet_code());
										pageContentCapture.setHeavyCount(jsonObject.getHeavyCount());
									}
									//错敏词校验-----结束
								}

								pageContentCapture.setModifyState("1");
								String id = pageContentCapture.getId();
								TenantContext.setTenant(TenantId);
								PageContentCapture byId = iPageContentCaptureService.getById(id);

								if (!org.springframework.util.ObjectUtils.isEmpty(byId)) {
									Integer numberScans = byId.getNumberScans();
									if (!org.springframework.util.ObjectUtils.isEmpty(numberScans)) {
										numberScans = numberScans + 1;
										pageContentCapture.setNumberScans(numberScans);
										pageContentCapture.setCreateTime(byId.getCreateTime());
									}
								} else {
									pageContentCapture.setNumberScans(1);
								}
								pageContentCapture.setIsIgnore("2");
								pageContentCapture.setCheckState("2");
								if (
										!org.springframework.util.ObjectUtils.isEmpty(pageContentCapture)
												&&
												!org.springframework.util.ObjectUtils.isEmpty(pageContentCapture.getContent())
												&&
												pageContentCapture.getContent() != ""
												&&
												pageContentCapture.getContent() != null
								) {
									pageContentCapture.setTenantId(record.getTenantId());
									TenantContext.setTenant(TenantId);
									System.out.println(pageContentCapture.getTitle());
									iPageContentCaptureService.saveOrUpdate(pageContentCapture);
									String sensitiveList = pageContentCapture.getSensitiveList();
									if (!org.springframework.util.ObjectUtils.isEmpty(sensitiveList)) {
										iLinkCaptureErrorLogServicesave(pageContentCapture, LinkCaptureErrorLogId);
									}
								}
							}

						}
					}
				};
				threadVector.add(thread);
				thread.start();
				threadIndex++;
			}
			for (Thread thread : threadVector) {
				thread.join();
			}

		}
	}

	public void QueryList(ExternalInternalSite externalInternalSite) {
		String mainUrl = externalInternalSite.getMainUrl();
		if (org.springframework.util.ObjectUtils.isEmpty(mainUrl) || mainUrl == "" || mainUrl == null) {
			mainUrl = "http://www.hetaodaxue.com/hyywxx/";
		}

		Integer count = Integer.parseInt(externalInternalSiteService.count()+"");
		System.out.println(count);
		Integer pageSize = 100;
		Integer pageNum = (int) Math.ceil((double) count / pageSize);
		for (int i = 1; i <= pageNum; i += 1) {
			Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
			IPage<ExternalInternalSite> pageList = externalInternalSiteService.page(page);
			for (ExternalInternalSite record : pageList.getRecords()) {
				record.setTenantId(1000);
				System.out.println(record);
				externalInternalSiteService.updateById(record);
			}

		}
	}

	public boolean iLinkCaptureErrorLogServicesave(PageContentCapture pageContentCapture,String LinkCaptureErrorLogId ){
		LinkCaptureErrorLog linkCaptureErrorLog = new LinkCaptureErrorLog();
		linkCaptureErrorLog.setId(LinkCaptureErrorLogId);
		linkCaptureErrorLog.setCheckTime(new Date());
		linkCaptureErrorLog.setUpdateTime(new Date());
		try {
			linkCaptureErrorLog.setLinkPageId(pageContentCapture.getId());
		}catch (Exception e){
		}
		linkCaptureErrorLog.setSource(pageContentCapture.getSource());
		linkCaptureErrorLog.setTitle(pageContentCapture.getTitle());
		linkCaptureErrorLog.setTypeSource(pageContentCapture.getTypeSource());
		linkCaptureErrorLog.setStateSource(pageContentCapture.getStateSource());
		linkCaptureErrorLog.setMainUrl(pageContentCapture.getMainUrl());
		linkCaptureErrorLog.setSensitiveList(pageContentCapture.getSensitiveList());
		linkCaptureErrorLog.setSensitiveCount(pageContentCapture.getSensitiveCount());
		linkCaptureErrorLog.setRetCode(pageContentCapture.getRetCode());
		linkCaptureErrorLog.setErrorCount(pageContentCapture.getErrorCount());
		linkCaptureErrorLog.setHeavyCount(pageContentCapture.getHeavyCount());
		linkCaptureErrorLog.setModifyState(pageContentCapture.getModifyState());
		linkCaptureErrorLog.setArticleReleaseTime(pageContentCapture.getArticleReleaseTime());
		linkCaptureErrorLog.setNumberScans(pageContentCapture.getNumberScans());
		linkCaptureErrorLog.setIsIgnore(pageContentCapture.getIsIgnore());
		linkCaptureErrorLog.setCheckTime(pageContentCapture.getCheckTime());
		linkCaptureErrorLog.setCheckState(pageContentCapture.getCheckState());
		linkCaptureErrorLog.setCheckErrorCount(pageContentCapture.getCheckErrorCount());
		linkCaptureErrorLog.setTenantId(pageContentCapture.getTenantId());
		//扫描日志记录
		boolean save = iLinkCaptureErrorLogService.saveOrUpdate(linkCaptureErrorLog);
		return save;
	}
}
