package org.jeecg.modules.demo.pagecontentcapture.controller;

import java.net.URL;
import java.util.*;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import lombok.Data;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.system.query.QueryGenerator;
import org.jeecg.common.util.oConvertUtils;
import org.jeecg.modules.demo.externalinternalsite.entity.ExternalInternalSite;
import org.jeecg.modules.demo.externalinternalsite.service.IExternalInternalSiteService;
import org.jeecg.modules.demo.linkanchorlist.entity.LinkAnchorList;
import org.jeecg.modules.demo.linkanchorlist.service.ILinkAnchorListService;
import org.jeecg.modules.demo.linkerrorwordstatistics.entity.LinkErrorWordStatistics;
import org.jeecg.modules.demo.linkerrorwordstatistics.service.ILinkErrorWordStatisticsService;
import org.jeecg.modules.demo.linkpublishingtimegrasping.entity.LinkPublishingTimeGrasping;
import org.jeecg.modules.demo.linkpublishingtimegrasping.service.ILinkPublishingTimeGraspingService;
import org.jeecg.modules.demo.linkpublishingtitle.entity.LinkPublishingTitle;
import org.jeecg.modules.demo.linkpublishingtitle.service.ILinkPublishingTitleService;
import org.jeecg.modules.demo.linkwhiteword.entity.LinkWhiteWord;
import org.jeecg.modules.demo.linkwhiteword.service.ILinkWhiteWordService;
import org.jeecg.modules.demo.mainsite.entity.MainSite;
import org.jeecg.modules.demo.mainsite.service.IMainSiteService;
import org.jeecg.modules.demo.pagecontentcapture.entity.JsResult;
import org.jeecg.modules.demo.pagecontentcapture.entity.PageContentCapture;
import org.jeecg.modules.demo.pagecontentcapture.entity.PageContentCaptureVo;
import org.jeecg.modules.demo.pagecontentcapture.service.IPageContentCaptureService;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.extern.slf4j.Slf4j;

import org.jeecg.modules.demo.pagecontentcapture.service.IPageContentCaptureVoService;
import org.jeecg.modules.spyderUtils.spider.*;
import org.jeecg.common.system.base.controller.JeecgController;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.DigestUtils;
import org.springframework.util.ObjectUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.ModelAndView;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.jeecg.common.aspect.annotation.AutoLog;
import org.apache.shiro.authz.annotation.RequiresPermissions;

 /**
 * @Description: 页面内容抓取
 * @Author: jeecg-boot
 * @Date:   2023-04-17
 * @Version: V1.0
 */
@Api(tags="页面内容抓取")
@RestController
@RequestMapping("/pagecontentcapture/pageContentCapture")
@Slf4j
public class PageContentCaptureController extends JeecgController<PageContentCapture, IPageContentCaptureService> {

	 @Autowired
	 private IPageContentCaptureService pageContentCaptureService;

	 @Autowired
	 private IPageContentCaptureVoService pageContentCaptureVoService;

	 @Autowired
	 private IExternalInternalSiteService externalInternalSiteService;

	 @Autowired
	 private IPageContentCaptureService iPageContentCaptureService;

	 @Autowired
	 private ILinkPublishingTitleService linkPublishingTitleService;

	 @Autowired
	 private ILinkPublishingTimeGraspingService iLinkPublishingTimeGraspingService;

	 @Autowired
	 private ILinkWhiteWordService iLinkWhiteWordService;

	 @Autowired
	 private ILinkErrorWordStatisticsService iLinkErrorWordStatisticsService;

	 @Autowired
	 private IMainSiteService iMainSiteService;

	 @Autowired
	 private TrendsSpyder trendsSpyder;

	 @Autowired
	 private FangCunUtils fangCunUtils;

	 @Autowired
	 private LinkOrWordSaveUtils linkOrWordSaveUtils;

	 @Autowired
	 private JsoupUtils jsoupUtils;

	 @Autowired
	 private CrawlingUtils crawlingUtils;

	 @Autowired
	 private CrawlingPhoneIdNumberCardNumber crawlingPhoneIdNumberCardNumber;


	 @Autowired
	 private ILinkAnchorListService iLinkAnchorListService;

	/**
	 * 分页列表查询
	 *
	 * @param pageContentCapture
	 * @param pageNo
	 * @param pageSize
	 * @param req
	 * @return
	 */
	@ApiOperation(value="页面内容抓取-分页列表查询", notes="页面内容抓取-分页列表查询")
	@GetMapping(value = "/list")
	public Result<IPage<PageContentCapture>> queryPageList(PageContentCapture pageContentCapture,
								   @RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
								   @RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
								   HttpServletRequest req) {
		QueryWrapper<PageContentCapture> queryWrapper = QueryGenerator.initQueryWrapper(pageContentCapture, req.getParameterMap());
		Page<PageContentCapture> page = new Page<PageContentCapture>(pageNo, pageSize);
		IPage<PageContentCapture> pageList = pageContentCaptureService.page(page, queryWrapper);
		return Result.OK(pageList);
	}

	 @ApiOperation(value="页面内容抓取-分页列表查询", notes="页面内容抓取-分页列表查询")
	 @GetMapping(value = "/listVo")
	 public Result<IPage<PageContentCaptureVo>> queryPageListVo(PageContentCaptureVo pageContentCaptureVo,
																@RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
																@RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
																HttpServletRequest req) {
		 QueryWrapper<PageContentCaptureVo> queryWrapper = QueryGenerator.initQueryWrapper(pageContentCaptureVo, req.getParameterMap());
		 Page<PageContentCaptureVo> page = new Page<PageContentCaptureVo>(pageNo, pageSize);
		 IPage<PageContentCaptureVo> pageList = pageContentCaptureVoService.page(page, queryWrapper);
		 return Result.OK(pageList);
	 }


	 @ApiOperation(value="错链信息-分页列表查询", notes="错链信息-分页列表查询")
	 @GetMapping(value = "/listError")
	 public Result<IPage<PageContentCapture>> queryPageListError(PageContentCapture pageContentCapture,
															@RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
															@RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
															HttpServletRequest req) {
		 Map<String, String[]> parameterMap = req.getParameterMap();
		 QueryWrapper<PageContentCapture> queryWrapper = QueryGenerator.initQueryWrapper(pageContentCapture, req.getParameterMap());
		 queryWrapper.isNotNull("sensitive_list");
		 queryWrapper.ne("sensitive_list","");
		 queryWrapper.gt("error_count",0);
		 Page<PageContentCapture> page = new Page<PageContentCapture>(pageNo, pageSize);
		 IPage<PageContentCapture> pageList = pageContentCaptureService.page(page, queryWrapper);
		 for (PageContentCapture record : pageList.getRecords()) {
			 Document parse1 = Jsoup.parse(record.getSensitiveList());
			 Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
			 String sensitiveErrorList = "";
			 String errorMessage = "";
			 for(Element element : elements1){
				 String title = element.attr("title")+"<br>";
				 errorMessage += Jsoup.parse(title).text() +"<br>";
				 String stringError = element.toString();
				 int i = record.getRetMsg().indexOf(stringError);
				 String t1 = "";
				 String t2 = "";
				 try {
					 t1 = record.getRetMsg().substring(i-20,i);
					 t2 = record.getRetMsg().substring(i+(stringError.length()),i+(stringError.length())+20);
				 }catch (Exception e){
					 e.printStackTrace();
				 }
				 sensitiveErrorList += t1 + stringError +t2+"<br>";
			 }
			 record.setErrorMessage(errorMessage);
			 record.setSensitiveErrorList(sensitiveErrorList);
		 }
		 return Result.OK(pageList);
	 }

	 @ApiOperation(value="错链信息-分页列表查询", notes="错链信息-分页列表查询")
	 @GetMapping(value = "/listVoError")
	 public Result<IPage<PageContentCaptureVo>> queryPageListVoError(PageContentCaptureVo pageContentCaptureVo,
																 @RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
																 @RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
																 HttpServletRequest req) {
		 Map<String, String[]> parameterMap = req.getParameterMap();
		 QueryWrapper<PageContentCaptureVo> queryWrapper = QueryGenerator.initQueryWrapper(pageContentCaptureVo, req.getParameterMap());
		 queryWrapper.isNotNull("sensitive_list");
		 queryWrapper.ne("sensitive_list","");
		 queryWrapper.gt("error_count",0);
		 Page<PageContentCaptureVo> page = new Page<PageContentCaptureVo>(pageNo, pageSize);
		 IPage<PageContentCaptureVo> pageList = pageContentCaptureVoService.page(page, queryWrapper);
		 for (PageContentCaptureVo record : pageList.getRecords()) {
			 PageContentCapture byId = pageContentCaptureService.getById(record.getId());
			 Document parse1 = Jsoup.parse(record.getSensitiveList());
			 Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
			 String sensitiveErrorList = "";
			 String errorMessage = "";
			 for(Element element : elements1){
				 String title = element.attr("title")+"<br>";
				 errorMessage += Jsoup.parse(title).text() +"<br>";
				 String stringError = element.toString();
				 int i = byId.getRetMsg().indexOf(stringError);
				 String t1 = "";
				 String t2 = "";
				 try {
					 t1 = byId.getRetMsg().substring(i-20,i);
					 t2 = byId.getRetMsg().substring(i+(stringError.length()),i+(stringError.length())+20);
				 }catch (Exception e){
					 e.printStackTrace();
				 }
				 sensitiveErrorList += t1 + stringError +t2+"<br>";
			 }
			 record.setErrorMessage(errorMessage);
			 record.setSensitiveErrorList(sensitiveErrorList);
		 }
		 return Result.OK(pageList);
	 }

	 /*
	  * 错敏词复核
	  */
	 @ApiOperation(value="错敏词复核信息-分页列表查询", notes="错敏词复核信息-分页列表查询")
	 @GetMapping(value = "/listErrorCheck")
	 public Result<String> queryPageListErrorCheck(PageContentCapture pageContentCapture1) {
		 if(!ObjectUtils.isEmpty(pageContentCapture1)){
			 String mainUrl = pageContentCapture1.getMainUrl();
			 if(ObjectUtils.isEmpty(mainUrl) && mainUrl != "" && mainUrl != null){
				 QueryWrapper<PageContentCapture> queryWrapper = new QueryWrapper<PageContentCapture>();
				 queryWrapper.isNotNull("sensitive_list");
				 queryWrapper.ne("sensitive_list","");
				 queryWrapper.gt("error_count",0);
				 queryWrapper.eq("main_url",mainUrl);
				 long count = pageContentCaptureService.count(queryWrapper);
				 Integer pageSize = 100;
				 Integer pageNum = (int)Math.ceil((double)count/pageSize);
				 Integer tenantId = null;
				 for(int i = 1; i <= pageNum;i+=1){
					 Page<PageContentCapture> page = new Page<PageContentCapture>(i, pageSize);
					 IPage<PageContentCapture> pageList = pageContentCaptureService.page(page, queryWrapper);
					 for (PageContentCapture contentCapture : pageList.getRecords()) {
						 tenantId = contentCapture.getTenantId();
						 String source = contentCapture.getSource();
						 contentCapture.setUpdateTime(new Date());
						 String LinkCaptureErrorLogId = IdWorker.getIdStr();
						 if(!ObjectUtils.isEmpty(source)){
							 //爬取页面
							 Document document = null;
							 try {
								 document = Jsoup.parse(new URL(source), 10000);
								 contentCapture.setIsInvalid("2");
							 }catch (Exception e){
								 contentCapture.setIsInvalid("1");
							 }
							 if(!ObjectUtils.isEmpty(document)){
								 String baseUri = document.baseUri();
								 document = jsoupUtils.setUrl(document, baseUri);
								 String sHtml = document.toString();
								 sHtml = sHtml.replaceAll("amp;","");
								 contentCapture.setContent(sHtml);
								 //获取文章发布的标题
								 QueryWrapper<LinkPublishingTitle> linkPublishingTitleQueryWrapper =new QueryWrapper<LinkPublishingTitle>();
								 linkPublishingTitleQueryWrapper.eq("main_site_url",contentCapture.getMainUrl());
								 List<LinkPublishingTitle> linkPublishingTitles = linkPublishingTitleService.list(linkPublishingTitleQueryWrapper);
								 String title = null;
								 title = crawlingUtils.getPublishingTitle(contentCapture.getMainUrl(), document);

								 if(title != null && title != "" && title.length() != 0){
									 contentCapture.setTitle(title);
								 }else{
									 //抓取网页title的内容
									 Elements eles = document.getElementsByTag("title");
									 String string = eles.text();
									 contentCapture.setTitle(string);
								 }
								 QueryWrapper<LinkPublishingTimeGrasping> linkPublishingTimeGraspingQueryWrapper =new QueryWrapper<LinkPublishingTimeGrasping>();
								 linkPublishingTimeGraspingQueryWrapper.eq("main_site_url",contentCapture.getMainUrl());
								 //获取文章发布的时间
								 List<LinkPublishingTimeGrasping> linkPublishingTimeGraspings = iLinkPublishingTimeGraspingService.list(linkPublishingTimeGraspingQueryWrapper);

								 Date parse = crawlingUtils.getPublishingTimeGrasping(contentCapture.getMainUrl(), document);

								 if(!ObjectUtils.isEmpty(parse)){
									 contentCapture.setArticleReleaseTime(parse);
								 }
								 //父地址类型  typeSource
								 contentCapture.setTypeSource(contentCapture.getTypeSource());
								 contentCapture.setStateSource(contentCapture.getStateSource());
								 contentCapture.setMainUrl(contentCapture.getMainUrl());
								 //错敏词校验
								 JsResult jsonObject = fangCunUtils.getJsResult(contentCapture.getContent());

								 String ret_msg = jsonObject.getRet_msg();
								 contentCapture.setErrorCount(jsonObject.getErrorCount());
								 contentCapture.setRetMsg(jsonObject.getRet_msg());
								 contentCapture.setRetCode(jsonObject.getRet_code());
								 if(!ObjectUtils.isEmpty(ret_msg)){
									 //跳过白名单词汇表
									 contentCapture = crawlingUtils.JumpLinkWhiteWord(ret_msg,contentCapture.getMainUrl(),contentCapture);
									 Document parse1 = Jsoup.parse(contentCapture.getRetMsg());
									 Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
									 linkOrWordSaveUtils.saveLinkCaptureErrorMessage(LinkCaptureErrorLogId,contentCapture);
									 if(elements1.size() > 0){
										 contentCapture.setCheckErrorCount(elements1.size());
										 contentCapture.setErrorCount(1);
									 }
								 }
								 contentCapture.setRetCode(jsonObject.getRet_code());
								 contentCapture.setHeavyCount(jsonObject.getHeavyCount());
							 }
							 //错敏词校验-----结束
							 //判断是否修改完成
							 String id = contentCapture.getId();
							 PageContentCapture byId = iPageContentCaptureService.getById(id);
							 if(!ObjectUtils.isEmpty(byId)){
								 Integer numberScans = byId.getNumberScans();
								 if(!ObjectUtils.isEmpty(numberScans)){
									 numberScans = numberScans + 1;
									 contentCapture.setNumberScans(numberScans);
									 contentCapture.setCreateTime(byId.getCreateTime());
								 }
							 }else{
								 contentCapture.setNumberScans(1);
							 }
							 contentCapture.setIsIgnore("2");
							 contentCapture.setCheckState("1");
							 contentCapture.setCheckTime(new Date());
							 contentCapture.setTenantId(contentCapture.getTenantId());
							 iPageContentCaptureService.saveOrUpdate(contentCapture);
							 linkOrWordSaveUtils.iLinkCaptureErrorLogServicesave(contentCapture,LinkCaptureErrorLogId);
						 }
					 }
					 try {
						 //扫描结束对信息进行统计
						 PageContentCapture pageContentCapture = iPageContentCaptureService.listCountByMainUrl(mainUrl);
						 //聚合错敏感词信息，将其结果保存到
						 if(!ObjectUtils.isEmpty(pageContentCapture)){
							 LinkErrorWordStatistics linkErrorWordStatistics = new LinkErrorWordStatistics();
							 linkErrorWordStatistics.setId(IdWorker.getIdStr());
							 linkErrorWordStatistics.setCreateBy(pageContentCapture.getCreateBy());
							 linkErrorWordStatistics.setCreateTime(pageContentCapture.getCreateTime());
							 linkErrorWordStatistics.setUpdateTime(pageContentCapture.getUpdateTime());
							 linkErrorWordStatistics.setUpdateBy(pageContentCapture.getUpdateBy());
							 String mainUrl1 = pageContentCapture.getMainUrl();
							 //通过站点id查询对应的站点信息
							 linkErrorWordStatistics.setMainSiteUrl(mainUrl1);
							 QueryWrapper<MainSite> mainSiteQueryWrapper = new QueryWrapper<MainSite>();
							 mainSiteQueryWrapper.eq("main_site_url",mainUrl1);
							 List<MainSite> mainSiteList = iMainSiteService.list(mainSiteQueryWrapper);
							 if(mainSiteList.size() >= 1){
								 MainSite mainSite = mainSiteList.get(0);
								 linkErrorWordStatistics.setMainSiteId(mainSite.getId());
								 linkErrorWordStatistics.setMainSiteName(mainSite.getMainSiteName());
							 }
							 linkErrorWordStatistics.setTestType("2");
							 linkErrorWordStatistics.setScanTime(pageContentCapture.getUpdateTime());
							 linkErrorWordStatistics.setErrorWordCount(pageContentCapture.getHeavyCount());
							 linkErrorWordStatistics.setTenantId(tenantId);
							 iLinkErrorWordStatisticsService.save(linkErrorWordStatistics);
						 }
					 }catch (Exception e){
						 e.printStackTrace();
					 }
				 }
			 }
		 }
		 return Result.OK("复核成功");
	 }

	 /**
	 * 添加
	 * @param pageContentCapture
	 * @return
	 */
	@AutoLog(value = "页面内容抓取-添加")
	@ApiOperation(value="页面内容抓取-添加", notes="页面内容抓取-添加")
	@RequiresPermissions("pagecontentcapture:page_content_capture:add")
	@PostMapping(value = "/add")
	public Result<String> add(@RequestBody PageContentCapture pageContentCapture) {
		pageContentCaptureService.save(pageContentCapture);
		return Result.OK("添加成功！");
	}
	
	/**
	 *  编辑
	 *
	 * @param pageContentCapture
	 * @return
	 */
	@AutoLog(value = "页面内容抓取-编辑")
	@ApiOperation(value="页面内容抓取-编辑", notes="页面内容抓取-编辑")
	@RequiresPermissions("pagecontentcapture:page_content_capture:edit")
	@RequestMapping(value = "/edit", method = {RequestMethod.PUT,RequestMethod.POST})
	public Result<String> edit(@RequestBody PageContentCapture pageContentCapture) {
		pageContentCaptureService.updateById(pageContentCapture);
		return Result.OK("编辑成功!");
	}

	 /**
	  *  编辑
	  * @return
	  */
	 @RequestMapping(value = "/ignore", method = {RequestMethod.PUT,RequestMethod.POST})
	 public Result<String> ignore(@RequestBody Params params) {
		 Page<MainSite> pageMainSite = new Page<MainSite>(1, 1);
		 IPage<MainSite> pageListMainSite = iMainSiteService.page(pageMainSite);
		 MainSite mainSite = null;
		 if(pageListMainSite.getRecords().size() >= 1){
			 mainSite = pageListMainSite.getRecords().get(0);
		 }
		 if(!ObjectUtils.isEmpty(mainSite) && params.getPlainOptions().length > 0){
			 for (String plainOption : params.getPlainOptions()) {
				 QueryWrapper<LinkWhiteWord> whiteWordQueryWrapper = new QueryWrapper<LinkWhiteWord>();
				 whiteWordQueryWrapper.eq("word",plainOption);
				 try {
					 iLinkWhiteWordService.remove(whiteWordQueryWrapper);
				 }catch (Exception e){
					 e.printStackTrace();
				 }
			 }
		 }
		 if(!ObjectUtils.isEmpty(mainSite) && params.getCheckedList().length > 0){
			 for (String plainOption : params.getCheckedList()) {
				 LinkWhiteWord linkWhiteWord = new LinkWhiteWord();
				 linkWhiteWord.setId(IdWorker.getIdStr());
				 linkWhiteWord.setMainSiteId(mainSite.getId());
				 linkWhiteWord.setMainSiteName(mainSite.getMainSiteName());
				 linkWhiteWord.setMainSiteUrl(mainSite.getMainSiteUrl());
				 linkWhiteWord.setTenantId(mainSite.getTenantId());
				 linkWhiteWord.setWord(plainOption);
				 if(!ObjectUtils.isEmpty(plainOption) && plainOption != ""){
					 try {
						 iLinkWhiteWordService.save(linkWhiteWord);
					 }catch (Exception e){
						 e.printStackTrace();
					 }
				 }
			 }
		 }
		 return Result.OK("编辑成功!");
	 }

	 /**
	  *  编辑
	  *
	  * @return
	  */
	 @DeleteMapping(value = "/ignoreAll")
	 public Result<String> ignoreAllBatch(@RequestParam(name="ids",required=true) String ids) {
		 Page<MainSite> pageMainSite = new Page<MainSite>(1, 1);
		 IPage<MainSite> pageListMainSite = iMainSiteService.page(pageMainSite);
		 MainSite mainSite = null;
		 if(pageListMainSite.getRecords().size() >= 1){
			 mainSite = pageListMainSite.getRecords().get(0);
		 }
		 String[] split = ids.split(",");
		 if(split.length > 0){
			 for (String id : split) {
				 PageContentCapture pageContentCapture = pageContentCaptureService.getById(id);
				 String sensitiveList = pageContentCapture.getSensitiveList();
				 Document parseHtml = Jsoup.parse(sensitiveList);
				 Elements elements = parseHtml.select("span.proofreadSignClass.errorWordMark");
				 for (Element element : elements) {
					 String text = element.text();
					 QueryWrapper<LinkWhiteWord> queryWrapper = new QueryWrapper<LinkWhiteWord>();
					 queryWrapper.eq("word",text);
					 Page<LinkWhiteWord> page = new Page<LinkWhiteWord>(1, 1);
					 Page<LinkWhiteWord> page1 = iLinkWhiteWordService.page(page, queryWrapper);
					 if(page1.getRecords().size() <= 0){
						 LinkWhiteWord linkWhiteWord = new LinkWhiteWord();
						 linkWhiteWord.setId(IdWorker.getIdStr());
						 linkWhiteWord.setMainSiteId(mainSite.getId());
						 linkWhiteWord.setMainSiteName(mainSite.getMainSiteName());
						 linkWhiteWord.setMainSiteUrl(mainSite.getMainSiteUrl());
						 linkWhiteWord.setTenantId(mainSite.getTenantId());
						 linkWhiteWord.setWord(text);
						 try {
							 iLinkWhiteWordService.save(linkWhiteWord);
						 }catch (Exception e){
							 e.printStackTrace();
						 }
					 }
				 }
			 }
		 }
		 return Result.OK("编辑成功!");
	 }

	 @DeleteMapping(value = "/ignoreAllCancell")
	 public Result<String> ignoreAllBatchCancell(@RequestParam(name="ids",required=true) String ids) {
		 Page<MainSite> pageMainSite = new Page<MainSite>(1, 1);
		 IPage<MainSite> pageListMainSite = iMainSiteService.page(pageMainSite);
		 MainSite mainSite = null;
		 if(pageListMainSite.getRecords().size() >= 1){
			 mainSite = pageListMainSite.getRecords().get(0);
		 }
		 String[] split = ids.split(",");
		 if(split.length > 0){
			 for (String id : split) {
				 PageContentCapture pageContentCapture = pageContentCaptureService.getById(id);
				 String sensitiveList = pageContentCapture.getSensitiveList();
				 Document parseHtml = Jsoup.parse(sensitiveList);
				 Elements elements = parseHtml.select("span.proofreadSignClass.errorWordMark");
				 for (Element element : elements) {
					 String text = element.text();
					 System.out.println(text);
					 QueryWrapper<LinkWhiteWord> queryWrapper = new QueryWrapper<LinkWhiteWord>();
					 queryWrapper.eq("word",text);
					 try {
						 iLinkWhiteWordService.remove(queryWrapper);
					 }catch (Exception e){
					 }

				 }
			 }
		 }
		 return Result.OK("编辑成功!");
	 }


	 @DeleteMapping(value = "/ModifyTrueBatchDelete")
	 public Result<String> ModifyTrueBatchDelete(@RequestParam(name="ids",required=true) String ids) {
		 Page<MainSite> pageMainSite = new Page<MainSite>(1, 1);
		 IPage<MainSite> pageListMainSite = iMainSiteService.page(pageMainSite);
		 MainSite mainSite = null;
		 if(pageListMainSite.getRecords().size() >= 1){
			 mainSite = pageListMainSite.getRecords().get(0);
		 }
		 String[] split = ids.split(",");
		 if(split.length > 0){
			 for (String id : split) {
				 try {
					 PageContentCapture byId = pageContentCaptureService.getById(id);
					 byId.setModifyState("2");
					 pageContentCaptureService.updateById(byId);
				 }catch (Exception e){
					 e.printStackTrace();
				 }
			 }
		 }
		 return Result.OK("编辑成功!");
	 }

	 @DeleteMapping(value = "/ModifyFalseBatchDelete")
	 public Result<String> ModifyFalseBatchDelete(@RequestParam(name="ids",required=true) String ids) {
		 Page<MainSite> pageMainSite = new Page<MainSite>(1, 1);
		 IPage<MainSite> pageListMainSite = iMainSiteService.page(pageMainSite);
		 MainSite mainSite = null;
		 if(pageListMainSite.getRecords().size() >= 1){
			 mainSite = pageListMainSite.getRecords().get(0);
		 }
		 String[] split = ids.split(",");
		 if(split.length > 0){
			 for (String id : split) {
				 try {
					 PageContentCapture byId = pageContentCaptureService.getById(id);
					 byId.setModifyState("1");
					 pageContentCaptureService.updateById(byId);
				 }catch (Exception e){
					 e.printStackTrace();
				 }
			 }
		 }
		 return Result.OK("编辑成功!");
	 }

	 @DeleteMapping(value = "/whetherIgnoreAll")
	 public Result<String> whetherIgnoreAll(@RequestParam(name="ids",required=true) String ids) {
		 Page<MainSite> pageMainSite = new Page<MainSite>(1, 1);
		 IPage<MainSite> pageListMainSite = iMainSiteService.page(pageMainSite);
		 MainSite mainSite = null;
		 if(pageListMainSite.getRecords().size() >= 1){
			 mainSite = pageListMainSite.getRecords().get(0);
		 }
		 String[] split = ids.split(",");
		 if(split.length > 0){
			 for (String id : split) {
				 try {
					 PageContentCapture byId = pageContentCaptureService.getById(id);
					 byId.setIsIgnore("1");
					 pageContentCaptureService.updateById(byId);
				 }catch (Exception e){
					 e.printStackTrace();
				 }
			 }
		 }
		 return Result.OK("编辑成功!");
	 }

	 @DeleteMapping(value = "/whetherIgnoreAllCancell")
	 public Result<String> whetherIgnoreAllCancell(@RequestParam(name="ids",required=true) String ids) {
		 Page<MainSite> pageMainSite = new Page<MainSite>(1, 1);
		 IPage<MainSite> pageListMainSite = iMainSiteService.page(pageMainSite);
		 MainSite mainSite = null;
		 if(pageListMainSite.getRecords().size() >= 1){
			 mainSite = pageListMainSite.getRecords().get(0);
		 }
		 String[] split = ids.split(",");
		 if(split.length > 0){
			 for (String id : split) {
				 try {
					 PageContentCapture byId = pageContentCaptureService.getById(id);
					 byId.setIsIgnore("2");
					 pageContentCaptureService.updateById(byId);
				 }catch (Exception e){
					 e.printStackTrace();
				 }
			 }
		 }
		 return Result.OK("编辑成功!");
	 }

	/**
	 *   通过id删除
	 *
	 * @param id
	 * @return
	 */
	@AutoLog(value = "页面内容抓取-通过id删除")
	@ApiOperation(value="页面内容抓取-通过id删除", notes="页面内容抓取-通过id删除")
	@RequiresPermissions("pagecontentcapture:page_content_capture:delete")
	@DeleteMapping(value = "/delete")
	public Result<String> delete(@RequestParam(name="id",required=true) String id) {
		pageContentCaptureService.removeById(id);
		return Result.OK("删除成功!");
	}
	
	/**
	 *  批量删除
	 *
	 * @param ids
	 * @return
	 */
	@AutoLog(value = "页面内容抓取-批量删除")
	@ApiOperation(value="页面内容抓取-批量删除", notes="页面内容抓取-批量删除")
	@RequiresPermissions("pagecontentcapture:page_content_capture:deleteBatch")
	@DeleteMapping(value = "/deleteBatch")
	public Result<String> deleteBatch(@RequestParam(name="ids",required=true) String ids) {
		this.pageContentCaptureService.removeByIds(Arrays.asList(ids.split(",")));
		return Result.OK("批量删除成功!");
	}

	 @Data
	 class PageContentCaptureTrueOrFalse{
		private String wordName;
		private String wordValue;
		private boolean flag;
	 }
	 /**
	  * 通过id查询
	  *
	  * @param id
	  * @return
	  */
	 //@AutoLog(value = "页面内容抓取-通过id查询")
	 @ApiOperation(value="页面内容抓取-通过id查询", notes="页面内容抓取-通过id查询")
	 @GetMapping(value = "/queryByIdErrorList")
	 public Result<List<PageContentCaptureTrueOrFalse>> queryByIdErrorList(@RequestParam(name="id",required=true) String id) {
		 List<PageContentCaptureTrueOrFalse> list = new ArrayList<PageContentCaptureTrueOrFalse>();
		 PageContentCapture pageContentCapture = pageContentCaptureService.getById(id);
		 if(pageContentCapture==null) {
			 return Result.error("未找到对应数据");
		 }
		 String sensitiveList = pageContentCapture.getSensitiveList();
		 Document parseHtml = Jsoup.parse(sensitiveList);
		 Elements elements = parseHtml.select("span.proofreadSignClass.errorWordMark");
		 for (Element element : elements) {
			 PageContentCaptureTrueOrFalse captureTrueOrFalse = new PageContentCaptureTrueOrFalse();
			 String text = element.text();
			 QueryWrapper<LinkWhiteWord> queryWrapper = new QueryWrapper<LinkWhiteWord>();
			 queryWrapper.eq("word",text);
			 Page<LinkWhiteWord> page = new Page<LinkWhiteWord>(1, 1);
			 Page<LinkWhiteWord> page1 = iLinkWhiteWordService.page(page, queryWrapper);
			 if(page1.getRecords().size() > 0){
				 captureTrueOrFalse.setFlag(true);
				 captureTrueOrFalse.setWordName(text);
				 captureTrueOrFalse.setWordValue(element.toString());
				 list.add(captureTrueOrFalse);
			 }else{
				 captureTrueOrFalse.setFlag(false);
				 captureTrueOrFalse.setWordName(text);
				 captureTrueOrFalse.setWordValue(element.toString());
				 list.add(captureTrueOrFalse);
			 }
		 }
		 return Result.OK(list);
	 }
	/**
	 * 通过id查询
	 *
	 * @param id
	 * @return
	 */
	@ApiOperation(value="页面内容抓取-通过id查询", notes="页面内容抓取-通过id查询")
	@GetMapping(value = "/queryById")
	public Result<PageContentCapture> queryById(@RequestParam(name="id",required=true) String id) {
		PageContentCapture pageContentCapture = pageContentCaptureService.getById(id);
		if(pageContentCapture==null) {
			return Result.error("未找到对应数据");
		}
		return Result.OK(pageContentCapture);
	}

	 @ApiOperation(value="页面内容抓取-通过id查询", notes="页面内容抓取-通过id查询")
	 @GetMapping(value = "/queryByIdModifyTrue")
	 public Result<PageContentCapture> queryByIdModifyTrue(@RequestParam(name="id",required=true) String id) {
		 PageContentCapture pageContentCapture = pageContentCaptureService.getById(id);
		 if(pageContentCapture==null) {
			 return Result.error("未找到对应数据");
		 }
		 pageContentCapture.setModifyState("2");
		 pageContentCaptureService.updateById(pageContentCapture);
		 return Result.OK(pageContentCapture);
	 }

	 @ApiOperation(value="页面内容抓取-通过id查询", notes="页面内容抓取-通过id查询")
	 @GetMapping(value = "/queryByIdModifyFalse")
	 public Result<PageContentCapture> queryByIdModifyFalse(@RequestParam(name="id",required=true) String id) {
		 PageContentCapture pageContentCapture = pageContentCaptureService.getById(id);
		 if(pageContentCapture==null) {
			 return Result.error("未找到对应数据");
		 }
		 pageContentCapture.setModifyState("1");
		 pageContentCaptureService.updateById(pageContentCapture);
		 return Result.OK(pageContentCapture);
	 }

	 @ApiOperation(value="页面内容抓取-通过id查询", notes="页面内容抓取-通过id查询")
	 @GetMapping(value = "/queryWhetherToIgnore")
	 public Result<PageContentCapture> queryWhetherToIgnore(@RequestParam(name="id",required=true) String id) {
		 PageContentCapture pageContentCapture = pageContentCaptureService.getById(id);
		 if(pageContentCapture==null) {
			 return Result.error("未找到对应数据");
		 }
		 pageContentCapture.setIsIgnore("1");
		 pageContentCaptureService.updateById(pageContentCapture);
		 return Result.OK(pageContentCapture);
	 }
	 @ApiOperation(value="页面内容抓取-通过id查询", notes="页面内容抓取-通过id查询")
	 @GetMapping(value = "/queryWhetherToIgnoreCancel")
	 public Result<PageContentCapture> queryWhetherToIgnoreCancel(@RequestParam(name="id",required=true) String id) {
		 PageContentCapture pageContentCapture = pageContentCaptureService.getById(id);
		 if(pageContentCapture==null) {
			 return Result.error("未找到对应数据");
		 }
		 pageContentCapture.setIsIgnore("2");
		 pageContentCaptureService.updateById(pageContentCapture);
		 return Result.OK(pageContentCapture);
	 }

	 /**
	  * 导出excel
	  *
	  * @param request
	  * @param pageContentCapture
	  */
	 @RequestMapping(value = "/exportXlsError")
	 public ModelAndView exportXlsError(HttpServletRequest request, PageContentCapture pageContentCapture) {
		 // Step.1 组装查询条件
		 QueryWrapper<PageContentCapture> queryWrapper = QueryGenerator.initQueryWrapper(pageContentCapture, request.getParameterMap());
		 String selections = request.getParameter("selections");
		 if (oConvertUtils.isNotEmpty(selections)) {
			 List<String> selectionList = Arrays.asList(selections.split(","));
			 queryWrapper.in("id",selectionList);
		 }
		 // Step.2 获取导出数据
		 List<PageContentCapture> exportList = pageContentCaptureService.list(queryWrapper);
		 for (PageContentCapture record : exportList) {
			 Document parse1 = Jsoup.parse(record.getSensitiveList());
			 Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
			 String sensitiveErrorList = "";
			 String errorMessage = "";
			 String content = "";
			 for(Element element : elements1){
				 String title = element.attr("title")+"\n";
				 content += title;
				 errorMessage += Jsoup.parse(title).text() +"\n";
				 String stringError = element.toString();
				 int i = record.getRetMsg().indexOf(stringError);
				 String t1 = "";
				 String t2 = "";
				 try {
					 t1 = record.getRetMsg().substring(i-20,i);
					 t2 = record.getRetMsg().substring(i+(stringError.length()),i+(stringError.length())+20);
				 }catch (Exception e){
					 e.printStackTrace();
				 }
				 sensitiveErrorList += Jsoup.parse(t1).text() + stringError +Jsoup.parse(t2).text()+"\n";
			 }
			 record.setErrorMessage(errorMessage);
			 record.setRetMsg(sensitiveErrorList);
			 record.setContent(content);
		 }
		 return super.exportXlsList(exportList, pageContentCapture, PageContentCapture.class, "页面内容抓取");
	 }
    /**
    * 导出excel
    *
    * @param request
    * @param pageContentCapture
    */
    @RequiresPermissions("pagecontentcapture:page_content_capture:exportXls")
    @RequestMapping(value = "/exportXls")
    public ModelAndView exportXls(HttpServletRequest request, PageContentCapture pageContentCapture) {
        return super.exportXls(request, pageContentCapture, PageContentCapture.class, "页面内容抓取");
    }

    /**
      * 通过excel导入数据
    *
    * @param request
    * @param response
    * @return
    */
    @RequiresPermissions("pagecontentcapture:page_content_capture:importExcel")
    @RequestMapping(value = "/importExcel", method = RequestMethod.POST)
    public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
        return super.importExcel(request, response, PageContentCapture.class);
    }


	 /*
	  * 错敏词扫描
	  */
	 @ApiOperation(value="页面抓取以及错敏词校验", notes="页面抓取以及错敏词校验-分页列表查询")
	 @GetMapping(value = "/listCrawling")
	 public Result<String> queryPageListCrawling(
			 ExternalInternalSite externalInternalSite
	 ) throws Exception {
		 //如果没有选择任何内容，对所有内容进行校验
		 List<MainSite> list = iMainSiteService.list();
		 for (MainSite mainSite : list) {
			 crawlingContentCapture(mainSite);
		 }
		 return Result.OK("抓取成功");
	 }

	 public void  crawlingContentCapture(MainSite mainSite){

		 QueryWrapper<LinkAnchorList> listQueryWrapper = new QueryWrapper<LinkAnchorList>();
		 Page<LinkAnchorList> linkAnchorListPage = new Page<LinkAnchorList>(1, 1);
		 Page<LinkAnchorList> page1 = iLinkAnchorListService.page(linkAnchorListPage, listQueryWrapper);
		 List<LinkAnchorList> records = page1.getRecords();
		 LinkAnchorList linkAnchorList = null;
		 if(records.size() > 0){
			 linkAnchorList = records.get(0);
		 }
		 Integer count =  0;
		 if(!ObjectUtils.isEmpty(linkAnchorList)){
			 //查询链接不是锚链接的链接
			 count = externalInternalSiteService.countSourceNoAnchor(mainSite.getMainSiteUrl(),linkAnchorList.getAnchorLink());
		 }else{
			 count = externalInternalSiteService.countSource(mainSite.getMainSiteUrl());
		 }
		 Integer pageSize = 100;
		 Integer pageNum = (int)Math.ceil((double)count/pageSize);
		 for(int i = 1; i <= pageNum;i+=1){
			 Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
			 List<ExternalInternalSite> recordsList = null;
			 if(!ObjectUtils.isEmpty(linkAnchorList)){
				 //查询链接不是锚链接的链接
				 IPage<ExternalInternalSite> pageList = externalInternalSiteService.pageSourceMainUrlNoAnchor(page,mainSite.getMainSiteUrl(),linkAnchorList.getAnchorLink());
				 recordsList = pageList.getRecords();
			 }else{
				 IPage<ExternalInternalSite> pageList = externalInternalSiteService.pageSourceMainUrl(page,mainSite.getMainSiteUrl());
				 recordsList = pageList.getRecords();
			 }
			 if(!ObjectUtils.isEmpty(recordsList)){
				 for (ExternalInternalSite record : recordsList) {
					 String LinkCaptureErrorLogId = IdWorker.getIdStr();
					 PageContentCapture pageContentCapture = new PageContentCapture();
					 pageContentCapture.setId(DigestUtils.md5DigestAsHex(record.getSource().getBytes()));
					 //查询当前id有没有对应的内容
					 try {
						 PageContentCapture byId = iPageContentCaptureService.getById(DigestUtils.md5DigestAsHex(record.getSource().getBytes()));
						 if(!ObjectUtils.isEmpty(byId)){
							 pageContentCapture = byId;
						 }
					 }catch (Exception e){
						 e.printStackTrace();
					 }
					 pageContentCapture.setCreateTime(new Date());
					 pageContentCapture.setUpdateTime(new Date());
					 pageContentCapture.setSource(record.getSource());
					 if(!ObjectUtils.isEmpty(record.getSource())){
						 try {
							 //爬取页面
							 Document document = null;
							 String baseUri = null;
							 String grabPage = mainSite.getGrabPage();
							 try {
								 SpyderReturn spyderReturn = trendsSpyder.companyTrendsSpyderMessage(record.getSource(), grabPage);
								 document = spyderReturn.getDocument();
								 baseUri = spyderReturn.getBaseUri();
								 pageContentCapture.setIsInvalid("2");
							 }catch (Exception e){
								 pageContentCapture.setIsInvalid("1");
								 document = null;
							 }
							 if(!ObjectUtils.isEmpty(document)){
								 document= jsoupUtils.setUrl(document, baseUri);
								 String sHtml = document.toString();
								 sHtml = sHtml.replaceAll("amp;","");
								 pageContentCapture.setContent(sHtml);
								 //获取文章发布的标题
								 QueryWrapper<LinkPublishingTitle> linkPublishingTitleQueryWrapper =new QueryWrapper<LinkPublishingTitle>();
								 linkPublishingTitleQueryWrapper.eq("main_site_url",record.getMainUrl());
								 String title = null;
								 title = crawlingUtils.getPublishingTitle(record.getMainUrl(), document);
								 if(title != null && title != "" && title.length() != 0){
									 pageContentCapture.setTitle(title);
								 }else{
									 //抓取网页title的内容
									 Elements eles = document.getElementsByTag("title");
									 String string = eles.text();
									 pageContentCapture.setTitle(string);
								 }
								 QueryWrapper<LinkPublishingTimeGrasping> linkPublishingTimeGraspingQueryWrapper =new QueryWrapper<LinkPublishingTimeGrasping>();
								 linkPublishingTimeGraspingQueryWrapper.eq("main_site_url",record.getMainUrl());
								 //获取文章发布的时间
								 Date parse = crawlingUtils.getPublishingTimeGrasping(record.getMainUrl(), document);
								 if(!ObjectUtils.isEmpty(parse)){
									 pageContentCapture.setArticleReleaseTime(parse);
								 }
								 //父地址类型  typeSource
								 pageContentCapture.setTypeSource(record.getTypeSource());
								 pageContentCapture.setStateSource(record.getStateSource());
								 pageContentCapture.setMainUrl(record.getMainUrl());
								 //错敏词校验-----开始
								 JsResult jsonObject = fangCunUtils.getJsResult(pageContentCapture.getContent());
								 String ret_msg = jsonObject.getRet_msg();
								 pageContentCapture.setErrorCount(jsonObject.getErrorCount());
								 pageContentCapture.setRetMsg(jsonObject.getRet_msg());
								 pageContentCapture.setRetCode(jsonObject.getRet_code());
								 if(!ObjectUtils.isEmpty(ret_msg)){
									 //跳过白名单词汇
									 pageContentCapture =  crawlingUtils.JumpLinkWhiteWord(ret_msg,record.getMainUrl(),pageContentCapture);
									 //保存错词信息
									 linkOrWordSaveUtils.saveLinkCaptureErrorMessage(LinkCaptureErrorLogId,pageContentCapture);
									 Document parse1 = Jsoup.parse(pageContentCapture.getRetMsg());
									 Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
									 pageContentCapture.setSensitiveList(elements1.toString());
									 pageContentCapture.setSensitiveCount(0);
									 pageContentCapture.setSensitiveCount(elements1.size());
									 if(elements1.size() > 0){
										 pageContentCapture.setCheckErrorCount(elements1.size());
										 pageContentCapture.setErrorCount(1);
										 pageContentCapture.setModifyState("1");
									 }
								 }
								 pageContentCapture.setRetCode(jsonObject.getRet_code());
								 pageContentCapture.setHeavyCount(jsonObject.getHeavyCount());
							 }
							 //错敏词校验-----结束
						 }catch (Exception eee){
							 eee.printStackTrace();
						 }
					 }
					 pageContentCapture.setIsIgnore("2");
					 String id = pageContentCapture.getId();
					 PageContentCapture byId = iPageContentCaptureService.getById(id);
					 if(!ObjectUtils.isEmpty(byId)){
						 Integer numberScans = byId.getNumberScans();
						 if(!ObjectUtils.isEmpty(numberScans)){
							 numberScans = numberScans + 1;
							 pageContentCapture.setNumberScans(numberScans);
							 pageContentCapture.setCreateTime(byId.getCreateTime());
						 }
						 pageContentCapture.setIsIgnore(byId.getIsIgnore());
					 }else{
						 pageContentCapture.setNumberScans(1);
					 }
					 pageContentCapture.setCheckState("2");
					 if(!ObjectUtils.isEmpty(pageContentCapture))
					 {
						 pageContentCapture.setTenantId(record.getTenantId());
						 iPageContentCaptureService.saveOrUpdate(pageContentCapture);
						 //抓取对应的手机号，身份证号，银行卡号
						 crawlingPhoneIdNumberCardNumber.crawling(
								 mainSite.getId(),
								 mainSite.getMainSiteName(),
								 mainSite.getMainSiteUrl(),
								 pageContentCapture.getSource(),
								 pageContentCapture.getTitle(),
								 pageContentCapture.getArticleReleaseTime(),
								 pageContentCapture.getContent(),
								 pageContentCapture.getIsInvalid(),
								 String.valueOf(pageContentCapture.getTenantId())
						 );
						 String sensitiveList = pageContentCapture.getSensitiveList();
						 if(!ObjectUtils.isEmpty(sensitiveList)){
							 linkOrWordSaveUtils.iLinkCaptureErrorLogServicesave(pageContentCapture,LinkCaptureErrorLogId);
						 }
					 }
				 }
			 }

		 }
		 try {
			 //扫描结束对信息进行统计
			 linkOrWordSaveUtils.LinkErrorWordStatisticsSave(mainSite,"1");
		 }catch (Exception e){
			 e.printStackTrace();
		 }
	 }


	 @RequestMapping(value = "/onHandleCheck")
	 public Result<String> onHandleCheck(HttpServletRequest request, PageContentCapture pageContentCapture) {
		 String tenantId = request.getHeader("X-Tenant-Id");
		 // Step.1 组装查询条件 QueryGenerator.initQueryWrapper(pageContentCapture, req.getParameterMap());
		 QueryWrapper<PageContentCapture> queryWrapper = QueryGenerator.initQueryWrapper(pageContentCapture, request.getParameterMap());
		 queryWrapper.isNotNull("sensitive_list");
		 queryWrapper.ne("sensitive_list","");
		 queryWrapper.eq("is_ignore","2");
		 queryWrapper.gt("error_count",0);
		 String selections = request.getParameter("selections");
		 System.out.println(selections);
		 //如果选择了对应的内容只对对应内容进行校验
		 String mainSiteUrl = null;
		 if (oConvertUtils.isNotEmpty(selections)) {
			 List<String> selectionList = Arrays.asList(selections.split(","));
			 queryWrapper.in("id",selectionList);
			 List<PageContentCapture> exportList = pageContentCaptureService.list(queryWrapper);
			 for (PageContentCapture contentCapture : exportList)
			 {
				 mainSiteUrl = contentCapture.getMainUrl();
				 checkPageContentCapture(contentCapture);
			 }
		 }else{
			 //如果没有选择任何内容，对所有内容进行校验
			 List<MainSite> list = iMainSiteService.list();
			 for (MainSite mainSite : list) {
				 pageContentCapture.setMainUrl(mainSite.getMainSiteUrl());
				 queryPageListErrorCheckAll(pageContentCapture,mainSite);
				 //扫描结束对信息进行统计
			 }
		 }
		 //扫描结束对信息进行统计
		 for (MainSite mainSite : iMainSiteService.list()) {
			 linkOrWordSaveUtils.LinkErrorWordStatisticsSave(mainSite,"2");
		 }
		 return Result.OK("完成");
	 }



	 /*
	  * 复核选择信息
	  * */
	public void checkPageContentCapture(PageContentCapture contentCapture){
		String source = contentCapture.getSource();
		contentCapture.setUpdateTime(new Date());
		String LinkCaptureErrorLogId = IdWorker.getIdStr();
		if(!ObjectUtils.isEmpty(source)){
			//爬取页面
			Document document = null;
			try {
				document = Jsoup.parse(new URL(source), 10000);
				contentCapture.setIsInvalid("2");
			}catch (Exception e){
				contentCapture.setIsInvalid("1");
			}

			if(!ObjectUtils.isEmpty(document)){
				String baseUri = document.baseUri();
				document = jsoupUtils.setUrl(document, baseUri);
				String sHtml = document.toString();
				sHtml = sHtml.replaceAll("amp;","");
				contentCapture.setContent(sHtml);
				//获取文章发布的标题
				QueryWrapper<LinkPublishingTitle> linkPublishingTitleQueryWrapper =new QueryWrapper<LinkPublishingTitle>();
				linkPublishingTitleQueryWrapper.eq("main_site_url",contentCapture.getMainUrl());
				String title = null;
				title = crawlingUtils.getPublishingTitle(contentCapture.getMainUrl(), document);
				if(title != null && title != "" && title.length() != 0){
					contentCapture.setTitle(title);
				}else{
					//抓取网页title的内容
					Elements eles = document.getElementsByTag("title");
					String string = eles.text();
					contentCapture.setTitle(string);
				}
				contentCapture.setTitle(title);
				QueryWrapper<LinkPublishingTimeGrasping> linkPublishingTimeGraspingQueryWrapper =new QueryWrapper<LinkPublishingTimeGrasping>();
				linkPublishingTimeGraspingQueryWrapper.eq("main_site_url",contentCapture.getMainUrl());
				//获取文章发布的时间
				Date parse = crawlingUtils.getPublishingTimeGrasping( contentCapture.getMainUrl(), document);
				if(!ObjectUtils.isEmpty(parse)){
					contentCapture.setArticleReleaseTime(parse);
				}
				//父地址类型  typeSource
				contentCapture.setTypeSource(contentCapture.getTypeSource());
				contentCapture.setStateSource(contentCapture.getStateSource());
				contentCapture.setMainUrl(contentCapture.getMainUrl());
				//错敏词校验-----开始
				JsResult jsonObject = fangCunUtils.getJsResult(contentCapture.getContent());
				String ret_msg = jsonObject.getRet_msg();
				contentCapture.setErrorCount(jsonObject.getErrorCount());
				contentCapture.setRetMsg(jsonObject.getRet_msg());
				contentCapture.setRetCode(jsonObject.getRet_code());
				if(!ObjectUtils.isEmpty(ret_msg)){
					//跳过白名单词汇表
					contentCapture = crawlingUtils.JumpLinkWhiteWord(ret_msg, contentCapture.getMainUrl(), contentCapture);

					linkOrWordSaveUtils.saveLinkCaptureErrorMessage(LinkCaptureErrorLogId,contentCapture);
					Document parse1 = Jsoup.parse(contentCapture.getRetMsg());
					Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
					if(elements1.size() > 0){
						contentCapture.setSensitiveList(elements1.toString());
						contentCapture.setCheckErrorCount(elements1.size());
						contentCapture.setErrorCount(1);
						contentCapture.setModifyState("1");
					}else{
						contentCapture.setModifyState("2");
					}
				}
				contentCapture.setRetCode(jsonObject.getRet_code());
				contentCapture.setHeavyCount(jsonObject.getHeavyCount());
			}
			String id = contentCapture.getId();
			PageContentCapture byId = iPageContentCaptureService.getById(id);
			if(!ObjectUtils.isEmpty(byId)){
				Integer numberScans = byId.getNumberScans();
				if(!ObjectUtils.isEmpty(numberScans)){
					numberScans = numberScans + 1;
					contentCapture.setNumberScans(numberScans);
					contentCapture.setCreateTime(byId.getCreateTime());
				}
			}else{
				contentCapture.setNumberScans(1);
			}
			contentCapture.setIsIgnore(contentCapture.getIsIgnore());
			contentCapture.setCheckState("1");
			contentCapture.setCheckTime(new Date());
			contentCapture.setTenantId(contentCapture.getTenantId());
			iPageContentCaptureService.saveOrUpdate(contentCapture);
			linkOrWordSaveUtils.iLinkCaptureErrorLogServicesave(contentCapture,LinkCaptureErrorLogId);
		}
	}

	/*
	* 复核全部信息
	* */

	public Result<String> queryPageListErrorCheckAll(PageContentCapture pageContentCapture1,MainSite mainSite) {
		String mainUrl = pageContentCapture1.getMainUrl();
		if(!ObjectUtils.isEmpty(mainUrl) && mainUrl != "" && mainUrl != null){




			QueryWrapper<PageContentCapture> queryWrapper = new QueryWrapper<PageContentCapture>();
			queryWrapper.isNotNull("sensitive_list");
			queryWrapper.ne("sensitive_list","");
			queryWrapper.gt("error_count",0);
			queryWrapper.eq("main_url",mainUrl);



			long count = pageContentCaptureService.count(queryWrapper);
			System.out.println(count);
			Integer pageSize = 100;
			Integer pageNum = (int)Math.ceil((double)count/pageSize);
			Integer tenantId = null;
			for(int i = 1; i <= pageNum;i+=1){
				Page<PageContentCapture> page = new Page<PageContentCapture>(i, pageSize);
				IPage<PageContentCapture> pageList = pageContentCaptureService.page(page, queryWrapper);
				for (PageContentCapture contentCapture : pageList.getRecords()) {
					tenantId = contentCapture.getTenantId();
					String source = contentCapture.getSource();
					contentCapture.setUpdateTime(new Date());
					String LinkCaptureErrorLogId = IdWorker.getIdStr();
					if(!ObjectUtils.isEmpty(source)){
						//爬取页面
						Document document = null;
						String baseUri = null;
						String grabPage = mainSite.getGrabPage();
						try {
							SpyderReturn spyderReturn = trendsSpyder.companyTrendsSpyderMessage(source, grabPage);
							document = spyderReturn.getDocument();
							baseUri = spyderReturn.getBaseUri();
							contentCapture.setIsInvalid("2");
						}catch (Exception e){
							contentCapture.setIsInvalid("1");
							document = null;
						}
						if(!ObjectUtils.isEmpty(document)){
							document = jsoupUtils.setUrl(document, baseUri);
							String sHtml = document.toString();
							sHtml = sHtml.replaceAll("amp;","");
							contentCapture.setContent(sHtml);
							//获取文章发布的标题
							String title = null;
							title = crawlingUtils.getPublishingTitle(contentCapture.getMainUrl(), document);
							if(title != null && title != "" && title.length() != 0){
								contentCapture.setTitle(title);
							}else{
								//抓取网页title的内容
								Elements eles = document.getElementsByTag("title");
								String string = eles.text();
								contentCapture.setTitle(string);
							}
							QueryWrapper<LinkPublishingTimeGrasping> linkPublishingTimeGraspingQueryWrapper =new QueryWrapper<LinkPublishingTimeGrasping>();
							linkPublishingTimeGraspingQueryWrapper.eq("main_site_url",contentCapture.getMainUrl());
							//获取文章发布的时间
							Date parse = crawlingUtils.getPublishingTimeGrasping(contentCapture.getMainUrl(),document);
							if(!ObjectUtils.isEmpty(parse)){
								contentCapture.setArticleReleaseTime(parse);
							}
							//父地址类型  typeSource
							contentCapture.setTypeSource(contentCapture.getTypeSource());
							contentCapture.setStateSource(contentCapture.getStateSource());
							contentCapture.setMainUrl(contentCapture.getMainUrl());
							//错敏词校验-----开始
							JsResult jsonObject = fangCunUtils.getJsResult(contentCapture.getContent());
							String ret_msg = jsonObject.getRet_msg();
							contentCapture.setErrorCount(jsonObject.getErrorCount());
							contentCapture.setRetMsg(jsonObject.getRet_msg());
							contentCapture.setRetCode(jsonObject.getRet_code());
							if(!ObjectUtils.isEmpty(ret_msg)){
								//跳过白名单词汇表
								contentCapture = crawlingUtils.JumpLinkWhiteWord(ret_msg, contentCapture.getMainUrl(), contentCapture);
								Document parse1 = Jsoup.parse(contentCapture.getRetMsg());
								Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
								if(elements1.size() > 0){
									contentCapture.setCheckErrorCount(elements1.size());
									contentCapture.setErrorCount(1);
									contentCapture.setModifyState("1");
								}else{
									contentCapture.setModifyState("2");
								}
							}
							contentCapture.setRetCode(jsonObject.getRet_code());
							contentCapture.setHeavyCount(jsonObject.getHeavyCount());
						}
						//错敏词校验-----结束
						//判断是否修改完成
						String id = contentCapture.getId();
						PageContentCapture byId = iPageContentCaptureService.getById(id);
						if(!ObjectUtils.isEmpty(byId)){
							Integer numberScans = byId.getNumberScans();
							if(!ObjectUtils.isEmpty(numberScans)){
								numberScans = numberScans + 1;
								contentCapture.setNumberScans(numberScans);
								contentCapture.setCreateTime(byId.getCreateTime());
							}
						}else{
							contentCapture.setNumberScans(1);
						}
						contentCapture.setIsIgnore(contentCapture.getIsIgnore());
						contentCapture.setCheckState("1");
						contentCapture.setCheckTime(new Date());
						contentCapture.setTenantId(contentCapture.getTenantId());
						System.out.println(contentCapture);
						if(!ObjectUtils.isEmpty(contentCapture)){
							iPageContentCaptureService.saveOrUpdate(contentCapture);
						}
						linkOrWordSaveUtils.iLinkCaptureErrorLogServicesave(contentCapture,LinkCaptureErrorLogId);
					}
				}
			}
		}
		 return Result.OK("复核成功");
	 }

 }
