package org.jeecg.modules.demo.externalinternalsite.controller;

import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import kong.unirest.HttpResponse;
import kong.unirest.JsonNode;
import kong.unirest.Unirest;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.system.query.QueryGenerator;
import org.jeecg.modules.demo.externalinternalsite.entity.ExternalInternalSite;
import org.jeecg.modules.demo.externalinternalsite.service.IExternalInternalSiteService;

import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.extern.slf4j.Slf4j;

import org.jeecg.modules.demo.linkcaptureerrorlog.entity.LinkCaptureErrorLog;
import org.jeecg.modules.demo.linkcaptureerrorlog.service.ILinkCaptureErrorLogService;
import org.jeecg.modules.demo.linkcaptureerrormessage.entity.LinkCaptureErrorMessage;
import org.jeecg.modules.demo.linkcaptureerrormessage.service.ILinkCaptureErrorMessageService;
import org.jeecg.modules.demo.linkerrorwordstatistics.entity.LinkErrorWordStatistics;
import org.jeecg.modules.demo.linkerrorwordstatistics.service.ILinkErrorWordStatisticsService;
import org.jeecg.modules.demo.linkpublishingtimegrasping.entity.LinkPublishingTimeGrasping;
import org.jeecg.modules.demo.linkpublishingtimegrasping.service.ILinkPublishingTimeGraspingService;
import org.jeecg.modules.demo.linkpublishingtitle.entity.LinkPublishingTitle;
import org.jeecg.modules.demo.linkpublishingtitle.service.ILinkPublishingTitleService;
import org.jeecg.modules.demo.linkwhiteword.entity.LinkWhiteWord;
import org.jeecg.modules.demo.linkwhiteword.service.ILinkWhiteWordService;
import org.jeecg.modules.demo.mainsite.entity.MainSite;
import org.jeecg.modules.demo.mainsite.service.IMainSiteService;
import org.jeecg.modules.demo.pagecontentcapture.entity.JsResult;
import org.jeecg.modules.demo.pagecontentcapture.entity.PageContentCapture;
import org.jeecg.modules.demo.pagecontentcapture.service.IPageContentCaptureService;
import org.jeecg.common.system.base.controller.JeecgController;
import org.jeecg.modules.spyderUtils.spider.*;
import org.jeecg.modules.system.entity.SysDict;
import org.jeecg.modules.system.entity.SysDictItem;
import org.jeecg.modules.system.service.ISysDictItemService;
import org.jeecg.modules.system.service.ISysDictService;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.DigestUtils;
import org.springframework.util.ObjectUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.ModelAndView;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.jeecg.common.aspect.annotation.AutoLog;
import org.apache.shiro.authz.annotation.RequiresPermissions;

 /**
 * @Description: 扫描链接表
 * @Author: jeecg-boot
 * @Date:   2023-04-03
 * @Version: V1.0
 */
@Api(tags="扫描链接表")
@RestController
@RequestMapping("/externalinternalsite/externalInternalSite")
@Slf4j
public class ExternalInternalSiteController extends JeecgController<ExternalInternalSite, IExternalInternalSiteService> {
	@Autowired
	private IExternalInternalSiteService externalInternalSiteService;

	@Autowired
	private IPageContentCaptureService iPageContentCaptureService;

	@Autowired
	private ILinkPublishingTitleService linkPublishingTitleService;

	@Autowired
	private ILinkPublishingTimeGraspingService iLinkPublishingTimeGraspingService;

	 @Autowired
	 private ISysDictService sysDictService;

	 @Autowired
	 private ISysDictItemService iSysDictItemService;

	 @Autowired
	 private ILinkWhiteWordService iLinkWhiteWordService;

	 @Autowired
	 private ILinkCaptureErrorLogService iLinkCaptureErrorLogService;

	 @Autowired
	 private ILinkCaptureErrorMessageService iLinkCaptureErrorMessageService;

	 @Autowired
	 private ILinkErrorWordStatisticsService iLinkErrorWordStatisticsService;

	 @Autowired
	 private IMainSiteService iMainSiteService;

	 @Autowired
	 private ReportSpyder reportSpyder;

	 @Autowired
	 private FangCunUtils fangCunUtils;

	 @Autowired
	 private LinkOrWordSaveUtils linkOrWordSaveUtils;

	 @Autowired
	 private JsoupUtils jsoupUtils;

	 @Autowired
	 private CrawlingUtils crawlingUtils;
	/**
	 * 分页列表查询
	 *
	 * @param externalInternalSite
	 * @param pageNo
	 * @param pageSize
	 * @param req
	 * @return
	 */
	//@AutoLog(value = "扫描链接表-分页列表查询")
	@ApiOperation(value="扫描链接表-分页列表查询", notes="扫描链接表-分页列表查询")
	@GetMapping(value = "/list")
	public Result<IPage<ExternalInternalSite>> queryPageList(ExternalInternalSite externalInternalSite,
								   @RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
								   @RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
								   HttpServletRequest req) {
		QueryWrapper<ExternalInternalSite> queryWrapper = QueryGenerator.initQueryWrapper(externalInternalSite, req.getParameterMap());
		Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(pageNo, pageSize);
		IPage<ExternalInternalSite> pageList = externalInternalSiteService.page(page, queryWrapper);
		return Result.OK(pageList);
	}
	//@AutoLog(value = "扫描链接表-分页列表查询")
	@ApiOperation(value="扫描链接表-分页列表查询", notes="扫描链接表-分页列表查询")
	@GetMapping(value = "/listErr")
	public Result<IPage<ExternalInternalSite>> queryPageListErr(ExternalInternalSite externalInternalSite,
								   @RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
								   @RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
								   HttpServletRequest req) {
		QueryWrapper<ExternalInternalSite> queryWrapper = QueryGenerator.initQueryWrapper(externalInternalSite, req.getParameterMap());
		queryWrapper.ge("error_count",0);
		Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(pageNo, pageSize);
		IPage<ExternalInternalSite> pageList = externalInternalSiteService.page(page, queryWrapper);
		return Result.OK(pageList);
	}


	 //@AutoLog(value = "扫描链接表-分页列表查询")
	 @ApiOperation(value="扫描链接表-分页列表查询", notes="扫描链接表-分页列表查询")
	 @GetMapping(value = "/listGroupByWai")
	 public Result<IPage<ExternalInternalSite>> queryPageListGroupByWai(ExternalInternalSite externalInternalSite,
															  @RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
															  @RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
															  HttpServletRequest req) {
		 Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(pageNo, pageSize);
		 IPage<ExternalInternalSite> pageList = externalInternalSiteService.pageGroupByWai(page, externalInternalSite);
		 return Result.OK(pageList);
	 }

	 /*
	 * 错敏词扫描
	 */
	 @ApiOperation(value="页面抓取以及错敏词校验", notes="页面抓取以及错敏词校验-分页列表查询")
	 @GetMapping(value = "/listCrawling")
	 public Result<String> queryPageListCrawling(
			 ExternalInternalSite externalInternalSite
	 ) throws Exception {

		 String mainUrl = externalInternalSite.getMainUrl();
		 if(org.springframework.util.ObjectUtils.isEmpty(mainUrl) || mainUrl == "" || mainUrl == null){
			 mainUrl = "http://www.bynrsw.gov.cn/sites/hq/index.jsp";
		 }
		 QueryWrapper<MainSite> mainSiteQueryWrapperQuery = new QueryWrapper<MainSite>();
		 Page<MainSite> pageMainSite = new Page<MainSite>(1, 1);
		 System.out.println(iMainSiteService.list());
		 mainSiteQueryWrapperQuery.eq("main_site_url",mainUrl);
		 Page<MainSite> page1 = iMainSiteService.page(pageMainSite, mainSiteQueryWrapperQuery);
		 log.info(page1.getRecords()+"");
		 MainSite mainSiteQuery = null;
		 if(page1.getRecords().size() >= 1){
			 mainSiteQuery = page1.getRecords().get(0);
		 }
		 Integer count =  externalInternalSiteService.countSource(mainUrl);
		 System.out.println(count);
		 Integer pageSize = 10;
		 Integer pageNum = (int)Math.ceil((double)count/pageSize);
		 for(int i = 1; i <= pageNum;i+=1){
			 Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
			 IPage<ExternalInternalSite> pageList = externalInternalSiteService.pageSourceMainUrl(page,mainUrl);
			 System.out.println(pageList.getRecords().size());
			 for (ExternalInternalSite record : pageList.getRecords()) {
				 System.out.println(record);
				 String LinkCaptureErrorLogId = IdWorker.getIdStr();
				 PageContentCapture pageContentCapture = new PageContentCapture();
				 pageContentCapture.setId(DigestUtils.md5DigestAsHex(record.getSource().getBytes()));
				 Integer tenantId = record.getTenantId();
				 //查询当前id有没有对应的内容
				 try {
					 PageContentCapture byId = iPageContentCaptureService.getById(DigestUtils.md5DigestAsHex(record.getSource().getBytes()));
					 if(!ObjectUtils.isEmpty(byId)){
						 pageContentCapture = byId;
					 }
				 }catch (Exception e){
					 e.printStackTrace();
				 }
				 pageContentCapture.setCreateTime(new Date());
				 pageContentCapture.setUpdateTime(new Date());
				 pageContentCapture.setSource(record.getSource());
				 if(!ObjectUtils.isEmpty(record.getSource())){
					 //爬取页面
					 Document document = null;
					 String baseUri = null;
					 try {

						 String grabPage = mainSiteQuery.getGrabPage();
						 if(grabPage.equals("1")){
							 document = Jsoup.parse(new URL(record.getSource()), 10000);
						 }else{
							 try {
								 String regex = "^(https?://)?([\\w.-]+)\\.([a-zA-Z]{2,})(/|$)";
								 Pattern pattern = Pattern.compile(regex);
								 Matcher matcher = pattern.matcher(record.getSource());
								 if (matcher.find()) {
									 String mainDomain = matcher.group(2) + "." + matcher.group(3);
									 String newUrl = (matcher.group(1) == null ? "http://" : matcher.group(1)) + mainDomain;
									 baseUri = newUrl+"/";
								 }
							 }catch (Exception e){
								 e.printStackTrace();
							 }
							 String s = reportSpyder.companyReportMessage(record.getSource());
							 document = Jsoup.parse(s);
						 }
						 pageContentCapture.setIsInvalid("2");
					 }catch (Exception e){
						 pageContentCapture.setIsInvalid("1");
					 }
					 if(!ObjectUtils.isEmpty(document)){
						 Elements tags = document.select("link[href],img[src],*[style*=background-image],script[src],td[background],a[href]");
						 try {
							 for (Element tag : tags) {
								 try {
									 System.out.println(tag);
									 String attr = tag.attr("href");
									 System.out.println(attr);
									 if (attr == null || attr.isEmpty()) continue;
									 if (!attr.startsWith("http") && !attr.startsWith("data")) {
										 String absUrl = new URI(baseUri).resolve(attr).toString();
										 absUrl = new URI(absUrl).normalize().toString();
										 tag.attr("href", absUrl);
									 }
								 }catch (Exception e){
									 e.printStackTrace();
								 }

							 }
						 }catch (Exception e){
							 e.printStackTrace();
						 }
						 try {
							 for (Element tag : tags) {
								 try {
									 System.out.println(tag);
									 String attr = tag.attr("src");
									 System.out.println(attr);
									 if (attr == null || attr.isEmpty()) continue;
									 if (!attr.startsWith("http") && !attr.startsWith("data")) {
										 String absUrl = new URI(baseUri).resolve(attr).toString();
										 absUrl = new URI(absUrl).normalize().toString();
										 tag.attr("src", absUrl);
									 }
								 }catch (Exception e){
									 e.printStackTrace();
								 }
							 }
						 }catch (Exception e){
							 e.printStackTrace();
						 }
						 try {
							 for (Element tag : tags) {
								 try {
									 System.out.println(tag);
									 String attr = tag.attr("background");
									 System.out.println(attr);
									 if (attr == null || attr.isEmpty()) continue;
									 if (!attr.startsWith("http") && !attr.startsWith("data")) {
										 String absUrl = new URI(baseUri).resolve(attr).toString();
										 absUrl = new URI(absUrl).normalize().toString();
										 tag.attr("background", absUrl);
									 }
								 }catch (Exception e){
									 e.printStackTrace();
								 }
							 }
						 }catch (Exception e){
							 e.printStackTrace();
						 }
						 try {
							 for (Element tag : tags) {
								 try {
									 String attr = tag.attr("iframe");
									 if (attr == null || attr.isEmpty()) continue;
									 if (!attr.startsWith("http") && !attr.startsWith("data")) {
										 String absUrl = new URI(baseUri).resolve(attr).toString();
										 absUrl = new URI(absUrl).normalize().toString();
										 tag.attr("iframe", absUrl);
									 }
								 }catch (Exception e){
									 e.printStackTrace();
								 }
							 }
						 }catch (Exception e){
							 e.printStackTrace();
						 }
						 String sHtml = document.toString();
						 sHtml = sHtml.replaceAll("amp;","");
						 pageContentCapture.setContent(sHtml);
						 //获取文章发布的标题
						 QueryWrapper<LinkPublishingTitle> linkPublishingTitleQueryWrapper =new QueryWrapper<LinkPublishingTitle>();
						 linkPublishingTitleQueryWrapper.eq("main_site_url",record.getMainUrl());
						 List<LinkPublishingTitle> linkPublishingTitles = linkPublishingTitleService.list(linkPublishingTitleQueryWrapper);
						 String title = null;
						 for (LinkPublishingTitle linkPublishingTitle : linkPublishingTitles) {
							 String typeCssXpath = linkPublishingTitle.getTypeCssXpath();
							 try {
								 if(typeCssXpath.equals("selector")){
									 Elements select = document.select(linkPublishingTitle.getElementPositioningType());
									 String string = select.toString();
									 String text = Jsoup.parse(string).text();
									 if(text != null && text != ""){
										 title = text;
									 }
								 }
								 if(typeCssXpath.equals("xpath")){
									 // 5.将document转换成xpath能解析的文档树结构
									 JXDocument jdoc = new JXDocument(document.getAllElements()); // 此处可以使用xpath语法
									 // 6.使用xpath语法来解析html文档来完成数据清洗
									 List<JXNode> elNodes = jdoc.selN(linkPublishingTitle.getElementPositioningType());
									 for (JXNode elNode : elNodes) {
										 String string = elNode.toString();
										 String text = Jsoup.parse(string).text();
										 if(text != null && text != ""){
											 title = text;
										 }
									 }
								 }
							 }catch (Exception e){
							 }
						 }
						 if(title != null && title != "" && title.length() != 0){
							 pageContentCapture.setTitle(title);
						 }else{
							 //抓取网页title的内容
							 Elements eles = document.getElementsByTag("title");
							 String string = eles.text();
							 pageContentCapture.setTitle(string);
						 }
						 QueryWrapper<LinkPublishingTimeGrasping> linkPublishingTimeGraspingQueryWrapper =new QueryWrapper<LinkPublishingTimeGrasping>();
						 linkPublishingTimeGraspingQueryWrapper.eq("main_site_url",record.getMainUrl());
						 //获取文章发布的时间
						 List<LinkPublishingTimeGrasping> linkPublishingTimeGraspings = iLinkPublishingTimeGraspingService.list(linkPublishingTimeGraspingQueryWrapper);
						 String time = null;
						 for (LinkPublishingTimeGrasping linkPublishingTimeGrasping : linkPublishingTimeGraspings) {
							 String typeCssXpathTime = linkPublishingTimeGrasping.getTypeCssXpath();
							 try {
								 if(typeCssXpathTime.equals("selector")){
									 Elements select = document.select(linkPublishingTimeGrasping.getElementPositioningType());
									 String string = select.toString();
									 String text = Jsoup.parse(string).text();
									 if(text != null && text != ""){
										 time = text;
									 }
								 }
								 if(typeCssXpathTime.equals("xpath")){
									 // 5.将document转换成xpath能解析的文档树结构
									 JXDocument jdoc = new JXDocument(document.getAllElements()); // 此处可以使用xpath语法
									 // 6.使用xpath语法来解析html文档来完成数据清洗
									 List<JXNode> elNodes = jdoc.selN(linkPublishingTimeGrasping.getElementPositioningType());
									 for (JXNode elNode : elNodes) {
										 String string = elNode.toString();
										 String text = Jsoup.parse(string).text();
										 if(text != null && text != ""){
											 time = text;
										 }
									 }
								 }
							 }catch (Exception e){
							 }
						 }
						 System.out.println(time);
						 //格式化时间
						 Date parse = null;
						 if(time != null && time != "" && time.length() != 0){
							 for (LinkPublishingTimeGrasping linkPublishingTimeGrasping : linkPublishingTimeGraspings) {
								 String timeFormat = linkPublishingTimeGrasping.getTimeFormat();
								 QueryWrapper<SysDict> sysDictQueryWrapper = new QueryWrapper<SysDict>();
								 sysDictQueryWrapper.eq("dict_code","time_type");
								 List<SysDict> listSysDict = sysDictService.list(sysDictQueryWrapper);
								 SysDict sysDict = null;
								 if(listSysDict.size() >= 1){
									 sysDict = listSysDict.get(0);
								 }
								 QueryWrapper<SysDictItem> sysDictItemQueryWrapper = new QueryWrapper<SysDictItem>();
								 sysDictItemQueryWrapper.eq("dict_id",sysDict.getId());
								 sysDictItemQueryWrapper.eq("item_value",timeFormat);
								 List<SysDictItem> sysDictItems = iSysDictItemService.list(sysDictItemQueryWrapper);
								 for (SysDictItem sysDictItem : sysDictItems) {
									 try {
										 String itemText = sysDictItem.getItemText();
										 //正则表达式
										 String description = sysDictItem.getDescription();
										 Pattern r = Pattern.compile(description);
										 Matcher m = r.matcher(time);
										 String dateStr = null;
										 if (m.find()) {
											 dateStr = m.group(0);
										 }
										 if(dateStr != null && dateStr != ""){
											 SimpleDateFormat simpleDateFormat = new SimpleDateFormat(itemText);
											 parse = simpleDateFormat.parse(dateStr);
										 }
									 }catch (Exception e){
									 }
								 }
							 }
						 }
						 if(!ObjectUtils.isEmpty(parse)){
							 pageContentCapture.setArticleReleaseTime(parse);
						 }
						 //父地址类型  typeSource
						 pageContentCapture.setTypeSource(record.getTypeSource());
						 pageContentCapture.setStateSource(record.getStateSource());
						 pageContentCapture.setMainUrl(record.getMainUrl());
						 //错敏词校验-----开始
						 Integer[] IntStr = {1000,1001,1002,1004,2000,2001,3000,3001,4000,5000,7001,7002};
						 Object oIntStr = JSONArray.toJSON(IntStr);
						 Map<String, Object> paramMap = new HashMap<>();
						 paramMap.put("userid", "quyanhong");
						 paramMap.put("ukey", "K103fhnBhXak");
						 paramMap.put("text", URLEncoder.encode( pageContentCapture.getContent()));
						 paramMap.put("error_types",oIntStr);
						 paramMap.put("threshold",0.2);
						 paramMap.put("strict_degree","m");
						 paramMap.put("domains","test");
						 HttpResponse<JsonNode> response = Unirest.post("http://api.ifuncun.cn:8081/v1/proofreadHtml")
								 .header("Content-Type", "application/json")
								 .body( JSONObject.toJSONString(paramMap)
								 )
								 .asJson();
						 String string = response.getBody().toString();
						 JsResult jsonObject = JSONObject.parseObject(string, JsResult.class);
						 String ret_msg = jsonObject.getRet_msg();
						 pageContentCapture.setErrorCount(jsonObject.getErrorCount());
						 pageContentCapture.setRetMsg(jsonObject.getRet_msg());
						 pageContentCapture.setRetCode(jsonObject.getRet_code());
						 if(!ObjectUtils.isEmpty(ret_msg)){
							 //跳过白名单词汇表
							 QueryWrapper<LinkWhiteWord> linkWhiteWordQueryWrapper = new QueryWrapper<LinkWhiteWord>();
							 linkWhiteWordQueryWrapper.eq("main_site_url",record.getMainUrl());
							 List<LinkWhiteWord> linkWhiteWordList = iLinkWhiteWordService.list(linkWhiteWordQueryWrapper);
							 for (LinkWhiteWord linkWhiteWord : linkWhiteWordList) {
								 int index = ret_msg.indexOf(linkWhiteWord.getWord());
								 if(index > 0){
									 Document parseHtml = Jsoup.parse(ret_msg);
									 Elements elements = parseHtml.select("span.proofreadSignClass.errorWordMark");
									 for (Element element : elements) {
										 String text = element.text();
										 if(text.equals(linkWhiteWord.getWord())){
											 String string1 = parseHtml.toString();
											 String replace = string1.replace(element.toString(), linkWhiteWord.getWord());
											 pageContentCapture.setRetMsg(replace);
											 pageContentCapture.setErrorCount(0);
										 }
									 }
								 }
							 }
							 Document parse1 = Jsoup.parse(pageContentCapture.getRetMsg());
							 Elements elements1 = parse1.select("span.proofreadSignClass.errorWordMark");
							 System.out.println("==============="+elements1.size());
							 for (Element element : elements1) {
								 LinkCaptureErrorMessage linkCaptureErrorMessage = new LinkCaptureErrorMessage();
								 linkCaptureErrorMessage.setErrorMessage(element.toString());
								 linkCaptureErrorMessage.setErrerWord(element.text());
								 linkCaptureErrorMessage.setLinkCaptureErrorLogId(LinkCaptureErrorLogId);
								 linkCaptureErrorMessage.setId(DigestUtils.md5DigestAsHex((linkCaptureErrorMessage.getErrorMessage()+pageContentCapture.getId()).getBytes()));
								 linkCaptureErrorMessage.setLinkPageId(pageContentCapture.getId());
								 linkCaptureErrorMessage.setTenantId(pageContentCapture.getTenantId());
								 iLinkCaptureErrorMessageService.saveOrUpdate(linkCaptureErrorMessage);
							 }
							 pageContentCapture.setSensitiveList(elements1.toString());
							 pageContentCapture.setSensitiveCount(0);
							 pageContentCapture.setSensitiveCount(elements1.size());
							 if(elements1.size() > 0){
								 pageContentCapture.setErrorCount(1);
							 }
						 }
						 pageContentCapture.setRetCode(jsonObject.getRet_code());
						 pageContentCapture.setHeavyCount(jsonObject.getHeavyCount());
					 }
					 //错敏词校验-----结束
				 }

				 pageContentCapture.setModifyState("1");
				 String id = pageContentCapture.getId();
				 PageContentCapture byId = iPageContentCaptureService.getById(id);

				 if(!ObjectUtils.isEmpty(byId)){
					 Integer numberScans = byId.getNumberScans();
					 if(!ObjectUtils.isEmpty(numberScans)){
						 numberScans = numberScans + 1;
						 pageContentCapture.setNumberScans(numberScans);
						 pageContentCapture.setCreateTime(byId.getCreateTime());
					 }
				 }else{
					 pageContentCapture.setNumberScans(1);
				 }
				 pageContentCapture.setIsIgnore("2");
				 pageContentCapture.setCheckState("2");

				 if(
						 !ObjectUtils.isEmpty(pageContentCapture)
								 &&
						 !ObjectUtils.isEmpty(pageContentCapture.getContent())
				 )
				 {
					 pageContentCapture.setTenantId(record.getTenantId());
					 iPageContentCaptureService.saveOrUpdate(pageContentCapture);
					 String sensitiveList = pageContentCapture.getSensitiveList();
					 if(!ObjectUtils.isEmpty(sensitiveList)){
						 iLinkCaptureErrorLogServicesave(pageContentCapture,LinkCaptureErrorLogId);
					 }
				 }
			 }
		 }

		 try {
			 //扫描结束对信息进行统计
			 PageContentCapture pageContentCapture = iPageContentCaptureService.listCountByMainUrl(mainUrl);
			 //聚合错敏感词信息，将其结果保存到
			 if(!ObjectUtils.isEmpty(pageContentCapture)){
				 LinkErrorWordStatistics linkErrorWordStatistics = new LinkErrorWordStatistics();
				 linkErrorWordStatistics.setId(IdWorker.getIdStr());
				 linkErrorWordStatistics.setCreateBy(pageContentCapture.getCreateBy());
				 linkErrorWordStatistics.setCreateTime(pageContentCapture.getCreateTime());
				 linkErrorWordStatistics.setUpdateTime(pageContentCapture.getUpdateTime());
				 linkErrorWordStatistics.setUpdateBy(pageContentCapture.getUpdateBy());
				 String mainUrl1 = pageContentCapture.getMainUrl();
				 //通过站点id查询对应的站点信息
				 linkErrorWordStatistics.setMainSiteUrl(mainUrl1);
				 QueryWrapper<MainSite> mainSiteQueryWrapper = new QueryWrapper<MainSite>();
				 mainSiteQueryWrapper.eq("main_site_url",mainUrl1);
				 List<MainSite> mainSiteList = iMainSiteService.list(mainSiteQueryWrapper);
				 if(mainSiteList.size() >= 1){
					 MainSite mainSite = mainSiteList.get(0);
					 linkErrorWordStatistics.setMainSiteId(mainSite.getId());
					 linkErrorWordStatistics.setMainSiteName(mainSite.getMainSiteName());
				 }
				 linkErrorWordStatistics.setTestType("1");
				 linkErrorWordStatistics.setScanTime(pageContentCapture.getUpdateTime());
				 linkErrorWordStatistics.setErrorWordCount(pageContentCapture.getHeavyCount());
				 linkErrorWordStatistics.setTenantId(pageContentCapture.getTenantId());
				 iLinkErrorWordStatisticsService.save(linkErrorWordStatistics);
			 }
		 }catch (Exception e){
			 e.printStackTrace();
		 }
		 return Result.OK("抓取成功");
	 }


	 public void QueryList(ExternalInternalSite externalInternalSite) {
		 String mainUrl = externalInternalSite.getMainUrl();
		 if (ObjectUtils.isEmpty(mainUrl) || mainUrl == "" || mainUrl == null) {
			 mainUrl = "http://www.hetaodaxue.com/hyywxx/";
		 }
		 Integer count = Integer.parseInt(externalInternalSiteService.count()+"");
		 System.out.println(count);
		 Integer pageSize = 100;
		 Integer pageNum = (int) Math.ceil((double) count / pageSize);
		 for (int i = 1; i <= pageNum; i += 1) {
			 Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
			 IPage<ExternalInternalSite> pageList = externalInternalSiteService.page(page);
			 for (ExternalInternalSite record : pageList.getRecords()) {
				 record.setTenantId(1000);
				 System.out.println(record);
				 externalInternalSiteService.updateById(record);
			 }

		 }
	 }



	 public boolean iLinkCaptureErrorLogServicesave(PageContentCapture pageContentCapture,String LinkCaptureErrorLogId ){
		 LinkCaptureErrorLog linkCaptureErrorLog = new LinkCaptureErrorLog();
		 linkCaptureErrorLog.setId(LinkCaptureErrorLogId);
		 linkCaptureErrorLog.setCheckTime(new Date());
		 linkCaptureErrorLog.setUpdateTime(new Date());
		 try {
			 linkCaptureErrorLog.setLinkPageId(pageContentCapture.getId());
		 }catch (Exception e){
		 }
		 linkCaptureErrorLog.setSource(pageContentCapture.getSource());
		 linkCaptureErrorLog.setTitle(pageContentCapture.getTitle());
		 linkCaptureErrorLog.setTypeSource(pageContentCapture.getTypeSource());
		 linkCaptureErrorLog.setStateSource(pageContentCapture.getStateSource());
		 linkCaptureErrorLog.setMainUrl(pageContentCapture.getMainUrl());
		 linkCaptureErrorLog.setSensitiveList(pageContentCapture.getSensitiveList());
		 linkCaptureErrorLog.setSensitiveCount(pageContentCapture.getSensitiveCount());
		 linkCaptureErrorLog.setRetCode(pageContentCapture.getRetCode());
		 linkCaptureErrorLog.setErrorCount(pageContentCapture.getErrorCount());
		 linkCaptureErrorLog.setHeavyCount(pageContentCapture.getHeavyCount());
		 linkCaptureErrorLog.setModifyState(pageContentCapture.getModifyState());
		 linkCaptureErrorLog.setArticleReleaseTime(pageContentCapture.getArticleReleaseTime());
		 linkCaptureErrorLog.setNumberScans(pageContentCapture.getNumberScans());
		 linkCaptureErrorLog.setIsIgnore(pageContentCapture.getIsIgnore());
		 linkCaptureErrorLog.setCheckTime(pageContentCapture.getCheckTime());
		 linkCaptureErrorLog.setCheckState(pageContentCapture.getCheckState());
		 linkCaptureErrorLog.setCheckErrorCount(pageContentCapture.getCheckErrorCount());
		 linkCaptureErrorLog.setTenantId(pageContentCapture.getTenantId());
		 //扫描日志记录
		 boolean save = iLinkCaptureErrorLogService.saveOrUpdate(linkCaptureErrorLog);
		return save;
	 }
	/**
	 *   添加
	 *
	 * @param externalInternalSite
	 * @return
	 */
	@AutoLog(value = "扫描链接表-添加")
	@ApiOperation(value="扫描链接表-添加", notes="扫描链接表-添加")
	@RequiresPermissions("externalinternalsite:external_internal_site:add")
	@PostMapping(value = "/add")
	public Result<String> add(@RequestBody ExternalInternalSite externalInternalSite) {
		externalInternalSiteService.save(externalInternalSite);
		return Result.OK("添加成功！");
	}
	
	/**
	 *  编辑
	 *
	 * @param externalInternalSite
	 * @return
	 */
	@AutoLog(value = "扫描链接表-编辑")
	@ApiOperation(value="扫描链接表-编辑", notes="扫描链接表-编辑")
	@RequiresPermissions("externalinternalsite:external_internal_site:edit")
	@RequestMapping(value = "/edit", method = {RequestMethod.PUT,RequestMethod.POST})
	public Result<String> edit(@RequestBody ExternalInternalSite externalInternalSite) {
		externalInternalSiteService.updateById(externalInternalSite);
		return Result.OK("编辑成功!");
	}
	
	/**
	 *   通过id删除
	 *
	 * @param id
	 * @return
	 */
	@AutoLog(value = "扫描链接表-通过id删除")
	@ApiOperation(value="扫描链接表-通过id删除", notes="扫描链接表-通过id删除")
	@RequiresPermissions("externalinternalsite:external_internal_site:delete")
	@DeleteMapping(value = "/delete")
	public Result<String> delete(@RequestParam(name="id",required=true) String id) {
		externalInternalSiteService.removeById(id);
		return Result.OK("删除成功!");
	}
	
	/**
	 *  批量删除
	 *
	 * @param ids
	 * @return
	 */
	@AutoLog(value = "扫描链接表-批量删除")
	@ApiOperation(value="扫描链接表-批量删除", notes="扫描链接表-批量删除")
	@RequiresPermissions("externalinternalsite:external_internal_site:deleteBatch")
	@DeleteMapping(value = "/deleteBatch")
	public Result<String> deleteBatch(@RequestParam(name="ids",required=true) String ids) {
		this.externalInternalSiteService.removeByIds(Arrays.asList(ids.split(",")));
		return Result.OK("批量删除成功!");
	}
	
	/**
	 * 通过id查询
	 *
	 * @param id
	 * @return
	 */
	//@AutoLog(value = "扫描链接表-通过id查询")
	@ApiOperation(value="扫描链接表-通过id查询", notes="扫描链接表-通过id查询")
	@GetMapping(value = "/queryById")
	public Result<ExternalInternalSite> queryById(@RequestParam(name="id",required=true) String id) {
		ExternalInternalSite externalInternalSite = externalInternalSiteService.getById(id);
		if(externalInternalSite==null) {
			return Result.error("未找到对应数据");
		}
		return Result.OK(externalInternalSite);
	}

    /**
    * 导出excel
    *
    * @param request
    * @param externalInternalSite
    */
    @RequiresPermissions("externalinternalsite:external_internal_site:exportXls")
    @RequestMapping(value = "/exportXls")
    public ModelAndView exportXls(HttpServletRequest request, ExternalInternalSite externalInternalSite) {
        return super.exportXls(request, externalInternalSite, ExternalInternalSite.class, "扫描链接表");
    }

    /**
      * 通过excel导入数据
    *
    * @param request
    * @param response
    * @return
    */
    @RequiresPermissions("externalinternalsite:external_internal_site:importExcel")
    @RequestMapping(value = "/importExcel", method = RequestMethod.POST)
    public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
        return super.importExcel(request, response, ExternalInternalSite.class);
    }

}
