package org.jeecg.modules.demo.linkchainerrordetection.controller;

import java.net.URI;
import java.net.URL;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.system.query.QueryGenerator;
import org.jeecg.common.util.oConvertUtils;
import org.jeecg.modules.demo.externalinternalsite.entity.ExternalInternalSite;
import org.jeecg.modules.demo.externalinternalsite.service.IExternalInternalSiteService;
import org.jeecg.modules.demo.linkcaptureerrorlog.service.ILinkCaptureErrorLogService;
import org.jeecg.modules.demo.linkchainerrordetection.entity.LinkChainErrorDetection;
import org.jeecg.modules.demo.linkchainerrordetection.service.ILinkChainErrorDetectionService;

import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.extern.slf4j.Slf4j;

import org.jeecg.modules.demo.linkchainerrorlog.entity.LinkChainErrorLog;
import org.jeecg.modules.demo.linkchainerrorlog.service.ILinkChainErrorLogService;
import org.jeecg.modules.demo.linkerrorstatistics.entity.LinkErrorStatistics;
import org.jeecg.modules.demo.linkerrorstatistics.service.ILinkErrorStatisticsService;
import org.jeecg.modules.demo.linkouterchaindetection.service.ILinkOuterChainDetectionService;
import org.jeecg.modules.demo.linkwhitelist.entity.LinkWhiteList;
import org.jeecg.modules.demo.linkwhitelist.service.ILinkWhiteListService;
import org.jeecg.modules.demo.mainsite.entity.MainSite;
import org.jeecg.modules.demo.mainsite.service.IMainSiteService;
import org.jeecg.modules.spyderUtils.spider.*;
import org.jeecg.common.system.base.controller.JeecgController;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.DigestUtils;
import org.springframework.util.ObjectUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.ModelAndView;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.jeecg.common.aspect.annotation.AutoLog;
import org.apache.shiro.authz.annotation.RequiresPermissions;

/**
 * @Description: 错链检测
 * @Author: jeecg-boot
 * @Date:   2023-04-13
 * @Version: V1.0
 */
@Api(tags="错链检测")
@RestController
@RequestMapping("/linkchainerrordetection/linkChainErrorDetection")
@Slf4j
public class LinkChainErrorDetectionController extends JeecgController<LinkChainErrorDetection, ILinkChainErrorDetectionService> {
	@Autowired
	private ILinkChainErrorDetectionService linkChainErrorDetectionService;
	@Autowired
	private IMainSiteService iMainSiteService;

	@Autowired
	private IExternalInternalSiteService externalInternalSiteService;

	@Autowired
	private ILinkOuterChainDetectionService linkOuterChainDetectionService;

	@Autowired
	private ILinkCaptureErrorLogService iLinkCaptureErrorLogService;

	@Autowired
	private ILinkChainErrorLogService iLinkChainErrorLogService;

	@Autowired
	private ILinkErrorStatisticsService iLinkErrorStatisticsService;

	@Autowired
	private ILinkWhiteListService linkWhiteListService;

	@Autowired
	private TrendsSpyder trendsSpyder;

	@Autowired
	private FangCunUtils fangCunUtils;

	@Autowired
	private LinkOrWordSaveUtils linkOrWordSaveUtils;

	@Autowired
	private JsoupUtils jsoupUtils;

	@Autowired
	private CrawlingUtils crawlingUtils;
	/**
	 * 分页列表查询
	 *
	 * @param linkChainErrorDetection
	 * @param pageNo
	 * @param pageSize
	 * @param req
	 * @return
	 */
	//@AutoLog(value = "错链检测-分页列表查询")
	@ApiOperation(value="错链检测-分页列表查询", notes="错链检测-分页列表查询")
	@GetMapping(value = "/list")
	public Result<IPage<LinkChainErrorDetection>> queryPageList(LinkChainErrorDetection linkChainErrorDetection,
																@RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
																@RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
																HttpServletRequest req) {
		QueryWrapper<LinkChainErrorDetection> queryWrapper = QueryGenerator.initQueryWrapper(linkChainErrorDetection, req.getParameterMap());
		Page<LinkChainErrorDetection> page = new Page<LinkChainErrorDetection>(pageNo, pageSize);
		IPage<LinkChainErrorDetection> pageList = linkChainErrorDetectionService.page(page, queryWrapper);
		return Result.OK(pageList);
	}

	//@AutoLog(value = "同步错链数据-分页列表查询")
	@ApiOperation(value="同步错链数据-分页列表查询", notes="同步错链数据-分页列表查询")
	@GetMapping(value = "/spyderListErrorLink")
	public Result<String> updateListErrorLink(HttpServletRequest request, LinkChainErrorDetection linkChainErrorDetection) {
		String tenantId = request.getHeader("X-Tenant-Id");
		//如果没有选择任何内容，对所有内容进行校验
		List<MainSite> list = iMainSiteService.list();
		for (MainSite mainSite : list) {
			crawlingErrorLink(mainSite);
		}
		return Result.OK("成功");
	}
	private void crawlingErrorLink(MainSite mainSite) {
		String mainUrl = mainSite.getMainSiteUrl();
		//只统计错链
		Integer count =  externalInternalSiteService.countSourceLinkError(mainUrl);
		System.out.println(count);
		Integer pageSize = 10;
		Integer pageNum = (int)Math.ceil((double)count/pageSize);
		Integer tenantId = null;
		for(int i = 1; i <= pageNum;i+=1){
			Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
			//只统计错链
			IPage<ExternalInternalSite> pageList = externalInternalSiteService.pageSourceErrorMainUrl(page,mainUrl);
			for (ExternalInternalSite internalSite : pageList.getRecords()) {
				//爬取页面，判断当前链接是否是错链
				Document document = null;
				try {
					document = Jsoup.parse(new URL(internalSite.getUrl()), 10000);
				}catch (Exception e){
				}
				if(!org.apache.commons.lang3.ObjectUtils.isEmpty(document)){
					continue;
				}
				tenantId = internalSite.getTenantId();
				LinkChainErrorDetection errorDetection = new LinkChainErrorDetection();
				errorDetection.setId(internalSite.getId());
				errorDetection.setCreateBy(internalSite.getMainUrl());
				errorDetection.setCreateTime(new Date());
				errorDetection.setUpdateBy(internalSite.getMainUrl());
				errorDetection.setUpdateTime(new Date());
				QueryWrapper<MainSite> mainSiteQueryWrapper = new QueryWrapper<MainSite>();
				String mainUrl1 = internalSite.getMainUrl();
				mainSiteQueryWrapper.eq("main_site_url",mainUrl1);
				List<MainSite> mainSiteList = iMainSiteService.list(mainSiteQueryWrapper);
				if(mainSiteList.size() >= 1){
					MainSite mainSiteIn = mainSiteList.get(0);
					errorDetection.setMainSiteId(mainSiteIn.getId());
					errorDetection.setMainSiteName(mainSiteIn.getMainSiteName());
					errorDetection.setMainSiteUrl(mainSiteIn.getMainSiteUrl());
				}
				errorDetection.setErrorUrl(internalSite.getUrl());
				errorDetection.setErrorUrlType(internalSite.getTypeUrl());
				errorDetection.setErrorUrlState(internalSite.getStateUrl());
				//设置引用页数
				QueryWrapper<ExternalInternalSite> siteQueryWrapper = new QueryWrapper<ExternalInternalSite>();
				siteQueryWrapper.eq("url",internalSite.getUrl());
				long countSiteQuery = externalInternalSiteService.count(siteQueryWrapper);
				errorDetection.setNumberApplicationPages(Integer.parseInt(String.valueOf(countSiteQuery)));

				QueryWrapper<LinkChainErrorDetection> linkChainErrorDetectionQueryWrapper = new QueryWrapper<LinkChainErrorDetection>();
				linkChainErrorDetectionQueryWrapper.eq("id",internalSite.getId());
				Page<LinkChainErrorDetection> pageListS = new Page<LinkChainErrorDetection>(1, 1);
				Page<LinkChainErrorDetection> page1 = linkChainErrorDetectionService.page(pageListS,linkChainErrorDetectionQueryWrapper);
				List<LinkChainErrorDetection> chainErrorDetections = page1.getRecords();
				LinkChainErrorDetection chainDetection = null;
				if(chainErrorDetections.size() >=1){
					chainDetection = chainErrorDetections.get(0);
				}
				if(!ObjectUtils.isEmpty(chainDetection)){
					errorDetection.setUpdateTime(chainDetection.getUpdateTime());
					Integer numberScans = chainDetection.getNumberScans();
					numberScans = numberScans + 1;
					errorDetection.setNumberScans(numberScans);
					errorDetection.setIsIgnore(chainDetection.getIsIgnore());
					errorDetection.setCheckState(chainDetection.getCheckState());
				}else{
					errorDetection.setNumberScans(1);
					errorDetection.setIsIgnore("2");
					errorDetection.setCheckState("2");
				}
				errorDetection.setIsRevise("2");
				errorDetection.setTenantId(internalSite.getTenantId());
				linkChainErrorDetectionService.saveOrUpdate(errorDetection);

				saveLinkChainErrorLog(errorDetection);

			}
		}
		try {
			//统计数据
			LinkErrorStatistics linkErrorStatistics = iLinkErrorStatisticsService.listCountByMainUrl(mainUrl);
			if(!ObjectUtils.isEmpty(linkErrorStatistics)){
				linkErrorStatistics.setId(IdWorker.getIdStr());
				linkErrorStatistics.setTestType("1");
				linkErrorStatistics.setScanTime(linkErrorStatistics.getUpdateTime());
				linkErrorStatistics.setTenantId(tenantId);
				iLinkErrorStatisticsService.save(linkErrorStatistics);
			}
		}catch (Exception e){
			e.printStackTrace();
		}
	}

	/*
	* 复核错链数据
	* */
	@RequestMapping(value = "/onHandleCheckListErrorLink")
	public Result<String> onHandleCheckListErrorLink(HttpServletRequest request,LinkChainErrorDetection linkChainErrorDetection) {
		String selections = request.getParameter("selections");
		QueryWrapper<LinkChainErrorDetection> queryWrapper = QueryGenerator.initQueryWrapper(linkChainErrorDetection, request.getParameterMap());
		if (oConvertUtils.isNotEmpty(selections)) {
			List<String> selectionList = Arrays.asList(selections.split(","));
			queryWrapper.in("id",selectionList);
			List<LinkChainErrorDetection> exportList = linkChainErrorDetectionService.list(queryWrapper);
			//List<List<LinkChainErrorDetection>> lists = groupList(exportList);
			checkErrorLink(exportList);
		}else{
			//只统计错链
			Long count =  linkChainErrorDetectionService.count();
			Integer pageSize = 10;
			Integer pageNum = (int)Math.ceil((double)count/pageSize);
			for(int i = 1; i <= pageNum;i+=1){
				Page<LinkChainErrorDetection> page = new Page<LinkChainErrorDetection>(i, pageSize);
				IPage<LinkChainErrorDetection> pageList = linkChainErrorDetectionService.page(page, queryWrapper);
				//并发执行
				//List<List<LinkChainErrorDetection>> lists = groupList(pageList.getRecords());
				checkErrorLink(pageList.getRecords());
			}
		}
		try {
			//如果没有选择任何内容，对所有内容进行校验
			List<MainSite> list = iMainSiteService.list();
			for (MainSite mainSite : list) {
				//统计数据
				LinkErrorStatistics linkErrorStatistics = iLinkErrorStatisticsService.listCountByMainUrl(mainSite.getMainSiteUrl());
				if(!ObjectUtils.isEmpty(linkErrorStatistics)){
					linkErrorStatistics.setId(IdWorker.getIdStr());
					linkErrorStatistics.setTestType("2");
					linkErrorStatistics.setScanTime(linkErrorStatistics.getUpdateTime());
					linkErrorStatistics.setTenantId(mainSite.getTenantId());
					iLinkErrorStatisticsService.save(linkErrorStatistics);
				}
			}
		}catch (Exception e){
			e.printStackTrace();
		}
		return Result.OK("成功");
	}


	/**
	 *   添加
	 *
	 * @param linkChainErrorDetection
	 * @return
	 */
	@AutoLog(value = "错链检测-添加")
	@ApiOperation(value="错链检测-添加", notes="错链检测-添加")
	@RequiresPermissions("linkchainerrordetection:link_chain_error_detection:add")
	@PostMapping(value = "/add")
	public Result<String> add(@RequestBody LinkChainErrorDetection linkChainErrorDetection) {
		try {
			String mainSiteId = linkChainErrorDetection.getMainSiteId();
			MainSite mainSite = iMainSiteService.getById(mainSiteId);
			linkChainErrorDetection.setMainSiteName(mainSite.getMainSiteName());
			linkChainErrorDetection.setMainSiteUrl(mainSite.getMainSiteUrl());
		}catch (Exception e){
			e.printStackTrace();
		}
		linkChainErrorDetectionService.save(linkChainErrorDetection);
		return Result.OK("添加成功！");
	}

	/**
	 *  编辑
	 *
	 * @param linkChainErrorDetection
	 * @return
	 */
	@AutoLog(value = "错链检测-编辑")
	@ApiOperation(value="错链检测-编辑", notes="错链检测-编辑")
	@RequiresPermissions("linkchainerrordetection:link_chain_error_detection:edit")
	@RequestMapping(value = "/edit", method = {RequestMethod.PUT,RequestMethod.POST})
	public Result<String> edit(@RequestBody LinkChainErrorDetection linkChainErrorDetection) {
		try {
			String mainSiteId = linkChainErrorDetection.getMainSiteId();
			MainSite mainSite = iMainSiteService.getById(mainSiteId);
			linkChainErrorDetection.setMainSiteName(mainSite.getMainSiteName());
			linkChainErrorDetection.setMainSiteUrl(mainSite.getMainSiteUrl());
		}catch (Exception e){
			e.printStackTrace();
		}
		linkChainErrorDetectionService.updateById(linkChainErrorDetection);
		return Result.OK("编辑成功!");
	}
	/**
	 *   通过id删除
	 *
	 * @param id
	 * @return
	 */
	@AutoLog(value = "错链检测-通过id删除")
	@ApiOperation(value="错链检测-通过id删除", notes="错链检测-通过id删除")
	@RequiresPermissions("linkchainerrordetection:link_chain_error_detection:delete")
	@DeleteMapping(value = "/delete")
	public Result<String> delete(@RequestParam(name="id",required=true) String id) {
		linkChainErrorDetectionService.removeById(id);
		return Result.OK("删除成功!");
	}
	/**
	 *  批量删除
	 *
	 * @param ids
	 * @return
	 */
	@AutoLog(value = "错链检测-批量删除")
	@ApiOperation(value="错链检测-批量删除", notes="错链检测-批量删除")
	@RequiresPermissions("linkchainerrordetection:link_chain_error_detection:deleteBatch")
	@DeleteMapping(value = "/deleteBatch")
	public Result<String> deleteBatch(@RequestParam(name="ids",required=true) String ids) {
		this.linkChainErrorDetectionService.removeByIds(Arrays.asList(ids.split(",")));
		return Result.OK("批量删除成功!");
	}

	@DeleteMapping(value = "/batchHIgnoreHandle")
	public Result<String> IgnoreBatchTrue(@RequestParam(name="ids",required=true) String ids) {
		String[] split = ids.split(",");
		for (String sId : split) {
			try {
				LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(sId);
				linkChainErrorDetection.setIsIgnore("1");
				linkChainErrorDetectionService.updateById(linkChainErrorDetection);
			}catch (Exception e){
				e.printStackTrace();
			}
		}
		return Result.OK("批量设置成功!");
	}

	@DeleteMapping(value = "/batchHIgnoreHandleDelete")
	public Result<String> IgnoreBatchFalse(@RequestParam(name="ids",required=true) String ids) {
		String[] split = ids.split(",");
		for (String sId : split) {
			try {
				LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(sId);
				linkChainErrorDetection.setIsIgnore("2");
				linkChainErrorDetectionService.updateById(linkChainErrorDetection);
			}catch (Exception e){
				e.printStackTrace();
			}
		}
		return Result.OK("批量设置成功!");
	}

	@DeleteMapping(value = "/batchHModifyTrue")
	public Result<String> batchHModifyTrue(@RequestParam(name="ids",required=true) String ids) {
		String[] split = ids.split(",");
		for (String sId : split) {
			try {
				LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(sId);
				linkChainErrorDetection.setIsRevise("1");
				linkChainErrorDetectionService.updateById(linkChainErrorDetection);
			}catch (Exception e){
				e.printStackTrace();
			}
		}
		return Result.OK("批量设置成功!");
	}

	@DeleteMapping(value = "/batchHModifyFalse")
	public Result<String> batchHModifyFalse(@RequestParam(name="ids",required=true) String ids) {
		String[] split = ids.split(",");
		for (String sId : split) {
			try {
				LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(sId);
				linkChainErrorDetection.setIsRevise("2");
				linkChainErrorDetectionService.updateById(linkChainErrorDetection);
			}catch (Exception e){
				e.printStackTrace();
			}
		}
		return Result.OK("批量设置成功!");
	}

	@DeleteMapping(value = "/batchHAddWhiteLink")
	public Result<String> batchHAddWhiteLink(@RequestParam(name="ids",required=true) String ids) {
		String[] split = ids.split(",");
		for (String sId : split) {
			try {
				LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(sId);
				String errorUrl = linkChainErrorDetection.getErrorUrl();
				QueryWrapper<LinkWhiteList> linkWhiteListQueryWrapper = new QueryWrapper<LinkWhiteList>();
				linkWhiteListQueryWrapper.eq("white_link",errorUrl);
				linkWhiteListService.remove(linkWhiteListQueryWrapper);
				if(!ObjectUtils.isEmpty(errorUrl)){
					LinkWhiteList whiteList = new LinkWhiteList();
					whiteList.setId(DigestUtils.md5DigestAsHex(errorUrl.getBytes()));
					whiteList.setCreateTime(new Date());
					whiteList.setUpdateTime(new Date());
					whiteList.setMainSiteId(linkChainErrorDetection.getId());
					whiteList.setMainSiteName(linkChainErrorDetection.getMainSiteName());
					whiteList.setMainSiteUrl(linkChainErrorDetection.getMainSiteUrl());
					whiteList.setWhiteLink(errorUrl);
					boolean save = linkWhiteListService.saveOrUpdate(whiteList);
				}
			}catch (Exception e){
				e.printStackTrace();
			}
		}
		return Result.OK("批量设置成功!");
	}

	@DeleteMapping(value = "/batchHRemoveWhiteLink")
	public Result<String> batchHRemoveWhiteLink(@RequestParam(name="ids",required=true) String ids) {
		String[] split = ids.split(",");
		for (String sId : split) {
			try {
				LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(sId);
				String errorUrl = linkChainErrorDetection.getErrorUrl();
				QueryWrapper<LinkWhiteList> linkWhiteListQueryWrapper = new QueryWrapper<LinkWhiteList>();
				linkWhiteListQueryWrapper.eq("white_link",errorUrl);
				linkWhiteListService.remove(linkWhiteListQueryWrapper);
			}catch (Exception e){
				e.printStackTrace();
			}
		}
		return Result.OK("批量设置成功!");
	}
	/**
	 * 通过id查询
	 *
	 * @param id
	 * @return
	 */
	//@AutoLog(value = "错链检测-通过id查询")
	@ApiOperation(value="错链检测-通过id查询", notes="错链检测-通过id查询")
	@GetMapping(value = "/queryById")
	public Result<LinkChainErrorDetection> queryById(@RequestParam(name="id",required=true) String id) {
		LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(id);
		if(linkChainErrorDetection==null) {
			return Result.error("未找到对应数据");
		}
		return Result.OK(linkChainErrorDetection);
	}

	//@AutoLog(value = "错链检测-通过id查询")
	@ApiOperation(value="错链检测-通过id查询", notes="错链检测-通过id查询")
	@GetMapping(value = "/queryByIdIgnoreTrue")
	public Result<LinkChainErrorDetection> queryByIdIgnoreTrue(@RequestParam(name="id",required=true) String id) {
		LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(id);
		if(linkChainErrorDetection==null) {
			return Result.error("未找到对应数据");
		}
		linkChainErrorDetection.setIsIgnore("1");
		linkChainErrorDetectionService.updateById(linkChainErrorDetection);
		return Result.OK(linkChainErrorDetection);
	}

	//@AutoLog(value = "错链检测-通过id查询")
	@ApiOperation(value="错链检测-通过id查询", notes="错链检测-通过id查询")
	@GetMapping(value = "/queryByIdIgnoreFalse")
	public Result<LinkChainErrorDetection> queryByIdIgnoreFalse(@RequestParam(name="id",required=true) String id) {
		LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(id);
		if(linkChainErrorDetection==null) {
			return Result.error("未找到对应数据");
		}
		linkChainErrorDetection.setIsIgnore("2");
		linkChainErrorDetectionService.updateById(linkChainErrorDetection);
		return Result.OK(linkChainErrorDetection);
	}


	//@AutoLog(value = "错链检测-通过id查询")
	@ApiOperation(value="错链检测-通过id查询", notes="错链检测-通过id查询")
	@GetMapping(value = "/queryByIdModifyTrue")
	public Result<LinkChainErrorDetection> queryByIdModifyTrue(@RequestParam(name="id",required=true) String id) {
		LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(id);
		if(linkChainErrorDetection==null) {
			return Result.error("未找到对应数据");
		}
		linkChainErrorDetection.setIsRevise("1");
		linkChainErrorDetectionService.updateById(linkChainErrorDetection);
		return Result.OK(linkChainErrorDetection);
	}

	//@AutoLog(value = "错链检测-通过id查询")
	@ApiOperation(value="错链检测-通过id查询", notes="错链检测-通过id查询")
	@GetMapping(value = "/queryByIdModifyFalse")
	public Result<LinkChainErrorDetection> queryByIdModifyFalse(@RequestParam(name="id",required=true) String id) {
		LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(id);
		if(linkChainErrorDetection==null) {
			return Result.error("未找到对应数据");
		}
		linkChainErrorDetection.setIsRevise("2");
		linkChainErrorDetectionService.updateById(linkChainErrorDetection);
		return Result.OK(linkChainErrorDetection);
	}

		//@AutoLog(value = "错链检测-通过id查询")
	@ApiOperation(value="错链检测-通过id查询", notes="错链检测-通过id查询")
	@GetMapping(value = "/addWhiteLink")
	public Result<LinkChainErrorDetection> addWhiteLink(@RequestParam(name="id",required=true) String id) {
		LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(id);
		String errorUrl = linkChainErrorDetection.getErrorUrl();
		if(!ObjectUtils.isEmpty(errorUrl)){
			LinkWhiteList whiteList = new LinkWhiteList();
			whiteList.setId(DigestUtils.md5DigestAsHex(errorUrl.getBytes()));
			whiteList.setCreateTime(new Date());
			whiteList.setUpdateTime(new Date());
			whiteList.setMainSiteId(linkChainErrorDetection.getId());
			whiteList.setMainSiteName(linkChainErrorDetection.getMainSiteName());
			whiteList.setMainSiteUrl(linkChainErrorDetection.getMainSiteUrl());
			whiteList.setWhiteLink(errorUrl);
			boolean save = linkWhiteListService.saveOrUpdate(whiteList);
		}
		if(linkChainErrorDetection==null) {
			return Result.error("未找到对应数据");
		}
		return Result.OK(linkChainErrorDetection);
	}

	@ApiOperation(value="错链检测-通过id查询", notes="错链检测-通过id查询")
	@GetMapping(value = "/removeWhiteLink")
	public Result<LinkChainErrorDetection> removeWhiteLink(@RequestParam(name="id",required=true) String id) {
		LinkChainErrorDetection linkChainErrorDetection = linkChainErrorDetectionService.getById(id);
		String errorUrl = linkChainErrorDetection.getErrorUrl();
		if(!ObjectUtils.isEmpty(errorUrl)){
			QueryWrapper<LinkWhiteList> linkWhiteListQueryWrapper = new QueryWrapper<LinkWhiteList>();
			linkWhiteListQueryWrapper.eq("white_link",errorUrl);
			linkWhiteListService.remove(linkWhiteListQueryWrapper);
		}
		if(linkChainErrorDetection==null) {
			return Result.error("未找到对应数据");
		}
		return Result.OK(linkChainErrorDetection);
	}

	/**
	 * 导出excel
	 *
	 * @param request
	 * @param linkChainErrorDetection
	 */
	@RequiresPermissions("linkchainerrordetection:link_chain_error_detection:exportXls")
	@RequestMapping(value = "/exportXls")
	public ModelAndView exportXls(HttpServletRequest request, LinkChainErrorDetection linkChainErrorDetection) {
		return super.exportXls(request, linkChainErrorDetection, LinkChainErrorDetection.class, "错链检测");
	}

	/**
	 * 通过excel导入数据
	 *
	 * @param request
	 * @param response
	 * @return
	 */
	@RequiresPermissions("linkchainerrordetection:link_chain_error_detection:importExcel")
	@RequestMapping(value = "/importExcel", method = RequestMethod.POST)
	public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
		return super.importExcel(request, response, LinkChainErrorDetection.class);
	}

	public void saveLinkChainErrorLog(LinkChainErrorDetection errorDetection){
		//记录日志信息
		LinkChainErrorLog linkChainErrorLog = new LinkChainErrorLog();
		linkChainErrorLog.setId(IdWorker.getIdStr());
		linkChainErrorLog.setCreateBy(errorDetection.getCreateBy());
		linkChainErrorLog.setCreateTime(errorDetection.getCreateTime());
		linkChainErrorLog.setUpdateBy(errorDetection.getUpdateBy());
		linkChainErrorLog.setUpdateTime(errorDetection.getUpdateTime());
		linkChainErrorLog.setErrorDetectionId(errorDetection.getId());
		linkChainErrorLog.setMainSiteId(errorDetection.getMainSiteId());
		linkChainErrorLog.setMainSiteName(errorDetection.getMainSiteName());
		linkChainErrorLog.setMainSiteUrl(errorDetection.getMainSiteUrl());
		linkChainErrorLog.setErrorUrl(errorDetection.getErrorUrl());
		linkChainErrorLog.setErrorUrlType(errorDetection.getErrorUrlType());
		linkChainErrorLog.setErrorUrlState(errorDetection.getErrorUrlState());
		linkChainErrorLog.setNumberApplicationPages(errorDetection.getNumberApplicationPages());
		linkChainErrorLog.setNumberScans(errorDetection.getNumberScans());
		linkChainErrorLog.setNumberScans(errorDetection.getNumberScans());
		linkChainErrorLog.setIsRevise(errorDetection.getIsRevise());
		linkChainErrorLog.setIsIgnore(errorDetection.getIsIgnore());
		linkChainErrorLog.setCheckTime(errorDetection.getCheckTime());
		linkChainErrorLog.setCheckState(errorDetection.getCheckState());
		linkChainErrorLog.setCheckMessage(errorDetection.getCheckMessage());
		linkChainErrorLog.setTenantId(errorDetection.getTenantId());
		boolean save = iLinkChainErrorLogService.save(linkChainErrorLog);
	}
	//并发执行
	public void ConcurrencyThread(List<List<LinkChainErrorDetection>> lists){
		ExecutorService threadPool = Executors.newSingleThreadExecutor();
		int i = 0;
		for (List<LinkChainErrorDetection> list : lists) {
			threadPool.execute(new Runnable() {
				public void run() {
					checkErrorLink(list);
				}
			});
			i++;
		}
		threadPool.shutdown();// 任务执行完毕，关闭线程池
	}
	//List集合分组
	public  List<List<LinkChainErrorDetection>> groupList(List<LinkChainErrorDetection> list) {
		List<List<LinkChainErrorDetection>> listGroup = new ArrayList<List<LinkChainErrorDetection>>();
		int listSize = list.size();
		//子集合的长度
		int toIndex = 2;
		for (int i = 0; i < list.size(); i += 2) {
			if (i + 2 > listSize) {
				toIndex = listSize - i;
			}
			List<LinkChainErrorDetection> newList = list.subList(i, i + toIndex);
			listGroup.add(newList);
		}
		return listGroup;
	}
	private void checkErrorLink(List<LinkChainErrorDetection> exportList){
		for (LinkChainErrorDetection errorDetection : exportList) {
			String errorUrl = errorDetection.getErrorUrl();
			String mainSiteId = errorDetection.getMainSiteId();
			MainSite mainSite = iMainSiteService.getById(mainSiteId);
			QueryWrapper<ExternalInternalSite> internalSiteQueryWrapper = new QueryWrapper<ExternalInternalSite>();
			internalSiteQueryWrapper.eq("url",errorUrl);
			long count = externalInternalSiteService.count(internalSiteQueryWrapper);
			Integer pageSize = 100;
			Integer pageNum = (int)Math.ceil((double)count/pageSize);
			int countError = 0;
			for(int i = 1; i <= pageNum;i+=1){
				Page<ExternalInternalSite> page = new Page<ExternalInternalSite>(i, pageSize);
				IPage<ExternalInternalSite> pageList = externalInternalSiteService.page(page,internalSiteQueryWrapper);
				List<ExternalInternalSite> records = pageList.getRecords();
				for (ExternalInternalSite record : records) {
					//爬取页面
					Document document = null;
					String grabPage = mainSite.getGrabPage();
					try {
						SpyderReturn spyderReturn = trendsSpyder.companyTrendsSpyderMessage(record.getSource(), grabPage);
						document = spyderReturn.getDocument();
						String baseUri = spyderReturn.getBaseUri();
						Elements tags = document.select("link[href],img[src],*[style*=background-image],script[src],td[background],a[href],iframe[src]");
						try {
							for (Element tag : tags) {
								String attr = tag.attr("href");
								if (attr == null || attr.isEmpty()) continue;
								if (!attr.startsWith("http") && !attr.startsWith("data")) {
									String absUrl = new URI(baseUri).resolve(attr).toString();
									absUrl = new URI(absUrl).normalize().toString();
									tag.attr("href", absUrl);
								}
							}
						}catch (Exception e){
							e.printStackTrace();
						}
						try {
							for (Element tag : tags) {
								String attr = tag.attr("src");
								if (attr == null || attr.isEmpty()) continue;
								if (!attr.startsWith("http") && !attr.startsWith("data")) {
									String absUrl = new URI(baseUri).resolve(attr).toString();
									absUrl = new URI(absUrl).normalize().toString();
									tag.attr("src", absUrl);
								}
							}
						}catch (Exception e){
							e.printStackTrace();
						}
						try {
							for (Element tag : tags) {
								String attr = tag.attr("background");
								if (attr == null || attr.isEmpty()) continue;
								if (!attr.startsWith("http") && !attr.startsWith("data")) {
									String absUrl = new URI(baseUri).resolve(attr).toString();
									absUrl = new URI(absUrl).normalize().toString();
									tag.attr("background", absUrl);
								}
							}
						}catch (Exception e){
							e.printStackTrace();
						}
						try {
							for (Element tag : tags) {
								String attr = tag.attr("iframe");
								if (attr == null || attr.isEmpty()) continue;
								if (!attr.startsWith("http") && !attr.startsWith("data")) {
									String absUrl = new URI(baseUri).resolve(attr).toString();
									absUrl = new URI(absUrl).normalize().toString();
									tag.attr("iframe", absUrl);
								}
							}
						}catch (Exception e){
							e.printStackTrace();
						}
					}catch (Exception e){
						document = null;
					}
					if(!ObjectUtils.isEmpty(document)){
						String sHtml = document.toString();
						sHtml = sHtml.replaceAll("amp;","");
						int i1 = sHtml.indexOf(errorUrl);
						if(i1 > 0){
							countError++;
						}

					}
				}
			}
			if(countError > 0){
				errorDetection.setIsRevise("2");
				errorDetection.setCheckNumberApplicationPages(countError);
			}
			if(countError == 0){
				errorDetection.setIsRevise("1");
			}
			errorDetection.setCreateTime(new Date());
			errorDetection.setUpdateTime(new Date());
			errorDetection.setCheckTime(new Date());
			errorDetection.setCheckState("1");
			log.info("复核日志信息："+errorDetection);
			boolean b = linkChainErrorDetectionService.updateById(errorDetection);
			saveLinkChainErrorLog(errorDetection);
			log.info(b+"");
		}
	}
}
