package org.jeecg.modules.spyderUtils;

import kong.unirest.GetRequest;
import kong.unirest.Unirest;
import lombok.extern.slf4j.Slf4j;
import org.jeecg.modules.spyderUtils.spider.*;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.ObjectUtils;

import java.net.URI;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Slf4j
@Component
public class SpyderUtils {

    //创建任务并抓取数据，设置好对应的页面初始路径
    @Autowired
    private CreateChromeDriver createChromeDriver;

    @Autowired
    private JsoupUtils jsoupUtils;

    @Autowired
    private JsoupUtilsService jsoupUtilsService;

    @Autowired
    private FangCunUtils fangCunUtils;

    private String scriptStrwebdriver = "window.navigator.webdriver=undefined";

    public String getSpyderGetHTMLByUrl(String url,String baseUrl,String type){
        String html = null;
        if(!ObjectUtils.isEmpty(url)){
            try {
                Document parse = null;
                if(type.equals("1")){
                    parse = Jsoup.parse(new URL(url), 10000);
                }else{
                    RemoteWebDriver driver = createChromeDriver.createChromeDriver();
                    try {
                        driver.executeScript(scriptStrwebdriver);
                        driver.get(url);
                        new WebDriverWait(driver, Duration.ofSeconds(5));
                        Thread.sleep(700);
                        try {
                            // 页面滚动到下方
                            Integer start = 0;
                            Integer end = 500;
                            //6000为最大值，模拟滚动
                            while (true){
                                if (end == 6000){
                                    break;
                                }
                                //设置滑动距离顶部的距离是多少到多少进行变化，
                                String scriptStr = "window.scrollTo("+ start + ","+ end +")";
                                driver.executeScript(scriptStr);
                                try {
                                    Thread.sleep(500);
                                } catch (InterruptedException e) {
                                    e.printStackTrace();
                                }
                                start+=500;
                                end+=500;
                            }
                        }catch (Exception e){
                            e.printStackTrace();
                        }
                        String pageSource = driver.getPageSource();
                        parse = Jsoup.parse(pageSource);
                    }catch (Exception e){
                        e.printStackTrace();
                    }
                    driver.quit();
                }
                if(!ObjectUtils.isEmpty(parse)){
                    if(ObjectUtils.isEmpty(baseUrl)){
                        baseUrl = url;
                    }
                    Document document = setUrl(parse,baseUrl);
                    if(!ObjectUtils.isEmpty(document)){
                        parse = document;
                    }
                }
                html = parse.toString();
            }catch (Exception e){
                e.printStackTrace();
            }
        }else{
            return null;
        }

        return html;
    }

    String styleCss = "background: yellow;border-style: solid;border-color: red;border-width: 2px;padding: 5px";

    public Document setUrl(Document document , String baseUrl){
        Elements tags = document.select("link[href],img[src],*[style*=background-image],script[src],td[background],a[href],iframe[src]");
        try {
            for (Element tag : tags) {
                String attr = tag.attr("a");
                if (attr == null || attr.isEmpty()) continue;
                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                    String absUrl = new URI(baseUrl).resolve(attr).toString();
                    absUrl = new URI(absUrl).normalize().toString();
                    tag.attr("href", absUrl);
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        try {
            for (Element tag : tags) {
                String attr = tag.attr("href");
                if (attr == null || attr.isEmpty()) continue;
                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                    String absUrl = new URI(baseUrl).resolve(attr).toString();
                    absUrl = new URI(absUrl).normalize().toString();
                    tag.attr("href", absUrl);
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        try {
            for (Element tag : tags) {
                String attr = tag.attr("src");
                if (attr == null || attr.isEmpty()) continue;
                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                    String absUrl = new URI(baseUrl).resolve(attr).toString();
                    absUrl = new URI(absUrl).normalize().toString();
                    tag.attr("src", absUrl);
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        try {
            for (Element tag : tags) {
                String attr = tag.attr("background");
                if (attr == null || attr.isEmpty()) continue;
                if (!attr.startsWith("http") && !attr.startsWith("data")) {
                    String absUrl = new URI(baseUrl).resolve(attr).toString();
                    absUrl = new URI(absUrl).normalize().toString();
                    tag.attr("background", absUrl);
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        try {
            for (Element tag : tags) {
                try {
                    String attr = tag.attr("iframe");
                    if (attr == null || attr.isEmpty()) continue;
                    if (!attr.startsWith("http") && !attr.startsWith("data")) {
                        String absUrl = new URI(baseUrl).resolve(attr).toString();
                        absUrl = new URI(absUrl).normalize().toString();
                        tag.attr("iframe", absUrl);
                    }
                }catch (Exception e){
                    e.printStackTrace();
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        try {
            for (Element tag : tags) {
                try {
                    String attr = tag.attr("iframe");
                    if (attr == null || attr.isEmpty()) continue;
                    if (!attr.startsWith("http") && !attr.startsWith("data")) {
                        String absUrl = new URI(baseUrl).resolve(attr).toString();
                        absUrl = new URI(absUrl).normalize().toString();
                        tag.attr("src", absUrl);
                    }
                }catch (Exception e){
                    e.printStackTrace();
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        try {
            for (Element tag : tags) {
                try {
                    String attr = tag.attr("img");
                    if (attr == null || attr.isEmpty()) continue;
                    if (!attr.startsWith("http") && !attr.startsWith("data")) {
                        String absUrl = new URI(baseUrl).resolve(attr).toString();
                        absUrl = new URI(absUrl).normalize().toString();
                        tag.attr("src", absUrl);
                    }
                }catch (Exception e){
                    e.printStackTrace();
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return document;
    }
    /**
     * 获取错敏词信息
     * @return String
     */



    /**
     * 获取页面当中的的银行卡号，和标注号的HTML文本
     * @return String
     */
    public JsoupReturn getBankCardNumber(String html){
        JsoupReturn jsoupReturn = new JsoupReturn();
        //获取所有身份证号
        List<String> htmlList = new ArrayList<String>();
        List<String> aList = new ArrayList<String>();
        List<String> strList = new ArrayList<String>();
        Pattern pattern = Pattern.compile("\\b(62[0-9]{14})\\b");
        Matcher matcher = pattern.matcher(html);
        String rHtml = html;
        Integer index = 0;
        while (matcher.find()) {
            String cardNumber = matcher.group(1);
            if (!ObjectUtils.isEmpty(cardNumber)){
                String cardNumberHtml =  "<span id='bankCard"+index+"' title='"+cardNumber+"' style='"+this.styleCss+"'>"+cardNumber+"</span>";
                String ahtml = "<a href='#bankCard"+index+"'>"+cardNumber+"</a>";
                rHtml = rHtml.replaceAll(cardNumber,cardNumberHtml);
                htmlList.add(cardNumberHtml);
                aList.add(ahtml);
                strList.add(cardNumber);
                index++;
            }
        }
        jsoupReturn.setHtml(rHtml);
        jsoupReturn.setHtmlList(htmlList);
        jsoupReturn.setAList(aList);
        jsoupReturn.setStrList(strList);
        return jsoupReturn;
    }
    /**
     * 获取页面当中的的身份证号，和标注号的HTML文本
     * @return String
     */
    public JsoupReturn getIdNumber(String html){
        JsoupReturn jsoupReturn = new JsoupReturn();
        //获取所有身份证号
        List<String> htmlList = new ArrayList<String>();
        List<String> aList = new ArrayList<String>();
        List<String> strList = new ArrayList<String>();
        String regex = "(?<![\\d\\w])\\d{17}[\\dX](?![\\d\\w])";
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(html);
        String rHtml = html;
        Integer index = 0;
        log.info("==============获取页面当中的的身份证号，和标注号的HTML文本==============");
        while (matcher.find()) {
            String idNumber = matcher.group();
            if(!ObjectUtils.isEmpty(idNumber)){
                String idNumberHtml =  "<span id='idNumber"+index+"' title='"+idNumber+"' style='"+this.styleCss+"'>"+idNumber+"</span>";
                String ahtml = "<a href='#idNumber"+index+"'>"+idNumber+"</a>";
                rHtml = rHtml.replaceAll(idNumber,idNumberHtml);
                htmlList.add(idNumberHtml);
                aList.add(ahtml);
                strList.add(idNumber);
                index++;
            }

        }
        jsoupReturn.setHtml(rHtml);
        jsoupReturn.setHtmlList(htmlList);
        jsoupReturn.setAList(aList);
        jsoupReturn.setStrList(strList);
        return jsoupReturn;
    }
    /**
     * 获取页面当中的的电话号，和标注号的HTML文本
     * @return String
     */
    public JsoupReturn getPhone(String html){
        JsoupReturn jsoupReturn = new JsoupReturn();
        String regex = "\\b((\\+?86|\\(\\+86\\))?(1[3456789]\\d{9}|0[1-9][0-9]{1,2}-\\d{7,8}))\\b";
        // String regex = "\\b((\\+?86|\\(\\+86\\))?(1[3456789]\\d{9}|0[1-9][0-9]{1,2}-\\d{7,8}|\\d{7,8})(-\\d{1,4})?)\\b";
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(html);
        //获取所有手机号和电话号
        List<String> htmlList = new ArrayList<String>();
        List<String> aList = new ArrayList<String>();
        List<String> strList = new ArrayList<String>();
        String rHtml = html;
        Integer index = 0;
        while (matcher.find()) {
            String groupPhone = matcher.group();
            if(!ObjectUtils.isEmpty(groupPhone)){
                String groupPhoneHtml =  "<span id='phoneNumber"+index+"' title='"+groupPhone+"' style='"+this.styleCss+"'>"+groupPhone+"</span>";
                String ahtml = "<a href='#phoneNumber"+index+"'>"+groupPhone+"</a>";
                rHtml = rHtml.replaceAll(groupPhone,groupPhoneHtml);
                htmlList.add(groupPhoneHtml);
                aList.add(ahtml);
                strList.add(groupPhone);
                index++;
            }
        }
        jsoupReturn.setHtml(rHtml);
        jsoupReturn.setHtmlList(htmlList);
        jsoupReturn.setAList(aList);
        jsoupReturn.setStrList(strList);
        return jsoupReturn;
    }
    /**
     * 获取页面当中的所有链链，和标注号的HTML文本
     * @return String
     */
    public JsoupReturn getLink(String html){
        JsoupReturn jsoupReturn = new JsoupReturn();

        List<String> htmlList = new ArrayList<String>();
        List<String> aList = new ArrayList<String>();
        List<String> strList = new ArrayList<String>();


        List<String> LinkListHtml = new ArrayList<String>();
        List<String> LinkListSrc = new ArrayList<String>();

        Document doc = Jsoup.parse(html);
        Elements links = doc.select("a[href]");
        Elements images = doc.select("img[src]");
        Elements imageDataS = doc.select("img[data-src]");
        Elements videos = doc.select("video[src]");
        Elements videoDataS = doc.select("video[data-src]");
        Elements js = doc.select("script[src]");
        Elements css = doc.select("link[rel=stylesheet]");
        Elements iframes = doc.select("iframe[src]");
        Elements iframeDataS = doc.select("iframe[data-src]");


        for (Element link : links) {
            LinkListHtml.add(link.toString());
            String href = link.attr("href");
            LinkListSrc.add(href);
        }
        for (Element image : images) {
            LinkListHtml.add(image.toString());
            String src = image.attr("src");
            LinkListSrc.add(src);
        }
        for (Element imageData : imageDataS) {
            LinkListHtml.add(imageData.toString());
            String dataSrc = imageData.attr("data-src");
            LinkListSrc.add(dataSrc);
        }
        for (Element video : videoDataS) {
            LinkListHtml.add(video.toString());
            String src = video.attr("data-src");
            LinkListSrc.add(src);
        }
        for (Element video : videos) {
            LinkListHtml.add(video.toString());
            String src = video.attr("src");
            LinkListSrc.add(src);
        }
        for (Element jss : js) {
            LinkListHtml.add(jss.toString());
            String jssSrc = jss.attr("src");
            LinkListSrc.add(jssSrc);
        }
        for (Element csss : css) {
            LinkListHtml.add(csss.toString());
            String rel = csss.attr("rel");
            if(!ObjectUtils.isEmpty(rel)){
                LinkListSrc.add(rel);
            }
            String href = csss.attr("href");
            if(!ObjectUtils.isEmpty(href)){
                LinkListSrc.add(href);
            }
        }
        for (Element iframe : iframes) {
            LinkListHtml.add(iframe.toString());
            String src = iframe.attr("src");
            if(!ObjectUtils.isEmpty(src)){
                LinkListSrc.add(src);
            }
        }
        for (Element iframe : iframeDataS) {
            LinkListHtml.add(iframe.toString());
            String src = iframe.attr("data-src");
            if(!ObjectUtils.isEmpty(src)){
                LinkListSrc.add(src);
            }
        }

        Integer index = 0;
        for (String link : LinkListHtml) {
            String linka = LinkListSrc.get(index);

            if(!ObjectUtils.isEmpty(link)){
                String linkHtml =  "<span id='linkNumber"+index+"' title='"+link+"' style='"+this.styleCss+"'>"+link+"</span>";
                html = html.replaceAll(link,linkHtml);
                String ahtml = "<a href='#linkNumber"+index+"'>"+linka+"</a>";
                htmlList.add(linkHtml);
                aList.add(ahtml);
                strList.add(link);
                index++;
            }
            String linkStr = "<span title='"+link+"' style='"+this.styleCss+"'>"+link+"</span>";


            html = html.replaceAll(link,linkStr);
        }
        jsoupReturn.setHtmlList(htmlList);
        jsoupReturn.setAList(aList);
        jsoupReturn.setStrList(strList);
        jsoupReturn.setHtml(html);
        return jsoupReturn;
    }
    /**
     * 获取页面当中的所有错链，和标注号的HTML文本
     * @return String
     */
    public JsoupReturn getErrorLink(String html){
        JsoupReturn jsoupReturn = new JsoupReturn();

        List<String> htmlList = new ArrayList<String>();
        List<String> aList = new ArrayList<String>();
        List<String> strList = new ArrayList<String>();


        List<String> LinkListHtml = new ArrayList<String>();
        List<String> LinkListSrc = new ArrayList<String>();

        Document doc = Jsoup.parse(html);
        Elements links = doc.select("a[href]");
        Elements images = doc.select("img[src]");
        Elements imageDataS = doc.select("img[data-src]");
        Elements videos = doc.select("video[src]");
        Elements videoDataS = doc.select("video[data-src]");
        Elements js = doc.select("script[src]");
        Elements css = doc.select("link[rel=stylesheet]");
        Elements iframes = doc.select("iframe[src]");
        Elements iframeDataS = doc.select("iframe[data-src]");


        for (Element link : links) {
            LinkListHtml.add(link.toString());
            String href = link.attr("href");
            LinkListSrc.add(href);
        }
        for (Element image : images) {
            LinkListHtml.add(image.toString());
            String src = image.attr("src");
            LinkListSrc.add(src);
        }
        for (Element imageData : imageDataS) {
            LinkListHtml.add(imageData.toString());
            String dataSrc = imageData.attr("data-src");
            LinkListSrc.add(dataSrc);
        }
        for (Element video : videoDataS) {
            LinkListHtml.add(video.toString());
            String src = video.attr("data-src");
            LinkListSrc.add(src);
        }
        for (Element video : videos) {
            LinkListHtml.add(video.toString());
            String src = video.attr("src");
            LinkListSrc.add(src);
        }
        for (Element jss : js) {
            LinkListHtml.add(jss.toString());
            String jssSrc = jss.attr("src");
            LinkListSrc.add(jssSrc);
        }
        for (Element csss : css) {
            LinkListHtml.add(csss.toString());
            String rel = csss.attr("rel");
            if(!ObjectUtils.isEmpty(rel)){
                LinkListSrc.add(rel);
            }
            String href = csss.attr("href");
            if(!ObjectUtils.isEmpty(href)){
                LinkListSrc.add(href);
            }
        }
        for (Element iframe : iframes) {
            LinkListHtml.add(iframe.toString());
            String src = iframe.attr("src");
            if(!ObjectUtils.isEmpty(src)){
                LinkListSrc.add(src);
            }
        }
        for (Element iframe : iframeDataS) {
            LinkListHtml.add(iframe.toString());
            String src = iframe.attr("data-src");
            if(!ObjectUtils.isEmpty(src)){
                LinkListSrc.add(src);
            }
        }
        Integer index = 0;
        for (String link : LinkListHtml) {
            String linka = LinkListSrc.get(index);
            GetRequest getRequest = Unirest.get(linka);
            int code = getRequest.hashCode();
            if(code != 200){
                if(!ObjectUtils.isEmpty(link)){
                    String linkHtml =  "<span id='linkNumber"+index+"' title='"+link+"' style='"+this.styleCss+"'>"+link+"</span>";
                    html = html.replaceAll(link,linkHtml);
                    String ahtml = "<a href='#linkNumber"+index+"'>"+linka+"</a>";
                    htmlList.add(linkHtml);
                    aList.add(ahtml);
                    strList.add(link);
                    index++;
                }
                String linkStr = "<span title='"+link+"' style='"+this.styleCss+"'>"+link+"</span>";
                html = html.replaceAll(link,linkStr);
            }
        }
        jsoupReturn.setHtmlList(htmlList);
        jsoupReturn.setAList(aList);
        jsoupReturn.setStrList(strList);
        jsoupReturn.setHtml(html);
        return jsoupReturn;
    }
    /**
     * 获取页面当中的所有错链，和标注号的HTML文本
     * @return String
     */
    public JsoupReturn getOuterLink(String html,String baseLink){
        JsoupReturn jsoupReturn = new JsoupReturn();

        List<String> htmlList = new ArrayList<String>();
        List<String> aList = new ArrayList<String>();
        List<String> strList = new ArrayList<String>();


        List<String> LinkListHtml = new ArrayList<String>();
        List<String> LinkListSrc = new ArrayList<String>();

        Document doc = Jsoup.parse(html);
        Elements links = doc.select("a[href]");
        Elements images = doc.select("img[src]");
        Elements imageDataS = doc.select("img[data-src]");
        Elements videos = doc.select("video[src]");
        Elements videoDataS = doc.select("video[data-src]");
        Elements js = doc.select("script[src]");
        Elements css = doc.select("link[rel=stylesheet]");
        Elements iframes = doc.select("iframe[src]");
        Elements iframeDataS = doc.select("iframe[data-src]");


        for (Element link : links) {
            LinkListHtml.add(link.toString());
            String href = link.attr("href");
            LinkListSrc.add(href);
        }
        for (Element image : images) {
            LinkListHtml.add(image.toString());
            String src = image.attr("src");
            LinkListSrc.add(src);
        }
        for (Element imageData : imageDataS) {
            LinkListHtml.add(imageData.toString());
            String dataSrc = imageData.attr("data-src");
            LinkListSrc.add(dataSrc);
        }
        for (Element video : videoDataS) {
            LinkListHtml.add(video.toString());
            String src = video.attr("data-src");
            LinkListSrc.add(src);
        }
        for (Element video : videos) {
            LinkListHtml.add(video.toString());
            String src = video.attr("src");
            LinkListSrc.add(src);
        }
        for (Element jss : js) {
            LinkListHtml.add(jss.toString());
            String jssSrc = jss.attr("src");
            LinkListSrc.add(jssSrc);
        }
        for (Element csss : css) {
            LinkListHtml.add(csss.toString());
            String rel = csss.attr("rel");
            if(!ObjectUtils.isEmpty(rel)){
                LinkListSrc.add(rel);
            }
            String href = csss.attr("href");
            if(!ObjectUtils.isEmpty(href)){
                LinkListSrc.add(href);
            }
        }
        for (Element iframe : iframes) {
            LinkListHtml.add(iframe.toString());
            String src = iframe.attr("src");
            if(!ObjectUtils.isEmpty(src)){
                LinkListSrc.add(src);
            }
        }
        for (Element iframe : iframeDataS) {
            LinkListHtml.add(iframe.toString());
            String src = iframe.attr("data-src");
            if(!ObjectUtils.isEmpty(src)){
                LinkListSrc.add(src);
            }
        }
        Integer index = 0;
        for (String link : LinkListHtml) {
        }
        jsoupReturn.setHtmlList(htmlList);
        jsoupReturn.setAList(aList);
        jsoupReturn.setStrList(strList);
        jsoupReturn.setHtml(html);
        return jsoupReturn;
    }

}
