htmlunit 学习笔记

htmlunit 是一款开源的java 页面分析工具，读取页面后，可以有效的使用htmlunit分析页面上的内容。项目可以模拟浏览器运行，被誉为java浏览器的开源实现。是一个没有界面的浏览器，运行速度迅速。是junit的扩展之一

采用的是Rhinojs引擎。模拟js运行

常规意义上，该项目可以用来进行页面的测试工作，实现网页自动化测试，（包括JS）

但是一般来说，在小型爬虫项目中，这种框架十分常用，可以有效的分析出 dom的标签，并且有效的运行页面上的js以便得到一些需要执行JS才能得到的值。

我们研究的爬虫应用中 仅仅用Httpclient+jsoup是不够的，因为有些页面数据是js加载出来的 httpclient解析不出来，比如百度云用户页面，淘宝页面，连开源中国博客的博客内容都是Js加载或者处理的，所以httpClient是解析不出来的，这里htmlunit就是一个很好的方案，内嵌js浏览器，模拟Js运行，把结果执行出来。这个就是我们想要的。所以搞爬虫,htmlunit必须要掌握好；
Htlmlunit我使用了最新版，最新版需要jdk1.8不然会报错jdk版本对应不上。

package com.myactiviti.HelloActiviti;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HtmlDivision;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSpan;
import com.gargoylesoftware.htmlunit.html.HtmlTable;

/**
 * Hello world!
 *
 */
public class HtmlUnit1 {
    public static void main(String[] args) {
        Map<String, Integer> map = getHighIP.getInstance().map;
        getHtml(map);
    }
    public static void getHtml(Map<String, Integer> ips) {

        Object[] array = ips.keySet().toArray();
        String ip = array[0].toString();
        System.out.println(ip+"ip and port"+ips.get(ip));
        WebClient webclient = new WebClient(BrowserVersion.FIREFOX_52,ip,ips.get(ip));
        webclient.getOptions().setTimeout(10000);
        try {
            HtmlPage page = webclient.getPage("http://www.java1234.com/");

           //System.out.println("网页内容："+page.asXml());
            HtmlDivision htmlDivision = page.getHtmlElementById("navMenu");
            //System.out.println("网页内容txt："+htmlDivision.asText());

            DomNodeList<DomElement> aList = page.getElementsByTagName("a");
             for (DomElement domElement : aList) {
                 System.out.println("通过标签获取："+domElement.asXml());
            }

            HtmlSpan item = (HtmlSpan) page.getByXPath("//div[@id='navMenu'][1]/ul/li[2]/a/span").get(0); // xpath方式
            System.out.println("xpath方式获取某个元素值："+item.asText());


        } catch (FailingHttpStatusCodeException e) {
            // TODO Auto-generated catch block
            System.out.println("+++++++++++++++++++++++++++++++++++++++=");
            ips.remove(ip);
            getHtml(ips);
        } catch (MalformedURLException e) {
            // TODO Auto-generated catch block
            System.out.println("________________________");
            ips.remove(ip);
            getHtml(ips);
        } catch (Exception e) {
            ips.remove(ip);
            System.out.println("有异常啊！");
            getHtml(ips);
            // TODO Auto-generated catch block
        }finally {
            webclient.close();
        }
    }

}

通过单例模式爬取网站http://www.xicidaili.com/上的高匿IP放到Map中

package com.myactiviti.HelloActiviti;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HtmlDivision;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSpan;
import com.gargoylesoftware.htmlunit.html.HtmlTable;

/**
 * Hello world!
 *
 */
public class getHighIP {
    Map<String, Integer> map = new HashMap<String, Integer>();

    private static getHighIP instance=null;
    public static synchronized getHighIP getInstance(){
        if(instance==null){
               instance=new getHighIP();
        }
        return instance;
    }

    private getHighIP(){
        WebClient webclient = new WebClient(BrowserVersion.FIREFOX_52);
        //webclient.getOptions().setJavaScriptEnabled(false);//关闭js
        //webclient.getOptions().setCssEnabled(false);//关闭css
        //获取高匿
        try {
            HtmlPage ipPage = webclient.getPage("http://www.xicidaili.com/");
            HtmlTable htmlTable = ipPage.getHtmlElementById("ip_list");
            for (int i = 2; i < 10; i++) {
                String ip = htmlTable.getCellAt(i, 1).asText();
                String port =htmlTable.getCellAt(i, 2).asText();
                map.put(ip, Integer.parseInt(port));
            }
            //System.out.println(ips);

        } catch (FailingHttpStatusCodeException e1) {
            // TODO Auto-generated catch block
            e1.printStackTrace();
        } catch (MalformedURLException e1) {
            // TODO Auto-generated catch block
            e1.printStackTrace();
        } catch (IOException e1) {
            // TODO Auto-generated catch block
            e1.printStackTrace();
        }finally {
            webclient.close();
        }
    }
}