使用jsoup获取具有类的div元素

Question

我是Jsoup解析的新手，我想获得这个页面上所有公司的列表：https://angel.co/companies?company_types[]=startup现在，一种实现这一点的方法实际上是使用与我需要的相关的div标记来检查页面。但是，当我调用该方法时：

Document doc = Jsoup.connect("https://angel.co/companies?company_types[]=Startup").get();
System.out.println(doc.html());

首先，我甚至无法在我的consol html输出中找到那些DIV标记（这些标记应该给出公司的列表）；其次，即使我找到了它，我如何才能找到具有class name的某个DIV元素：

div class=" dc59 frw44 _a _jm"  

请原谅我的行话，我不知道该怎么做。

Answer 1

这些数据不嵌入到页面中,但使用后续API调用检索：

• 发布https://angel.co/company_filters/search_data以获取ids数组和名为hexdigest
的令牌
• 获取https://angel.co/companies/startups以使用前一个请求的输出检索公司数据

对每个页面重复上述操作（因此每个页面都需要一个新的令牌&一个ID列表）。可以使用Chrome dev console在“网络”选项卡中看到此过程。

第一个post请求提供JSON输出，但第二个请求(get)提供JSON对象属性中的HTML数据。

以下内容提取了公司筛选器：

private static CompanyFilter getCompanyFilter(final String filter, final int page) throws IOException {

    String response = Jsoup.connect("https://angel.co/company_filters/search_data")
            .header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8")
            .header("X-Requested-With", "XMLHttpRequest")
            .data("filter_data[company_types][]=", filter)
            .data("sort", "signal")
            .data("page", String.valueOf(page))
            .userAgent("Mozilla")
            .ignoreContentType(true)
            .post().body().text();

    GsonBuilder gsonBuilder = new GsonBuilder();
    Gson gson = gsonBuilder.create();
    return gson.fromJson(response, CompanyFilter.class);
}

那么下面摘录公司：

private static List<Company> getCompanies(final CompanyFilter companyFilter) throws IOException {

    List<Company> companies = new ArrayList<>();

    URLConnection urlConn = new URL("https://angel.co/companies/startups?" + companyFilter.buildRequest()).openConnection();
    urlConn.setRequestProperty("User-Agent", "Mozilla");
    urlConn.connect();
    BufferedReader reader = new BufferedReader(new InputStreamReader(urlConn.getInputStream(), "UTF-8"));
    HtmlContainer htmlObj = new Gson().fromJson(reader, HtmlContainer.class);

    Element doc = Jsoup.parse(htmlObj.getHtml());
    Elements data = doc.select("div[data-_tn]");

    if (data.size() > 0) {
        for (int i = 2; i < data.size(); i++) {
            companies.add(new Company(data.get(i).select("a").first().attr("title"),
                    data.get(i).select("a").first().attr("href"),
                    data.get(i).select("div.pitch").first().text()));
        }

    } else {
        System.out.println("no data");
    }
    return companies;
}

主要功能：

public static void main(String[] args) throws IOException {

    int pageCount = 1;
    List<Company> companies = new ArrayList<>();

    for (int i = 0; i < 10; i++) {

        System.out.println("get page n°" + pageCount);
        CompanyFilter companyFilter = getCompanyFilter("Startup", pageCount);
        pageCount++;
        System.out.println("digest     : " + companyFilter.getDigest());
        System.out.println("count      : " + companyFilter.getTotalCount());
        System.out.println("array size : " + companyFilter.getIds().size());
        System.out.println("page       : " + companyFilter.getpage());

        companies.addAll(getCompanies(companyFilter));

        if (companies.size() == 0) {
            break;
        } else {
            System.out.println("size     : " + companies.size());
        }
    }
}

Company、CompanyFilter&HTMLContainer是模型类：

class CompanyFilter {

    @SerializedName("ids")
    private List<Integer> mIds;

    @SerializedName("hexdigest")
    private String mDigest;

    @SerializedName("total")
    private String mTotalCount;

    @SerializedName("page")
    private int mPage;

    @SerializedName("sort")
    private String mSort;

    @SerializedName("new")
    private boolean mNew;

    public List<Integer> getIds() {
        return mIds;
    }

    public String getDigest() {
        return mDigest;
    }

    public String getTotalCount() {
        return mTotalCount;
    }

    public int getpage() {
        return mPage;
    }

    private String buildRequest() {
        String out = "total=" + mTotalCount + "&";
        out += "sort=" + mSort + "&";
        out += "page=" + mPage + "&";
        out += "new=" + mNew + "&";
        for (int i = 0; i < mIds.size(); i++) {
            out += "ids[]=" + mIds.get(i) + "&";
        }
        out += "hexdigest=" + mDigest + "&";
        return out;
    }
}

private static class Company {

    private String mLink;
    private String mName;
    private String mDescription;

    public Company(String name, String link, String description) {
        mLink = link;
        mName = name;
        mDescription = description;
    }

    public String getLink() {
        return mLink;
    }

    public String getName() {
        return mName;
    }

    public String getDescription() {
        return mDescription;
    }
} 

private static class HtmlContainer {

    @SerializedName("html")
    private String mHtml;

    public String getHtml() {
        return mHtml;
    }
}

完整的代码也可以在这里获得