<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>4.5.15</version>
</dependency>
<dependency>
<groupId>cn.wanghaomiao</groupId>
<artifactId>JsoupXpath</artifactId>
<version>0.1.1</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.8</version>
</dependency>
<dependency>
<groupId>com.gitee.jd-platform-opensource</groupId>
<artifactId>asyncTool</artifactId>
<version>V1.3-SNAPSHOT</version>
</dependency>
<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>
package com.xiaoyun.future;
import cn.hutool.http.HttpUtil;
import cn.wanghaomiao.xpath.exception.NoSuchAxisException;
import cn.wanghaomiao.xpath.exception.NoSuchFunctionException;
import cn.wanghaomiao.xpath.exception.XpathSyntaxErrorException;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.jd.platform.async.callback.ICallback;
import com.jd.platform.async.callback.IWorker;
import com.jd.platform.async.executor.Async;
import com.jd.platform.async.worker.WorkResult;
import com.jd.platform.async.wrapper.WorkerWrapper;
import lombok.Builder;
import lombok.Data;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
/**
* 执行顺序
* ....|=> B-01 => |
* A=>| => B-02 => | => C => D
* ...|=> B-03 => |
*/
public class AsyncToolTest {
private static Map<String, String> cacheMap = new ConcurrentHashMap<>();
private static List<String> urlList = new LinkedList<>();
public static void main(String[] args) throws ExecutionException, InterruptedException {
String url = "https://www.newcger.com/";
//A: xpath
String xpathNameFirst = "//ul[@id='mainNav']/li//a/@href";
//C: xpath
String xpathNameThree = "//div[@class='info']/a/@href";
//D: xpath
String xpathNameEnd = "//div[@class='fd_div']/ul//li/a/@href";
//第四步 D 根据单个页面 得出网盘下载链接
TaskFunction<String, Set<Object>> tEnd = new TaskFunction<String, Set<Object>>() {
@Override
public Set<Object> action(String xpathName, Map<String, WorkerWrapper> allWrappers) {
Set<Object> urls = new HashSet<>();
allWrappers.keySet().stream().parallel().forEach(key -> {
if (key.startsWith(TaskEnum.THREE_TASK.toString())) {
Set<Object> set = (Set<Object>) allWrappers.get(key).getWorkResult().getResult();
set.stream().parallel().forEach(item -> {
List<Object> html = getHtml(XpathEntity.builder().xpathName(xpathName).url(item.toString()).build());
Object down = html.get(1);
// System.out.println("单个页面下载地址:" + down.toString());
urls.add(html.get(1));
}
);
}
});
return urls;
}
};
WorkerWrapper<String, Set<Object>> tEndWork = new WorkerWrapper.Builder<String, Set<Object>>()
.worker(tEnd)
.callback(tEnd)
.id(TaskEnum.END_TASK.toString())
.param(xpathNameEnd)
.build();
//第三步 C 根据 所得标题url得出 单个页面地址
TaskFunction<String, Set<Object>> tThree = new TaskFunction<String, Set<Object>>() {
@Override
public Set<Object> action(String xpathName, Map<String, WorkerWrapper> allWrappers) {
Set<Object> urls = new HashSet<>();
allWrappers.keySet().stream().parallel().forEach(key -> {
if (key.startsWith(TaskEnum.SECOND_TASK.toString())) {
String urlResult = (String) allWrappers.get(key).getWorkResult().getResult();
List<Object> html = getHtml(XpathEntity.builder().xpathName(xpathName).url(urlResult).build());
html.stream().parallel().forEach(item -> {
// System.out.println("单个页面地址:" + item);
urls.add(item);
});
}
}
);
return urls;
}
};
WorkerWrapper<String, Set<Object>> tThreeWork = new WorkerWrapper.Builder<String, Set<Object>>()
.worker(tThree)
.callback(tThree)
.id(TaskEnum.THREE_TASK.toString())
.param(xpathNameThree)
.next(tEndWork)
.build();
//第二步 B-01 获取AE模板url地址:
TaskFunction<Integer, String> tSecond01 = new TaskFunction<Integer, String>() {
@Override
public String action(Integer num, Map<String, WorkerWrapper> allWrappers) {
List<Object> result = (List<Object>) allWrappers.get(TaskEnum.FIRST_TASK.toString()).getWorkResult().getResult();
if (result != null) {
String url = (String) result.get(num);
// System.out.println(url);
return url;
}
return null;
}
@Override
public void result(boolean success, Integer param, WorkResult<String> workResult) {
if (success) {
String result = workResult.getResult();
urlList.add(result);
}
}
};
WorkerWrapper<Integer, String> tSecond01Work = new WorkerWrapper.Builder<Integer, String>()
.worker(tSecond01)
.callback(tSecond01)
.param(2)
.id(TaskEnum.SECOND_TASK.toString() + "01")
.next(tThreeWork)
.build();
//第二步 B-02 获取视频素材url地址:
WorkerWrapper<Integer, String> tSecond02Work = new WorkerWrapper.Builder<Integer, String>()
.worker(tSecond01)
.callback(tSecond01)
.param(4)
.id(TaskEnum.SECOND_TASK.toString() + "02")
.next(tThreeWork)
.build();
//第二步 B-03 获取音频素材url地址:
WorkerWrapper<Integer, String> tSecond03Work = new WorkerWrapper.Builder<Integer, String>()
.worker(tSecond01)
.callback(tSecond01)
.param(5)
.id(TaskEnum.SECOND_TASK.toString() + "03")
.next(tThreeWork)
.build();
//第一步 A: 获取所有链接
TaskFunction<XpathEntity, List<Object>> tFirst = new TaskFunction<XpathEntity, List<Object>>() {
@Override
public List<Object> action(XpathEntity object, Map<String, WorkerWrapper> allWrappers) {
List<Object> html = getHtml(object);
return html;
}
};
WorkerWrapper<XpathEntity, List<Object>> tFirstWork = new WorkerWrapper.Builder<XpathEntity, List<Object>>()
.worker(tFirst)
.callback(tFirst)
.param(XpathEntity.builder().url(url).xpathName(xpathNameFirst).build())
.id(TaskEnum.FIRST_TASK.toString())
.next(tSecond01Work, tSecond02Work, tSecond03Work)
.build();
Async.beginWork(500000, tFirstWork);
//获取所有结果
tEndWork.getWorkResult().getResult().stream().forEach(item -> {
System.out.println(item);
});
Async.shutDown();
// System.out.println(cacheMap);
}
private static List<Object> getHtml(XpathEntity xpathEntity) {
String body = null;
if (cacheMap.get(xpathEntity.getUrl()) != null) {
body = cacheMap.get(xpathEntity.getUrl());
} else {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
e.printStackTrace();
}
body = HttpUtil.
createGet(xpathEntity.getUrl()).
execute().body();
cacheMap.put(xpathEntity.getUrl(), body);
}
JXDocument jxDocument = new JXDocument(body);
List<Object> sel = new ArrayList<>();
try {
sel = jxDocument.sel(xpathEntity.getXpathName());
} catch (NoSuchAxisException e) {
e.printStackTrace();
} catch (NoSuchFunctionException e) {
e.printStackTrace();
} catch (XpathSyntaxErrorException e) {
e.printStackTrace();
}
return sel;
}
}
@Data
@Builder
class XpathEntity {
private String url;
private String xpathName;
}
enum TaskEnum {
FIRST_TASK,
SECOND_TASK,
THREE_TASK,
END_TASK;
}
class TaskFunction<T, V> implements IWorker<T, V>, ICallback<T, V> {
@Override
public void begin() {
}
@Override
public void result(boolean success, T param, WorkResult<V> workResult) {
}
@Override
public V action(T object, Map<String, WorkerWrapper> allWrappers) {
return null;
}
@Override
public V defaultValue() {
return null;
}
}