重学Elasticsearch第8章 : SpringBoot整合Jest客户端
邓兴为
官方文档 : https://www.elastic.co/guide/en/elasticsearch/reference/8.3/index.html
JestClient介绍
任何使用过Elasticsearch的人都知道，使用基于rest的搜索API构建查询可能是单调乏味且容易出错的。
Jest，一个用于Elasticsearch的HTTP Java客户端。Elasticsearch提供了自己原生的Java客户端，然而 Jest提供了更流畅的API和更容易使用的接口。
JestClient非官方提供的ES客户端, 现在该客户端已经不再维护, 所以我们主要还是使用RestHighLevelClient
- 由于博主有个项目中采用的客户端是Jest, 所以基于ES系列的完整性, 将这种JestClient的API也记录一下
引入依赖
<!-- JEST -->
<dependency>
<groupId>io.searchbox</groupId>
<artifactId>jest</artifactId>
<version>6.3.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.6.2</version>
</dependency>
ES的配置
(1)、application.yml 配置文件
elasticsearch:
host: http://127.0.0.1:9200
spring:
elasticsearch:
jest:
uris: http://localhost:9200
(2)、java 连接配置类
/**
* ES的配置类
* ElasticSearchConfig
*/
@Configuration
@ConfigurationProperties(prefix = "elasticsearch")
public class JestClientConfig {
private String host;
@Bean
public JestClient jestClient() {
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(
new HttpClientConfig.Builder("http://localhost:9200")
.multiThreaded(true)
.defaultMaxTotalConnectionPerRoute(2)
.maxTotalConnection(10)
.build());
return factory.getObject();
}
}
JestClient操作ElasticSearch
客户端初始化
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import io.searchbox.core.SearchResult.Hit;
import io.searchbox.core.Suggest;
import io.searchbox.core.SuggestResult;
import io.searchbox.core.Update;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import cn.xgs.entity.CsdnBlog;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.cluster.Health;
import io.searchbox.cluster.NodesInfo;
import io.searchbox.cluster.NodesStats;
import io.searchbox.core.Bulk;
import io.searchbox.core.Count;
import io.searchbox.core.CountResult;
import io.searchbox.core.Delete;
import io.searchbox.core.DocumentResult;
import io.searchbox.core.Get;
import io.searchbox.core.Index;
import io.searchbox.core.Search;
import io.searchbox.core.SearchResult;
import io.searchbox.indices.ClearCache;
import io.searchbox.indices.CloseIndex;
import io.searchbox.indices.CreateIndex;
import io.searchbox.indices.DeleteIndex;
import io.searchbox.indices.Flush;
import io.searchbox.indices.IndicesExists;
import io.searchbox.indices.Optimize;
import io.searchbox.indices.mapping.PutMapping;
/**
* Elasticserach jestClient示例
*
*/
public class Jest {
private static JestClient jestClient;
private static String indexName = "article";
@Before
public void getClient() throws Exception{
System.out.println("连接");
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(new HttpClientConfig
.Builder("http://192.168.189.138:9200")
.gson(new GsonBuilder().setDateFormat("yyyy-MM-dd'T'hh:mm:ss").create())
.connTimeout(1500)
.readTimeout(3000)
.multiThreaded(true)
.build());
jestClient=factory.getObject();
}
@After
public void tearDown() throws Exception {
closeJestClient(jestClient);
}
/**
* 关闭JestClient客户端
* @param jestClient
* @throws Exception
*/
public void closeJestClient(JestClient jestClient) throws Exception {
if (jestClient != null) {
System.out.println("关闭");
jestClient.shutdownClient();
}
}
}
索引创建
/**
* 创建索引
* @throws Exception
*/
@Test
public void createIndex() throws Exception {
JestResult jr = jestClient.execute(new CreateIndex.Builder(indexName).build());
System.out.println(jr.isSucceeded());
}
Mapping创建
/**
* Put映射
* @throws Exception
*/
@Test
public void createIndexMapping() throws Exception {
String source = "{\"" + typeName + "\":{\"properties\":{"
+ "\"author\":{\"type\":\"string\",\"index\":\"not_analyzed\"}"
+ ",\"title\":{\"type\":\"string\"}"
+ ",\"content\":{\"type\":\"string\"}"
+ ",\"price\":{\"type\":\"string\"}"
+ ",\"view\":{\"type\":\"string\"}"
+ ",\"tag\":{\"type\":\"string\"}"
+ ",\"date\":{\"type\":\"date\",\"format\":\"yyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis\"}"
+ "}}}";
System.out.println(source);
PutMapping putMapping = new PutMapping.Builder(indexName, typeName, source).build();
JestResult jr = jestClient.execute(putMapping);
System.out.println(jr.isSucceeded());
}
Bulk批量操作
/**
* Bulk插入数据
* @throws IOException
*/
@Test
public void bulkIndex() throws IOException{
CsdnBlog csdnBlog1=new CsdnBlog();
csdnBlog1.setAuthor("1111");
csdnBlog1.setTitile("中国获租巴基斯坦瓜达尔港2000亩土地 为期43年");
csdnBlog1.setContent("据了解，瓜达尔港务局于今年6月完成了1500亩土地的征收工作，另外500亩的征收工作也将很快完成");
csdnBlog1.setDate(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").format(new Date()));
csdnBlog1.setView("100");
csdnBlog1.setTag("JAVA,ANDROID,C++,LINUX");
CsdnBlog csdnBlog2=new CsdnBlog();
csdnBlog2.setAuthor("2222");
csdnBlog2.setTitile("中国获租巴基斯坦瓜达尔港2000亩土地 为期43年");
csdnBlog2.setContent("据了解，瓜达尔港务局于今年6月完成了1500亩土地的征收工作，另外500亩的征收工作也将很快完成");
csdnBlog2.setDate(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").format(new Date()));
csdnBlog2.setView("200");
csdnBlog2.setTag("ANDROID,C++,LINUX");
CsdnBlog csdnBlog3=new CsdnBlog();
csdnBlog3.setAuthor("3333");
csdnBlog3.setTitile("中国获租巴基斯坦瓜达尔港2000亩土地 为期43年");
csdnBlog3.setContent("据了解，瓜达尔港务局于今年6月完成了1500亩土地的征收工作，另外500亩的征收工作也将很快完成");
csdnBlog3.setDate(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").format(new Date()));
csdnBlog3.setView("300");
csdnBlog3.setTag("JAVA,ANDROID,C++,LINUX");
CsdnBlog csdnBlog4=new CsdnBlog();
csdnBlog4.setAuthor("4444");
csdnBlog4.setTitile("中国获租巴基斯坦瓜达尔港2000亩土地 为期43年");
csdnBlog4.setContent("据了解，瓜达尔港务局于今年6月完成了1500亩土地的征收工作，另外500亩的征收工作也将很快完成");
csdnBlog4.setDate(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").format(new Date()));
csdnBlog4.setView("400");
csdnBlog4.setTag("JAVA,ANDROID,C++,LINUX");
Bulk bulk = new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(indexName)
.addAction(Arrays.asList(
new Index.Builder(csdnBlog1).build(),
new Index.Builder(csdnBlog2).build(),
new Index.Builder(csdnBlog3).build(),
new Index.Builder(csdnBlog4).build()
)).build();
JestResult jestResult=jestClient.execute(bulk);
System.out.println(jestResult.getJsonString());
}
单值/多值匹配(term,terms)
/**
* 单值完全匹配查询
* @throws Exception
*/
@Test
public void termQuery() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 检索字段author的值为aaaa的文档
QueryBuilder queryBuilder = QueryBuilders.termQuery("author", "aaaa");//单值完全匹配查询
searchSourceBuilder.query(queryBuilder);
// 分页
searchSourceBuilder.size(10);
searchSourceBuilder.from(0);
String query = searchSourceBuilder.toString();
System.out.println(query); // 打印的就是DSL
Search search = new Search.Builder(query).addIndex(indexName).build();
SearchResult result = jestClient.execute(search);
List<Hit<Object, Void>> hits = result.getHits(Object.class);
System.out.println("Size:" + hits.size());
for (Hit<Object, Void> hit : hits) {
Object news = hit.source;
System.out.println(news.toString());
}
}
/**
* 多值完全匹配查询
* @throws Exception
*/
@Test
public void termsQuery() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 检索字段author的值为1111或2222的文档
QueryBuilder queryBuilder = QueryBuilders
.termsQuery("author", new String[]{ "1111", "2222" });//多值完全匹配查询
searchSourceBuilder.query(queryBuilder);
searchSourceBuilder.size(10);
searchSourceBuilder.from(0);
String query = searchSourceBuilder.toString();
System.out.println(query);
Search search = new Search.Builder(query)
.addIndex(indexName)
.addType(indexName)
.build();
SearchResult result = jestClient.execute(search);
List<Hit<Object, Void>> hits = result.getHits(Object.class);
System.out.println("Size:" + hits.size());
for (Hit<Object, Void> hit : hits) {
Object news = hit.source;
System.out.println(news.toString());
}
}
通配符查询 (wildcardQuery)
/**
* 通配符和正则表达式查询
* @throws Exception
*/
@Test
public void wildcardQuery() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
QueryBuilder queryBuilder = QueryBuilders
.wildcardQuery("title", "*:*");//通配符和正则表达式查询
searchSourceBuilder.query(queryBuilder);
searchSourceBuilder.size(10);
searchSourceBuilder.from(0);
String query = searchSourceBuilder.toString();
System.out.println(query);
Search search = new Search.Builder(query)
.addIndex(indexName)
.build();
SearchResult result = jestClient.execute(search);
List<Hit<Object, Void>> hits = result.getHits(Object.class);
System.out.println("Size:" + hits.size());
for (Hit<Object, Void> hit : hits) {
Object news = hit.source;
System.out.println(news.toString());
}
}
前缀查询 (prefixQuery)
/**
* 前缀查询
* @throws Exception
*/
@Test
public void prefixQuery() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 字段author的值, 前缀为11的文档
QueryBuilder queryBuilder = QueryBuilders
.prefixQuery("author", "11");//前缀查询
searchSourceBuilder.query(queryBuilder);
searchSourceBuilder.size(10);
searchSourceBuilder.from(0);
String query = searchSourceBuilder.toString();
System.out.println(query);
Search search = new Search.Builder(query)
.addIndex(indexName)
.addType(indexName)
.build();
SearchResult result = jestClient.execute(search);
List<Hit<Object, Void>> hits = result.getHits(Object.class);
System.out.println("Size:" + hits.size());
for (Hit<Object, Void> hit : hits) {
Object news = hit.source;
System.out.println(news.toString());
}
}
Range范围查询
/**
* 区间查询
* @throws Exception
*/
@Test
public void rangeQuery() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
QueryBuilder queryBuilder = QueryBuilders
.rangeQuery("date")
.gte("2019-02-20T15:43:48")
.lte("2019-02-20T15:55:45")
.includeLower(true)
.includeUpper(true);//是否包含区间两边
searchSourceBuilder.query(queryBuilder);
searchSourceBuilder.size(10);
searchSourceBuilder.from(0);
String query = searchSourceBuilder.toString();
System.out.println(query);
Search search = new Search.Builder(query)
.addIndex(indexName)
.addType(indexName)
.build();
SearchResult result = jestClient.execute(search);
List<Hit<Object, Void>> hits = result.getHits(Object.class);
System.out.println("Size:" + hits.size());
for (Hit<Object, Void> hit : hits) {
Object news = hit.source;
System.out.println(news.toString());
}
}
queryString
/**
* 文本检索，应该是将查询的词先分成词库中存在的词，然后分别去检索，存在任一存在的词即返回，查询词分词后是OR的关系。需要转义特殊字符
* @throws Exception
*/
@Test
public void queryString() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 文本检索，应该是将查询的词先分成词库中存在的词，然后分别去检索，存在任一存在的词即返回，查询词分词后是OR的关系。
// 会对华为手机进行分词,没有设置检索的field,默认对mapping中字符串类型的filed进行检索;
QueryBuilder queryBuilder = QueryBuilders.queryStringQuery("华为手机")
.filed("author").defaultOperator(Operator.AND);;
searchSourceBuilder.query(queryBuilder);
searchSourceBuilder.size(10);
searchSourceBuilder.from(0);
String query = searchSourceBuilder.toString();
System.out.println(query);
Search search = new Search.Builder(query)
.addIndex(indexName)
.addType(typeName)
.build();
SearchResult result = jestClient.execute(search);
List<Hit<Object, Void>> hits = result.getHits(Object.class);
System.out.println("Size:" + hits.size());
for (Hit<Object, Void> hit : hits) {
Object news = hit.source;
System.out.println(news.toString());
}
}
Count(根据条件查询出文档数量)
/**
* Count文档
* @throws Exception
*/
@Test
public void count() throws Exception {
String[] name = new String[]{ "1111", "2222" };
String from = "2019-02-20T15:43:48";
String to = "2019-02-20T15:59:48";
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
.must(QueryBuilders.termsQuery("author", name))
.must(QueryBuilders.rangeQuery("date").gte(from).lte(to));
searchSourceBuilder.query(queryBuilder);
String query = searchSourceBuilder.toString();
System.out.println(query);
Count count = new Count.Builder()
.addIndex(indexName)
.addType(indexName)
.query(query)
.build();
CountResult results = jestClient.execute(count);
Double counts = results.getCount();
System.out.println("Count:" + counts);
}
索引和节点操作
/**
* 删除索引
* @throws Exception
*/
@Test
public void deleteIndex() throws Exception {
JestResult jr = jestClient.execute(new DeleteIndex.Builder(indexName).build());
boolean result = jr.isSucceeded();
System.out.println(result);
}
/**
* 将删除所有的索引
* @throws Exception
*/
@Test
public void deleteIndexAll() throws Exception {
DeleteIndex deleteIndex = new DeleteIndex.Builder("article").build();
JestResult result = jestClient.execute(deleteIndex);
System.out.println(result.getJsonString());
}
/**
* 清缓存
* @throws Exception
*/
@Test
public void clearCache() throws Exception {
ClearCache closeIndex = new ClearCache.Builder().build();
JestResult result = jestClient.execute(closeIndex);
System.out.println(result.getJsonString());
}
/**
* 关闭索引
* @throws Exception
*/
@Test
public void closeIndex() throws Exception {
CloseIndex closeIndex = new CloseIndex.Builder("article").build();
JestResult result = jestClient.execute(closeIndex);
System.out.println(result.getJsonString());
}
/**
* 优化索引
* @throws Exception
*/
@Test
public void optimize() throws Exception {
Optimize optimize = new Optimize.Builder().build();
JestResult result = jestClient.execute(optimize);
System.out.println(result.getJsonString());
}
/**
* 刷新索引
* @throws Exception
*/
@Test
public void flush() throws Exception {
Flush flush = new Flush.Builder().build();
JestResult result = jestClient.execute(flush);
System.out.println(result.getJsonString());
}
/**
* 判断索引目录是否存在
* @throws Exception
*/
@Test
public void indicesExists() throws Exception {
IndicesExists indicesExists = new IndicesExists.Builder("article").build();
JestResult result = jestClient.execute(indicesExists);
System.out.println(result.getJsonString());
}
/**
* 查看节点信息
* @throws Exception
*/
@Test
public void nodesInfo() throws Exception {
NodesInfo nodesInfo = new NodesInfo.Builder().build();
JestResult result = jestClient.execute(nodesInfo);
System.out.println(result.getJsonString());
}
/**
* 查看集群健康信息
* @throws Exception
*/
@Test
public void health() throws Exception {
Health health = new Health.Builder().build();
JestResult result = jestClient.execute(health);
System.out.println(result.getJsonString());
}
/**
* 节点状态
* @throws Exception
*/
@Test
public void nodesStats() throws Exception {
NodesStats nodesStats = new NodesStats.Builder().build();
JestResult result = jestClient.execute(nodesStats);
System.out.println(result.getJsonString());
}
文档的CRUD
/**
* 更新Document
* @param index
* @param type
* @param id
* @throws Exception
*/
@Test
public void updateDocument() throws Exception {
CsdnBlog article = new CsdnBlog();
article.setTitile("中国3333颗卫星拍到阅兵现场高清照");
article.setContent("据中国资源卫星应用中心报道，9月3日，纪念中国人民抗日战争暨世界反法西斯战争胜利70周年大阅兵在天安门广场举行。资源卫星中心针对此次盛事，综合调度在轨卫星，9月1日至3日连续三天持续观测首都北京天安门附近区域，共计安排5次高分辨率卫星成像。在阅兵当日，高分二号卫星、资源三号卫星及实践九号卫星实现三星联合、密集观测，捕捉到了阅兵现场精彩瞬间。为了保证卫星准确拍摄天安门及周边区域，提高数据处理效率，及时制作合格的光学产品，资源卫星中心运行服务人员从卫星观测计划制定、复核、优化到系统运行保障、光学产品图像制作，提前进行了周密部署，并拟定了应急预案，为圆满完成既定任务奠定了基础。");
article.setDate(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").format(new Date()));
article.setAuthor("匿名");
article.setView("110");
article.setTag("java,android");
String script = "{" +
" \"doc\" : {" +
" \"titile\" : \""+article.getTitile()+"\"," +
" \"content\" : \""+article.getContent()+"\"," +
" \"author\" : \""+article.getAuthor()+"\"," +
" \"view\" : \""+article.getView()+"\"," +
" \"tag\" : \""+article.getTag()+"\"," +
" \"date\" : \""+article.getDate()+"\"" +
" }" +
"}";
Update update = new Update.Builder(script).index(indexName).type(indexName).id("pUTcCWkBKPh010MkRo9h").build();
JestResult result = jestClient.execute(update);
System.out.println(result.getJsonString());
}
/**
* 删除Document
* @param index
* @param type
* @param id
* @throws Exception
*/
@Test
public void deleteDocument() throws Exception {
Delete delete = new Delete.Builder("").index(indexName).type(typeName).build();
JestResult result = jestClient.execute(delete);
System.out.println(result.getJsonString());
}
/**
* 获取Document
* @param index
* @param type
* @param id
* @throws Exception
*/
@Test
public void getDocument() throws Exception {
Get get = new Get.Builder(indexName,"").type(indexName).build();
JestResult result = jestClient.execute(get);
CsdnBlog article = result.getSourceAsObject(CsdnBlog.class);
System.out.println(article.getTitile()+","+article.getContent());
}
查询全部文档(matchAllQuery)
/**
* 查询全部
* @throws Exception
*/
@Test
public void searchAll() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchAllQuery());
Search search = new Search.Builder(searchSourceBuilder.toString())
.addIndex(indexName)
.build();
SearchResult result = jestClient.execute(search);
System.out.println("本次查询共查到："+result.getTotal()+"篇文章！");
List<Hit<CsdnBlog,Void>> hits = result.getHits(CsdnBlog.class);
for (Hit<CsdnBlog, Void> hit : hits) {
CsdnBlog source = hit.source;
System.out.println("标题："+source.getTitile());
System.out.println("内容："+source.getContent());
System.out.println("浏览数："+source.getView());
System.out.println("标签："+source.getTag());
System.out.println("作者："+source.getAuthor());
}
}
分页 (from, size)
/**
* 分页
* @throws IOException
*/
@Test
public void page() throws IOException{
int pageNumber=1;
int pageSize=10;
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.queryStringQuery("zshuai"));
searchSourceBuilder.from((pageNumber - 1) * pageSize);//设置起始页
searchSourceBuilder.size(pageSize);//设置页大小
Search search = new Search.Builder(searchSourceBuilder.toString())
.addIndex(indexName)// 索引名称
.build();
SearchResult result = jestClient.execute(search);
// 自动解析
JsonObject jsonObject = result.getJsonObject();
JsonObject hitsobject = jsonObject.getAsJsonObject("hits");
long took = jsonObject.get("took").getAsLong();
long total = hitsobject.get("total").getAsLong();
System.out.println("took:"+took+" "+"total:"+total);
}
高亮
/**
* 搜索高亮显示
* @throws Exception
*/
@Test
public void createSearch() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("view", "200"));
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.field("view");//高亮title
highlightBuilder.preTags("<em>").postTags("</em>");//高亮标签
highlightBuilder.fragmentSize(500);//高亮内容长度
searchSourceBuilder.highlighter(highlightBuilder);
System.out.println(searchSourceBuilder.toString());
Search search = new Search.Builder(searchSourceBuilder.toString())
.addIndex(indexName)
.build();
SearchResult result = jestClient.execute(search);
System.out.println(result.getJsonString());
System.out.println("本次查询共查到："+result.getTotal()+"篇文章！");
List<Hit<CsdnBlog,Void>> hits = result.getHits(CsdnBlog.class);
System.out.println(hits.size());
for (Hit<CsdnBlog, Void> hit : hits) {
CsdnBlog source = hit.source;
//获取高亮后的内容
Map<String, List<String>> highlight = hit.highlight;
List<String> views = highlight.get("view");//高亮后的title
if(views!=null){
source.setView(views.get(0));
}
System.out.println("标题："+source.getTitile());
System.out.println("内容："+source.getContent());
System.out.println("浏览数："+source.getView());
System.out.println("标签："+source.getTag());
System.out.println("作者："+source.getAuthor());
}
}
实体类
package cn.xgs.entity;
public class CsdnBlog {
private String author;
private String titile;
private String tag;
private String content;
private String view;
private String date;
public String getAuthor() {
return author;
}
public void setAuthor(String author) {
this.author = author;
}
public String getTitile() {
return titile;
}
public void setTitile(String titile) {
this.titile = titile;
}
public String getTag() {
return tag;
}
public void setTag(String tag) {
this.tag = tag;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public String getView() {
return view;
}
public void setView(String view) {
this.view = view;
}
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
public CsdnBlog(String author, String titile, String tag, String content, String view, String date) {
super();
this.author = author;
this.titile = titile;
this.tag = tag;
this.content = content;
this.view = view;
this.date = date;
}
public CsdnBlog() {
super();
// TODO Auto-generated constructor stub
}
}
实战
检索操作
@Builder
@Data
public class PageableResult<T> {
private Pageable page;
private List<T> data;
}
@Service
@Slf4j
public class EsServiceImpl {
public PageableResult<EsHLog> searchLogList(LogSearchReq req) throws IOException {
SearchSourceBuilder builder = new SearchSourceBuilder();
BoolQueryBuilder boolQuery = createBuilder(req);
builder.query(boolQuery)
.from((req.getPageIndex() - 1) * req.getPageSize())
.size(req.getPageSize())
.sort("logTime", req.getTimeSortFlag() != null && req.getTimeSortFlag() ? SortOrder.ASC : SortOrder.DESC)
.fetchSource(true)
.timeout(TimeValue.timeValueMinutes(2));
SearchResult searchResult = (SearchResult) this.execute(req.getAppId(), null, builder, req.getEndTime(), null);
PageableResult<EsHLog> result = this.convertPageableResult(searchResult, this::convert);
result.getPage().setPageIndex(req.getPageIndex());
result.getPage().setPageSize(req.getPageSize());
return result;
}
private BoolQueryBuilder createBuilder(LogSearchReq req) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
//1、appId
TermQueryBuilder appQuery = QueryBuilders.termQuery("appId", req.getAppId());
boolQuery.filter(appQuery);
//env
TermQueryBuilder envQuery = QueryBuilders.termQuery("env", env);
boolQuery.filter(envQuery);
//2、time
RangeQueryBuilder timeQuery = QueryBuilders.rangeQuery("logTime")
.from(req.getStartTime()).includeLower(true)
.to(req.getEndTime()).includeUpper(true);
boolQuery.filter(timeQuery);
//3、title
if (StringUtils.isNotBlank(req.getTitle())) {
QueryStringQueryBuilder titleQuery = QueryBuilders.queryStringQuery(req.getTitle())
.field("title")
.defaultOperator(Operator.AND);//by default, match with AND
boolQuery.filter(titleQuery);
}
//4、log level
if (CollectionUtils.isNotEmpty(req.getLevels())) {
TermsQueryBuilder levelQuery = QueryBuilders.termsQuery("level", req.getLevels());
boolQuery.filter(levelQuery);
}
//5、tags
if (StringUtils.isNotBlank(req.getTags())) {
if (req.getTags().contains(";") || !req.getTags().contains(":")) {
String[] split = req.getTags().split(";");
TermsQueryBuilder tagsQuery = QueryBuilders.termsQuery("tags", Arrays.asList(split));
boolQuery.filter(tagsQuery);
} else {
String[] tags = req.getTags().split(":");
for (String tag : tags) {
TermsQueryBuilder tagsQuery = QueryBuilders.termsQuery("tags", tag);
boolQuery.filter(tagsQuery);
}
}
}
//6、content
if (StringUtils.isNotBlank(req.getContent())) {
QueryStringQueryBuilder messageQuery = QueryBuilders.queryStringQuery(req.getContent())
.field("fullMessage")
.defaultOperator(Operator.AND);//by default, match with AND
boolQuery.filter(messageQuery);
}
//7、machine
if (CollectionUtils.isNotEmpty(req.getMachines())) {
TermsQueryBuilder machineQuery = QueryBuilders.termsQuery("machine", req.getMachines());
boolQuery.filter(machineQuery);
}
//8、ip
if (CollectionUtils.isNotEmpty(req.getIps())) {
TermsQueryBuilder ipQuery = QueryBuilders.termsQuery("machineIp", req.getIps());
boolQuery.filter(ipQuery);
}
//9、traceId
if (StringUtils.isNotBlank(req.getTraceId())) {
TermQueryBuilder traceQuery = QueryBuilders.termQuery("traceId", req.getTraceId());
boolQuery.filter(traceQuery);
}
//10、logName
if (StringUtils.isNotBlank(req.getLogName())) {
TermQueryBuilder logNameQuery = QueryBuilders.termQuery("logName", req.getLogName());
boolQuery.filter(logNameQuery);
}
if (StringUtils.isNotBlank(req.getPattern())) {
TermQueryBuilder patternQuery = QueryBuilders.termQuery("pattern", req.getPattern());
boolQuery.filter(patternQuery);
}
if (StringUtils.isNotBlank(req.getMessageSize())) {
ExistsQueryBuilder messageSize = QueryBuilders.existsQuery("messageSize");
boolQuery.filter(messageSize);
RangeQueryBuilder gteQuery = QueryBuilders.rangeQuery("messageSize")
.gte(req.getMessageSize());
boolQuery.filter(gteQuery);
}
if (StringUtils.isNotBlank(req.getUnitRouteCode())) {
TermQueryBuilder routeCodeQuery = QueryBuilders.termQuery("routeCode", req.getUnitRouteCode());
boolQuery.filter(routeCodeQuery);
}
return boolQuery;
}
private JestResult execute(String appId, Long time, SearchSourceBuilder builder,
LocalDateTime queryEndTime, String esCluster) throws IOException {
JestClient jestClient = EsApplication.getInstance().getJestClient(StringUtils.isNotBlank(esCluster) ? esCluster :
getCluster(appId, queryEndTime)); // 集群名
SearchResult result = jestClient.execute(
new Search.Builder(builder.toString())
// 索引名
.addIndex(getIndex(appId, time))
.build());
return result;
}
/**
* convert search result
*/
private <R> PageableResult<R> convertPageableResult(SearchResult searchResult, Function<SearchResult.Hit<EsHLog, Void>, R> converter) {
Pageable page = new Pageable();
page.setTotalCount(getTotal(searchResult));
List<SearchResult.Hit<EsHLog, Void>> hits = searchResult.getHits(EsHLog.class, false);
return PageableResult.<R>builder()
.page(page)
.data(Optional.ofNullable(hits).orElse(new ArrayList<>())
.stream()
.map(hit -> converter.apply(hit))
.collect(Collectors.toList()))
.build();
}
}
聚合操作
public List<MachineEntity> searchMachineList(String appId, LocalDateTime startTime, LocalDateTime endTime) throws IOException {
SearchSourceBuilder builder = new SearchSourceBuilder();
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
//1、appId
TermQueryBuilder appQuery = QueryBuilders.termQuery("appId", appId);
//env
TermQueryBuilder envQuery = QueryBuilders.termQuery("env", env);
boolQuery.filter(envQuery);
//2、time
RangeQueryBuilder timeQuery = QueryBuilders.rangeQuery("logTime")
.from(startTime).includeLower(true)
.to(endTime).includeUpper(true);
boolQuery.filter(appQuery)
.filter(timeQuery);
//3、machine agg
TermsAggregationBuilder machineAgg = AggregationBuilders.terms("agg_by_machine").field("machine").size(Integer.MAX_VALUE);
builder.query(boolQuery)
.aggregation(machineAgg)
.fetchSource(false)
.size(0)
.timeout(TimeValue.timeValueMinutes(1));
SearchResult searchResult = (SearchResult) this.execute(appId, null, builder, endTime, null);
MetricAggregation aggregations = searchResult.getAggregations();
TermsAggregation machineAggResult = aggregations.getTermsAggregation("agg_by_machine");
return this.convertTermsAggResult(machineAggResult, bucket -> {
MachineEntity machine = MachineEntity.builder().label(bucket.getKey()).name(bucket.getKey()).build();
return machine;
});
}
private JestResult execute(String appId, Long time, SearchSourceBuilder builder,
LocalDateTime queryEndTime, String esCluster) throws IOException {
JestClient jestClient = EsApplication.getInstance().getJestClient(StringUtils.isNotBlank(esCluster) ? esCluster :
getCluster(appId, queryEndTime)); // 集群名
SearchResult result = jestClient.execute(
new Search.Builder(builder.toString())
// 索引名
.addIndex(getIndex(appId, time))
.build());
return result;
}
/**
* convert terms aggregation
*
* @return
*/
protected <R> List<R> convertTermsAggResult(TermsAggregation termsAgg, Function<TermsAggregation.Entry, R> converter) {
if (termsAgg != null && CollectionUtils.isNotEmpty(termsAgg.getBuckets())) {
return termsAgg.getBuckets().stream().map(converter).collect(Collectors.toList());
}
return new ArrayList<>(0);
}
索引信息
public class searchIndexMsg(LocalDate localDate) throws IOException {
String indexDate = DateUtil.format(DateTimeUtils.toDate(localDate), PURE_DATE_FORMAT);
Cat.IndicesBuilder indicesBuilder = new Cat.IndicesBuilder().setParameter("bytes", "mb");
CatResult result = executeByCat(indexDate, indicesBuilder, cluster);
private CatResult executeByCat(String indexDate, Cat.IndicesBuilder builder, String esCluster) throws IOException {
if(StringUtils.isBlank(esCluster)){
return null;
}
JestClient jestClient = EsApplication.getInstance().getJestClient(esCluster);
return jestClient.execute(builder.addIndex(getIndexByCat(indexDate)).build());
}
}
类似资料：