最近项目中有文件信息需要快速索引,就打算把原来存储到MYSQL里面的数据全部存储到ES中
一下是代码
yml配置
```yaml
elasticsearch:
bboss:
elasticUser:
elasticPassword:
elasticsearch:
rest:
hostNames: 192.168.0.117:29200
dateFormat: yyyy-MM-dd hh:mm:ss
timeZone: Asia/Shanghai
ttl: 2d
showTemplate: true
discoverHost: false
dslfile:
refreshInterval: -1
http:
timeoutConnection: 400000
timeoutSocket: 400000
connectionRequestTimeout: 400000
retryTime: 1
maxLineLength: -1
maxHeaderCount: 200
maxTotal: 400
defaultMaxPerRoute: 200
soReuseAddress: false
soKeepAlive: false
timeToLive: 3600000
keepAlive: 3600000
keystore:
keyPassword:
hostnameVerifier:
工具类
```java
package com.tudou.potato.datagaea.manager.emapper;
import org.frameworkset.elasticsearch.ElasticSearchHelper;
import org.frameworkset.elasticsearch.client.ClientInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class FileModelESMapper {
public Logger logger = LoggerFactory.getLogger(FileModelESMapper.class);
private static String mapper;
private static ClientInterface restClient;
private static ClientInterface restNoClient;
public static ClientInterface restClient() {
if (restClient == null) {
restClient = ElasticSearchHelper.getConfigRestClientUtil(mapper);
}
return restClient;
}
public static ClientInterface restClientNoConfig() {
if (restNoClient == null) {
restNoClient = ElasticSearchHelper.getRestClientUtil();
}
return restNoClient;
}
@Value("${es.mappers.fileModel}")
public void setMapper(String basePath) {
FileModelESMapper.mapper = basePath;
}
}
Service
package com.tudou.potato.datagaea.manager.repo;
import com.tudou.potato.datagaea.manager.esentity.FileModel;
import com.tudou.potato.datagaea.manager.http.req.FileQuary;
import com.tudou.potato.datagaea.manager.http.req.FileQuaryPage;
import java.util.List;
public interface FileStorageService {
void insertFileModel(FileModel fileModel);
FileModel searchFileModelByID(Integer id);
void deleteFileModel(FileModel fileModel);
List<FileModel> queryBaseSpotByPage(FileQuaryPage filepage);
List<FileModel> selectByFileInfo(FileQuary params);
}
package com.tudou.potato.datagaea.manager.repo.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.tudou.potato.datagaea.manager.emapper.FileModelESMapper;
import com.tudou.potato.datagaea.manager.esentity.FileModel;
import com.tudou.potato.datagaea.manager.http.req.FileQuary;
import com.tudou.potato.datagaea.manager.http.req.FileQuaryPage;
import com.tudou.potato.datagaea.manager.repo.FileStorageService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.frameworkset.elasticsearch.client.ClientInterface;
import org.frameworkset.elasticsearch.entity.ESDatas;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Service
public class FileStorageServiceImpl implements FileStorageService {
@Value("${es.function.fileModel.getById}")
private String getById;
@Value("${es.function.fileModel.createModel}")
private String createModel;
@Value("${es.function.fileModel.queryBaseSpotByPage}")
private String queryBaseSpotByPage;
@Value("${es.function.fileModel.selectByFileInfo}")
private String selectByFileInfo;
@Value("${es.index}")
private String filemodel_index;
@Value("${es.table}")
private String table_str;
@Override
public void insertFileModel(FileModel fileModel) {
fileModel.setFileModelId(JSONObject.toJSONString(fileModel.getCollectData()).hashCode());
ClientInterface basemapper = FileModelESMapper.restClient();
if (!basemapper.existIndice(filemodel_index)) {
basemapper.createIndiceMapping(filemodel_index, createModel);
}
//删除ID重复的数据
basemapper
.deleteDocument(filemodel_index, table_str, String.valueOf(fileModel.getFileModelId()));
//新增doc
basemapper.addDocument(filemodel_index, table_str, fileModel);
}
@Override
public FileModel searchFileModelByID(Integer id) {
ClientInterface basemapper = FileModelESMapper.restClient();
Map<String, java.lang.Object> paraMap = new HashMap<>();
paraMap.put("id", id);
ESDatas<FileModel> datas = basemapper
.searchList(filemodel_index + "/_search", getById, paraMap, FileModel.class);
return JSON.parseObject(JSON.toJSONString(datas.getDatas().get(0)), FileModel.class);
}
@Override
public void deleteFileModel(FileModel fileModel) {
ClientInterface basemapper = FileModelESMapper.restClient();
basemapper
.deleteDocument(filemodel_index, table_str, String.valueOf(fileModel.getFileModelId()));
}
@Override
public List<FileModel> queryBaseSpotByPage(FileQuaryPage param) {
ClientInterface basemapper = FileModelESMapper.restClient();
ESDatas<FileModel> datas = basemapper
.searchList(filemodel_index + "/_search", queryBaseSpotByPage, param, FileModel.class);
return JSON.parseArray(JSON.toJSONString(datas.getDatas()), FileModel.class);
}
@Override
public List<FileModel> selectByFileInfo(FileQuary params) {
ClientInterface basemapper = FileModelESMapper.restClient();
ESDatas<FileModel> datas = basemapper
.searchList(filemodel_index + "/_search", selectByFileInfo, params, FileModel.class);
return JSON.parseArray(JSON.toJSONString(datas.getDatas()), FileModel.class);
}
}
xml 实现
<properties>
<!--
一个简单的检索dsl,中有四个变量
name 全文检索字段
startTime
endTime
通过map传递变量参数值
变量语法参考文档:https://my.oschina.net/bboss/blog/1556866
-->
<property name="createfileModelIndice">
<![CDATA[{
"settings": {
"number_of_shards": 6,
"index.refresh_interval": "5s"
},
"mappings": {
"file_model": {}
}
}]]>
</property>
<property name="queryBaseSpotByPage">
<![CDATA[
{
"query": {
"bool": {
"must":[
#if($workspaceId)
{ "match":
{
"workspaceId": $workspaceId
}
}
#end
#foreach($item in $collecParams.entrySet())
#if($item.value),
{ "match":
{
"collectData.$item.key": "$item.value"
}
}
#end
#end
]
}
},
## 分页起点
"from":#[from],
## 最多返回size条记录
"size":#[pageSize]
}
]]>
</property>
<property name="selectByFileInfo">
<![CDATA[
{
"query": {
"bool": {
"must":[
#if($workspaceId)
{ "match":{ "workspaceId": $workspaceId } }
#end
#foreach($item in $collecParams.entrySet())
#if($item.value),
{"match": { "collectData.$item.key": "$item.value" } }
#end
#end
#if($storageType),
{ "match":{ "fileLinkInfo.dataSourceType": "$storageType" } }
#end
]
}
}
}
]]>
</property>
<property name="searchFileModelByID">
<![CDATA[{
"query": {
"match": {
"fileModelId": #[id]
}
}
}]]>
</property>
</properties>