SVN可视化平台(7) -- ElasticSearch 全文检索，Spring Data ES 和 RestHighLevelClient 的两种方式，解决最多查询一万条数据的问题

avatar 2021年12月22日08:50:51 6 5778 views
博主分享免费Java教学视频，B站账号：Java刘哥 ，长期提供技术问题解决、项目定制：本站商品点此

本文介绍 ES 查询数据的2种方式

最近在做SVN web化中，需要把数百万的SVN文件信息存储到数据库，然后存储到ES，实现全文检索功能。

最开始我用的是 Spring Data ES，后面出现了一个 Bug，是最多返回10000条数据。

Spring Data ES 不支持，我后面才改用 RestHighLevelClient 的方式

一、解决最多查询10000条数据的问题

在测试环境发现，分页返回总数最多10000条

原因是 es 为了提高查询效率，hits默认是10000

 

解决办法是

第一步、

  1. PUT http://127.0.0.1:9200/_all/_settings?preserve_existing=true
  2. {
  3.   "index":{
  4.     "max_result_window":2000000000
  5.   }
  6. }

第二步、

在请求里加   "track_total_hits":true

 

二、代码准备

1、依赖

  1. <dependency>
  2. 	<groupId>org.springframework.boot</groupId>
  3. 	<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
  4. 	<version>2.3.4.RELEASE</version>
  5. </dependency>

 

2、配置文件

  1. spring:
  2. # ES 配置
  3.   elasticsearch:
  4.     rest:
  5.       uris: ["http://127.0.0.1:9200"]
  6.       connection-timeout: 10s
  7.       username:
  8.       password:

 

3、索引

  1. PUT http://127.0.0.1:9200/common_file
  2. {
  3.     "settings": {
  4.         "index": {
  5.             "number_of_shards": "1",
  6.             "number_of_replicas": "1"
  7.         },
  8.         "index.max_ngram_diff": 5,
  9.         "analysis": {
  10.             "analyzer": {
  11.                 "ngram_analyzer": {
  12.                     "tokenizer": "ngram_tokenizer"
  13.                 }
  14.             },
  15.             "tokenizer": {
  16.                 "ngram_tokenizer": {
  17.                     "type": "ngram",
  18.                     "min_gram": 1,
  19.                     "max_gram": 5,
  20.                     "token_chars": [
  21.                         "letter",
  22.                         "digit"
  23.                     ]
  24.                 }
  25.             }
  26.         }
  27.     },
  28.     "mappings": {
  29.         "properties": {
  30.             "repoPrefix": {
  31.                 "type": "keyword"
  32.             },
  33.             "fileName": {
  34.                 "type": "text",
  35.                 "analyzer": "ngram_analyzer"
  36.             },
  37.             "level": {
  38.                 "type": "long"
  39.             },
  40.             "leafFlag": {
  41.                 "type": "boolean"
  42.             },
  43.             "filePath": {
  44.             	"type": "text",
  45.                 "fields": {
  46.                     "keyword": {
  47.                         "type": "keyword"
  48.                     },
  49.                     "text": {
  50.                         "type": "text",
  51.                         "analyzer": "ngram_analyzer"
  52.                     }
  53.                 }
  54.             },
  55.             "creatorName": {
  56.                 "type": "text"
  57.             },
  58.             "description": {
  59.                 "type": "text"
  60.             },
  61.             "updateTime": {
  62.                 "type": "date"
  63.             },
  64.             "revision": {
  65.                 "type": "long"
  66.             },
  67.             "createTime": {
  68.                 "type": "date"
  69.             },
  70.             "fileSize": {
  71.                 "type": "long"
  72.             },
  73.             "updaterName": {
  74.                 "type": "text"
  75.             },
  76.             "_class": {
  77.                 "type": "keyword"
  78.             },
  79.             "id": {
  80.                 "type": "long"
  81.             },
  82.             "projectId": {
  83.                 "type": "long"
  84.             },
  85.             "fileType": {
  86.                 "type": "long"
  87.             },
  88.             "roleAuthList": {
  89.             	"type": "nested",
  90.                 "properties": {
  91.                 	"roleCode" : {
  92.                 		"type": "keyword"
  93.                 	},
  94.                 	"auth" : {
  95.                 		"type": "text"
  96.                 	}
  97.                 }
  98.             }
  99.         }
  100.     }
  101. }

 

4、实体类

  1. import lombok.Data;
  2. import org.springframework.data.elasticsearch.annotations.Document;
  3. import org.springframework.data.elasticsearch.annotations.Field;
  4. import org.springframework.data.elasticsearch.annotations.FieldType;
  5. import org.springframework.stereotype.Component;
  8. import javax.persistence.Id;
  9. import java.io.Serializable;
  10. import java.util.Date;
  11. import java.util.List;
  13. /**
  14.  * 文件类对应索引
  15.  */
  16. @Component
  17. @Document(indexName = "common_file", indexStoreType = "common_file")
  18. @Data
  19. public class EsCommonFile implements Serializable
  20. {
  23.     /**
  24.      * ID
  25.      */
  26.     @Id
  27.     private Long id;
  29.     /**
  30.      * 项目id
  31.      */
  32.     private Long projectId;
  34.     /**
  35.      * 文件名称
  36.      */
  37.     private String fileName;
  39.     /**
  40.      * 文件类型(1 file,2 folder)
  41.      */
  42.     private Integer fileType;
  44.     /**
  45.      * 文件路径
  46.      */
  47.     private String repoPrefix;
  49.     /**
  50.      * 文件路径
  51.      */
  52.     private String filePath;
  54.     /**
  55.      * 文件大小，单位字节
  56.      */
  57.     private Long fileSize;
  60.     /**
  61.      * 文件版本
  62.      */
  63.     private Long revision;
  65.     /**
  66.      * 创建人账号
  67.      */
  68.     private String creatorName;
  70.     /**
  71.      * 修改人账号
  72.      */
  73.     private String updaterName;
  75.     /**
  76.      * 描述
  77.      */
  78.     private String description;
  80.     /**
  81.      * 文件层级，仓库目录为1，子目录依次+1
  82.      */
  83.     private Integer level;
  85.     /**
  86.      * 是否存在子目录(1不存在，0存在)
  87.      */
  88.     private Boolean leafFlag;
  90.     /**
  91.      * 创建时间
  92.      */
  93.     private Date createTime;
  96.     /**
  97.      * 更新时间
  98.      */
  99.     private Date updateTime;
  101.     /**
  102.      * 授权列表
  103.      */
  104.     @Field(type = FieldType.Nested)
  105.     private List<CommonFileRoleAuth> roleAuthList;
  107. }

 

 

三、Spring Data ES 查询方式 (不推荐，无法解决查询限制10000的问题)

Spring Data ES简单，可以用来做一些添加，删除，修改、简单查询

复杂查询有点力不从心，比如我这里的限制10000的问题

代码我也贴上

  1. @Autowired
  2. private EsCommonFileRepository esCommonFileRepository;
  4. private LayPage<CommonFileVO> searchEsFile(FileSearchDTO fileSearchDTO)
  5. {
  8. 	//查询对象
  9. 	BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
  10. 	// level查询
  11. 	if (CommonUtil.isNotEmpty(fileSearchDTO.getLevel()))
  12. 	{
  13. 		boolQueryBuilder.must(QueryBuilders.rangeQuery("level").from(fileSearchDTO.getLevel() + 1));
  14. 	}
  15. 	// repoPrefix查询
  16. 	if (CommonUtil.isNotEmpty(fileSearchDTO.getRepoPrefix()))
  17. 	{
  18. 		boolQueryBuilder.must(QueryBuilders.matchQuery("repoPrefix", fileSearchDTO.getRepoPrefix()).operator(Operator.AND));
  19. 	}
  21. 	// 关键字查询
  22. 	if (CommonUtil.isNotEmpty(fileSearchDTO.getSearchKeywords()))
  23. 	{
  24. 		// 搜文件路径
  25. 		if ("filePath".equals(fileSearchDTO.getSearchType()))
  26. 		{
  27. 			boolQueryBuilder.must(QueryBuilders.matchQuery("filePath.text", fileSearchDTO.getSearchKeywords()).operator(Operator.AND));
  29. 		}
  30. 		// 搜文件名称
  31. 		else
  32. 		{
  33. 			boolQueryBuilder.must(QueryBuilders.matchQuery("fileName", fileSearchDTO.getSearchKeywords()).operator(Operator.AND));
  34. 		}
  35. 	}
  36. 	// 权限
  37. 	List<String> loginUserRoleCodes = prjUserRoleService.getLoginUserRoleCodeList();
  39. 	List<String> authList = new ArrayList<>();
  40. 	authList.add("rw");
  41. 	authList.add("r");
  42. 	loginUserRoleCodes.add("*");
  43. 	boolQueryBuilder.must(QueryBuilders.nestedQuery("roleAuthList", QueryBuilders.termsQuery("roleAuthList.roleCode", loginUserRoleCodes), ScoreMode.None));
  44. 	boolQueryBuilder.must(QueryBuilders.nestedQuery("roleAuthList", QueryBuilders.termsQuery("roleAuthList.auth", authList), ScoreMode.None));
  47. 	PageRequest pageRequest = PageRequest.of(fileSearchDTO.getPage() - 1, fileSearchDTO.getLimit(),
  48. 			Sort.Direction.DESC, "fileType", "_score", "updateTime");
  49. 	org.springframework.data.domain.Page<EsCommonFile> esCommonFilePage = esCommonFileRepository.search(boolQueryBuilder, pageRequest);
  52. 	List<EsCommonFile> content = esCommonFilePage.getContent();
  53. 	List<CommonFileVO> commonFileVOList = new ArrayList<>();
  54. 	for (EsCommonFile esCommonFile : content)
  55. 	{
  56. 		CommonFileVO commonFileVO = new CommonFileVO();
  57. 		BeanUtils.copyProperties(esCommonFile, commonFileVO);
  58. 		commonFileVOList.add(commonFileVO);
  59. 	}
  60. 	LayPage<CommonFileVO> result = new LayPage<>();
  61. 	result.setCount(esCommonFilePage.getTotalElements());
  62. 	result.setData(commonFileVOList);
  63. 	return result;
  64. }

需要自己创建一个 repository 和开启扫包

具体可以看我之前文章

 

四、RestHighLevelClient 查询方式 (推荐)

  1. @Autowired
  2. private RestHighLevelClient restHighLevelClient;
  4. private LayPage<CommonFileVO> newSearchEsFile(FileSearchDTO fileSearchDTO) throws Exception
  5. {
  8. 	//查询对象
  9. 	BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
  10. 	// level查询 范围查询，level > 1 + #{level}
  11. 	if (CommonUtil.isNotEmpty(fileSearchDTO.getLevel()))
  12. 	{
  13. 		boolQueryBuilder.must(QueryBuilders.rangeQuery("level").from(fileSearchDTO.getLevel() + 1));
  14. 	}
  15. 	// repoPrefix查询，等值查询
  16. 	if (CommonUtil.isNotEmpty(fileSearchDTO.getRepoPrefix()))
  17. 	{
  18. 		boolQueryBuilder.must(QueryBuilders.matchQuery("repoPrefix", fileSearchDTO.getRepoPrefix()).operator(Operator.AND));
  19. 	}
  20. 	// filePath查询，模糊查询，类似 like '/项目/测试项目%'
  21. 	if (CommonUtil.isNotEmpty(fileSearchDTO.getFilePath()))
  22. 	{
  23. 		boolQueryBuilder.must(QueryBuilders.wildcardQuery("filePath.keyword", fileSearchDTO.getFilePath() + "*"));
  24. 	}
  26. 	// 关键字查询
  27. 	if (CommonUtil.isNotEmpty(fileSearchDTO.getSearchKeywords()))
  28. 	{
  29. 		// 搜文件路径，分词查询
  30. 		if ("filePath".equals(fileSearchDTO.getSearchType()))
  31. 		{
  32. 			boolQueryBuilder.must(QueryBuilders.matchQuery("filePath.text", fileSearchDTO.getSearchKeywords()).operator(Operator.AND));
  34. 		}
  35. 		// 搜文件名称，分词查询
  36. 		else
  37. 		{
  38. 			boolQueryBuilder.must(QueryBuilders.matchQuery("fileName", fileSearchDTO.getSearchKeywords()).operator(Operator.AND));
  39. 		}
  40. 	}
  41. 	// 根据权限过滤
  42. 	// 查询登录用户的角色编码
  43. 	List<String> loginUserRoleCodes = prjUserRoleService.getLoginUserRoleCodeList();
  44. 	List<String> authList = new ArrayList<>();
  45. 	// 只允许 auth 为 rw 或 r
  46. 	authList.add("rw");
  47. 	authList.add("r");
  48. 	// 只允许 roleCode 为登录用户的角色编码或 *  (roleCodeList 是嵌套索引)
  49. 	loginUserRoleCodes.add("*");
  50. 	boolQueryBuilder.must(QueryBuilders.nestedQuery("roleAuthList", QueryBuilders.termsQuery("roleAuthList.roleCode", loginUserRoleCodes), ScoreMode.None));
  51. 	boolQueryBuilder.must(QueryBuilders.nestedQuery("roleAuthList", QueryBuilders.termsQuery("roleAuthList.auth", authList), ScoreMode.None));
  53. 	SearchRequest request = new SearchRequest();
  54. 	// 指定索引
  55. 	request.indices("common_file");
  56. 	SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
  57. 	// 传 "track_total_hits":true
  58. 	sourceBuilder.trackTotalHits(true);
  59. 	// 排序
  60. 	sourceBuilder.sort("fileType", SortOrder.DESC);
  61. 	sourceBuilder.sort("_score", SortOrder.DESC);
  62. 	sourceBuilder.sort("updateTime", SortOrder.DESC);
  63. 	// 设置从第几页开始，一页查询多少条
  64. 	int pageNum = fileSearchDTO.getPage();
  65. 	int start = (( pageNum < 1 ? 1 : pageNum) - 1) * fileSearchDTO.getLimit();
  66. 	sourceBuilder.from(start);
  67. 	sourceBuilder.size(fileSearchDTO.getLimit());
  69. 	sourceBuilder.query(boolQueryBuilder);
  70. 	request.source(sourceBuilder);
  71. 	SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
  72. 	SearchHits hits = response.getHits();
  73. 	List<CommonFileVO> commonFileVOList = new ArrayList<>();
  74. 	
  75. 	// 遍历查询结果
  76. 	for (SearchHit searchHit : hits) {
  77. 		EsCommonFile esCommonFile = JSON.parseObject(searchHit.getSourceAsString(),EsCommonFile.class);
  78. 		CommonFileVO commonFileVO = new CommonFileVO();
  79. 		BeanUtils.copyProperties(esCommonFile, commonFileVO);
  80. 		commonFileVOList.add(commonFileVO);
  81. 	}
  83. 	LayPage<CommonFileVO> result = new LayPage<>();
  84. 	// 设置查询总数
  85. 	result.setCount(hits.getTotalHits().value);
  86. 	result.setData(commonFileVOList);
  87. 	return result;
  88. }

 

 

历史上的今天
十二月
22日
  • 微信
  • 交流学习，资料分享
  • weinxin
  • 个人淘宝
  • 店铺名：言曌博客咨询部

  • (部分商品未及时上架淘宝)
avatar

发表评论

avatar 登录者：匿名
匿名评论，评论回复后会有邮件通知

  

已通过评论：0   待审核评论数：0