网站导航
> 文章列表 > springboot实现ES多种分页方式

springboot实现ES多种分页方式

网友: 文章列表 2024-03-22 03:11:47

springboot实现ES多种分页方式

es有两种分页方式

  • from+size 浅分页
  • scroll 深分页

在这里就不展开介绍了，网络上有很多相关介绍。
我们还可以通过维护自增主键来实现分页
下面结合Springboot，通过代码展现实现es分页的三种实现方式

需要引入依赖：

        <dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-data-elasticsearch</artifactId><version>2.2.12.RELEASE</version></dependency>

代码：

ESBase.java

@Service
@Slf4j
public class ESBase<T, U extends UserIdxESReq> {//    @Autowired
//    ElasticsearchRepository<T, String> elasticsearchRepository;@AutowiredElasticsearchTemplate elasticsearchTemplate;@Value("${scroll.timeout.ms:60000}")private long scrollTimeoutMs;public NativeSearchQueryBuilder getNativeSearchQueryBuilder(U esReq) {/*NativeSearchQueryBuilder searchQueryBuilder = new NativeSearchQueryBuilder();BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();if (StrUtil.isNotBlank(taskItemAddReq.getSettleNo())) {TermsQueryBuilder deptNoTermsQb = QueryBuilders.termsQuery(CommonConstant.SETTLE_NO, portBillDetailFactoringFreightReq.getSettleNo());boolQueryBuilder.must(deptNoTermsQb);}searchQueryBuilder.withQuery(boolQueryBuilder);*/return new NativeSearchQueryBuilder();}protected NativeSearchQuery getNativeSearchQuery(U esReq) {String reqStr = JSONUtil.toJsonStr(esReq);String reqMd5 = SecureUtil.md5(reqStr);NativeSearchQuery searchQuery = getNativeSearchQueryBuilder(esReq)
//                .withSort(SortBuilders.fieldSort("pin").order(SortOrder.ASC)).withPageable(PageRequest.of(esReq.getPageNo(),esReq.getPageSize())).build();searchQuery.setPreference(reqMd5);return searchQuery;}/*** 通过自增主键来实现分页* @param start* @param end* @param clazz* @return*/public List<T> autoIncrementKeyPage(long start, long end, Class<T> clazz) {NativeSearchQueryBuilder searchQueryBuilder = new NativeSearchQueryBuilder();BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("key").gte(start).lte(end);boolQueryBuilder.must(rangeQueryBuilder);searchQueryBuilder.withQuery(boolQueryBuilder);NativeSearchQuery searchQuery = searchQueryBuilder.withSort(SortBuilders.fieldSort("key").order(SortOrder.ASC)).build();List<T> list = elasticsearchTemplate.queryForList(searchQuery, clazz);return list;}/*** from+size 浅分页* @param esReq* @return*/public EsQueryPojo<List<T>> fromSizePage(U esReq, Class<T> clazz) {Page<T> page = elasticsearchTemplate.queryForPage(getNativeSearchQuery(esReq), clazz);
//        Page<T> page = elasticsearchRepository.search(getNativeSearchQuery(esReq));EsQueryPojo<List<T>> listEsQueryPojo = new EsQueryPojo<>();listEsQueryPojo.setRows(page.getContent());//当前分页返回的结果listEsQueryPojo.setPage(page.getNumber());//当前页码listEsQueryPojo.setTotal(page.getTotalPages());//分页总数listEsQueryPojo.setRecords(page.getTotalElements());//元素总数listEsQueryPojo.setSize(page.getSize());//当前页返回的数量return listEsQueryPojo;}/*** scroll 深分页,另一种方式是通过维护自增主键* @param esReq* @return*/public EsScrollQueryPojo<List<T>> scrollPage(U esReq, Class<T> clazz) {EsScrollQueryPojo<List<T>> listEsQueryPojo = new EsScrollQueryPojo<>();ScrolledPage<T> scrolledPage;try {if (StringUtils.isEmpty(esReq.getScrollId())) {//第一次查询scrolledPage = elasticsearchTemplate.startScroll(scrollTimeoutMs, getNativeSearchQuery(esReq), clazz);//游标的过期时间是指两个查询之间的间隔，而不是整个查询过程} else {//带scrollId查询scrolledPage = elasticsearchTemplate.continueScroll(esReq.getScrollId(), scrollTimeoutMs, clazz);}listEsQueryPojo.setRows(scrolledPage.getContent());//当前分页返回的结果listEsQueryPojo.setPage(scrolledPage.getNumber());//当前页码，在深度分页不生效listEsQueryPojo.setTotal(scrolledPage.getTotalPages());//分页总数listEsQueryPojo.setRecords(scrolledPage.getTotalElements());//元素总数listEsQueryPojo.setSize(scrolledPage.getSize());//当前返回的数量listEsQueryPojo.setScrollId(scrolledPage.getScrollId());} catch (Exception e) {log.error(ExceptionEnum.ES_SCROLL_PAGE_EXCEPTION.getMsg() + e.getMessage());throw e;}return listEsQueryPojo;}public void clearScroll(String scrollId) {if (scrollId != null) {elasticsearchTemplate.clearScroll(scrollId);}}public void deleteIndexData(Class<T> clazz) {if (elasticsearchTemplate.indexExists(clazz)) {DeleteQuery deleteQuery = new DeleteQuery();deleteQuery.setQuery(QueryBuilders.matchAllQuery());elasticsearchTemplate.delete(deleteQuery, clazz);//删除语句log.info("clearing index data");} else {log.warn("index not exsit");}}public long countIndex(Class<T> clazz) {if (elasticsearchTemplate.indexExists(clazz)) {NativeSearchQueryBuilder searchQueryBuilder = new NativeSearchQueryBuilder();searchQueryBuilder.withQuery(QueryBuilders.matchAllQuery());return elasticsearchTemplate.count(searchQueryBuilder.build(), clazz);} else {log.warn("index not exsit");return 0L;}}
}

UserIdxES.java

@Service
@Slf4j
public class UserIdxES extends ESBase<UserIdx, UserIdxESReq> {@Overrideprotected NativeSearchQuery getNativeSearchQuery(UserIdxESReq esReq) {String reqStr = JSONUtil.toJsonStr(esReq);String reqMd5 = SecureUtil.md5(reqStr);NativeSearchQuery searchQuery = getNativeSearchQueryBuilder(esReq)
//                .withSort(SortBuilders.fieldSort("pin").order(SortOrder.ASC)).withPageable(PageRequest.of(esReq.getPageNo(),esReq.getPageSize())).build();searchQuery.setPreference(reqMd5);return searchQuery;}
}

UserIdx.java

@AllArgsConstructor
@NoArgsConstructor
@Data
@Document(indexName = "user_idx", type = "user_type")
public class UserIdx {@Idprivate String id;@Field(name = "pin", type = FieldType.Keyword)private String pin;@Field(name = "name", type = FieldType.Keyword)private String name;@Field(name = "sex", type = FieldType.Keyword)private String sex;@Field(name = "age", type = FieldType.Keyword)private String age;@Field(name = "score", type = FieldType.Keyword)private String score;
}

UserIdxESReq.java

@NoArgsConstructor
@AllArgsConstructor
@Data
@ApiModel(value = "UserIdxESReq", description = "UserIdxESReq request bean")
public class UserIdxESReq {@ApiModelProperty(example = "0")private int pageNo;@ApiModelProperty(example = "10")private int pageSize;@ApiModelProperty(value = "分页ID", name = "scrollId")private String scrollId;
}

测试
ESBaseTest.java

@SpringBootTest
@RunWith(SpringRunner.class)
//TODO
public class ESBaseTest {@AutowiredESBase<UserIdx, UserIdxESReq> esBase;@AutowiredESBase<UserWithKeyIdx, UserIdxESReq> esBase1;@Testpublic void fromSizePageTest() {UserIdxESReq UserIdxEsReq = new UserIdxESReq();UserIdxEsReq.setPageNo(0);UserIdxEsReq.setPageSize(10);esBase.fromSizePage(UserIdxEsReq, UserIdx.class);}@Testpublic void scrollPageTest() {UserIdxESReq UserIdxEsReq = new UserIdxESReq();UserIdxEsReq.setPageNo(0);UserIdxEsReq.setPageSize(10);esBase.scrollPage(UserIdxEsReq, UserIdx.class);}@Testpublic void autoIncrementIdPageTest() {List<UserIdx> list = esBase.autoIncrementKeyPage(0, 1000, UserIdx.class);Assert.assertNotNull(list);}@Testpublic void deleteIndexDataTest() {esBase.deleteIndexData(UserIdx.class);}@Testpublic void countIndexTest() {long cnt = esBase1.countIndex(UserWithKeyIdx.class);Assert.assertEquals(cnt,100);}
}

UserIdxESBaseTest.java

@SpringBootTest
@RunWith(SpringRunner.class)
public class UserIdxESBaseTest {@AutowiredUserIdxES esService;@Testpublic void fromSizePageTest() {UserIdxESReq esReq = new UserIdxESReq();
//        esReq.setAge(123);
//        esReq.setPageNo(0);esReq.setPageSize(40);int pages = 0;int currentPage = 0;do {esReq.setPageNo(currentPage);EsQueryPojo<List<UserIdx>> esScrollQueryPojo = esService.fromSizePage(esReq,UserIdx.class);pages = esScrollQueryPojo.getTotal();currentPage = esScrollQueryPojo.getPage();List<UserIdx> list = esScrollQueryPojo.getRows();
//            currentPage++;} while (currentPage++ < pages);}@Testpublic void scrollPageTest() {UserIdxESReq esReq = new UserIdxESReq();esReq.setPageNo(0);esReq.setPageSize(10);
//        esService.scrollPage(esReq, TaskItemAddIndex.class);EsScrollQueryPojo<List<UserIdx>> esScrollQueryPojo = esService.scrollPage(esReq, UserIdx.class);List<UserIdx> taskItemAddDtoList = (List<UserIdx>) esScrollQueryPojo.getRows();Assert.assertNotNull(taskItemAddDtoList);}
}

附录：
2. http 测试笔记

测试es:
spring.data.elasticsearch.cluster-name=docker-cluster
spring.data.elasticsearch.cluster-nodes=x.x.x.x:9300查看：curl -X GET http://x.x.x.x:9200/_cat
创建索引：curl -X PUT http://x.x.x.x:9200/user_with_key_idx
删除索引：curl -XDELETE http://x.x.x.x:9200/user_with_key_idx
查看mappingcurl -XGET http://x.x.x.x:9200/user_with_key_idx?prettycurl -XGET http://x.x.x.x:9200/user_with_key_idx?pretty
查询数据：curl -XGET http://x.x.x.x:9200/user_with_key_idx/user_with_key_type/_search?pretty深度分页scroll游标使用curl -XGET http://x.x.x.x:9200/user_with_key_idx/_search?scroll=1m \\-H "Content-Type: application/json" \\-d '{"query": { "match_all": {}},"sort" : ["_doc"],"size":  1}'curl -XGET http://x.x.x.x:9200/_search/scroll?pretty \\
-H "Content-Type: application/json" \\
-d '{"scroll": "1m","scroll_id" : "DnF1ZXJ5VGhlbkZldGNoBQAAAAAAAAJFFlNsSXFUcS1qUzFlYWFFWGpWUkE5RncAAAAAAAACRxZTbElxVHEtalMxZWFhRVhqVlJBOUZ3AAAAAAAAAkYWU2xJcVRxLWpTMWVhYUVYalZSQTlGdwAAAAAAAAJIFlNsSXFUcS1qUzFlYWFFWGpWUkE5RncAAAAAAAACSRZTbElxVHEtalMxZWFhRVhqVlJBOUZ3"
}'创建mapping
curl -XPOST 'http://x.x.x.x:9200/user_with_key_idx/user_with_key_type/_mapping?pretty' -H "Content-Type: application/json"  \\
-d '
{"user_with_key_type": {"properties": {"pin": {"type": "keyword"},"name": {"type": "keyword"},"sex": {"type": "keyword"},"age": {"type": "keyword"},"score": {"type": "keyword"},"key": {"type": "long"}}}
}'user_with_key_idx/user_with_key_type/_mappingPUT user_with_key_idx
{"mappings": {"user_with_key_type": {"properties": {"pin": {"type": "keyword"},"name": {"type": "keyword"},"sex": {"type": "keyword"},"age": {"type": "keyword"},"score": {"type": "keyword"},"key": {"type": "long"}}}}
}插入数据：curl -XPOST http://x.x.x.x:9200/user_with_key_idx/user_with_key_type?pretty -H "Content-Type: application/json" \\-d '{"pin": "pin_001","phone": "1234567890","name": "lisi","bizData": "","sex": 1,"age": 18}'curl -XPOST http://x.x.x.x:9200/user_with_key_idx/user_with_key_type?pretty -H "Content-Type: application/json" \\-d '{"pin": "pin_004","phone": "1234567890","name": "lisi02","bizData": "","sex": 1,"age": 18}'查询数据：curl -XGET http://x.x.x.x:9200/user_with_key_idx/user_with_key_type/_search?pretty
  1. 深分页和浅分页的使用区别
  • scroll 深分页
curl -XGET http://11.51.197.4:9200/user_idx/user_type/_search?pretty&scroll=3m \\
-d '{"query":{"match_all": {}},"from": 0,"size": 3,"sort": [{"id": {"order": "asc"}}]
}'curl -XGET http://11.51.197.4:9200/_search/scroll \\
-d '{"scroll_id":"DnF1ZXJ5VGhlbkZldGNoBQAAAAAAAABvFlNsSXFUcS1qUzFlYWFFWGpWUkE5RncAAAAAAAAAcRZTbElxVHEtalMxZWFhRVhqVlJBOUZ3AAAAAAAAAHAWU2xJcVRxLWpTMWVhYUVYalZSQTlGdwAAAAAAAAByFlNsSXFUcS1qUzFlYWFFWGpWUkE5RncAAAAAAAAAcxZTbElxVHEtalMxZWFhRVhqVlJBOUZ3","scroll": "3m"
}'

-浅分页，调整索引的配置项index.max_result_window

PUT index_name/_settings
{
"index.max_result_window":10000
}
网络标签：

相关问题