How to perform a range query in elastic search with java Hight level rest client?

Question

Hi Currently I am having an elastic search index and i am trying to perform a date range query on the documents within that index using java high level rest client library . However it doesn't work as expected. so the following is my index .I have to perform a range query on the field @timestamp.

{
  "took" : 6,
  "timed_out" : false,
  "_shards" : {
    "total" : 2,
    "successful" : 2,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 229081,
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "mep-reports-2019.09.11",
        "_type" : "doc",
        "_id" : "68e8e03f-baf8-4bfc-a920-58e26edf835c-353899837500",
        "_score" : 1.0,
        "_source" : {
          "account_id" : "270d13e6-2f4f-4d51-99d5-92ffba5f0cb6",
          "inventory" : "SMS",
          "flight_name" : "test flight 001",
          "status" : "ENROUTE",
          "msg_text" : "Test !!!!!!!!!!!!!!1 elastic search:is.gd/YfzotY",
          "flight_id" : "68e8e03f-baf8-4bfc-a920-58e26edf835c",
          "submission_ts" : "1568197286",
          "recipient" : "353899837500",
          "o_error" : null,
          "nof_segments" : "-1",
          "campaign_id" : "0fae8662-bee9-46ac-9b3e-062f4ba55966",
          "campaign_name" : "Index search petri11",
          "@version" : "1",
          "sender" : "800111",
          "delivery_ts" : "0",
          "@timestamp" : "2019-09-11T10:21:26.000Z"
        }
      },
      {
        "_index" : "mep-reports-2019.09.11",
        "_type" : "doc",
        "_id" : "68e8e03f-baf8-4bfc-a920-58e26edf835c-353899837503",
        "_score" : 1.0,
        "_source" : {
          "account_id" : "270d13e6-2f4f-4d51-99d5-92ffba5f0cb6",
          "inventory" : "SMS",
          "flight_name" : "test flight 001",
          "status" : "ENROUTE",
          "msg_text" : "Test !!!!!!!!!!!!!!1 elastic searchh",
          "flight_id" : "68e8e03f-baf8-4bfc-a920-58e26edf835c",
          "submission_ts" : "1568197286",
          "recipient" : "353899837503",
          "o_error" : null,
          "nof_segments" : "-1",
          "campaign_id" : "0fae8662-bee9-46ac-9b3e-062f4ba55966",
          "campaign_name" : "Index search petri11",
          "@version" : "1",
          "sender" : "800111",
          "delivery_ts" : "0",
          "@timestamp" : "2019-09-11T10:21:26.000Z"
        }
      }
}

the followign is my java code that tries to do a range query on the @timestamp and it doesnt work . I am only pasting a part of the range query snippet below from my builder class named SearchRequestBuilder.

SearchRequestBuilder range(ZonedDateTime startDate, ZonedDateTime endDate) {

    System.out.println(startDate); // 2019-08-31T23:00Z
    System.out.println(endDate);   // 2019-09-17T13:25:18.154Z



    RangeQueryBuilder startRangeQueryBuilder = QueryBuilders.rangeQuery("@timestamp").gte(startDate);
    startRangeQueryBuilder.format("yyyy-MM-dd'T'HH:mm:ss.SSSZ");

    boolQueryBuilder.should(startRangeQueryBuilder);

    RangeQueryBuilder endRangeQueryBuilder = QueryBuilders.rangeQuery("@timestamp").lte(endDate);
    boolQueryBuilder.should(endRangeQueryBuilder);
    endRangeQueryBuilder.format("yyyy-MM-dd'T'HH:mm:ss.SSSZ");

    return this;
}

any idea how can i fix my range query to in order to work it correctly

the followign is a SearchRequestBuilder that builds an elastic search index

static class SearchRequestBuilder {

        private SearchRequest searchRequest;

        private SearchSourceBuilder searchSourceBuilder;

        private BoolQueryBuilder boolQueryBuilder;

        SearchRequestBuilder(String elasticSearchIndex) {
            this.searchRequest = new SearchRequest(elasticSearchIndex);
            this.searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
            this.searchSourceBuilder = new SearchSourceBuilder();
            this.searchRequest.source(searchSourceBuilder);
            this.boolQueryBuilder = QueryBuilders.boolQuery();
            this.searchSourceBuilder.query(this.boolQueryBuilder);
        }

        SearchRequestBuilder from(Integer pageNumber) {
            this.searchSourceBuilder.from(pageNumber);
            return this;
        }

        SearchRequestBuilder size(Integer pageSize) {
            this.searchSourceBuilder.size(pageSize);
            return this;
        }

        SearchRequestBuilder timeOut(long duration, TimeUnit timeUnit) {
            this.searchSourceBuilder.timeout(new TimeValue(duration, TimeUnit.SECONDS));
            return this;
        }

        SearchRequestBuilder range(ZonedDateTime startDate, ZonedDateTime endDate) {

            System.out.println(startDate);
            System.out.println(endDate);
        //  this.boolQueryBuilder.must(QueryBuilders.rangeQuery("@timestamp").from(startDate,true).to(endDate));

            //QueryBuilders.rangeQuery("FieldName").from(startDate,true).to(endDate));


            RangeQueryBuilder startRangeQueryBuilder = QueryBuilders.rangeQuery("@timestamp").gte(startDate);
            startRangeQueryBuilder.format("yyyy-MM-dd'T'HH:mm:ss.SSSZ");

            boolQueryBuilder.should(startRangeQueryBuilder);

            RangeQueryBuilder endRangeQueryBuilder = QueryBuilders.rangeQuery("@timestamp").lte(endDate);
            boolQueryBuilder.should(endRangeQueryBuilder);
            endRangeQueryBuilder.format("yyyy-MM-dd'T'HH:mm:ss.SSSZ");



            return this;
        }

        SearchRequestBuilder msisdn(String msisdn) {
            if (msisdn != null) {
                if (msisdn.indexOf("*") >= 0) {
                    WildcardQueryBuilder msisdnWildCardQueryBuilder = new WildcardQueryBuilder("recipient", msisdn);
                    boolQueryBuilder.must(msisdnWildCardQueryBuilder);
                } else {
                    MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("recipient", msisdn);
                    boolQueryBuilder.must(matchQueryBuilder);
                }
            }
            return this;
        }

        SearchRequestBuilder accountId(UUID accountId) {
            if (accountId != null) {
                MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("account_id", accountId.toString());
                boolQueryBuilder.must(matchQueryBuilder);
            }
            return this;
        }

        SearchRequestBuilder campaignId(UUID campaignId) {
            if (campaignId != null) {
                MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("campaign_id", campaignId.toString());
                boolQueryBuilder.must(matchQueryBuilder);
            }
            return this;
        }

        SearchRequestBuilder flightId(UUID flightId) {
            if (flightId != null) {
                MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("flight_id", flightId.toString());
                boolQueryBuilder.must(matchQueryBuilder);
            }
            return this;
        }

        SearchRequestBuilder cdrStatus(String cdrStatus) {
            if (cdrStatus != null) {
                MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("status", cdrStatus);
                boolQueryBuilder.must(matchQueryBuilder);
            }
            return this;
        }

        SearchRequestBuilder inventoryCode(String inventoryCode) {
            if (inventoryCode != null) {
                MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("inventory", inventoryCode);
                boolQueryBuilder.must(matchQueryBuilder);
            }
            return this;
        }

        SearchRequestBuilder sort() {
            searchSourceBuilder.sort(new FieldSortBuilder("@timestamp").order(SortOrder.DESC));
            return this;
        }

        SearchRequest build() {
            return this.searchRequest;
        }

    }

thank you so much

Answer 1

The main problem is that the should() clauses are not taken into account because the other clauses are must() (See why here). If you use filter() clauses instead of must() then it will work.

Example with the flightId method follows:

    SearchRequestBuilder flightId(UUID flightId) {
        if (flightId != null) {
            MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("flight_id", flightId.toString());
            boolQueryBuilder.filter(matchQueryBuilder);
        }                       ^
        return this;            |
    }                           |
                           change this