Web service only let me get 1000 rows at a time but the total is over 30000

Question

I am using some web services provided by Netsuite https://system.netsuite.com/help/helpcenter/en_US/Output/Help/SuiteFlex/WebServices/STP_searchMore.html#1087957

It only let me get 1000 rows at a time and then I need to perform a second search for the next set of 1000 rows and so on. There's some example code but it only returns the second set of rows, I am not sure on how to get the third, fourth and so on.

My code so far is:

private void getAllCustomers()
{
    // Instantiate a search object for customers.
    CustomerSearch custSearch = new CustomerSearch();
    CustomerSearchBasic custSearchBasic = new CustomerSearchBasic();


    // Search the customer status which is a list field (16,13,15)
    String statusKeysValue = "16,13,15";
    SearchMultiSelectField status = null;
    if (statusKeysValue != null && !statusKeysValue.Trim().Equals(""))
    {
        status = new SearchMultiSelectField();
        status.@operator = SearchMultiSelectFieldOperator.anyOf;
        status.operatorSpecified = true;
        string[] nskeys = statusKeysValue.Split(new Char[] { ',' });

        RecordRef[] recordRefs = new RecordRef[statusKeysValue.Length];
        for (int i = 0; i < nskeys.Length; i++)
        {
            RecordRef recordRef = new RecordRef();
            recordRef.internalId = nskeys[i];
            recordRefs[i] = recordRef;
        }
        status.searchValue = recordRefs;
        custSearchBasic.entityStatus = status;
    }

    custSearch.basic = custSearchBasic;

    // Invoke search() web services operation
    SearchResult response = _service.search(custSearch);            

     // Process response
    if (response.status.isSuccess)
    {
        // Process the records returned in the response
        // Here I get the first 1000 records
        processGetAllCustomersResponse(response);

        // Since pagination controls what is returned, check to see
        // if there are anymore pages to retrieve.

        SearchResult seachMoreResult = searchMore(response);

        if (seachMoreResult != null)
        {
            // Process response
            if (seachMoreResult.status.isSuccess)
            {
                // Here I get the next 1000 records
                    processGetAllCustomersResponse(seachMoreResult);

                // My problem now is to get the third set of 1000 customers, then the fourth and so on till I got all 34500 something
            }
            else
            {

            }

        }
    }
    else
    {

    }
}

private SearchResult searchMore(SearchResult response)
{
    // Keep getting pages until there are no more pages to get
    while (response.totalRecords > (response.pageSize * response.pageIndex))
    {
        return _service.searchMore(response.pageIndex + 1);
    }

    return null;
}

In processGetAllCustomersResponse I simply insert the rows in another database which works fine (apart from not getting all the rows I want).

Answer 1

I wrote this alternative to the NetSuite-supplied examples. This example retrieves TimeBill's based on their create date.

    /// <summary>
    /// Return the list of time bills whose last modified date is within 
    /// the indicated date range.
    /// </summary>
    /// <param name="from">Required from date</param>
    /// <param name="to">Optional to date</param>
    /// <returns>List of time bills</returns>
    public IEnumerable<TimeBill> GetTimeBills(DateTime from, DateTime to)
    {
        _log.Debug(String.Format("Enter TimeBill(DateTime from='{0}', DateTime to='{1}')", from, to));

        // Build search criteria.
        TimeBillSearch search = new TimeBillSearch();
        TimeBillSearchBasic searchBasic = new TimeBillSearchBasic();
        SearchDateField searchDateField = new SearchDateField();
        searchDateField.@operator = SearchDateFieldOperator.within;
        searchDateField.operatorSpecified = true;
        searchDateField.searchValue = from;
        searchDateField.searchValueSpecified = true;
        searchDateField.searchValue2 = to;
        searchDateField.searchValue2Specified = true;
        searchBasic.dateCreated = searchDateField;
        search.basic = searchBasic;

        return this.Get<TimeBill>(search);
    }  

    /// <summary>
    /// Perform a paged search and convert the returned record to the indicated type.
    /// </summary>
    private IEnumerable<T> Get<T>(SearchRecord searchRecord)
    {
        _log.Debug("Enter Get<T>(SearchRecord searchRecord)");

        // This is returned.
        List<T> list = new List<T>();

        // The suitetalk service return this.
        SearchResult result = null;

        using (ISuiteTalkService service = SuiteTalkFactory.Get<SuiteTalkService>())
        {
            do
            {
                // .search returns the first page of data.
                if (result == null)
                {
                    result = service.search(searchRecord);
                }
                else // .searchMore returns the next page(s) of data.
                {
                    result = service.searchMoreWithId(result.searchId, result.pageIndex + 1);
                }

                if (result.status.isSuccess)
                {
                    foreach (Record record in result.recordList)
                    {
                        if (record is T)
                        {
                            list.Add((T)Convert.ChangeType(record, typeof(T)));
                        }
                    }
                }
            }
            while (result.pageIndex < result.totalPages);
        }
        return list;
    }

Answer 2

i have changed SearchMore function ,now it will return a list of all responses , you need to change getAllCustomer function accordingly

EDIT : updated getAllCustomer Also

 private void getAllCustomers()
    {
        // Instantiate a search object for customers. 
        CustomerSearch custSearch = new CustomerSearch();
        CustomerSearchBasic custSearchBasic = new CustomerSearchBasic();


        // Search the customer status which is a list field (16,13,15) 
        String statusKeysValue = "16,13,15";
        SearchMultiSelectField status = null;
        if (statusKeysValue != null && !statusKeysValue.Trim().Equals(""))
        {
            status = new SearchMultiSelectField();
            status.@operator = SearchMultiSelectFieldOperator.anyOf;
            status.operatorSpecified = true;
            string[] nskeys = statusKeysValue.Split(new Char[] { ',' });

            RecordRef[] recordRefs = new RecordRef[statusKeysValue.Length];
            for (int i = 0; i < nskeys.Length; i++)
            {
                RecordRef recordRef = new RecordRef();
                recordRef.internalId = nskeys[i];
                recordRefs[i] = recordRef;
            }
            status.searchValue = recordRefs;
            custSearchBasic.entityStatus = status;
        }

        custSearch.basic = custSearchBasic;

        // Invoke search() web services operation 
        SearchResult response = _service.search(custSearch);

        // Process response 
        if (response.status.isSuccess)
        {
            // Process the records returned in the response 
            // Here I get the first 1000 records 
            processGetAllCustomersResponse(response);

            // Since pagination controls what is returned, check to see 
            // if there are anymore pages to retrieve. 

            List<SearchResult> seachMoreResult = searchMore(response);

            if (seachMoreResult != null)
            {
                foreach (SearchResult sr in seachMoreResult)
                {
                    if (sr.status.isSuccess)
                    {
                        // Here I get the next 1000 records 
                        processGetAllCustomersResponse(sr);

                        // My problem now is to get the third set of 1000 customers, then the fourth and so on till I got all 34500 something 
                    }
                    else
                    {

                    }
                }
                // Process response 


            }
        }
        else
        {

        }
    }

private IEnumerable<SearchResult> searchMore(SearchResult response)
    {
        // Keep getting pages until there are no more pages to get    
        int tempTotalRecords = response.totalRecords;
        int pageSize = response.pageSize * response.pageIndex;
        SearchResult tempResponse = null;
        List<SearchResult> records = new List<SearchResult>();
        while (tempTotalRecords > (pageSize))
        {
            SearchResult tempResponse = _service.searchMore(response.pageIndex + 1); 

            if (tempResponse.totalRecords > tempResponse.pageSize * tempResponse.pageIndex)
            {
                tempTotalRecords = tempResponse.totalRecords;
                pageSize = tempResponse.pageSize * tempResponse.pageIndex;
                records.Add(response);
            }
            else
                records.Add(response);


        }

        return records;
    }