Hi Klipfolio!

For a number of services, we are unfortunately unable to paginate them because they use something called cursor-based pagination, which is where the datasource is paginated by passing in a value from the current page, and is not a predictable number, like a numeric offset or page number.

To get around this, you can actually set up your own custom script, and push this data to a Google Sheet, where Klipfolio can create a datasource out of this sheet.

Now, as a forewarning, the below example only works for this one Hubspot endpoint here, and to modify it at all would require someone with at least basic knowledge of web programming(JavaScript) and APIs.

However, this can be used as a starting point for moving forward.

Hope this helps!

This example was setup using Google Scripts.
Note that you need to first upload a file to your Google Drive that has .json extension. Whatever the name of the file, you must fill this in the top of the script, along with the API key, and the maximum number of pages you want the datasource to have.

function runQuery() {

  /* Fill out this section here with your own parameters*/


  var apiKey="demo";//Test account API key


  var max=5;//Max number of pages to iterate through, so script does not hang forever


  var fileName='Test JSON actual file.json';//The actual filename of the existing file, in your Google Drive


  /* End of section*/




  //Run query once, to see if there are more pages, and to get offset value if so


  var url = "https://api.hubapi.com/companies/v2/companies/paged?hapikey="+apiKey;


  var options = {


      "async": true,


      "crossDomain": true,


      "method" : "GET",


      'muteHttpExceptions': true,


      "headers" : {}


    };


  var response = UrlFetchApp.fetch(url, options);





  //Get specific info from response


  var text=response.getContentText();//Get full query response


  var offset=text.substring(//Get first offset, if there is one


    text.indexOf('"offset":')+9, 


    text.indexOf(',',text.indexOf('"offset":'))


  );


  var hasMore=text.substring(//Find out if there are more pages


    text.indexOf('"has-more":')+11, 


    text.indexOf(',',text.indexOf('"has-more":'))


  );





  var iteration=0;//start at 0th iteration





  Logger.log(hasMore);


  Logger.log(offset);


  Logger.log(iteration<max && hasMore);





  var startText='{  "companies": [';//Text at start of file. This will be specific to the service and API query used


  var endText=']}';//Text at the end of file. This will be specific to the service and API query used





  var fileText=startText;//File contents, to append data to





  while( iteration<max && hasMore)


  {


    url = "https://api.hubapi.com/companies/v2/companies/paged?hapikey="+apiKey+"&offset="+offset;


    response = UrlFetchApp.fetch(url, options);


    text=response.getContentText();


    offset=text.substring(


      text.indexOf('"offset":')+9, 


      text.indexOf(',',text.indexOf('"offset":'))


    );





    hasMore=text.substring(//Find out if there is more data


      text.indexOf('"has-more":')+11, 


      text.indexOf(',',text.indexOf('"has-more":'))


    );





    var content=text.substring(


      text.indexOf('[')+1, 


      text.lastIndexOf(']')


    );





    if(iteration==0){ Logger.log(content);}else{}





    fileText+=content+",";


    iteration++;    


  }





  fileText=fileText.substring(0, fileText.length - 1);//remove last ,


  fileText+=endText;//Add end text





  DriveApp.getFilesByName(fileName).next().setContent(fileText);//Update file

}