问题描述
我必须从 API 获取数据到 google 表格,该表格应该绘制近 1600 个条目。但是,执行在 6 分钟时停止,只绘制了大约 1000 个条目。我在 Apps 脚本中的初始代码是:
function myFunction() {
var spreadsheet=SpreadsheetApp.getActive();
var sheet=spreadsheet.getActiveSheet();
var nextPage=1;
sheet.clear();
var headerRow=["ID","NAME","SOURCE","STATUS","PRICING_MIN_PRICE","PRICING_MAX_PRICE","LOACTION_COUNTRY","LOACTION_LOCALITY","IMAGES_COUNT","VIDEOS_COUNT","FEATURES_COUNT"];
sheet.appendRow(headerRow);
sheet.getRange(
spreadsheet.getCurrentCell().getRow(),1,sheet.getMaxColumns()).activate();
spreadsheet.getActiveRangeList().setFontWeight('bold');
var curr;
while(nextPage){
curr=nextPage;
var apiURL=`https://base.Amberstudent.com/api/v0/inventories?p=${curr}&limit=10&sort_key=relevance&sort_order=desc&statuses=active`;
var response=UrlFetchApp.fetch(apiURL);
var json=response.getContentText();
var dataPoints=JSON.parse(json);
var resArray=dataPoints.data.result;
for(var i=0;i<resArray.length;i++){
var id=resArray[i].id!=null?resArray[i].id:"";
var name=resArray[i].name!=null?resArray[i].name:"";
var source=resArray[i].source!=null?resArray[i].source:"";
var status=resArray[i].status!=null?resArray[i].status:"";
var pricing_min_price=resArray[i].pricing?resArray[i].pricing.min_price:"";
var pricing_max_price=resArray[i].pricing?resArray[i].pricing.max_price:"";
var location_country=(resArray[i].location&&resArray[i].location.country)?resArray[i].location.country.long_name:"";
var location_locality=(resArray[i].location&&resArray[i].location.locality)?resArray[i].location.locality.long_name:"";
var images_count=resArray[i].images.length;
var vid_count=resArray[i].videos.length;
var feature_count=resArray[i].features.length;
var row=[id,name,source,status,pricing_min_price,pricing_max_price,location_country,location_locality,images_count,vid_count,feature_count];
sheet.appendRow(row);
}
nextPage=dataPoints.data.Meta.next; //for every page the nextPage stores the value of the next page,and for the last page (159 approx),nextPage=null
}
}
正如我已经提到的那样,这不起作用。在互联网上搜索后,我找到了一些绕过执行时间的方法,并修改了我的代码,如下所示:
var spreadsheet=SpreadsheetApp.getActive();
var sheet=spreadsheet.getActiveSheet();
var nextPage=1; //set nextPage as a global variable so that it can be accessed by all functions
function isTimeUp(today) {
var Now = new Date();
return Now.getTime() - today.getTime() > 300000; //setting up a limit of 5 minutes
}
function myFunction() {
sheet.clear();
var today=new Date();
var headerRow=["ID",sheet.getMaxColumns()).activate();
spreadsheet.getActiveRangeList().setFontWeight('bold');
var curr;
while(nextPage){
if (isTimeUp(today)) {
// schedule a trigger for a different function
ScriptApp.newTrigger("repeatFunction")
.timeBased()
.everyMinutes(5)
.create();
break;
}
curr=nextPage;
var apiURL=`https://base.Amberstudent.com/api/v0/inventories?p=${curr}&limit=10&sort_key=relevance&sort_order=desc&statuses=active`;
var response=UrlFetchApp.fetch(apiURL);
var json=response.getContentText();
var dataPoints=JSON.parse(json);
var resArray=dataPoints.data.result;
for(var i=0;i<resArray.length;i++){
var id=resArray[i].id!=null?resArray[i].id:"";
var name=resArray[i].name!=null?resArray[i].name:"";
var source=resArray[i].source!=null?resArray[i].source:"";
var status=resArray[i].status!=null?resArray[i].status:"";
var pricing_min_price=resArray[i].pricing?resArray[i].pricing.min_price:"";
var pricing_max_price=resArray[i].pricing?resArray[i].pricing.max_price:"";
var location_country=(resArray[i].location&&resArray[i].location.country)?resArray[i].location.country.long_name:"";
var location_locality=(resArray[i].location&&resArray[i].location.locality)?resArray[i].location.locality.long_name:"";
var images_count=resArray[i].images.length;
var vid_count=resArray[i].videos.length;
var feature_count=resArray[i].features.length;
var row=[id,nextPage=null
}
}
function repeatFunction(){
while(nextPage){
var curr=nextPage;
var apiURL=`https://base.Amberstudent.com/api/v0/inventories?p=${curr}&limit=10&sort_key=relevance&sort_order=desc&statuses=active`;
var response=UrlFetchApp.fetch(apiURL);
var json=response.getContentText();
var dataPoints=JSON.parse(json);
var resArray=dataPoints.data.result;
for(var i=0;i<resArray.length;i++){
var id=resArray[i].id!=null?resArray[i].id:"";
var name=resArray[i].name!=null?resArray[i].name:"";
var source=resArray[i].source!=null?resArray[i].source:"";
var status=resArray[i].status!=null?resArray[i].status:"";
var pricing_min_price=resArray[i].pricing?resArray[i].pricing.min_price:"";
var pricing_max_price=resArray[i].pricing?resArray[i].pricing.max_price:"";
var location_country=(resArray[i].location&&resArray[i].location.country)?resArray[i].location.country.long_name:"";
var location_locality=(resArray[i].location&&resArray[i].location.locality)?resArray[i].location.locality.long_name:"";
var images_count=resArray[i].images.length;
var vid_count=resArray[i].videos.length;
var feature_count=resArray[i].features.length;
var row=[id,nextPage=null
if (nextPage==null) {
var triggers = ScriptApp.getProjectTriggers();
for (var i = 0; i < triggers.length; i++) {
// delete all triggers
ScriptApp.deleteTrigger(triggers[i]);
}
break;
}
}
}
我尝试将 nextPage 设置为全局变量并设置一个触发器,该触发器每 5 分钟调用一次 repeatFunction 方法。然而,这会产生类似无限循环的东西。数据不断被添加到电子表格中。我不知道如何解决这个问题,因为我对 Google App Scripts 的概念及其用法不熟悉。请帮我解决这个问题。如有需要,请询问更多详情。谢谢!
解决方法
我相信你的目标如下。
- 您的
myFunction()
工作正常。但是,您希望降低脚本的流程成本。
修改点:
- 在您的脚本中,循环中使用了
appendRow
。我认为这可能是您出现问题的原因之一。- 这也可以在 the benchmark 处看到。
- 虽然我不确定您要使用的 API 的详细信息,但在端点的查询参数中,使用了
limit=10
。并且,从I have to fetch data from an API to google sheets which is supposed to plot nearly 1600 entries.
开始,如果limit=10
是一个 API 调用的值数,则需要完成 160 个请求。我认为这将是您问题的另一个原因。在这种情况下,我想建议修改limit
的值。- 在此修改中,使用了
limit=2000
。在我的测试中,没有发生错误。但如果出现错误,请修改此值并再次测试。
- 在此修改中,使用了
当以上几点反映到你的脚本中时,它变成如下。
修改后的脚本:
在此修改中,请按如下方式修改您的 myFunction()
。
function myFunction2() {
var spreadsheet = SpreadsheetApp.getActive();
var sheet = spreadsheet.getActiveSheet();
var nextPage = 1;
sheet.clear();
var headerRow = ["ID","NAME","SOURCE","STATUS","PRICING_MIN_PRICE","PRICING_MAX_PRICE","LOACTION_COUNTRY","LOACTION_LOCALITY","IMAGES_COUNT","VIDEOS_COUNT","FEATURES_COUNT"];
sheet.appendRow(headerRow);
sheet.getRange(spreadsheet.getCurrentCell().getRow(),1,sheet.getMaxColumns()).activate();
spreadsheet.getActiveRangeList().setFontWeight('bold');
var ar = [];
var curr;
while (nextPage) {
curr = nextPage;
var apiURL = `https://base.amberstudent.com/api/v0/inventories?p=${curr}&limit=2000&sort_key=relevance&sort_order=desc&statuses=active`;
var response = UrlFetchApp.fetch(apiURL);
var json = response.getContentText();
var dataPoints = JSON.parse(json);
var resArray = dataPoints.data.result;
for (var i = 0; i < resArray.length; i++) {
var id = resArray[i].id != null ? resArray[i].id : "";
var name = resArray[i].name != null ? resArray[i].name : "";
var source = resArray[i].source != null ? resArray[i].source : "";
var status = resArray[i].status != null ? resArray[i].status : "";
var pricing_min_price = resArray[i].pricing ? resArray[i].pricing.min_price : "";
var pricing_max_price = resArray[i].pricing ? resArray[i].pricing.max_price : "";
var location_country = (resArray[i].location && resArray[i].location.country) ? resArray[i].location.country.long_name : "";
var location_locality = (resArray[i].location && resArray[i].location.locality) ? resArray[i].location.locality.long_name : "";
var images_count = resArray[i].images.length;
var vid_count = resArray[i].videos.length;
var feature_count = resArray[i].features.length;
var row = [id,name,source,status,pricing_min_price,pricing_max_price,location_country,location_locality,images_count,vid_count,feature_count];
ar.push(row);
}
nextPage = dataPoints.data.meta.next; //for every page the nextPage stores the value of the next page,and for the last page (159 approx),nextPage=null
}
sheet.getRange(sheet.getLastRow() + 1,ar.length,ar[0].length).setValues(ar);
}
注意:
- 在此修改中,使用
setValues
代替appendRow
,并使用limit=2000
。在这种情况下,1,585 个值由一个 API 调用检索。而在我的环境中,上述修改脚本的处理时间约为 20 秒。