如何批处理行数据并发送单个 JSON 有效负载？

Question

我目前在 Google Sheet 上使用 Google Apps Script，它将单个行数据发送到 AWS API Gateway 以生成屏幕截图。目前，多个单个 JSON 负载请求导致一些 Lambda 函数失败。所以我想对行数据进行批处理，然后作为单个有效负载发送，这样单个 AWS Lambda 函数就可以执行并完成多个屏幕截图。

在迭代下面代码中每一行的数据后，如何对 JSON 有效负载进行批处理？

function S3payload () {
  var PAYLOAD_SENT = "S3 SCREENSHOT DATA SENT";
  
  var sheet = SpreadsheetApp.getActiveSheet(); // Use data from the active sheet
  
  // Add temporary column header for Payload Status new column entries
  sheet.getRange('E1').activate();
  sheet.getCurrentCell().setValue('payload status');
  
  var startRow = 2;                            // First row of data to process
  var numRows = sheet.getLastRow() - 1;        // Number of rows to process
  var lastColumn = sheet.getLastColumn();      // Last column
  var dataRange = sheet.getRange(startRow, 1, numRows, lastColumn) // Fetch the data range of the active sheet
  var data = dataRange.getValues();            // Fetch values for each row in the range
  
  // Work through each row in the spreadsheet
  for (var i = 0; i < data.length; ++i) {
    var row = data[i];  
    // Assign each row a variable   
    var index = row[0];     // Col A: Index Sequence Number
    var img = row[1];   // Col B: Image Row
    var url = row[2];      // Col C: URL Row
    var payloadStatus = row[lastColumn - 1];  // Col E: Payload Status (has the payload been sent)
  
    var siteOwner = "email@example.com";
    
    // Prevent from sending payload duplicates
    if (payloadStatus !== PAYLOAD_SENT) {  
        
      /* Forward the Contact Form submission to the owner of the site
      var emailAddress = siteOwner; 
      var subject = "New contact form submission: " + name;
      var message = message;*/
      
      //Send payload body to AWS API GATEWAY
      //var sheetid = SpreadsheetApp.getActiveSpreadsheet().getId(); // get the actual id
      //var companyname = SpreadsheetApp.getActiveSpreadsheet().getName(); // get the name of the sheet (companyname)
      
      var payload = {
        "img": img,
        "url": url
      };
      
      var url = 'https://requestbin.herokuapp.com/vbxpsavc';
      var options = {
        'method': 'post',
        'payload': JSON.stringify(payload)
      };
      
      var response = UrlFetchApp.fetch(url,options);
      
      sheet.getRange(startRow + i, lastColumn).setValue(PAYLOAD_SENT); // Update the last column with "PAYLOAD_SENT"
      SpreadsheetApp.flush(); // Make sure the last cell is updated right away
      
      // Remove temporary column header for Payload Status    
      sheet.getRange('E1').activate();
      sheet.getCurrentCell().clear({contentsOnly: true, skipFilteredRows: true});
      
    }
  }
}

单个 JSON 负载示例

{"img":"https://s3screenshotbucket.s3.amazonaws.com/realitymine.com.png","url":"https://realitymine.com"}

示例所需的输出结果

[
    {"img":"https://s3screenshotbucket-useast1v5.s3.amazonaws.com/gavurin.com.png","url":"https://gavurin.com"},
    {"img":"https://s3screenshotbucket-useast1v5.s3.amazonaws.com/google.com.png","url":"https://google.com"},
    {"img":"https://s3screenshotbucket-useast1v5.s3.amazonaws.com/amazon.com","url":"https://www.amazon.com"},  
    {"img":"https://s3screenshotbucket-useast1v5.s3.amazonaws.com/stackoverflow.com","url":"https://stackoverflow.com"},
    {"img":"https://s3screenshotbucket-useast1v5.s3.amazonaws.com/duckduckgo.com","url":"https://duckduckgo.com"},
    {"img":"https://s3screenshotbucket-useast1v5.s3.amazonaws.com/docs.aws.amazon.com","url":"https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-features.html"},  
    {"img":"https://s3screenshotbucket-useast1v5.s3.amazonaws.com/github.com","url":"https://github.com"},  
    {"img":"https://s3screenshotbucket-useast1v5.s3.amazonaws.com/github.com/shelfio/chrome-aws-lambda-layer","url":"https://github.com/shelfio/chrome-aws-lambda-layer"},  
    {"img":"https://s3screenshotbucket-useast1v5.s3.amazonaws.com/gwww.youtube.com","url":"https://www.youtube.com"},   
    {"img":"https://s3screenshotbucket-useast1v5.s3.amazonaws.com/w3docs.com","url":"https://www.w3docs.com"}       
]

Answer 1

修改

1. UrlFetchApp除非别无他法，否则不要在循环中调用方法。尽管 Google 提供了慷慨的配额，但它不是无限的，您将在任何大量行和发送频率上快速消耗掉它。
2. 使用现代 ES6 功能，例如map将值的行转换为所需负载格式的对象。请注意，您必须启用 V8 运行时才能使用它们。

接下来是一个可运行的测试片段，展示了如何修改脚本。我选择从中排除状态更新逻辑，因为由您决定如何在批量更新失败的情况下更新状态：

//TEST MOCKS:
const SpreadsheetApp = {
  getActiveSheet() {
  
    const Sheet = {
      getLastRow() { return 3; },
      getLastColumn() { return 5; },
      getDataRange() {
      
        const Range = {
          getValues() {
            return new Array(Sheet.getLastRow())
            .fill([])
            .map(
              (r,ri) => new Array(Sheet.getLastColumn())
              .fill(`mock row ${ri}`)
              .map((c,ci) => `${c} cell ${ci}`)
            );
          }
        };
        
        return Range;
      }
    };
    
    return Sheet;
  }
};

const UrlFetchApp = {
  fetch(uri, options) {
    console.log({ uri, options });
  }
};
//END MOCKS;

const sendToS3 = () => {

  var PAYLOAD_SENT = "S3 SCREENSHOT DATA SENT";

  var sheet = SpreadsheetApp.getActiveSheet();

  var startRow = 2;
  var numRows = sheet.getLastRow() - 1;
  var lastColumn = sheet.getLastColumn();

  var dataRange = sheet.getDataRange();
  var data = dataRange.getValues();

  var siteOwner = "email@example.com";

  const appURI = 'https://hfcrequestbin.herokuapp.com/vbxpsavb';

  const payloads = data.map(([index, img, url]) => ({ img, url }));

  const options = {
    'method': 'post',
    'payload': JSON.stringify(payloads)
  };

  const response = UrlFetchApp.fetch(appURI, options);
}

sendToS3();

笔记

1. 批处理 POST 有效负载时，请记住，每个请求的最大正文大小有配额（当前为 50 MB）。
2. 不叫I / O（输入/输出）的方法，例如getRange，getValue在一个循环中，它们是由慢的性质，使用批量方法，如getDataRange，getValues，setValues等，并只在存储器阵列执行的所有修改。
3. activate仅当您明确想要更改焦点时才使用方法，不要依赖它来确定范围。只需使用对通过诸如getRange.