如何在dynamodb中超过1mb的扫描数据限制

Question

我正在将 dynamodb 与 nodejs 一起使用，我有 3000 条记录，并且我正在代码中编写 60 多个段，每个段扫描 1mb 的数据并将结果显示为 60 多个 1 mb 限制的段。所以请提供解决方案如何在单步中获得 3000 条记录扫描，这意味着在一个段中。请快速提供解决方案，因为我在项目中间感到震惊。请帮我。下面是我的代码。

var AWS = require("aws-sdk");
var async = require("async");

    AWS.config.update({
      region: "---",
      endpoint: "-----------",
      accessKeyId: "-----------------",
      secretAccessKey:"----------"
    });


var db = new AWS.DynamoDB.DocumentClient()

var table = "rets_property_all";
var pstart =new Date () .getTime ();

async.parallel({
            0 : function(callback){
        db.scan ({TableName: table,
          ProjectionExpression: "#cityname,ListingKey ",
          FilterExpression: "#cityname = :v_id",
          ExpressionAttributeNames: {
              "#cityname": "CityName",
          },
          ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
            TotalSegments: 63,
            Segment:  0//by the worker who has been called
        },function (err , res) {
            callback (null , res.Items);
        });
            },
            1 : function(callback){
        db.scan ({TableName: table,
          ProjectionExpression: "#cityname,ListingKey ",
          FilterExpression: "#cityname = :v_id",
          ExpressionAttributeNames: {
              "#cityname": "CityName",
          },
          ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
            TotalSegments: 63,
            Segment: 1//by the worker who has been called
        }, function (err , res) {
            callback (null , res.Items);
        });
            },
      2 : function(callback){
        db.scan ({TableName: table,
          ProjectionExpression: "#cityname,ListingKey ",
          FilterExpression: "#cityname = :v_id",
          ExpressionAttributeNames: {
              "#cityname": "CityName",
          },
          ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
            TotalSegments: 63,
            Segment: 2//by the worker who has been called
        }, function (err , res) {
            callback (null , res.Items);
        });
            },
--------
---------
------

      62 : function(callback){
        db.scan ({TableName: table,
          ProjectionExpression: "#cityname,ListingKey ",
          FilterExpression: "#cityname = :v_id",
          ExpressionAttributeNames: {
              "#cityname": "CityName",
          },
          ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
            TotalSegments: 63,
            Segment: 62//by the worker who has been called
        }, function (err , res) {
            callback (null , res.Items);
        });
      },

        },function(err,results){
      if (err) {throw err; }
      var pend = new Date () .getTime ();

      console.log (results);
        })

Answer 1

实际上，无法覆盖1 MB的扫描限制。这是 DynamoDB 设计限制，不能被任何 API 覆盖。您可能需要了解架构或 AWS 服务设计的限制。

您可以LastEvaluatedKey在后续扫描中使用从上次扫描结束的位置开始。

扫描的结果集限制为每次调用 1 MB。您可以使用扫描响应中的 LastEvaluatedKey 来检索更多结果。

用例不清楚为什么要在一次扫描中获取所有 3000 条记录。甚至，如果您有特定的用例，在 DynamoDB 扫描上根本无法实现。

甚至，在关系数据库中，您获取游标并通过迭代顺序获取所有记录。同样，在 DynamoDB 中，您必须递归地使用 Scan 直到LastEvaluatedKey为空。

递归单次扫描的示例代码，直到 LastEvaluatedKey 为空：-

var docClient = new AWS.DynamoDB.DocumentClient();
var params = {
    TableName: table,
    ProjectionExpression: "#cityname,ListingKey ",
    FilterExpression: "#cityname = :v_id",
    ExpressionAttributeNames: {
        "#cityname": "CityName",
    },
    ExpressionAttributeValues: { ":v_id": 'BALTIMORE' }
};

docClient.scan(params, onScan);

function onScan(err, data) {
    if (err) {
        console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
    } else {
        // print all the movies
        console.log("Scan succeeded.");
        data.Items.forEach(function (movie) {
            console.log("Item :", JSON.stringify(movie));
        });

        // continue scanning if we have more movies
        if (typeof data.LastEvaluatedKey != "undefined") {
            console.log("Scanning for more...");
            params.ExclusiveStartKey = data.LastEvaluatedKey;
            docClient.scan(params, onScan);
        }
    }
}