标签: node.js-stream

我在Node Js中读取一个大型XML文件(~1.5gb).我正在尝试对其进行流式处理并使用大量数据执行某些操作,但我发现很难理解文档.

我目前的简单代码是:

var fs = require('fs');

var stream = fs.createReadStream('xml/bigxmlfile.xml');

stream.on('data', function(chunk){ 
    console.log(chunk)
});

控制台提供了一堆buffer十六进制(我认为)代码,如下所示:

<Buffer 65 61 6e 2d 63 75 74 20 67 72 69 64 20 6c 69 6e 65 73 20 74 68 65 20 73 70 72 65 61 64 20 63 6f 6c 6c 61 72 20 61 6e 64 20 6d 69 74 65 72 65 64 2c 20 74 ...>
<Buffer 65 79 77 6f 72 64 73 3e …

http.IncomingMessage何时触发其"关闭"事件？

根据文档,它应该在底层连接关闭时发生.但是,它永远不会被调用以下示例代码(我确保它不是由keep-alive引起的):

var http = require('http'),
    fs = require('fs');

var server = http.createServer(function(req, res) {
    res.shouldKeepAlive = false;
    req.on("end", function() {
        console.log("request end");
    });
    req.on("close", function() {
        console.log("request close");    // Never called
    });
    res.end("Close connection");
});
server.listen(5555);

我正在使用node.js v0.10.22.

由Node的文件系统解析混淆.这是我的代码:

var fs = require('fs'),
    xml2js = require('xml2js');

var parser = new xml2js.Parser();

var stream = fs.createReadStream('xml/bigXML.xml');
stream.setEncoding('utf8');

stream.on('data', function(chunk){ 

    parser.parseString(chunk, function (err, result) {
        console.dir(result);
        console.log('Done');
    });
});


stream.on('end', function(chunk){
    // file have been read over,do something...
    console.log("IT'S OVER")
});

这导致......没有任何事情发生.根本没有来自XML2JS /解析器的输出.当我尝试console.log(chunk)它似乎chunks没有输出任何类型的有意义的块基于除字节大小以外的任何东西.一个'块'的输出是:

<?xml version="1.0" encoding="UTF-8"?>
    <merchandiser xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="merchandiser.xsd">
    <header><merchantId>1237</merchantId><merchantName>NORDSTROM.com</merchantName><createdOn>12/13/2013 23:50:57</createdOn></header>
    <product product_id="52863929">// product info</product>
    <product product_id="26537849">// product info</product>
    <product product_id="25535647">// product info</product>

这个块里面有很多很多<product>来自XML 的条目.块将在<product>条目中间的某处结束,下一个块将从此处停止的位置开始.

主要问题是如何createReadStream从输出<product结束处开始输出块</product> …

我在标准输入流上不断收到 EPIPE 错误，但找不到原因：

这是我的代码：

var checkFile = function(data, callback){
   var child_process = spawn('ffprobe', ['-print_format', 'json', '-show_format', 'pipe:0']);

   var stdInError = function(e) {
       console.log(e);
   }
   child_process.stdin.on('error', stdInError);

   var generalError = function() {
       console.log("general Error" + "\n");
   }
   child_process.on('error', generalError);

   child_process.stdout.on('data', function(data){
        console.log("data" + "\n");
        console.log(data);
        console.log("\n");
   });

   child_process.on('close', function(){
       console.log("close" + "\n");
   }

   var exit = function(){
       console.log("exit");
   }
   child_process.on('exit', exit);

   console.log("write" + "\n");
   child_process.stdin.write(data);
   child_process.stdin.end();
};

这是我的输出：

write

data

<Buffer 7b 0a 20 20 20 20 22 66 6f 72 …

(下面的新信息)我正在尝试设置一个lambda函数,通过解压缩并将结果写回S3,对上传的tgz文件作出反应.解压缩和解压缩工作正常,但上传到S3失败:

/Users/russell/lambda/gzip/node_modules/aws-sdk/lib/s3/managed_upload.js:350
    var buf = self.body.read(self.partSize - self.partBuffer.length) ||
                        ^
TypeError: undefined is not a function
    at ManagedUpload.fillStream (/Users/russell/lambda/gzip/node_modules/aws-sdk/lib/s3/managed_upload.js:350:25)
    at Entry.<anonymous> (/Users/russell/lambda/gzip/node_modules/aws-sdk/lib/s3/managed_upload.js:167:28)
    at Entry.emit (events.js:104:17)
    at Entry._read (/Users/russell/lambda/gzip/node_modules/tar/lib/entry.js:123:12)
    at Entry.end (/Users/russell/lambda/gzip/node_modules/tar/lib/entry.js:82:8)
    at Parse._process (/Users/russell/lambda/gzip/node_modules/tar/lib/parse.js:107:13)
    at BlockStream.<anonymous> (/Users/russell/lambda/gzip/node_modules/tar/lib/parse.js:47:8)
    at BlockStream.emit (events.js:107:17)
    at BlockStream._emitChunk (/Users/russell/lambda/gzip/node_modules/tar/node_modules/block-stream/block-stream.js:145:10)
    at BlockStream.write (/Users/russell/lambda/gzip/node_modules/tar/node_modules/block-stream/block-stream.js:45:10)

当我写入S3时会发生此错误,但如果我将文件本地写入磁盘则可以正常工作,因此管道是正确的.

以下是演示此问题的代码:

var aws = require('aws-sdk');
var s3 = new aws.S3({apiVersion: '2006-03-01'});
var zlib = require('zlib');
var tar = require('tar');
var fstream = require('fstream');

fstream.Reader({'path': 'testdata.tar.gz'})
    .pipe(zlib.Unzip())
    .pipe(tar.Parse())
    .on('entry', function(entry) {
        var filename = …

我正在使用node.js作为游戏服务器.

这是我的服务器脚本

 var net = require('net');
 var http = require('http');

var host =  '192.168.1.77';
var portNum = 12345;//


  function policy() 
{
    var xml = '<?xml version="1.0"?>\n<!DOCTYPE cross-domain-policy SYSTEM' +
            '"http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd">\n<cross-domain-policy>\n';

    xml += '<allow-access-from domain="*" to-ports="*"/>\n';
    xml += '</cross-domain-policy>\n\0' + "\0";
    return xml;
}



var server = net.createServer(function (stream) 
{
stream.setEncoding('utf8');

stream.on('data', function (data) { 

    if (data == '<policy-file-request/>\0') 
    {
        var x = policy();
        stream.write(x);
        var server_date = new Date();
        var serv_sec = server_date.getSeconds();
        return;
    }

    var comm = JSON.parse(data); 
    if …

在 Expressjs 中，我上传了一张图片。它是一个缓冲区编码 7 位：

 { fieldname: 'file',
  originalname: 'img.JPG',
  encoding: '7bit',
  mimetype: 'image/jpeg',
  buffer: <Buffer ff d8 ff e0 00 10 4a 46 49 46 00 01 01 01 00 64 00 64 00 00 ff db 00 43 00 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 0
1 01 01 01 01 01 ... >,
  size: 873066 }

如何将其作为图像保存到文件系统？保存了一些东西，但它不是图像：

Windows 照片查看器无法打开此图片，因为该文件似乎已损坏、损坏过大。

这些不起作用：

fs.writeFile('uploaded-img.jpg', req.file, 'ascii', function(err) …