小编tec*_*ous的帖子

我在Node.JS中使用Async模块来跟踪我的异步调用.但是,我收到一个错误 - "已经调用了Callback".有人可以帮帮我吗？

async.each(data['results'], function(result, done) { 
    if (result['twitter_id'] !== null) { //Isolate twitter handle
        var param = { "user.screen_name": result['twitter_id']}
        db.test4.find( param, function(err, users) {
            if( err ) {
                return done(err);
            } else if (!users) {
                res.send("No user found");
            } else {
                users.forEach( function(Result) { 
                    twitter_ids.push(Result);
                    //console.log(Result);
                    done();
                });
            }
        });
    }
}, function(err) {  
    if (err) {
        throw err
    }
    res.send(twitter_ids);
});

我们希望能够部署我们的 Spark 作业，以便在部署期间处理数据时不会出现任何停机时间（目前大约有 2-3 分钟的窗口）。在我看来，最简单的方法是模拟“蓝/绿部署”理念，即启动新版本的 Spark 作业，让它预热，然后关闭旧作业。但是，使用结构化流和检查点，我们不能这样做，因为新的 Spark 作业看到最新的检查点文件已经存在（来自旧作业）。我在下面附上了一个示例错误。有没有人对潜在的解决方法有任何想法？

我想过将现有的检查点目录复制到新创建的作业的另一个检查点目录 - 虽然这应该作为一种解决方法（一些数据可能会被重新处理，但我们的数据库应该去重复），这似乎超级hacky，我宁愿不追求。

Caused by: org.apache.hadoop.fs.FileAlreadyExistsException: rename destination /user/checkpoint/job/offsets/3472939 already exists
    at org.apache.hadoop.hdfs.server.namenode.FSDirRenameOp.validateOverwrite(FSDirRenameOp.java:520)
    at org.apache.hadoop.hdfs.server.namenode.FSDirRenameOp.unprotectedRenameTo(FSDirRenameOp.java:364)
    at org.apache.hadoop.hdfs.server.namenode.FSDirRenameOp.renameTo(FSDirRenameOp.java:282)
    at org.apache.hadoop.hdfs.server.namenode.FSDirRenameOp.renameToInt(FSDirRenameOp.java:247)
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.renameTo(FSNamesystem.java:3677)
    at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.rename2(NameNodeRpcServer.java:914)
    at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.rename2(ClientNamenodeProtocolServerSideTranslatorPB.java:587)
    at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
    at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616)
    at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:982)
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2049)
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2045)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:422)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
    at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2045)

    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
    at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
    at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73)
    at org.apache.hadoop.hdfs.DFSClient.rename(DFSClient.java:1991)
    at org.apache.hadoop.fs.Hdfs.renameInternal(Hdfs.java:335)
    at org.apache.hadoop.fs.AbstractFileSystem.rename(AbstractFileSystem.java:678)
    at org.apache.hadoop.fs.FileContext.rename(FileContext.java:958)
    at org.apache.spark.sql.execution.streaming.HDFSMetadataLog$FileContextManager.rename(HDFSMetadataLog.scala:356)
    at org.apache.spark.sql.execution.streaming.HDFSMetadataLog.org$apache$spark$sql$execution$streaming$HDFSMetadataLog$$writeBatch(HDFSMetadataLog.scala:160)
    ... 20 …

快速提问,但有没有人知道为什么MongoDB不会保存以下文件？我在NodeJS中使用MongoJS连接到MongoDB并使用以下代码行进行保存(在我的JS文件中,函数调用位于变量定义下面).保存功能不保存,但会自动转到回调功能.

有什么想法吗？非常感谢 :).

编辑:将错误日志添加到回调后,我收到以下错误:

{ [MongoError: error parsing element 0 of field documents :: caused by :: wrong type for '0' field, expected object, found 0: "{"Word":"Count","read":1,"letter":1,"wh":1,"urging":2,"swift":1,"approval":1,"add'l":1,"lease":1,"space":1,"fayetteville":1,"vamc":1,"vets":2,"care":1..."]
  name: 'MongoError',
  ok: 0,
  errmsg: 'error parsing element 0 of field documents :: caused by :: wrong type for \'0\' field, expected object, found 0: "{"Word":"Count","read":1,"letter":1,"wh":1,"urging":2,"swift":1,"approval":1,"add\'l":1,"lease":1,"space":1,"fayetteville":1,"vamc":1,"vets":2,"care":1..."',
  code: 9 }

db.collection.save(json_buffer, function() {
    console.log("Complete");
});

  var json_buffer = {"Word":"Count","read":1,"letter":1,"wh":1,"urging":2,"swift":1,"approval":1,"add'l":1,"lease":1,"space":1,"fayetteville":1,"vamc":1,"vets":2,"care":1,"@gnip":3,"--":3,"delivering":3,"data":3,"happy":3,"customers":3,"outrageous":1,"france":1,"sell":1,"warships":1,"putin":1,"@senatorkirk":1,"@repkinzinger":1,"@usrepkeating":1,"&amp":5,"urge":1,"strong":1,"action":3,"polling":1,"shows":1,"race":1,"close":1,"pitch":1,"pre-primary":1,"deadline":1,"goal":2,"joining":1,"@teamcavuto":1,"shortly":1,"discuss":1,"can’t":1,"continue":1,"punt":1,"debt":2,"crisis":1,"road":1,"watch":1,"opening":1,"remarks":1,"today’s":1,"senate":1,"committee":1,"hearing":1,"mcdonald":1,"nomination":1,"urged":1,"passage":1,"#summermealsact":2,"yesterday":1,"#monticello":1,"#ny--so":1,"impt":1,"expand":1,"@usda's":1,"summer":2,"nutrition":1,"program":1,"great":2,"catch":1,"high":1,"school":2,"friend":1,"john":1,"choate":1,"today":7,"family":1,"capitol":1,"hill":1,"child":1,"america":1,"wake":1,"day":1,"wondering":1,"eat":1,"nebraska":1,"communities":1,"access":1,"local":1,"tv":1,"programming":1,"introduced":1,"bill":4,"work":2,"past":1,"time":2,"congress":1,"meaningful":1,"reduce":1,"threat":1,"cyber":1,"attacks":1,"@mercnews":1,"op-ed":1,"fitting":1,"@us_sfrc":1,"passed":1,"#crpd":1,"bob":1,"dole's":1,"bday":1,"#disabilitytreaty":1,"advocate":1,"beginning":1,"#isupportcrpd":1,"senator":1,"mcconnell":2,"co-sponsors":1,"protect":2,"correctional":1,"officers":2,"daily":1,"independent":2,"#ashland":1,"#kentucky":1,"millions":1,"children":2,"recv":1,"free":1,"reduced":1,"meals":1,"year":1,"left":1,"hungry":1,"months":1,"unacceptable":1,"rt":5,"@aterkel":1,"record":1,"phone":1,"@senschumer":1,"…":1,"good":1,"meeting":1,"anne":1,"rung":1,"nominee":1,"@ombpress":1,"office":2,"fed":1,"procurement":1,"policy":1,"cc":1,"@senatehsgac":1,"@federalreserve":1,"divert":1,"leftover":1,"funds":1,"foreclosure":1,"review":1,"hardest":1,"hit":1,"fund":1,"#ri":1,"supports":1,"corrections":1,"prisons":1,"manchester":1,"pine":1,"knot":1,"ashland":1,"inez":1,"lex":1,"miami":1,"valley":1,"well-run":1,"base":1,"makes":1,"wright-patt":1,"#1":1,"airmen":1,"@gopoversight":1,"release":1,"#irs":2,"needed":1,"days":1,"confirm":1,"lerner":1,"hard":2,"drive":1,"crash":2,"provide":1,"support":3,"runaway":1,"homeless":1,"youth":1,"victims":1,"trafficking":1,"@housefloor":1,"tomorrow":1,"glad":1,"signature":1,"industry":1,"@kydistillers":1,"@repandybarr":1,"obama":1,"meant":1,"flexibility":1,"re-elected":1,"thoughts":1,"prayers":1,"affected":1,"fires":1,"tooele":1,"pray":1,"staying":1,"safe":1,"#utpol":1,"hr":1,"passes":1,"house":1,"@repderekkilmer":1,"modernize":1,"labs":1,"fyi":1,"fun":1,"activities":1,"hosted":1,"google":1,"young":1,"inventors":1,"age":1,"13+":1,"choose":1,"projects":1,"virtual":1,"field":1,"trips":1,"joined":1,"nyc":1,"workforce":2,"development":2,"hosting":1,"roundtable":1,"#bronx":1,"failure":1,"disclose":1,"timely":1,"manner":1,"destroyed":1,"critical":1,"evidence":1,"reason":2,"special":1,"prosecutor":1,"26%":1,"texas":1,"live":1,"poverty":1,"#raisethewage":1,"#honorflightact":2,"codify":1,"process":1,"tsa":1,"expedited":1,"dignified":1,"screening":1,"veterans":1,"visiting":1,"war":1,"memorials":1,"humbled":1,"join":2,"medal":1,"honor":1,"recipient":1,"staff":1,"sergeant":1,"ryan":1,"pitts":1,"nashua":1,"ceremony":1,"pentagon":1,"icymi":1,"statement":1,"halbig":1,"burwell":1,"@ecpzachevans":1,"save":1,"date":1,"mt":1,"@shrinersfest":1,"dates":1,"announced":1,"shrinersfest":1,"june":1,"25-28":1,"feat":1,"@blueangels":1,"htt…":1,"unleash":1,"america's":1,"energy":2,"abundance":1,"create":1,"#jobs":1,"economic":1,"growth":1,"affordable":1,"#yes2energy":1,"marks":1,"#100days":1,"nigerian":1,"schoolgirls":1,"kidnapping":1,"forgotten":1,"#bringbackourgirls":1,"pleased":1,"mayor":1,"@rahmemanuel":1,"taking":1,"@fly2ohare":1,"noise":1,"measure":1,"seeks":1,"pay":1,"gratitude":1,"group":1,"americans":1,"make":1,"ultimate":1,"sacrifice":1,"stopped":1,"dutch":1,"embassy":1,"offer":1,"condolences":1,"lost":1,"loved":1,"#mh17":1,"#obamafailures":1,"min":1,"#131":1,"odds":1,"drives":1,"irs":1,"claims":1,"win":1,"fl":1,"lottery":1,"times":1,"#tcot":1}