在API中将34000条记录从API插入到sqlite中

Question

我必须从从JSON API获取的数据创建一个sqlite数据库.代码工作正常,并通过for循环逐个添加它们,但api响应时间为每次命中1秒,因此34000秒加上通过代码在sqlite中插入它们将花费大约9个小时.有没有办法加快这个？

编辑:我正在使用Objective C/sqlite3框架/ Xcode 4.2

继承守法典......

 dbPath=[self.databasePath UTF8String];
if(sqlite3_open(dbPath,&database)==SQLITE_OK)
{
  //   sqlite3_exec(database, "BEGIN", 0, 0, 0);
    const char *sqlstatement="insert into artist values(?,?,?,?,?)";
    sqlite3_stmt *compiledstatement;

    if(sqlite3_prepare_v2(database,sqlstatement , -1, &compiledstatement, NULL)==SQLITE_OK)
    {    
for(i=4611;i<=34803;i++)
{  
    NSURLResponse *response;
    NSError *err;
  NSData *data= [NSURLConnection sendSynchronousRequest:[NSURLRequest requestWithURL:[NSURL URLWithString:[NSString stringWithFormat:@"API&id=%i",i]]] returningResponse:&response error:&err];
    if(data.length>0)
    {
        NSError *err;
        NSDictionary *jsonDict=[NSJSONSerialization JSONObjectWithData:data options:NSJSONReadingAllowFragments error:&err];
        // sqlite3_exec(database, "BEGIN", 0, 0, 0);




                sqlite3_bind_text(compiledstatement,1,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"id"] UTF8String], -1, SQLITE_TRANSIENT);
                sqlite3_bind_text(compiledstatement,2,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"name"] UTF8String], -1, SQLITE_TRANSIENT);
                if([[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"description"])
                    sqlite3_bind_text(compiledstatement,3,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"description"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,3,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if([[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"links"]objectForKey:@"website"])
                    sqlite3_bind_text(compiledstatement,4,[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"links"]objectForKey:@"website"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,4,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if([[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"media"]objectForKey:@"low_res_images"]objectAtIndex:0]objectForKey:@"url"])
                    sqlite3_bind_text(compiledstatement,5,[[[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"media"]objectForKey:@"low_res_images"]objectAtIndex:0] objectForKey:@"url"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,5,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if(sqlite3_step(compiledstatement)==SQLITE_DONE)
                {
                    NSLog(@"done %i",i);
                }
                else NSLog(@"ERROR");


        }
        sqlite3_reset(compiledstatement);
                }

    }

}

else
    NSLog(@"error");

sqlite3_close(database);

Answer 1

是否有可能重构您的代码,以便您不会在每次迭代时打开数据库？

• 打开数据库
• 开始事务sqlite3_exec(...,"BEGIN",...)
• 编译声明
• 迭代数据集
  • 插入记录
• 完成编译语句
• 提交事务sqlite3_exec(...,{"ROLLBACK"或"COMMIT"},...)
• 关闭数据库

这与你现在的情况形成鲜明对比

• 迭代数据集
  • 打开数据库
  • 编译声明
  • 插入记录
  • 完成编译语句
  • 关闭数据库

以您的方式执行此操作的开销会影响性能.尝试重构我上面概述的方法,看看你是怎么做的.

编辑

我重新格式化了你的代码,以表明我在说什么.另外,我认为你所采取的其他性能(如另一个用户所示)是JSON调用.这可能是真的让你如此放慢速度.

dbPath=[self.databasePath UTF8String];
if(sqlite3_open(dbPath,&database)==SQLITE_OK)
{
    sqlite3_exec(database, "BEGIN", 0, 0, 0);
    const char *sqlstatement="insert into artist values(?,?,?,?,?)";
    sqlite3_stmt *compiledstatement;

    if(sqlite3_prepare_v2(database,sqlstatement , -1, &compiledstatement, NULL)==SQLITE_OK)
    {
        int hasError= 0;
        for(i=4611; hasError == 0 && i<=34803; i++)
        {  
            NSURLResponse *response;
            NSError *err;
            NSData *data= [NSURLConnection sendSynchronousRequest:[NSURLRequest requestWithURL:[NSURL URLWithString:[NSString stringWithFormat:@"API&id=%i",i]]] returningResponse:&response error:&err];
            if(data.length>0)
            {
                NSDictionary *jsonDict=[NSJSONSerialization JSONObjectWithData:data options:NSJSONReadingAllowFragments error:&err];
                // sqlite3_exec(database, "BEGIN", 0, 0, 0);
                sqlite3_bind_text(compiledstatement,1,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"id"] UTF8String], -1, SQLITE_TRANSIENT);
                sqlite3_bind_text(compiledstatement,2,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"name"] UTF8String], -1, SQLITE_TRANSIENT);
                if([[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"description"])
                    sqlite3_bind_text(compiledstatement,3,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"description"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,3,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if([[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"links"]objectForKey:@"website"])
                    sqlite3_bind_text(compiledstatement,4,[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"links"]objectForKey:@"website"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,4,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if([[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"media"]objectForKey:@"low_res_images"]objectAtIndex:0]objectForKey:@"url"])
                    sqlite3_bind_text(compiledstatement,5,[[[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"media"]objectForKey:@"low_res_images"]objectAtIndex:0] objectForKey:@"url"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,5,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if(sqlite3_step(compiledstatement)==SQLITE_DONE)
                {
                    NSLog(@"done %i",i);
                }
                else {
                    NSLog(@"ERROR");
                    hasError= 1;
                }
            }
            sqlite3_reset(compiledstatement);
        }
        // Really need to check error conditions with commit/rollback
        if( hasError == 0 ) {
            sqlite3_exec(database, "COMMIT", 0, 0, 0);
        }
        else {
            sqlite3_exec(database, "ROLLBACK", 0, 0, 0);
        }
    }
    sqlite3_close(database);
}
else {
    NSLog(@"error");
}