使用"data.table"包合并数据帧时出错

Question

以下是我正在经历和坚持的情况的可重现的示例(它是我用于评估合并数据集的各种方法的测试客户端,用于我的论文研究).

testData <- "https://github.com/abnova/test/blob/master/mergeTestData.zip?raw=true"

tmpFile <- tempfile()
tmpDir <- tempdir()

download.file(testData, tmpFile, method = 'curl',
              extra = '-L', quiet = TRUE)
testFiles <- unzip(tmpFile, exdir = tmpDir)

# To enable desired merge option, uncomment corresponding line

#MERGE_OPTION <- "lapply_merge"
#MERGE_OPTION <- "lapply_merge2"
#MERGE_OPTION <- "reduce_merge"
#MERGE_OPTION <- "reduce_merge2"
#MERGE_OPTION <- "reshape"
#MERGE_OPTION <- "plyr"
#MERGE_OPTION <- "dplyr"
MERGE_OPTION <- "data.table"
#MERGE_OPTION <- "data.table2"

loadData <- function (dataFile) {

  if (file.exists(dataFile)) {
    data <- readRDS(dataFile)
  }
  else { # error() undefined - replaced for stop() for now
    stop("Data file \'", dataFile, "\' not found! Run 'make' first.")
  }
  return (data)
}

loadDataSets <- function (dataDir) {

  dataSets <- list()

  dataFiles <- dir(dataDir, pattern='\\.rds$')
  dataSets <- lapply(seq_along(dataFiles),
                     function(i) {
                       nameSplit <- strsplit(dataFiles[i], "\\.")
                       dataset <- nameSplit[[1]][1]
                       assign(dataset,
                              loadData(file.path(dataDir, dataFiles[i])))
                       return (get(dataset))
                     })
  return (dataSets)
}

# load the datasets of transformed data
dataSets <- loadDataSets(tmpDir)

if (MERGE_OPTION == "lapply_merge") { # Option 1

  flossData <- data.frame(dataSets[[1]][1])

  # merge all loaded datasets by common column ("Project ID")
  silent <- lapply(seq(2, length(dataSets)),
                   function(i) {merge(flossData, dataSets[[1]][i],
                                      by = "Project ID",
                                      all = TRUE)})
}

if (MERGE_OPTION == "lapply_merge2") { # Option 1

  pids <- which(sapply(dataSets,
                       FUN=function(x) {'Project ID' %in% names(x)}))

  flossData <- dataSets[[pids[1]]]

  for (id in pids[2:length(pids)]) {
    flossData <- merge(flossData, dataSets[[id]],
                       by='Project ID', all = TRUE)
  }
}

if (MERGE_OPTION == "reduce_merge") { # Option 2

  flossData <- Reduce(function(...) 
    merge(..., by.x = "row.names", by.y = "Project ID", all = TRUE),
    dataSets)
}

# http://r.789695.n4.nabble.com/merge-multiple-data-frames-tt4331089.html#a4333772
if (MERGE_OPTION == "reduce_merge2") { # Option 2

    mergeAll <- function(..., by = "Project ID", all = TRUE) {
    dotArgs <- list(...)
    dotNames <- lapply(dotArgs, names)
    repNames <- Reduce(intersect, dotNames)
    repNames <- repNames[repNames != by]
    for(i in seq_along(dotArgs)){
      wn <- which( (names(dotArgs[[i]]) %in% repNames) &
                     (names(dotArgs[[i]]) != by))
      names(dotArgs[[i]])[wn] <- paste(names(dotArgs[[i]])[wn],
                                       names(dotArgs)[[i]], sep = ".")
    }
    Reduce(function(x, y) merge(x, y, by = by, all = all), dotArgs)
  }

  flossData <- mergeAll(dataSets)
}

if (MERGE_OPTION == "reshape") { # Option 3

  if (!suppressMessages(require(reshape))) install.packages('reshape')
  library(reshape)
  flossData <- reshape::merge_all(dataSets)
}

if (MERGE_OPTION == "plyr") { # Option 4

  if (!suppressMessages(require(plyr))) install.packages('plyr')
  library(plyr)
  flossData <- plyr::join_all(dataSets)
}

if (MERGE_OPTION == "dplyr") { # Option 5

  if (!suppressMessages(require(dplyr))) install.packages('dplyr')
  library(dplyr)

  flossData <- dataSets[[1]][1]
  flossData <- lapply(dataSets[[1]][-1],
                      function(x) {dplyr::left_join(x, flossData)})
}

if (MERGE_OPTION == "data.table") { # Option 6

  if (!suppressMessages(require(data.table))) 
    install.packages('data.table')
  library(data.table)

  flossData <- data.table(dataSets[[1]], key="Project ID")

  for (id in 2:length(dataSets)) {
    flossData <- merge(flossData, data.table(dataSets[[id]]),
                       by='Project ID', all.x = TRUE, all.y = FALSE)
  }
}

# http://stackoverflow.com/a/17458887/2872891
if (MERGE_OPTION == "data.table2") { # Option 6

  if (!suppressMessages(require(data.table))) 
    install.packages('data.table')
  library(data.table)

  DT <- data.table(dataSets[[1]], key="Project ID")
  flossData <- lapply(dataSets[[1]][-1], function(x) DT[.(x)])
}

# Additional Transformations (see TODO above)

# convert presence of Repo URL to integer
flossData[["Repo URL"]] <- as.integer(flossData[["Repo URL"]] != "")

# convert License Restrictiveness' factor levels to integers
#flossData[["License Restrictiveness"]] <- 
#  as.integer(flossData[["License Restrictiveness"]])

# convert User Community Size from character to integer
flossData[["User Community Size"]] <- 
  as.integer(flossData[["User Community Size"]])

# remove NAs
#flossData <- flossData[complete.cases(flossData[,3]),]
rowsNA <- apply(flossData, 1, function(x) {any(is.na(x))})
flossData <- flossData[!rowsNA,]

print(str(flossData))

该错误消息是如下:

Starting bmerge ...done in 0.001 secs
Starting bmerge ...done in 0.002 secs
Error in vecseq(f__, len__, if (allow.cartesian) NULL else as.integer(max(nrow(x),  : 

加入121229行的结果; 超过100000 = max(nrow(x),nrow(i)).检查i中的重复键值,每个键值一遍又一遍地连接到x中的同一组.如果没关系,请尝试包含j和删除by(by-without-by),以便为每个组运行j以避免大量分配.如果您确定要继续,请使用allow.cartesian = TRUE重新运行.否则,请在FAQ,Wiki,Stack Overflow和datatable-help中搜索此错误消息以获取建议.

当前的问题是启用data.table选项,但是,因为它是相同的包,我也很感激下一个选项的建议,它使用替代 data.table 语法进行合并(尽管我觉得它太混乱,但为了知识的完整性) ).先感谢您!

Answer 1

我会以这种方式处理这个问题:

首先,有一条错误消息.它说什么？

加入121229行的结果; 超过100000 = max(nrow(x),nrow(i)).检查i中的重复键值,每个键值一遍又一遍地连接到x中的同一组.如果没关系,请尝试包含j并按(逐个)删除,以便为每个组运行j以避免大量分配.如果您确定要继续,请使用allow.cartesian = TRUE重新运行.否则,请在FAQ,Wiki,Stack Overflow和datatable-help中搜索此错误消息以获取建议.

大!但是我正在使用这么多的数据集,以及很多软件包和很多功能.我必须将其缩小到哪个数据集产生此错误.

逐个测试:

ans1 = merge(as.data.table(dataSets[[1]]), as.data.table(dataSets[[2]]), 
                all.x=TRUE, all.y=FALSE, by="Project ID")
## works fine.

ans2 = merge(as.data.table(dataSets[[1]]), as.data.table(dataSets[[3]]), 
                all.x=TRUE, all.y=FALSE, by="Project ID")
## same error

啊哈,得到了同样的错误.

阅读错误消息的第二行:

所以,似乎发生了一些事情dataSets[[3]].它说要检查重复的键值i.我们这样做:

dim(dataSets[[3]])
# [1] 81487     3
dim(unique(as.data.table(dataSets[[3]]), by="Project ID"))
# [1] 49999     3

因此,dataSets[[3]]重复了"项目ID"值,因此对于每个重复值,dataSets[[1]]返回所有匹配的行- 这是第二行的第二部分解释的:each of which join to the same group in x over and over again.

尝试allow.cartesian=TRUE:

我知道有重复的密钥,仍然希望继续.但错误消息提到我们如何继续,添加"allow.cartesian = TRUE".

ans2 = merge(as.data.table(dataSets[[1]]), as.data.table(dataSets[[3]]), 
                all.x=TRUE, all.y=FALSE, by="Project ID", allow.cartesian=TRUE)

啊哈,现在工作正常!那怎么allow.cartesian = TRUE办？或者为什么要添加？错误消息说在stackoverflow上搜索消息(在其他事情中).

搜索allow.cartesian=TRUESO:

搜索引导我进入这个为什么在加入带有重复键的data.tables时有时需要allow.cartesian？问题,解释了目的,并且在评论下还包含来自@Roland的另一个链接:合并data.tables使用超过10 GB的RAM,这指出了所有启动它的初始问题.让我现在阅读这些帖子.

被base::merge赋予不同的结果呢？

现在,base :: merge是否会返回不同的结果(100,000行)？

dim(merge(dataSets[[1]], dataSets[[3]], all.x=TRUE, all.y=FALSE, by="Project ID"))
# [1] 121229      4

并不是的.它提供与使用时相同的尺寸data.table,但它并不关心是否存在重复的键,而是data.table警告您合并结果的潜在爆炸并允许您做出明智的决定.