减少R中的内存消耗 - 通过reference/data.table传递

Question

通过将子集操作从基本data.frame操作转移到data.table操作,我已经实现了大幅加速(~6.5倍).但我想知道我是否可以在记忆方面得到任何改善.

我的理解是R本身并没有通过参考传递(例如,见这里).所以,我正在寻找一种方法(没有重写复杂的函数Rcpp)来做到这一点.data.table提供了一些改进[ 在编辑我的问题后包括@joshua ulrich在下面捕获的拼写错误 ].但是如果可能的话,我正在寻找更大的改进.

• 另一种选择可能是R.oo包,虽然我还没有找到一个好的教程.(我还是需要读这个.
• 将引用类帮助呢？

在我的实际使用案例中,我正在通过模拟退火进行多个数据集的并行仿真.由于开发时间的增加和技术债务的增加,我宁愿不在Rcpp中重写模拟退火和损失函数计算.

问题示例:

我主要关心的是从数据集中删除一些观察子集并添加另一个观察子集.这里给出了一个非常简单(荒谬)的例子.有没有办法减少内存使用量？我当前的用法似乎是按值传递,因此内存使用量(RAM)大约翻了一番.

library(data.table)
set.seed(444L)

df1 <- data.frame(matrix(rnorm(1e7), ncol= 10))
df2 <- data.table(matrix(rnorm(1e7), ncol= 10))

prof_func <- function(df) {
  s1 <- sample(1:nrow(df), size= 500, replace=F)
  s2 <- sample(1:nrow(df), size= 500, replace=F)
  return(rbind(df[-s1,], df[s2,]))
}

dt_m <- df_m <- vector("numeric", length= 500L)

for (i in 1:500) {

  Rprof("./DF_mem.out", memory.profiling = TRUE)
  y <- prof_func(df1)
  Rprof(NULL)
  df <- summaryRprof("./DF_mem.out", memory= "both")
  df_m[i] <- df$by.self$mem.total[which(rownames(df$by.self) == "\"rbind\"")]


  Rprof("./DT_mem.out", memory.profiling = TRUE)
  y2 <- prof_func(df2)
  Rprof(NULL)
  dt <- summaryRprof("./DT_mem.out", memory = "both")
  dt_m[i] <- dt$by.self$mem.total[which(rownames(dt$by.self) == "\"rbind\"")]

}
pryr::object_size(df1)
80 MB
pryr::object_size(df2)
80 MB

# EDITED: via typo / fix from @Joshua Ulrich.
# improvement in memory usage via DT. still not pass-by-reference
quantile(df_m, seq(0,1,.1))
    0%    10%    20%    30%    40%    50%    60%    70%    80%    90%   100% 
379.00 428.60 440.10 447.70 455.36 459.20 466.48 469.89 474.40 482.10 512.60 
quantile(dt_m, seq(0,1,.1))
    0%    10%    20%    30%    40%    50%    60%    70%    80%    90%   100% 
 76.80  84.50  84.50  92.10  92.10  92.10  92.10 107.30 116.46 130.20 157.00 

附录:

### speed improvement:
#-----------------------------------------------
library(data.table)
library(microbenchmark)

set.seed(444L)

df1 <- data.frame(matrix(rnorm(1e7), ncol= 10))
df2 <- data.table(matrix(rnorm(1e7), ncol= 10))

microbenchmark(
  df= {
    s1 <- sample(1:nrow(df1), size= 500, replace=F)
    s2 <- sample(1:nrow(df1), size= 500, replace=F)
    df1 <- rbind(df1[-s1,], df1[s2,])
  },
  dt= {
    s1 <- sample(1:nrow(df2), size= 500, replace=F)
    s2 <- sample(1:nrow(df2), size= 500, replace=F)
    df2 <- rbind(df2[-s1,], df2[s2,])

  }, times= 100L)

Unit: milliseconds
 expr      min        lq     mean   median       uq      max neval cld
   df 672.5106 757.65188 814.1582 809.6346 864.6668 998.2290   100   b
   dt  68.1254  85.73178 139.1256 120.3613 148.8243 397.7359   100  a 

Answer 1

prof_func有错误.它要求rbind对df1代替它的论点(df).修复此问题,您将看到data.table对象减少了内存使用量.

library(data.table)
set.seed(444L)

df1 <- data.frame(matrix(rnorm(1e7), ncol= 10))
df2 <- data.table(matrix(rnorm(1e7), ncol= 10))

prof_func <- function(df) {
  s1 <- sample(1:nrow(df), size= 500, replace=F)
  s2 <- sample(1:nrow(df), size= 500, replace=F)
  return(rbind(df[-s1,], df[s2,]))
}

dt_m <- df_m <- vector("numeric", length= 500L)

for (i in 1:100) {
  Rprof("./DF_mem.out", memory.profiling = TRUE, interval=0.01)
  y <- prof_func(df1)
  Rprof(NULL)
  df <- summaryRprof("./DF_mem.out", memory= "both")
  df_m[i] <- df$by.total["\"rbind\"","mem.total"]

  Rprof("./DT_mem.out", memory.profiling = TRUE, interval=0.01)
  y2 <- prof_func(df2)
  Rprof(NULL)
  dt <- summaryRprof("./DT_mem.out", memory = "both")
  dt_m[i] <- dt$by.total["\"rbind\"","mem.total"]
}
quantile(df_m, seq(0,1,.1))
#    0%   10%   20%   30%   40%   50%   60%   70%   80%   90%  100% 
#   0.0   0.0   0.0   0.0   0.0   0.0   0.0 413.4 432.5 455.0 485.9 
quantile(dt_m, seq(0,1,.1))
#    0%   10%   20%   30%   40%   50%   60%   70%   80%   90%  100% 
#   0.0   0.0   0.0   0.0   0.0   0.0   0.0  53.9  84.5 122.6 153.1