Tom*_*ers 26 opengl graphics qt sdl r
对于我在R&Rcpp + OpenMP&Shiny中制作的实时交互式Mandelbrot查看器我正在寻找一种高性能的方式来显示1920x1080矩阵作为光栅图像,以期能够实现ca. 5-10 fps(计算Mandelbrot图像本身现在在中等变焦时达到约20-30 fps,当然滚动应该快速).使用image()with选项useRaster=TRUE,plot.raster或者甚至grid.raster()还没有完全削减它,所以我正在寻找更高性能的选项,理想情况下使用OpenGL加速.
我注意到有qt 包装器qtutils和qtpaint
http://finzi.psych.upenn.edu/R/library/qtutils/html/sceneDevice.html
你可以在其中设置参数opengl=TRUE和
http://finzi.psych.upenn.edu/ R/library/qtpaint/html/qplotView.html
再次使用参数opengl=TRUE和http://finzi.psych.upenn.edu/R/library/qtpaint/html/painting.html.
我还注意到应该能够使用rdyncall软件包调用SDL和GL/OpenGL函数(从https://cran.r-project.org/src/contrib/Archive/rdyncall/安装,从https://安装SDL)www.libsdl.org/download-1.2.php)`,演示可在http://hg.dyncall.org/pub/dyncall/bindings/file/87fd9f34eaa0/R/rdyncall/demo/00Index获得,例如http:// hg.dyncall.org/pub/dyncall/bindings/file/87fd9f34eaa0/R/rdyncall/demo/randomfield.R).
我是否正确使用这些软件包,应该能够使用opengl加速显示2D图像栅格？如果是这样,有没有人想过如何做到这一点(我问,因为我不是qt或SDL/OpenGL的专家)？
非OpenGL选项的某些时间对我的应用来说太慢了:
# some example data & desired colour mapping of [0-1] ranged data matrix
library(RColorBrewer)
ncol=1080
cols=colorRampPalette(RColorBrewer::brewer.pal(11, "RdYlBu"))(ncol)
colfun=colorRamp(RColorBrewer::brewer.pal(11, "RdYlBu"))
col = rgb(colfun(seq(0,1, length.out = ncol)), max = 255)
mat=matrix(seq(1:1080)/1080,nrow=1920,ncol=1080,byrow=TRUE)
mat2rast = function(mat, col) {
idx = findInterval(mat, seq(0, 1, length.out = length(col)))
colors = col[idx]
rastmat = t(matrix(colors, ncol = ncol(mat), nrow = nrow(mat), byrow = TRUE))
class(rastmat) = "raster"
return(rastmat)
}
system.time(mat2rast(mat, col)) # 0.24s
# plot.raster method - one of the best?
par(mar=c(0, 0, 0, 0))
system.time(plot(mat2rast(mat, col), asp=NA)) # 0.26s
# grid graphics - tie with plot.raster?
library(grid)
system.time(grid.raster(mat2rast(mat, col),interpolate=FALSE)) # 0.28s
# base R image()
par(mar=c(0, 0, 0, 0))
system.time(image(mat,axes=FALSE,useRaster=TRUE,col=cols)) # 0.74s # note Y is flipped to compared to 2 options above - but not so important as I can fill matrix the way I want
# magick - browser viewer, so no good....
# library(magick)
# image_read(mat2rast(mat, col))
# imager - doesn't plot in base R graphics device, so this one won't work together with Shiny
# If you wouldn't have to press ESC to return control to R this
# might have some potential though...
library(imager)
display(as.cimg(mat2rast(mat, col)))
# ggplot2 - just for the record...
df=expand.grid(y=1:1080,x=1:1920)
df$z=seq(1,1080)/1080
library(ggplot2)
system.time({q <- qplot(data=df,x=x,y=y,fill=z,geom="raster") +
scale_x_continuous(expand = c(0,0)) +
scale_y_continuous(expand = c(0,0)) +
scale_fill_gradientn(colours = cols) +
theme_void() + theme(legend.position="none"); print(q)}) # 11s
第一次发布这个问题五年后终于找到了答案：
比在 R 中显示光栅图像更快的选择image是使用包plusnativeRaster的格式。不过更快的是使用 SDL+OpenGl 解决方案，它可以通过外部函数接口调用- 该解决方案大约是。比结合使用仍快 10 倍。理论上，这应该允许在 640 x 480 分辨率、全屏或窗口中达到 200+ fps。我正在使用 OpenGL 纹理映射方法 - 我还没有尝试使用其他方法 - 这仍然会给出不同的时间。使用 SDL 或 OpenGL 进行强度到颜色映射大概也是可能的，尽管我无法立即弄清楚如何做到这一点（因为我现在事先这样做了，这仍然是一个很大的瓶颈）。如果有人知道如何在纯 OpenGL 中实现这一点（也许使用上面问题中提到的方法），请告诉我。使用SFML，通过 Rcpp 连接，而不是 SDL 也应该是可能的，并且应该比使用 SDL 更快，但我也没有尝试过。naragrid.rasterrdyncallnativeRastergrid.rasterSDL_BlitSurfaceglPixelMapfv
首先我们安装所需的包并计算一个很好的示例曼德尔布罗分形图像：
# 0. load required packages ####
library(remotes)
remotes::install_github("hongyuanjia/rdyncall")
library(rdyncall)
# install SDL libraries (SDL, SDL_image & SDL_mixer, version 1.2)
# from https://libsdl.org/release/,
# https://www.libsdl.org/projects/SDL_image/release/ and
# https://www.libsdl.org/projects/SDL_mixer/release/)
# 64 bit DLLs are to be put under R-4.2.3/bin/x64
# you can use
# source("https://raw.githubusercontent.com/Jean-Romain/lidRviewer/master/sdl.R")
# on Ubuntu install using sudo apt-get install libsdl1.2-dev libsdl-image1.2-dev libsdl-mixer1.2
remotes::install_github("coolbutuseless/nara")
library(nara)
remotes::install_github("tomwenseleers/mandelExplorer") # for nice example images
library(mandelExplorer)
library(microbenchmark)
# 1. create nice 640x480 image - here Mandelbrot fractal ####
xlims=c(-0.74877,-0.74872)
ylims=c(0.065053,0.065103)
x_res=640L
y_res=480L
nb_iter=as.double(nrofiterations(xlims))
m <- matrix(mandelRcpp2(as.double(xlims[[1]]), as.double(xlims[[2]]), # openmp+SIMD version
as.double(ylims[[1]]), as.double(ylims[[2]]),
x_res,
y_res,
nb_iter),
nrow=as.integer(x_res))
m[m==0] <- nb_iter
m <- equalizeman(m, nb_iter, rng = c(0, 0.95), levels = 1E4)^(1/8) # equalize colours & apply gamma correction
dim(m) # x_res x y_res, grayscale matrix normalized between 0 and 1
range(m) # 0 1
# A. display raster using image - not terribly fast (not fast enough for real-time rendering at good framerate) ####
x11(type = 'dbcairo', antialias = 'none', width = 6*x_res/y_res, height = 6) # Setup a fast graphics device that can render quickly
par(mar=c(0, 0, 0, 0))
microbenchmark(image(m, col=palettes[[2]], asp=y_res/x_res, axes=FALSE, useRaster=TRUE), unit='s')
# 0.08s, this includes colour mapping+raster display
# B. display raster user nara package, using nara::nativeRaster & grid.raster - 2.3x faster than image ####
microbenchmark({ col = palettes[[2]]
idx = findInterval(m, seq(0, 1, length.out = length(col)))
colors = col[idx] # matrix of hex colour values
natrast = nara::raster_to_nr(matrix(t(colors), ncol = x_res, # nativeRaster
nrow = y_res, byrow = TRUE))
grid::grid.raster(natrast, interpolate = FALSE) }, unit='s')
# 0.035s, 2.3x faster than image(), including the colour mapping + raster display
# timings just for colour mapping part & converting to nativeRaster
microbenchmark({ col = palettes[[2]]
idx = findInterval(m, seq(0, 1, length.out = length(col)))
colors = col[idx] # matrix of hex colour values
natrast = nara::raster_to_nr(matrix(t(colors), ncol = x_res, # nativeRaster
nrow = y_res, byrow = TRUE)) }, unit='s')
# 0.027s for colour mapping part & converting to nativeRaster
# timings just for display of nativeRaster
microbenchmark({ grid::grid.raster(natrast, interpolate = FALSE);
dev.flush() }, unit='s')
# 0.004s just for displaying nativeRaster
# C. display nativeRaster or rgb array using SDL & OpenGL and rdyncall: to display nativeRaster 10x faster than grid.graphics ####
dynport(SDL)
dynport(GL)
dynport(GLU)
# initialize SDL and create a window
if(SDL_Init(SDL_INIT_VIDEO) != 0) {
stop("Failed to initialize SDL")
}
# set OpenGL attributes
SDL_GL_SetAttribute(SDL_GL_RED_SIZE, 8)
SDL_GL_SetAttribute(SDL_GL_GREEN_SIZE, 8)
SDL_GL_SetAttribute(SDL_GL_BLUE_SIZE, 8)
SDL_GL_SetAttribute(SDL_GL_ALPHA_SIZE, 8)
SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER, 1)
# set up screen: graphics window or run full screen
screen <- SDL_SetVideoMode(x_res, y_res, 0, SDL_OPENGL+SDL_DOUBLEBUF) # for windowed mode
# screen <- SDL_SetVideoMode(x_res, y_res, 0, SDL_OPENGL+SDL_DOUBLEBUF+SDL_FULLSCREEN) # for fullscreen mode
if(is.null(screen)) {
stop("Failed to set video mode")
}
# initialize projection matrix
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
glOrtho(0, x_res, y_res, 0, -1, 1)
# initialize modelview matrix
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
# generate a texture
texId <- as.integer(0)
glGenTextures(1, texId)
glBindTexture(GL_TEXTURE_2D, texId)
# set texture parameters
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
# convert colour vector to matrix
colorsMatrix <- matrix(colors,
nrow = y_res,
ncol = x_res,
byrow = TRUE)
# create image matrix (created as nativeRaster to make sure it is a contiguous block of memory)
imageMatrix <- nara::raster_to_nr(colorsMatrix)
# timings just for nativeRaster display
microbenchmark({
# upload the image data to the texture
glTexImage2D(GL_TEXTURE_2D, 0, 4,
x_res, y_res, 0, GL_RGBA, GL_UNSIGNED_BYTE, imageMatrix)
# clear the window
glClear(GL_COLOR_BUFFER_BIT)
# enable textures
glEnable(GL_TEXTURE_2D)
# draw a quad with the texture
glBegin(GL_QUADS)
glTexCoord2f(0, 1); glVertex2f(0, 0)
glTexCoord2f(1, 1); glVertex2f(x_res, 0)
glTexCoord2f(1, 0); glVertex2f(x_res, y_res)
glTexCoord2f(0, 0); glVertex2f(0, y_res)
glEnd()
# disable textures
glDisable(GL_TEXTURE_2D)
# swap buffers to display the result
SDL_GL_SwapBuffers() },
unit='s')
# 0.0004s - 10x faster than displaying nativeRaster using grid.raster
# close window
SDL_Quit()
此处显示输出图像。