use*_*499 6 linux-device-driver linux-kernel rdma infiniband
我正在编写一个RDMA(InfiniBand)内核模块.
到目前为止,我已成功创建保护域,发送和接收队列的完成队列.
但每当我尝试通过调用ib_create_qp来创建队列对时,它都无法创建队列对.我写的代码如下所示:
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/err.h>
#include "myClient.h"
struct workqueue_struct *myClient_workqueue;
struct ib_sa_client myClient_sa_client;
/*
static void myClient_add_one(struct ib_device *device);
static void myClient_remove_one(struct ib_device *device);
*/
struct ib_pd *mypd;
struct ib_cq *myrcvcq;
struct ib_cq *myClientsendcq;
struct ib_qp *myClientqp;
void myClient_ib_recvcompletion(struct ib_cq *cq)
{
printk("A user-specified callback that is invoked when a completion event occurs on the CQ.\n");
}
void myClient_ib_sendcompletion(struct ib_cq *cq)
{
printk("A user-specified callback that is invoked when a completion event occurs on the CQ.\n");
}
static void my_qp_event_handler(struct ib_event *myqpAsyncEvent, void *anyPointer)
{
printk(KERN_INFO "Dummy affiliated asynchronous event occured function called \n");
}
static void myClient_add_one(struct ib_device *device)
{
union ib_gid tmp_gid;
int ret;
int hcaport = 1;
int result = -ENOMEM;
u16 port1Pkey;
struct ib_port_attr attr;
ret = ib_query_port(device,hcaport,&attr);
printk("ib query port result %d \n", ret);
// Creating the Protection Domain for RDMA
mypd = ib_alloc_pd(device);
if(IS_ERR(mypd)){
printk(KERN_INFO "Failed to allocate PD\n");
return;
}
else{
printk(KERN_INFO "1Successfully allocated the PD\n");
pdset = true;
}
// Creating the receive completion queue for RDMA
myrcvcq = ib_create_cq(device,myClient_ib_recvcompletion,NULL,NULL,myClient_recvq_size,0);
if(IS_ERR(myrcvcq)){
pr_err("%s:%d error code for receive cq%d\n", __func__, __LINE__, PTR_ERR(myrcvcq));
//printk("Error creating QP: %d \n",PTR_ERR(myClientqp));
}
else{
printk("Recieve CQ successfully created in address: %x \n",myrcvcq);
}
// Creating the send completion queue for RDMA
myClientsendcq = ib_create_cq(device,myClient_ib_sendcompletion, NULL, NULL,myClient_sendq_size,0 );
if(IS_ERR(myClientsendcq)){
pr_err("%s:%d scqerror code for send cq%d\n", __func__, __LINE__, PTR_ERR(myClientsendcq));
//printk("Error creating QP: %d \n",PTR_ERR(myClientqp));
}
else{
printk("1Send CQ successfully created in address: %x \n",myClientsendcq);
}
// Creating the queue pair
// Creating the queue pair
struct ib_qp_init_attr init_qpattr;
memset(&init_qpattr,0,sizeof(init_qpattr));
init_qpattr.event_handler = myClient_qp_event_handler;
init_qpattr.cap.max_send_wr = 2;
init_qpattr.cap.max_recv_wr = 2;
init_qpattr.cap.max_recv_sge = 1;
init_qpattr.cap.max_send_sge = 1;
init_qpattr.sq_sig_type = IB_SIGNAL_ALL_WR;
init_qpattr.qp_type = IB_QPT_UD;
init_qpattr.send_cq = myClientsendcq;
init_qpattr.recv_cq = myrcvcq;
myClientqp = ib_create_qp(mypd,&init_qpattr);
if(IS_ERR(myClientqp)){
pr_err("%s:%d error code %d\n", __func__, __LINE__, PTR_ERR(myClientqp));
//printk("Error creating QP: %d \n",PTR_ERR(myClientqp));
}
else{
printk(KERN_INFO "1The queue pair is successfully created \n");
qpcreated = true;
}
}
static void myClient_remove_one(struct ib_device *device)
{
}
static struct ib_client my_client = {
.name = "myRDMAclient",
.add = myClient_add_one,
.remove = myClient_remove_one
};
static int __init myRDMAclient_init(void)
{
int ret;
ret = ib_register_client(&my_client);
if(ret){
//printk(KERN_ALERT "KERN_ERR Failed to register IB client\n");
goto err_sa;
}
printk(KERN_ALERT "lKERN_INFO Successfully registered myRDMAclient module \n");
return 0;
err_sa:
return ret;
}
module_init(myRDMAclient_init);
这里除了ib_create_qp(mypd,&init_qpattr);无法创建队列对之外,所有查询都有效.
更新:在创建队列对之前注册了内存.但它仍然显示ib_create_qp的无效参数错误(错误代码-22)
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/err.h>
#include "myClient.h"
struct workqueue_struct *myClient_workqueue;
struct ib_sa_client myClient_sa_client;
/*
static void myClient_add_one(struct ib_device *device);
static void myClient_remove_one(struct ib_device *device);
*/
struct ib_pd *mypd;
struct ib_cq *myrcvcq;
struct ib_cq *myClientsendcq;
struct ib_qp *myClientqp;
struct ib_mr *mymr;
void myClient_ib_recvcompletion(struct ib_cq *cq)
{
printk("A user-specified callback that is invoked when a completion event occurs on the CQ.\n");
}
void myClient_ib_sendcompletion(struct ib_cq *cq)
{
printk("A user-specified callback that is invoked when a completion event occurs on the CQ.\n");
}
static void my_qp_event_handler(struct ib_event *myqpAsyncEvent, void *anyPointer)
{
printk(KERN_INFO "Dummy affiliated asynchronous event occured function called \n");
}
static void myClient_add_one(struct ib_device *device)
{
union ib_gid tmp_gid;
int ret;
int hcaport = 1;
int result = -ENOMEM;
u16 port1Pkey;
struct ib_port_attr attr;
ret = ib_query_port(device,hcaport,&attr);
printk("ib query port result %d \n", ret);
// Creating the Protection Domain for RDMA
mypd = ib_alloc_pd(device);
if(IS_ERR(mypd)){
printk(KERN_INFO "Failed to allocate PD\n");
return;
}
else{
printk(KERN_INFO "1Successfully allocated the PD\n");
pdset = true;
}
// Registering Memory
mymr = ib_get_dma_mr(mypd,IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ| IB_ACCESS_REMOTE_WRITE);
if(IS_ERR(mymr)){
printk("failed to register memory :( %d \n",PTR_ERR(mymr));
}else{
printk(KERN_INFO "Successfully registered memory region :) \n");
}
// End Registering Memory
// Creating the receive completion queue for RDMA
myrcvcq = ib_create_cq(device,myClient_ib_recvcompletion,NULL,NULL,myClient_recvq_size,0);
if(IS_ERR(myrcvcq)){
pr_err("%s:%d error code for receive cq%d\n", __func__, __LINE__, PTR_ERR(myrcvcq));
//printk("Error creating QP: %d \n",PTR_ERR(myClientqp));
}
else{
printk("Recieve CQ successfully created in address: %x \n",myrcvcq);
}
// Creating the send completion queue for RDMA
myClientsendcq = ib_create_cq(device,myClient_ib_sendcompletion, NULL, NULL,myClient_sendq_size,0 );
if(IS_ERR(myClientsendcq)){
pr_err("%s:%d scqerror code for send cq%d\n", __func__, __LINE__, PTR_ERR(myClientsendcq));
//printk("Error creating QP: %d \n",PTR_ERR(myClientqp));
}
else{
printk("1Send CQ successfully created in address: %x \n",myClientsendcq);
}
// Creating the queue pair
// Creating the queue pair
struct ib_qp_init_attr init_qpattr;
memset(&init_qpattr,0,sizeof(init_qpattr));
init_qpattr.event_handler = myClient_qp_event_handler;
init_qpattr.cap.max_send_wr = 2;
init_qpattr.cap.max_recv_wr = 2;
init_qpattr.cap.max_recv_sge = 1;
init_qpattr.cap.max_send_sge = 1;
init_qpattr.sq_sig_type = IB_SIGNAL_ALL_WR;
init_qpattr.qp_type = IB_QPT_UD;
init_qpattr.send_cq = myClientsendcq;
init_qpattr.recv_cq = myrcvcq;
myClientqp = ib_create_qp(mypd,&init_qpattr);
if(IS_ERR(myClientqp)){
pr_err("%s:%d error code %d\n", __func__, __LINE__, PTR_ERR(myClientqp));
//printk("Error creating QP: %d \n",PTR_ERR(myClientqp));
}
else{
printk(KERN_INFO "1The queue pair is successfully created \n");
qpcreated = true;
}
}
static void myClient_remove_one(struct ib_device *device)
{
}
static struct ib_client my_client = {
.name = "myRDMAclient",
.add = myClient_add_one,
.remove = myClient_remove_one
};
static int __init myRDMAclient_init(void)
{
int ret;
ret = ib_register_client(&my_client);
if(ret){
//printk(KERN_ALERT "KERN_ERR Failed to register IB client\n");
goto err_sa;
}
printk(KERN_ALERT "lKERN_INFO Successfully registered myRDMAclient module \n");
return 0;
err_sa:
return ret;
}
module_init(myRDMAclient_init);
Dor*_*cus -1
我认为您忘记注册内存区域。创建 QP 之前需要执行的操作是：
然后才创建 QP。
我不知道您正在使用什么设备和库，但在 Mellanox IB 库中它是：
char mr_buffer[REGION_SIZE];
//mypd its your protection domain that you allocated
struct ibv_mr *mr = ibv_reg_mr(mypd , mr_buffer, REGION_SIZE, 0);
if (!mr) {
//ERROR MSG
}