Ask for Help: drmPrimeHandleToFd failed on virtual machine env.
sry for my poor English. I'm a novice of libdrm, and I want to ask for help of it.
On my Ubuntu 22.04 test virtual machine installed on Vmware Workstation, change dumb buffer handle to prime fd only returns -1. But DRM has DRM DRM_PRIME_CAP_IMPORT
& DRM_PRIME_CAP_EXPORT
caps. On Ubuntu 22.04 physical machine, it works well.
here it's my test code depends on drm-howto:
struct drme_dumb_buffer {
int fd;
uint32_t handle;
uint32_t stride;
uint32_t width, height;
uint32_t size;
void *map = nullptr;
uint32_t format;
};
static std::shared_ptr<drme_dumb_buffer> alloc_buffer(int drm_fd, uint32_t width, uint32_t height)
{
std::shared_ptr<drme_dumb_buffer> buffer = std::make_shared<drme_dumb_buffer>();
// Step1: create dumb buffer
struct drm_mode_create_dumb create = {
.height = height,
.width = width,
.bpp = 32,
.flags = 0
};
if (drmIoctl(drm_fd, DRM_IOCTL_MODE_CREATE_DUMB, &create) != 0) {
fprintf(stderr, "[!] Failed to create DRM dumb buffer : (%d) %m\n",
errno);
return nullptr;
}
buffer->width = create.width;
buffer->height = create.height;
buffer->stride = create.pitch;
buffer->handle = create.handle;
buffer->size = create.size;
// Step2: map the dumb buffer for userspace drawing
// prepare buffer for mmap
struct drm_mode_map_dumb map = {
.handle = buffer->handle,
.pad = 0
};
if (drmIoctl(drm_fd, DRM_IOCTL_MODE_MAP_DUMB, &map) != 0) {
fprintf(stderr, "[!] Failed to map DRM dumb buffer : (%d) %m\n",
errno);
return nullptr;
}
// perform actual memory mapping
buffer->map = mmap(0, buffer->size, PROT_READ | PROT_WRITE,
MAP_SHARED, drm_fd, map.offset);
// clear buffer to 0
memset(buffer->map, std::numeric_limits<int>::max(), buffer->size);
// Step3: convert to DMA buffer fd
int prime_fd;
if (drmPrimeHandleToFD(drm_fd, buffer->handle, DRM_CLOEXEC,
&prime_fd) != 0) { // failed here!
fprintf(stderr, "[!] Failed to convert handle to prime fd : (%d) %m\n", errno);
return nullptr;
}
buffer->fd = prime_fd;
return buffer;
}
btw, through gbm_bo with GBM_BO_USE_WRITE
usage, it will create dumb buffer. In this case, gbm_bo_get_fd_for_plane
and gbm_bo_get_fd
also will return -1, even though DRM has DRM DRM_PRIME_CAP_IMPORT
& DRM_PRIME_CAP_EXPORT
caps.
int gbm_allocator_create_drm_fb(int fd, struct modeset_buf *buf)
{
uint32_t dmabuf_handles[4] = {0}, dmabuf_pitches[4] = {0}, dmabuf_offsets[4] = {0};
int dmabuf_fds[4] = {0};
/* check drm caps */
uint64_t cap;
if (drmGetCap(fd, DRM_CAP_PRIME, &cap) != 0 ||
!(cap & DRM_PRIME_CAP_IMPORT)) {
fprintf(stderr, "[!] PrimeFdToHandle not supported!\n");
return -1;
}
if (!(cap & DRM_PRIME_CAP_EXPORT)) {
fprintf(stderr, "[!] PrimeHandleToFd not supported!\n");
return -1;
}
printf("Created GBM device with backend: %s\n", gbm_device_get_backend_name(gbm));
char *drm_name = drmGetDeviceNameFromFd2(fd);
printf("Using DRM node: %s\n", drm_name);
free(drm_name);
/* create gbm bo */
struct gbm_bo* gbm_bo = gbm_bo_create(
gbm,
buf->width, buf->height,
GBM_FORMAT_XRGB8888,
GBM_BO_USE_RENDERING | GBM_BO_USE_SCANOUT | GBM_BO_USE_WRITE // create dumb buffer
);
if (gbm_bo == nullptr)
{
fprintf(stderr, "[!] failed to create gbm device.\n");
gbm_device_destroy(gbm);
return -1;
}
int n_planes = gbm_bo_get_plane_count(gbm_bo);
if (n_planes <= 0)
{
fprintf(stderr, "[!] GBM BO contains no planes.\n");
gbm_bo_destroy(gbm_bo);
gbm_device_destroy(gbm);
return -1;
}
dmabuf_handles[0] = gbm_bo_get_handle(gbm_bo).u32;
dmabuf_pitches[0] = gbm_bo_get_stride(gbm_bo);
dmabuf_offsets[0] = gbm_bo_get_offset(gbm_bo, 0);
int dma_fd = gbm_bo_get_fd(gbm_bo); // failed: return -1
if (dma_fd < 0) {
fprintf(stderr, "[!] gbm bo fd < 0.\n");
} else {
printf("[*] gbm bo fd: %d\n", dma_fd);
}
// mmap
uint32_t dst_stride = 0;
void* gbo_mapping = nullptr;
char* map = static_cast<char*>(gbm_bo_map(gbm_bo,
0, 0,
buf->width, buf->height,
GBM_BO_TRANSFER_READ_WRITE,
&dst_stride,
&gbo_mapping));
// create fb
uint32_t fb_id = 0;
int ret = drmModeAddFB2(fd, buf->width, buf->height, DRM_FORMAT_XRGB8888,
dmabuf_handles, dmabuf_pitches, dmabuf_offsets, &fb_id, 0);
if (ret) {
fprintf(stderr, "[!] cannot create framebuffer new (%d): %m\n",
errno);
ret = -errno;
gbm_bo_destroy(gbm_bo);
gbm_device_destroy(gbm);
return ret;
}
// close tmp handles
if(drmCloseBufferHandle(fd, dmabuf_handles[0]) != 0) {
fprintf(stderr, "[!] failed to drmCloseBufferHandle\n"); // also failed
}
buf->handle = gbm_bo_get_handle(gbm_bo).u32;
buf->stride = gbm_bo_get_stride(gbm_bo);
buf->map_data = static_cast<uint8_t*>(gbo_mapping);
// buf->size = buf->stride * buf->height;
buf->gbm_bo = gbm_bo;
buf->fb = fb_id;
printf("bo_get_stride: %u\n", buf->stride);
printf("dst_stride: %u\n", dst_stride);
return 0;
}
I need to change handle to fd for GPU Rendering & interprocess transfer. Did anyone know what's going wrong?