rdprail_gbm.c

#include <assert.h>
#include <stdlib.h>
#include <unistd.h>
#include <gbm.h>
#include <wlr/util/log.h>
#include "rdprail.h"

struct wlr_rdp_gbm_allocator {
    struct wlr_allocator base;
    GfxRedirServerContext *redir_ctx;
    struct gbm_device *gbm;
};

struct wlr_rdp_gbm_buffer {
    struct wlr_buffer buffer;
    uint32_t pool_id;
    uint32_t buffer_id;
    struct gbm_bo *bo;
    void *map_data;
    void *rdp_memory;      // Direct mapped memory
    size_t rdp_memory_size;
    int width;
    int height;
    int fd;              // -1 since we're using direct mapping
    bool data_access_in_progress;
    uint32_t stride;
};

static struct wlr_buffer *rdp_gbm_allocator_create_buffer(
        struct wlr_allocator *alloc,
        int width, int height,
        const struct wlr_drm_format *format) {
    struct wlr_rdp_gbm_allocator *rdp_alloc = (struct wlr_rdp_gbm_allocator *)alloc;
    
    // Create GBM buffer with known working settings
    struct gbm_bo *bo = gbm_bo_create(rdp_alloc->gbm, width, height,
        GBM_FORMAT_XRGB8888,  // Known working format
        GBM_BO_USE_SCANOUT);  // Known working flag
    
    if (!bo) {
        wlr_log(WLR_ERROR, "Failed to create GBM buffer");
        return NULL;
    }

    // Create RDP pool
    GFXREDIR_OPEN_POOL_PDU open_pool = {0};
    if (rdp_alloc->redir_ctx->OpenPool(rdp_alloc->redir_ctx, &open_pool) != 0) {
        wlr_log(WLR_ERROR, "Failed to open RDP pool");
        gbm_bo_destroy(bo);
        return NULL;
    }

    // Create buffer in RDP
    GFXREDIR_CREATE_BUFFER_PDU create_buffer = {
        .poolId = open_pool.poolId,
        .width = width,
        .height = height,
        .format = 0  // Format value from your RDP implementation
    };

    if (rdp_alloc->redir_ctx->CreateBuffer(rdp_alloc->redir_ctx, &create_buffer) != 0) {
        wlr_log(WLR_ERROR, "Failed to create RDP buffer");
        gbm_bo_destroy(bo);
        return NULL;
    }

    struct wlr_rdp_gbm_buffer *buffer = calloc(1, sizeof(*buffer));
    if (buffer == NULL) {
        gbm_bo_destroy(bo);
        return NULL;
    }

    buffer->pool_id = open_pool.poolId;
    buffer->buffer_id = create_buffer.bufferId;
    buffer->bo = bo;
    buffer->width = width;
    buffer->height = height;
    buffer->fd = -1;  // No fd since we're using direct mapping
    buffer->data_access_in_progress = false;

    // Map the buffer for CPU access
    void *data = NULL;
    void *map_data = gbm_bo_map(bo, 0, 0, width, height,
        GBM_BO_TRANSFER_WRITE | GBM_BO_TRANSFER_READ,
        &buffer->stride,
        &data);

    if (!map_data || !data) {
        wlr_log(WLR_ERROR, "Failed to map GBM buffer");
        free(buffer);
        gbm_bo_destroy(bo);
        return NULL;
    }

    buffer->map_data = map_data;
    buffer->rdp_memory = data;
    buffer->rdp_memory_size = buffer->stride * height;

    // Initialize wlr_buffer
    wlr_buffer_init(&buffer->buffer, &buffer_impl, width, height);

    return &buffer->buffer;
}

static void rdp_gbm_buffer_destroy(struct wlr_buffer *wlr_buffer) {
    struct wlr_rdp_gbm_buffer *buffer = (struct wlr_rdp_gbm_buffer *)wlr_buffer;

    if (buffer->data_access_in_progress) {
        wlr_rdp_buffer_end_data_ptr_access(wlr_buffer);
    }

    if (buffer->map_data) {
        gbm_bo_unmap(buffer->bo, buffer->map_data);
    }
    if (buffer->bo) {
        gbm_bo_destroy(buffer->bo);
    }

    free(buffer);
}

static bool rdp_gbm_buffer_begin_data_ptr_access(struct wlr_buffer *wlr_buffer,
        uint32_t flags, void **data, uint32_t *stride) {
    struct wlr_rdp_gbm_buffer *buffer = (struct wlr_rdp_gbm_buffer *)wlr_buffer;
    *data = buffer->rdp_memory;
    *stride = buffer->stride;
    buffer->data_access_in_progress = true;
    return true;
}

static void rdp_gbm_buffer_end_data_ptr_access(struct wlr_buffer *wlr_buffer) {
    struct wlr_rdp_gbm_buffer *buffer = (struct wlr_rdp_gbm_buffer *)wlr_buffer;
    buffer->data_access_in_progress = false;
}

static const struct wlr_buffer_impl buffer_impl = {
    .destroy = rdp_gbm_buffer_destroy,
    .begin_data_ptr_access = rdp_gbm_buffer_begin_data_ptr_access,
    .end_data_ptr_access = rdp_gbm_buffer_end_data_ptr_access,
};

static void rdp_gbm_allocator_destroy(struct wlr_allocator *wlr_alloc) {
    struct wlr_rdp_gbm_allocator *alloc = (struct wlr_rdp_gbm_allocator *)wlr_alloc;
    if (alloc == NULL) {
        return;
    }
    free(alloc);
}

static const struct wlr_allocator_interface rdp_gbm_allocator_impl = {
    .create_buffer = rdp_gbm_allocator_create_buffer,
    .destroy = rdp_gbm_allocator_destroy,
};

struct wlr_allocator *wlr_rdp_gbm_allocator_create(
        GfxRedirServerContext *redir_ctx, struct gbm_device *gbm) {
    struct wlr_rdp_gbm_allocator *alloc = calloc(1, sizeof(*alloc));
    if (alloc == NULL) {
        return NULL;
    }

    wlr_allocator_init(&alloc->base, &rdp_gbm_allocator_impl, WLR_BUFFER_CAP_DATA_PTR);
    alloc->redir_ctx = redir_ctx;
    alloc->gbm = gbm;

    return &alloc->base;
}