rdprail.c

/* rdprail.c */
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <linux/memfd.h>
#include <wlr/render/allocator.h>
#include <wlr/render/drm_format_set.h>
#include <wlr/types/wlr_buffer.h>
#include <wlr/interfaces/wlr_buffer.h>
#include <wlr/util/log.h>
#include <drm/drm_fourcc.h>
#include <stdio.h>
#include "rdprail.h"

// Logging function for debugging
static void log_buffer_state(struct wlr_buffer *buffer, const char *context) {
    fprintf(stderr, "Buffer state at %s:\n", context);
    fprintf(stderr, "  Width: %d\n", buffer->width);
    fprintf(stderr, "  Height: %d\n", buffer->height);
    fprintf(stderr, "  Impl: %p\n", (void*)buffer->impl);
    fprintf(stderr, "  Dropped: %s\n", buffer->dropped ? "true" : "false");
    fprintf(stderr, "  Locks: %zu\n", buffer->n_locks);
    fprintf(stderr, "  Accessing data ptr: %s\n", 
        buffer->accessing_data_ptr ? "true" : "false");
}

// Placeholder implementation for shared memory retrieval
/*void* rdp_get_shared_memory(uint32_t pool_id, size_t *size, int *fd) {
    // Use syscall for memfd_create as a fallback
    *fd = syscall(SYS_memfd_create, "rdp_buffer", MFD_CLOEXEC);
    
    // If syscall fails, fall back to temporary file
    if (*fd == -1) {
        char temp_path[] = "/tmp/rdp_buffer_XXXXXX";
        *fd = mkstemp(temp_path);
        if (*fd == -1) {
            wlr_log(WLR_ERROR, "Failed to create shared memory");
            return NULL;
        }
        // Unlink the temporary file so it's not visible in the filesystem
        unlink(temp_path);
    }
    
    // Set size
    *size = 1024 * 1024;  // 1MB
    
    // Resize the file descriptor
    if (ftruncate(*fd, *size) == -1) {
        wlr_log(WLR_ERROR, "Failed to resize shared memory");
        close(*fd);
        return NULL;
    }
    
    // Map the shared memory
    void *memory = mmap(NULL, *size, PROT_READ | PROT_WRITE, MAP_SHARED, *fd, 0);
    if (memory == MAP_FAILED) {
        wlr_log(WLR_ERROR, "Failed to map shared memory");
        close(*fd);
        return NULL;
    }
    
    return memory;
}*/

void* rdp_get_shared_memory(uint32_t pool_id, size_t *size, int *fd) {
    *size = 640 * 480 * 4;  // Calculate exact size needed
    
    // Create memfd with specific size
    *fd = syscall(SYS_memfd_create, "rdp_buffer", MFD_CLOEXEC);
    if (*fd == -1) {
        char temp_path[] = "/tmp/rdp_buffer_XXXXXX";
        *fd = mkstemp(temp_path);
        if (*fd == -1) {
            return NULL;
        }
        unlink(temp_path);
    }
    
    // Set the size
    if (ftruncate(*fd, *size) == -1) {
        close(*fd);
        return NULL;
    }
    
    // Map the memory with proper protection flags
    void *memory = mmap(NULL, *size, PROT_READ | PROT_WRITE, 
                       MAP_SHARED, *fd, 0);
    if (memory == MAP_FAILED) {
        close(*fd);
        return NULL;
    }
    
    // Clear the memory
    memset(memory, 0, *size);
    return memory;
}

// Buffer method implementations
/*static void wlr_rdp_buffer_destroy(struct wlr_buffer *buffer) {
    struct wlr_rdp_buffer *rdp_buffer = 
        wl_container_of(buffer, rdp_buffer, buffer);
    
    fprintf(stderr, "Destroying RDP buffer\n");
    
    // Ensure no data access in progress
    if (buffer->accessing_data_ptr) {
        wlr_log(WLR_ERROR, "Destroying buffer while data access in progress");
    }
    
    // Unmap and close shared memory
    if (rdp_buffer->rdp_memory) {
        munmap(rdp_buffer->rdp_memory, rdp_buffer->rdp_memory_size);
    }
    
    if (rdp_buffer->fd != -1) {
        close(rdp_buffer->fd);
    }
    
    // Free the buffer struct
    free(rdp_buffer);
}*/

static void wlr_rdp_buffer_destroy(struct wlr_buffer *buffer) {
    struct wlr_rdp_buffer *rdp_buffer = wl_container_of(buffer, rdp_buffer, buffer);
    
    // Wait for any ongoing data access to complete
    if (rdp_buffer->data_access_in_progress) {
        wlr_rdp_buffer_end_data_ptr_access(buffer);
    }
    
    // Clean up memory
    if (rdp_buffer->rdp_memory && rdp_buffer->rdp_memory_size > 0) {
        munmap(rdp_buffer->rdp_memory, rdp_buffer->rdp_memory_size);
        rdp_buffer->rdp_memory = NULL;
    }
    
    if (rdp_buffer->fd >= 0) {
        close(rdp_buffer->fd);
        rdp_buffer->fd = -1;
    }
    
    free(rdp_buffer);
}

static bool wlr_rdp_buffer_get_dmabuf(
    struct wlr_buffer *wlr_buffer, 
    struct wlr_dmabuf_attributes *attribs
) {
    // No DMA-BUF support for this buffer type
    return false;
}

static bool wlr_rdp_buffer_get_shm(
    struct wlr_buffer *wlr_buffer, 
    struct wlr_shm_attributes *attribs
) {
    struct wlr_rdp_buffer *rdp_buffer = 
        wl_container_of(wlr_buffer, rdp_buffer, buffer);
    
    if (!rdp_buffer->rdp_memory) {
        return false;
    }
    
    attribs->fd = rdp_buffer->fd;
    attribs->format = DRM_FORMAT_ARGB8888;
    attribs->width = rdp_buffer->width;
    attribs->height = rdp_buffer->height;
    attribs->stride = rdp_buffer->width * 4;
    attribs->offset = 0;
    
    return true;
}

static bool wlr_rdp_buffer_begin_data_ptr_access(
    struct wlr_buffer *wlr_buffer, 
    uint32_t flags, 
    void **data, 
    uint32_t *format, 
    size_t *stride
) {
    struct wlr_rdp_buffer *rdp_buffer = 
        wl_container_of(wlr_buffer, rdp_buffer, buffer);
    
    fprintf(stderr, "Beginning data ptr access\n");
    log_buffer_state(wlr_buffer, "Begin data ptr access");
    
    // Additional safety checks
    if (wlr_buffer->accessing_data_ptr) {
        wlr_log(WLR_ERROR, "Attempting to access buffer data while already in access");
        return false;
    }
    
    if (!rdp_buffer->rdp_memory) {
        wlr_log(WLR_ERROR, "No memory allocated for RDP buffer");
        return false;
    }
    
    if (data) {
        *data = rdp_buffer->rdp_memory;
        fprintf(stderr, "Data pointer set to %p\n", *data);
    }
    
    if (format) {
        *format = DRM_FORMAT_ARGB8888;
        fprintf(stderr, "Format set to %u\n", *format);
    }
    
    if (stride) {
        *stride = rdp_buffer->width * 4; // 4 bytes per pixel
        fprintf(stderr, "Stride set to %zu\n", *stride);
    }
    
    // Mark that data access is in progress
    rdp_buffer->data_access_in_progress = true;
    
    return true;
}

void wlr_rdp_buffer_end_data_ptr_access(struct wlr_buffer *wlr_buffer) {
    struct wlr_rdp_buffer *rdp_buffer = 
        wl_container_of(wlr_buffer, rdp_buffer, buffer);
    
    fprintf(stderr, "Ending data ptr access\n");
    log_buffer_state(wlr_buffer, "End data ptr access");
    
    // Reset data access flag
    rdp_buffer->data_access_in_progress = false;
}

// Define the buffer implementation matching wlr_buffer_impl
static const struct wlr_buffer_impl wlr_rdp_buffer_impl = {
    .destroy = wlr_rdp_buffer_destroy,
    .get_dmabuf = wlr_rdp_buffer_get_dmabuf,
    .get_shm = wlr_rdp_buffer_get_shm,
    .begin_data_ptr_access = wlr_rdp_buffer_begin_data_ptr_access,
    .end_data_ptr_access = wlr_rdp_buffer_end_data_ptr_access,
};

// Allocator creation methods
static struct wlr_buffer *rdp_allocator_create_buffer(
        struct wlr_allocator *alloc, int width, int height,
        const struct wlr_drm_format *format) {
    struct wlr_rdp_allocator *rdp_alloc =
        wl_container_of(alloc, rdp_alloc, base);

    fprintf(stderr, "Creating RDP buffer: %dx%d\n", width, height);

    // Create RDP pool and buffer
    GFXREDIR_OPEN_POOL_PDU open_pool = {0};
    if (rdp_alloc->redir_ctx->OpenPool(rdp_alloc->redir_ctx, &open_pool) != 0) {
        wlr_log(WLR_ERROR, "Failed to open RDP pool");
        return NULL;
    }

    GFXREDIR_CREATE_BUFFER_PDU create_buffer = {
        .poolId = open_pool.poolId,
        .width = width,
        .height = height,
        .format = 1 // Placeholder for GFXREDIR_BUFFER_PIXEL_FORMAT_ARGB_8888
    };

    if (rdp_alloc->redir_ctx->CreateBuffer(rdp_alloc->redir_ctx, &create_buffer) != 0) {
        wlr_log(WLR_ERROR, "Failed to create RDP buffer");
        return NULL;
    }

    // Allocate RDP buffer struct
    struct wlr_rdp_buffer *rdp_buffer = calloc(1, sizeof(struct wlr_rdp_buffer));
    if (!rdp_buffer) {
        wlr_log(WLR_ERROR, "Failed to allocate wlr_rdp_buffer");
        return NULL;
    }

    rdp_buffer->pool_id = open_pool.poolId;
    rdp_buffer->buffer_id = create_buffer.bufferId;
    rdp_buffer->width = width;
    rdp_buffer->height = height;
    rdp_buffer->fd = -1;
    rdp_buffer->data_access_in_progress = false;

    // Acquire shared memory
    rdp_buffer->rdp_memory = rdp_get_shared_memory(
        open_pool.poolId, &rdp_buffer->rdp_memory_size, &rdp_buffer->fd);
    if (!rdp_buffer->rdp_memory) {
        wlr_log(WLR_ERROR, "Failed to get shared memory for RDP buffer");
        free(rdp_buffer);
        return NULL;
    }

    // Initialize the buffer with our implementation
    wlr_buffer_init(
        &rdp_buffer->buffer,
        &wlr_rdp_buffer_impl,
        width, height
    );

    fprintf(stderr, "RDP buffer created successfully\n");
    return &rdp_buffer->buffer;
}

static void rdp_allocator_destroy(struct wlr_allocator *alloc) {
    struct wlr_rdp_allocator *rdp_alloc = 
        wl_container_of(alloc, rdp_alloc, base);
    
    fprintf(stderr, "Destroying RDP allocator\n");
    free(rdp_alloc);
}

static const struct wlr_allocator_interface rdp_allocator_impl = {
    .create_buffer = rdp_allocator_create_buffer,
    .destroy = rdp_allocator_destroy,
};

struct wlr_allocator *wlr_rdp_allocator_create(
        GfxRedirServerContext *redir_ctx) {
    fprintf(stderr, "Creating RDP allocator\n");
    
    if (!redir_ctx) {
        wlr_log(WLR_ERROR, "Invalid RDP context");
        return NULL;
    }
    
    struct wlr_rdp_allocator *alloc = 
        calloc(1, sizeof(struct wlr_rdp_allocator));
    if (!alloc) {
        wlr_log(WLR_ERROR, "Failed to allocate RDP allocator");
        return NULL;
    }
    
    // Initialize allocator
    wlr_allocator_init(&alloc->base, &rdp_allocator_impl, 
        WLR_BUFFER_CAP_DATA_PTR | WLR_BUFFER_CAP_SHM);
    
    // Store RDP context
    alloc->redir_ctx = redir_ctx;
    
    fprintf(stderr, "RDP allocator created successfully\n");
    return &alloc->base;
}