mapping memory reserved by mmap kernel boot param into user space

Question

As discussed in this question, i am reserving a memory chunk at the boot time using a kernel boot parameter memmap=8G$64G

I have written a character driver kernel module which , during initialization does a ioremap of this reserved memory chunk. As explained here , in my driver mmap all i need to do is remap_pfn_range for this memory chunk pointer returned by the ioremap.

I am running this on 3.0 linux kernel. My user space application opens this memory chunk as a device mounted by the driver. When i do mmap from the use space application i see a system hang. my dmesg don't provide me much information.

Any inputs ?

static int __init myDev_module_init(void)
{
   int retval;

   myDev_major = register_chrdev(0, DEVICE_NAME, &myDevfops);
   if (myDev_major < 0) 
   {
       err("failed to register device: error %d\n", myDev_major);
       retval = myDev_major;
       goto FAILED_CHRDEVREG;
   }

   myDev_class = class_create(THIS_MODULE, CLASS_NAME);
   if (IS_ERR(myDev_class)) 
   {   
       err("failed to register device class '%s'\n", CLASS_NAME);
       retval = PTR_ERR(myDev_class);
       goto FAILED_CLASSREG;
   }


   myDev_device = device_create(myDev_class, NULL, MKDEV(myDev_major, 0), NULL, CLASS_NAME "_" DEVICE_NAME);
   if (IS_ERR(myDev_device)) 
   {
       err("failed to create device '%s_%s'\n", CLASS_NAME, DEVICE_NAME);
       retval = PTR_ERR(myDev_device);
       goto FAILED_DEVREG;
   }

here the myDev.startOffset is initialized to #defined 64GB and myDev.memSize to 8GB.

 //myDev.startAddr = ioremap(myDev.startOffset,myDev.memSize);

 //memset_io(myDev.startAddr, 0, myDev.memSize);  
 return 0;

  FAILED_DEVREG:
   class_unregister(myDev_class);
   class_destroy(myDev_class);
  FAILED_CLASSREG:
   unregister_chrdev(myDev_major, DEVICE_NAME);
  FAILED_CHRDEVREG:
   return -1;
}

static int myDev_device_open(struct inode* inode, struct file* filp)
{
    dbg("");

    if ( ((filp->f_flags & O_ACCMODE) == O_WRONLY) || ((filp->f_flags & O_ACCMODE) == O_RDWR) ) 
    {
        warn(" Opening the device with write access\n");
        //return -EACCES;
    }

    info(" device Open is called\n");
    filp->private_data = &myDev;
    return 0;
 }

And the mmap is pretty straight forward.

static int myDev_device_mmap(struct file * f, struct vm_area_struct * vma)
{
 int retval = 0;
struct myDevDev * pDev = (struct myDevDev *)(f->private_data);

dbg("");
if(vma)
{
    if(f)
    {
        if(f->private_data)
            warn("mmap: f->private_data  : %p\n", f->private_data);
        else
            warn(" mmap :f->private_data  : NULL \n");
    }
    else
    {
        warn("mmap: f  :NULL\n");
    }
    warn(": mmap: vm start : %lu\n", vma->vm_start);
    warn(" mmap: vm end  : %lu\n", vma->vm_end);
    warn(" mmap: vm pg offset  : %lu\n", vma->vm_pgoff);


    
    //retval = remap_pfn_range(vma, vma->vm_start, pDev->startOffset >> PAGE_SHIFT,    pDev->memSize, PAGE_SHARED) ;
    // retval = remap_pfn_range(vma, vma->vm_start, pDev->startAddr >> PAGE_SHIFT,    pDev->memSize, PAGE_SHARED) ;
    //retval = remap_pfn_range(vma,pDev->startAddr ,pDev->startOffset >> PAGE_SHIFT,  pDev->memSize, PAGE_SHARED);
    retval = remap_pfn_range(vma,vma->vm_start ,pDev->startOffset >> PAGE_SHIFT,  pDev->memSize, PAGE_SHARED);
    if(retval <0)
    {
        warn(" ERROR : in mapping kernel virtual space to user space return value : %d \n",retval);
        return -EINVAL;
    }
    
    //if(0)
    {
        vma->vm_flags |=VM_LOCKED;
        vma->vm_ops = &myRemapVMOps;
        vma->vm_flags |= VM_RESERVED;

        vma->vm_private_data = f->private_data;
        myDevice_VMA_Open(vma);
    }
}
else
{
    warn ("vma is NULL");
}

dbg(" Done ");
warn("mmpaing done : \n");

return 0;
}

from my user space application i am doing the following :

int err, i=0;
void * mptr = NULL;
printf(" Access the reserved memory chunk \n  ");
int fd = open("/dev/myDevice", O_RDWR | O_SYNC);

if(fd <=0)
{
    printf("ERROR: my device driver is not loaded \n");
    return 1;
}

  printf("\n mmaping mem chunk size :%llu pagesize :%lu input mptr :%p\n", sz,getpagesize (), mptr);

  mptr = mmap(0, sz , PROT_READ | PROT_WRITE, MAP_SHARED | MAP_LOCKED, fd, 0);

if(mptr == MAP_FAILED) 
{
    close(fd);
    perror("Error mmapping the file");
    printf("\nmmapped mem address %p\n",mptr);
    exit(1);
}
printf("\nmmapped mem address %p\n",mptr);

//char * ptr = (char *)mptr;
//*ptr = 'a';

//int * pInt =  (int *) (((char *) mptr)+1); 
//for(;i<10000; ++i)
{
  //  pInt[i] = 2*i;
}

 /* free the mmapped memory
 */
if (munmap(mptr, sz) == -1) 
{
    perror("Error un-mmapping the file");
}

close(fd);

Observation :

I don't see the size getting reflected in the (vma->vm_end - vma->vm_start) which for some reason is ALWAYS 4K.

Answer 1

1. You can use standard phram driver in steps to access your memory from userspace. No coding is needed. Kernel recompilation at maximum.

2. Do You really have more then 64Gb of RAM? Does Your hardware really support it?

Answer 2

When i do mmap from the use space application i see a system hang.

Before you can use the remap_pfn_range() API, see documentation inlined inside mm/memory.c (of kernel source), and it indicated that you have to have held the mm semaphore first.

/**
 * remap_pfn_range - remap kernel memory to userspace
 * @vma: user vma to map to
 * @addr: target user address to start at
 * @pfn: physical address of kernel memory
 * @size: size of map area
 * @prot: page protection flags for this mapping
 *
 *  Note: this is only safe if the mm semaphore is held when called.
 */

And how to acquire the semaphore is via up_read() and down_read() API - just "grep" for up_read inside the mm directory and there are many examples:

mempolicy.c:    up_read(&mm->mmap_sem);
migrate.c:  up_read(&mm->mmap_sem);
mincore.c:      up_read(&current->mm->mmap_sem);
mmap.c:     up_read(&mm->mmap_sem);
msync.c:            up_read(&mm->mmap_sem);
nommu.c:    up_read(&mm->mmap_sem);