I created a modified BlockDist.chpl file that gets a C pointer and tries to give it to myElems in LocBlockArr. The parts I edited are denoted in the comment block. The original line here was this.myElems = this.locDom.myBlock.buildArray(eltType, initElts=initElts);
When creating a distributed array, it seems that a new LocBlockArr is created with domain 1..0 in addition to the local array for each locale, so I left the original line in for this case in the else statement.
//These modules are necessary for the changes I added to LocalBlockArr
use SysCTypes;
use CPtr;
class LocBlockArr {
type eltType;
param rank: int;
type idxType;
param stridable: bool;
const locDom: unmanaged LocBlockDom(rank, idxType, stridable);
var locRAD: unmanaged LocRADCache(eltType, rank, idxType, stridable)?; // non-nil if doRADOpt=true
pragma "local field" pragma "unsafe"
// may be initialized separately
var myPtr:c_ptr(eltType);
var myElems: [locDom.myBlock] eltType;
var locRADLock: chpl_LocalSpinlock;
proc init(type eltType,
param rank: int,
type idxType,
param stridable: bool,
const locDom: unmanaged LocBlockDom(rank, idxType, stridable),
param initElts: bool) {
this.eltType = eltType;
this.rank = rank;
this.idxType = idxType;
this.stridable = stridable;
this.locDom = locDom;
////////////////////////////////////////////////////////Start Changes/////////////////////////////////////////////////////////
if(this.locDom.myBlock.size)
{
var myPtr = createMmap();
var temp = trevorMakeArrayFromPtr(myPtr, this.locDom.myBlock.size:uint, this.locDom.myBlock);
writeln("Temp Domain: ", temp.domain);
this.myElems = temp;
writeln("myElems domain: ", this.myElems.domain);
writeln("Array pointer at: ", c_ptrTo(this.myElems), " on ", here.id);
if(myPtr != c_ptrTo(this.myElems)) //wont be able to close mmap later if it fails here
{
closemmap(myPtr);
}
}
else
{
this.myElems = this.locDom.myBlock.buildArray(eltType, initElts=initElts);
}
////////////////////////////////////////////////////////End Changes/////////////////////////////////////////////////////
}
// guard against dynamic dispatch resolution trying to resolve
// write()ing out an array of sync vars and hitting the sync var
// type's compilerError()
override proc writeThis(f) throws {
halt("LocBlockArr.writeThis() is not implemented / should not be needed");
}
proc deinit() {
// Elements in myElems are deinited in dsiDestroyArr if necessary.
// Here we need to clean up the rest of the array.
if locRAD != nil then
delete locRAD;
}
}
CreateMmap.chpl contains functions that are used in the modified block. I've listed it below.
The main takeaway is the function createMmap() returns a c_ptr(c_int) that I want myElems to use, and makeArrayFromPtr and makeArrayFromExternArray have been modified to take a domain as an argument.
use CPtr;
use SysCTypes;
use Sys;
use SysBasic;
use Time;
require "mmap.h";
require "sys/mman.h";
require "sys/stat.h";
require "fcntl.h";
extern proc shm_open(name:c_string, oflag:c_int, mode:c_int):c_int;
extern proc shm_unlink(name:c_string):c_int;
extern proc mmap(addr:c_void_ptr, length:size_t, prot:c_int, flags:c_int, fd:c_int, offset:off_t):c_void_ptr;
extern proc munmap(addr:c_void_ptr, length:size_t):c_int;
extern proc getpagesize():size_t;
extern proc ftruncate(fd:c_int, length:off_t):c_int;
extern const O_RDWR: c_int;
extern const O_CREAT: c_int;
extern const PROT_READ:c_int;
extern const PROT_WRITE:c_int;
extern const MAP_SHARED:c_int;
extern const MAP_FIXED:c_int;
//extern const nBytes:c_int;
const backingFile = "/test.bak".c_str();
const accessPerms = 644:c_int;
const nInts = 8:c_int;
const nBytes = 32:c_int; //limits mmap region to 32 bits
proc trevorMakeArrayFromExternArray(value:chpl_external_array, type eltType, dom:domain)
{
var mydom = defaultDist.dsiNewRectangularDom(rank=1,
idxType=int,
stridable=false,
inds=(dom.low..dom.high,));
mydom._free_when_no_arrs = true;
var arr = new unmanaged DefaultRectangularArr(eltType = eltType,
rank = 1,
idxType=mydom.idxType,
stridable=mydom.stridable,
dom=mydom,
data=value.elts:_ddata(eltType),
externFreeFunc=value.freer,
externArr=true,
_borrowed=true);
mydom.add_arr(arr, locking=false);
return _newArray(arr);
}
proc trevorMakeArrayFromPtr(value:c_ptr, num_elts:uint, dom:domain)
{
var data = chpl_make_external_array_ptr(value:c_void_ptr, num_elts);
return trevorMakeArrayFromExternArray(data, value.eltType, dom);
}
proc createMmap():c_ptr(c_int)
{
var fd = shm_open(backingFile, O_RDWR | O_CREAT, accessPerms):c_int;
ftruncate(fd, nBytes);
var region;
region = mmap(nil, nBytes:size_t, PROT_READ | PROT_WRITE, MAP_SHARED, fd:fd_t, 0:off_t);
var area = region:c_ptr(c_int);
writeln("mmap region: ", region, " on ", here.id);
return area;
}
proc closemmap(region:c_void_ptr)
{
munmap(region, nBytes:size_t);
shm_unlink(backingFile);
}
I've been using the following for a test case. The code above writes the memory address for myElems as well as the pointer returned from mmap(). The desired output would be for these addresses to be the same.
Compile with: chpl testMyBlock.chpl -lrt
use myBlockDist;
use CreateMmap;
use CPtr;
use SysCTypes;
use Time;
const Space = {1..8};
const D: domain(1) dmapped Block(boundingBox=Space) = Space;
var A: [D] c_int;
for i in Space
{
A[i] = (i):c_int;
}
writeln(A);
sleep(10); //time to read mmap data from another process
for loc in Locales
{
closemmap(c_ptrTo(A[A.localSubdomain().low]));
}
If the memory address for myElems matches the mmap pointer, then another process should be able to read the data written by chapel with mmap. I've been using this c program to test for that, but so far I haven't been able to get the memory addresses to match
/** Compilation: gcc -o memreader memreader.c -lrt **/
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <semaphore.h>
#include <string.h>
#include "mmap.h"
#define backingFile "/test.bak"
#define nInts 8
#define nBytes (nInts * sizeof(int))
#define accessPerms 0644
void report_and_exit(const char* msg) {
perror(msg);
exit(-1);
}
int main() {
int fd = shm_open(backingFile, O_RDWR, accessPerms); /* empty to begin */
if (fd < 0) report_and_exit("Can't get file descriptor...");
/* get a pointer to memory */
caddr_t memptr = mmap(NULL, /* let system pick where to put segment */
nBytes, /* how many bytes */
PROT_READ | PROT_WRITE, /* access protections */
MAP_SHARED, /* mapping visible to other processes */
fd, /* file descriptor */
0); /* offset: start at 1st byte */
if ((caddr_t) -1 == memptr) report_and_exit("Can't access segment...");
int *imemptr = (int *) memptr;
for (int i = 0; i < nInts; i++)
printf("%d %d\n", i, imemptr[i]);
printf("%p", imemptr);
/* cleanup */
munmap(memptr, nBytes);
close(fd);
unlink(backingFile);
return 0;
}