[MLton] Writing memory to disk ...

[Stephen Weeks]

I refactored the runtime's disk-backing code so it can be implemented
in the platform/* files.  The idea was to add three functions

  void diskBack_close (void *data);
  void diskBack_read (void *data, pointer buf, size_t size);
  void *diskBack_write (pointer buf, size_t size);

With these, the disk-backing code in gc/heap.c looks as follows.

    /* Write the heap to disk and try again. */
    void *data;

    data = diskBack_write (orig, size);
    releaseHeap (s, curHeapp);
    if (createHeap (s, curHeapp, desiredSize, minSize)) {
      diskBack_read (data, curHeapp->start, size);
      diskBack_close (data);
    } else {
      diskBack_close (data);
      if (s->controls.messages)
        GC_displayMem ();
      die ("Out of memory.  Unable to allocate %s bytes.\n",
           uintmaxToCommaString(minSize));
    }

On Unix platforms, disckBack_* use mkstemp + unlink, while on Windows,
they use GetTemp{FileName,Path} + FILE_FLAG_DELETE_ON_CLOSE as
suggested by Adam.  On Unix, the default temp dir is /var/tmp, and can
be overridden by environment variable TMP or TMPDIR.

All that remains is testing and adding the runtime control option.  I
haven't thought of anything better than disk-backed, so will go with
that in the absence of other suggestions.  (heap-on-disk?,
heap-to-disk?)


Here's the Unix implementation of the diskBack_* functions.

--------------------------------------------------------------------------------
static int tempFileDes (void) {
  int fd;
  char *template;
  const char *tmpDir;
  const char *tag = "/TempFileXXXXXXXXXX";
  mode_t m;

  tmpDir = getenv ("TMP");
  if (NULL == tmpDir) {
    tmpDir = getenv ("TMPDIR");
    if (NULL == tmpDir)
      tmpDir = "/var/tmp";
  }
  template = malloc_safe (strlen(tmpDir) + strlen(tag) + 1);
  strcpy (template, tmpDir);
  strcat (template, tag);
  m = umask(077);
  fd = mkstemp_safe (template);
  (void)umask(m);
  unlink_safe (template);
  free (template);
  return fd;
}

typedef struct {
  int fd;
} *WriteToDiskData;

void diskBack_read (void *data, pointer buf, size_t size) {
  int fd;

  fd = ((WriteToDiskData)data)->fd;
  lseek (fd, 0, SEEK_SET);
  read_safe (fd, buf, size);
}

void diskBack_close (void *data) {
  int fd;

  fd = ((WriteToDiskData)data)->fd;
  close_safe (fd);
  free (data);
}

void *diskBack_write (pointer buf, size_t size) {
  int fd;
  WriteToDiskData d;

  fd = tempFileDes ();
  write_safe (fd, buf, size);
  d = (WriteToDiskData)(malloc_safe (sizeof(*d)));
  d->fd = fd;
  return d;
}
--------------------------------------------------------------------------------

Here's the Windows implementation of the diskBack_* functions.

--------------------------------------------------------------------------------
#define BUFSIZE 65536

static HANDLE tempFileDes (void) {
  /* Based on http://msdn.microsoft.com/library/default.asp?url=/library/en-us/fileio/fs/creating_and_using_a_temporary_file.asp
   */  
  HANDLE hTempFile; 
  DWORD dwRetVal;
  DWORD dwBufSize=BUFSIZE;
  UINT uRetVal;
  char szTempName[BUFSIZE];  
  char lpPathBuffer[BUFSIZE];

  dwRetVal = GetTempPath(dwBufSize, lpPathBuffer);
  if (dwRetVal > dwBufSize)
    die ("GetTempPath failed with error %ld\n", GetLastError());
  uRetVal = GetTempFileName(lpPathBuffer, "TempFile", 0, szTempName);
  if (0 == uRetVal)
    die ("GetTempFileName failed with error %ld\n", GetLastError());
  hTempFile = CreateFile((LPTSTR) szTempName, GENERIC_READ | GENERIC_WRITE,
    0, NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE,
    NULL);                
  if (hTempFile == INVALID_HANDLE_VALUE)
    die ("CreateFile failed with error %ld\n", GetLastError());
  return hTempFile;
}

typedef struct {
  HANDLE handle;
} *WriteToDiskData;

void diskBack_read (void *data, pointer buf, size_t size) {
  HANDLE h;
  DWORD d;
  DWORD dwBytesRead;

  h = ((WriteToDiskData)data)->handle;
  d = SetFilePointer (h, 0, NULL, FILE_BEGIN);
  if (d == INVALID_SET_FILE_POINTER)
    die ("SetFilePointer failed with error %ld\n", GetLastError());
  unless (ReadFile(h, buf, size, &dwBytesRead, NULL))
    die ("ReadFile failed with error %ld\n", GetLastError());
}

void diskBack_close (void *data) {
  HANDLE h;

  h = ((WriteToDiskData)data)->handle;
  unless (CloseHandle (h))
    die ("CloseHandle failed with error %ld.", GetLastError());
  free (data);
}

void *diskBack_write (pointer buf, size_t size) {
  HANDLE h;
  WriteToDiskData d;
  DWORD dwBytesWritten;

  h = tempFileDes ();
  unless (WriteFile (h, buf, size, &dwBytesWritten, NULL))
    die ("WriteFile failed with error %ld\n", GetLastError());
  d = (WriteToDiskData)(malloc_safe (sizeof(*d)));
  d->handle = h;
  return d;
}