[MLton-commit] r4091
Matthew Fluet
MLton@mlton.org
Sun, 11 Sep 2005 19:23:18 -0700
Working on generational GC.
----------------------------------------------------------------------
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/Makefile
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/align.c
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/array.c
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/cheney-copy.c
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/debug.c
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/foreach.c
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/gc_state.h
A mlton/branches/on-20050822-x86_64-branch/runtime/gc/generational.c
A mlton/branches/on-20050822-x86_64-branch/runtime/gc/generational.h
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/heap.c
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/heap.h
A mlton/branches/on-20050822-x86_64-branch/runtime/gc/invariant.c
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/object.c
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/pointer.c
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/statistics.h
U mlton/branches/on-20050822-x86_64-branch/runtime/gc/util.h
A mlton/branches/on-20050822-x86_64-branch/runtime/gc/virtual-memory.h
----------------------------------------------------------------------
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/Makefile
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/Makefile 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/Makefile 2005-09-12 02:23:11 UTC (rev 4091)
@@ -51,7 +51,8 @@
CC = gcc -std=gnu99
CWFLAGS = -Wall -pedantic -Wextra -Wshadow -Wpointer-arith -Wcast-qual -Wcast-align -Wconversion -Wsign-compare -Wstrict-prototypes -Wredundant-decls -Winline
CWFLAGS = -pedantic -Wall -Wextra -Wno-unused \
- -Wshadow -Wredundant-decls \
+## -Wshadow \
+ -Wredundant-decls \
-Wpointer-arith -Wcast-qual -Wcast-align \
## -Wconversion \
-Wstrict-prototypes \
@@ -71,7 +72,9 @@
frame.c \
stack.c \
thread.c \
+ generational.c \
heap.c \
+ invariant.c \
foreach.c \
cheney-copy.c \
assumptions.c \
@@ -81,6 +84,7 @@
HFILES = \
gc_prefix.h \
util.h \
+ virtual-memory.h \
pointer.h \
model.h \
object.h \
@@ -90,6 +94,7 @@
thread.h \
weak.h \
major.h \
+ generational.h \
statistics.h \
heap.h \
gc_state.h \
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/align.c
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/align.c 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/align.c 2005-09-12 02:23:11 UTC (rev 4091)
@@ -27,7 +27,7 @@
}
*/
-static inline bool isAligned (uintptr_t a, size_t b) {
+static inline bool isAligned (size_t a, size_t b) {
return 0 == a % b;
}
@@ -55,16 +55,3 @@
pointer GC_alignFrontier (GC_state s, pointer p) {
return alignFrontier (s, p);
}
-
-/*
-static inline uint stackReserved (GC_state s, uint r) {
- uint res;
-
- res = pad (s, r, STACK_HEADER_SIZE + sizeof (struct GC_stack));
- if (DEBUG_STACKS)
- fprintf (stderr, "%s = stackReserved (%s)\n",
- uintToCommaString (res),
- uintToCommaString (r));
- return res;
-}
-*/
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/array.c
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/array.c 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/array.c 2005-09-12 02:23:11 UTC (rev 4091)
@@ -11,14 +11,13 @@
pointer a,
uint32_t arrayIndex,
uint32_t pointerIndex) {
- bool hasIdentity;
GC_header header;
uint16_t numNonObjptrs;
uint16_t numObjptrs;
GC_objectTypeTag tag;
header = getHeader (a);
- SPLIT_HEADER();
+ splitHeader(s, header, &tag, NULL, &numNonObjptrs, &numObjptrs);
assert (tag == ARRAY_TAG);
size_t nonObjptrBytesPerElement =
@@ -37,8 +36,8 @@
/* The number of bytes in an array, not including the header. */
static inline size_t arrayNumBytes (GC_state s,
pointer p,
- uint16_t numObjptrs,
- uint16_t numNonObjptrs) {
+ uint16_t numNonObjptrs,
+ uint16_t numObjptrs) {
size_t bytesPerElement;
GC_arrayLength numElements;
size_t result;
@@ -53,3 +52,32 @@
result = OBJPTR_SIZE;
return pad (s, result, GC_ARRAY_HEADER_SIZE);
}
+
+static inline size_t objectSize (GC_state s, pointer p) {
+ size_t headerBytes, objectBytes;
+ GC_header header;
+ GC_objectTypeTag tag;
+ uint16_t numNonObjptrs, numObjptrs;
+
+ header = getHeader (p);
+ splitHeader (s, header, &tag, NULL, &numNonObjptrs, &numObjptrs);
+ if (NORMAL_TAG == tag) { /* Fixed size object. */
+ headerBytes = GC_NORMAL_HEADER_SIZE;
+ objectBytes =
+ numNonObjptrsToBytes (numNonObjptrs, NORMAL_TAG)
+ + (numObjptrs * OBJPTR_SIZE);
+ } else if (ARRAY_TAG == tag) {
+ headerBytes = GC_ARRAY_HEADER_SIZE;
+ objectBytes = arrayNumBytes (s, p, numNonObjptrs, numObjptrs);
+ } else if (WEAK_TAG == tag) {
+ headerBytes = GC_NORMAL_HEADER_SIZE;
+ objectBytes =
+ numNonObjptrsToBytes (numNonObjptrs, NORMAL_TAG)
+ + (numObjptrs * OBJPTR_SIZE);
+ } else { /* Stack. */
+ assert (STACK_TAG == tag);
+ headerBytes = GC_STACK_HEADER_SIZE;
+ objectBytes = sizeof (struct GC_stack) + ((GC_stack)p)->reserved;
+ }
+ return headerBytes + objectBytes;
+}
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/cheney-copy.c
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/cheney-copy.c 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/cheney-copy.c 2005-09-12 02:23:11 UTC (rev 4091)
@@ -46,11 +46,10 @@
return pointerIsInToSpace (p);
}
-static inline void forward (GC_state s, objptr *opp) {
+static void forward (GC_state s, objptr *opp) {
objptr op;
pointer p;
GC_header header;
- GC_objectTypeTag tag;
op = *opp;
p = objptrToPointer (op, s->heap.start);
@@ -63,12 +62,14 @@
if (DEBUG_DETAILED and header == GC_FORWARDED)
fprintf (stderr, " already FORWARDED\n");
if (header != GC_FORWARDED) { /* forward the object */
- bool hasIdentity;
uint16_t numNonObjptrs, numObjptrs;
+ GC_objectTypeTag tag;
+
+ splitHeader(s, header, &tag, NULL, &numNonObjptrs, &numObjptrs);
+
size_t headerBytes, objectBytes, size, skip;
/* Compute the space taken by the header and object body. */
- SPLIT_HEADER();
if ((NORMAL_TAG == tag) or (WEAK_TAG == tag)) { /* Fixed size object. */
headerBytes = GC_NORMAL_HEADER_SIZE;
objectBytes =
@@ -77,7 +78,7 @@
skip = 0;
} else if (ARRAY_TAG == tag) {
headerBytes = GC_ARRAY_HEADER_SIZE;
- objectBytes = arrayNumBytes (s, p, numObjptrs, numNonObjptrs);
+ objectBytes = arrayNumBytes (s, p, numNonObjptrs, numObjptrs);
skip = 0;
} else { /* Stack. */
GC_stack stack;
@@ -122,7 +123,7 @@
size = headerBytes + objectBytes;
assert (forwardState.back + size + skip <= forwardState.toLimit);
/* Copy the object. */
- copy (p - headerBytes, forwardState.back, size);
+ GC_memcpy (p - headerBytes, forwardState.back, size);
/* If the object has a valid weak pointer, link it into the weaks
* for update after the copying GC is done.
*/
@@ -189,7 +190,7 @@
tempHeap = s->secondaryHeap;
s->secondaryHeap = s->heap;
s->heap = tempHeap;
- // setCardMapForMutator (s);
+ setCardMapAbsolute (s);
}
/* static inline bool detailedGCTime (GC_state s) { */
@@ -200,43 +201,204 @@
// struct rusage ru_start;
pointer toStart;
- assert (s->secondaryHeap.totalBytes >= s->heap.oldGenBytes);
+ assert (s->secondaryHeap.size >= s->heap.oldGenSize);
/* if (detailedGCTime (s)) */
/* startTiming (&ru_start); */
s->cumulative.numCopyingGCs++;
forwardState.toStart = s->secondaryHeap.start;
- forwardState.toLimit = s->secondaryHeap.start + s->secondaryHeap.totalBytes;
+ forwardState.toLimit = s->secondaryHeap.start + s->secondaryHeap.size;
if (DEBUG or s->messages) {
fprintf (stderr, "Major copying GC.\n");
fprintf (stderr, "fromSpace = "FMTPTR" of size %zd\n",
(uintptr_t) s->heap.start,
- /*uintToCommaString*/(s->heap.totalBytes));
+ /*uintToCommaString*/(s->heap.size));
fprintf (stderr, "toSpace = "FMTPTR" of size %zd\n",
(uintptr_t) s->secondaryHeap.start,
- /*uintToCommaString*/(s->secondaryHeap.totalBytes));
+ /*uintToCommaString*/(s->secondaryHeap.size));
}
assert (s->secondaryHeap.start != (pointer)NULL);
/* The next assert ensures there is enough space for the copy to
* succeed. It does not assert
- * (s->secondaryHeap.totalBytes >= s->heap.totalByes)
+ * (s->secondaryHeap.size >= s->heap.size)
* because that is too strong.
*/
- assert (s->secondaryHeap.totalBytes >= s->heap.oldGenBytes);
+ assert (s->secondaryHeap.size >= s->heap.oldGenSize);
toStart = alignFrontier (s, s->secondaryHeap.start);
forwardState.back = toStart;
foreachGlobalObjptr (s, forward);
foreachObjptrInRange (s, toStart, &forwardState.back, TRUE, forward);
updateWeaks (s);
- s->secondaryHeap.oldGenBytes = forwardState.back - s->secondaryHeap.start;
- s->cumulative.bytesCopied += s->secondaryHeap.oldGenBytes;
+ s->secondaryHeap.oldGenSize = forwardState.back - s->secondaryHeap.start;
+ s->cumulative.bytesCopied += s->secondaryHeap.oldGenSize;
if (DEBUG)
fprintf (stderr, "%zd bytes live.\n",
- /*uintToCommaString*/(s->secondaryHeap.oldGenBytes));
+ /*uintToCommaString*/(s->secondaryHeap.oldGenSize));
swapHeaps (s);
- // clearCrossMap (s);
+ clearCrossMap (s);
s->lastMajor.kind = GC_COPYING;
/* if (detailedGCTime (s)) */
/* stopTiming (&ru_start, &s->ru_gcCopy); */
if (DEBUG or s->messages)
fprintf (stderr, "Major copying GC done.\n");
}
+
+/* ---------------------------------------------------------------- */
+/* Minor Cheney Copying Collection */
+/* ---------------------------------------------------------------- */
+
+static inline void forwardIfInNursery (GC_state s, objptr *opp) {
+ objptr op;
+ pointer p;
+
+ op = *opp;
+ p = objptrToPointer (op, s->heap.start);
+ if (p < s->heap.nursery)
+ return;
+ if (DEBUG_GENERATIONAL)
+ fprintf (stderr,
+ "forwardIfInNursery opp = "FMTPTR" op = "FMTOBJPTR" p = "FMTPTR"\n",
+ (uintptr_t)opp, op, (uintptr_t)p);
+ assert (s->heap.nursery <= p and p < s->limitPlusSlop);
+ forward (s, opp);
+}
+
+/* Walk through all the cards and forward all intergenerational pointers. */
+static void forwardInterGenerationalObjptrs (GC_state s) {
+ uint8_t *cardMap;
+ uint8_t *crossMap;
+ size_t numCards;
+ pointer oldGenStart, oldGenEnd;
+
+ size_t cardIndex;
+ pointer cardStart, cardEnd;
+ pointer objectStart;
+
+ if (DEBUG_GENERATIONAL)
+ fprintf (stderr, "Forwarding inter-generational pointers.\n");
+ updateCrossMap (s);
+ /* Constants. */
+ cardMap = s->generational.cardMap;
+ crossMap = s->generational.crossMap;
+ numCards = sizeToCardIndex (align (s->heap.oldGenSize, s->generational.cardSize));
+ oldGenStart = s->heap.start;
+ oldGenEnd = oldGenStart + s->heap.oldGenSize;
+ /* Loop variables*/
+ objectStart = alignFrontier (s, s->heap.start);
+ cardIndex = 0;
+ cardStart = oldGenStart;
+checkAll:
+ assert (cardIndex <= numCards);
+ assert (isAlignedFrontier (s, objectStart));
+ if (cardIndex == numCards)
+ goto done;
+checkCard:
+ if (DEBUG_GENERATIONAL)
+ fprintf (stderr, "checking card %zu objectStart = "FMTPTR"\n",
+ cardIndex, (uintptr_t)objectStart);
+ assert (objectStart < oldGenStart + cardIndexToSize (cardIndex + 1));
+ if (cardMap[cardIndex]) {
+ pointer lastObject;
+ size_t size;
+
+ s->cumulative.markedCards++;
+ if (DEBUG_GENERATIONAL)
+ fprintf (stderr, "card %zu is marked objectStart = "FMTPTR"\n",
+ cardIndex, (uintptr_t)objectStart);
+ lastObject = objectStart;
+skipObjects:
+ assert (isAlignedFrontier (s, objectStart));
+ size = objectSize (s, objectData (s, objectStart));
+ if (objectStart + size < cardStart) {
+ objectStart += size;
+ goto skipObjects;
+ }
+ s->cumulative.minorBytesSkipped += objectStart - lastObject;
+ cardEnd = cardStart + s->generational.cardSize;
+ if (oldGenEnd < cardEnd)
+ cardEnd = oldGenEnd;
+ assert (objectStart < cardEnd);
+ lastObject = objectStart;
+ /* If we ever add Weak.set, then there could be intergenerational
+ * weak pointers, in which case we would need to link the weak
+ * objects into s->weaks. But for now, since there is no
+ * Weak.set, the foreachObjptrInRange will do the right thing on
+ * weaks, since the weak pointer will never be into the nursery.
+ */
+ objectStart = foreachObjptrInRange (s, objectStart, &cardEnd,
+ FALSE, forwardIfInNursery);
+ s->cumulative.minorBytesScanned += objectStart - lastObject;
+ if (objectStart == oldGenEnd)
+ goto done;
+ cardIndex = sizeToCardIndex (objectStart - oldGenStart);
+ cardStart = oldGenStart + cardIndexToSize (cardIndex);
+ goto checkCard;
+ } else {
+ unless (CROSS_MAP_EMPTY == crossMap[cardIndex])
+ objectStart = cardStart + (crossMap[cardIndex] >> CROSS_MAP_SCALE);
+ if (DEBUG_GENERATIONAL)
+ fprintf (stderr,
+ "card %zu is not marked"
+ " crossMap[%zu] == %"PRIu8
+ " objectStart = "FMTPTR"\n",
+ cardIndex, cardIndex,
+ crossMap[cardIndex], (uintptr_t)objectStart);
+ cardIndex++;
+ cardStart += s->generational.cardSize;
+ goto checkAll;
+ }
+ assert (FALSE);
+done:
+ if (DEBUG_GENERATIONAL)
+ fprintf (stderr, "Forwarding inter-generational pointers done.\n");
+}
+
+static void minorGC (GC_state s) {
+ size_t bytesAllocated;
+ size_t bytesCopied;
+ // struct rusage ru_start;
+
+ if (DEBUG_GENERATIONAL)
+ fprintf (stderr, "minorGC nursery = "FMTPTR" frontier = "FMTPTR"\n",
+ (uintptr_t)s->heap.nursery, (uintptr_t)s->frontier);
+ assert (invariant (s));
+ bytesAllocated = s->frontier - s->heap.nursery;
+ if (bytesAllocated == 0)
+ return;
+ s->cumulative.bytesAllocated += bytesAllocated;
+ if (not s->canMinor) {
+ s->heap.oldGenSize += bytesAllocated;
+ bytesCopied = 0;
+ } else {
+ if (DEBUG_GENERATIONAL or s->messages)
+ fprintf (stderr, "Minor GC.\n");
+/* if (detailedGCTime (s)) */
+/* startTiming (&ru_start); */
+ s->amInMinorGC = TRUE;
+ forwardState.toStart = s->heap.start + s->heap.oldGenSize;
+ if (DEBUG_GENERATIONAL)
+ fprintf (stderr, "toStart = "FMTPTR"\n", (uintptr_t)forwardState.toStart);
+ assert (isAlignedFrontier (s, forwardState.toStart));
+ forwardState.toLimit = forwardState.toStart + bytesAllocated;
+ assert (invariant (s));
+ s->cumulative.numMinorGCs++;
+ s->lastMajor.numMinorsGCs++;
+ forwardState.back = forwardState.toStart;
+ /* Forward all globals. Would like to avoid doing this once all
+ * the globals have been assigned.
+ */
+ foreachGlobalObjptr (s, forwardIfInNursery);
+ forwardInterGenerationalObjptrs (s);
+ foreachObjptrInRange (s, forwardState.toStart, &forwardState.back,
+ TRUE, forwardIfInNursery);
+ updateWeaks (s);
+ bytesCopied = forwardState.back - forwardState.toStart;
+ s->cumulative.bytesCopiedMinor += bytesCopied;
+ s->heap.oldGenSize += bytesCopied;
+ s->amInMinorGC = FALSE;
+/* if (detailedGCTime (s)) */
+/* stopTiming (&ru_start, &s->ru_gcMinor); */
+ if (DEBUG_GENERATIONAL or s->messages)
+ fprintf (stderr, "Minor GC done. %zd bytes copied.\n",
+ /*uintToCommaString*/(bytesCopied));
+ }
+}
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/debug.c
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/debug.c 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/debug.c 2005-09-12 02:23:11 UTC (rev 4091)
@@ -18,6 +18,7 @@
DEBUG_ENTER_LEAVE = FALSE,
DEBUG_GENERATIONAL = FALSE,
DEBUG_MARK_COMPACT = FALSE,
+ DEBUG_MEM = FALSE,
DEBUG_PROFILE = FALSE,
DEBUG_RESIZING = FALSE,
DEBUG_SHARE = FALSE,
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/foreach.c
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/foreach.c 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/foreach.c 2005-09-12 02:23:11 UTC (rev 4091)
@@ -45,14 +45,13 @@
pointer p,
bool skipWeaks,
GC_foreachObjptrFun f) {
- bool hasIdentity;
GC_header header;
uint16_t numNonObjptrs;
uint16_t numObjptrs;
GC_objectTypeTag tag;
header = getHeader (p);
- SPLIT_HEADER();
+ splitHeader(s, header, &tag, NULL, &numNonObjptrs, &numObjptrs);
if (DEBUG_DETAILED)
fprintf (stderr,
"foreachObjptrInObject ("FMTPTR")"
@@ -202,7 +201,7 @@
fprintf (stderr,
" front = "FMTPTR" *back = "FMTPTR"\n",
(uintptr_t)front, (uintptr_t)(*back));
- front = foreachObjptrInObject (s, toData (s, front), skipWeaks, f);
+ front = foreachObjptrInObject (s, objectData (s, front), skipWeaks, f);
}
b = *back;
}
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/gc_state.h
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/gc_state.h 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/gc_state.h 2005-09-12 02:23:11 UTC (rev 4091)
@@ -4,19 +4,23 @@
bool amInGC;
bool amInMinorGC;
objptr callFromCHandler; /* Handler for exported C calls (in heap). */
+ bool canMinor; /* TRUE iff there is space for a minor gc. */
struct GC_cumulativeStatistics cumulative;
objptr currentThread; /* Currently executing thread (in heap). */
GC_frameLayout *frameLayouts; /* Array of frame layouts. */
uint32_t frameLayoutsSize; /* Cardinality of frameLayouts array. */
pointer frontier; /* heap.start <= frontier < limit */
+ struct GC_generationalInfo generational;
objptr *globals;
uint32_t globalsSize;
struct GC_heap heap;
struct GC_lastMajorStatistics lastMajor;
pointer limit; /* limit = heap.start + heap.totalBytes */
+ pointer limitPlusSlop; /* limit + LIMIT_SLOP */
uint32_t maxFrameSize;
GC_objectType *objectTypes; /* Array of object types. */
uint32_t objectTypesSize; /* Cardinality of objectTypes array. */
+ size_t pageSize;
uint32_t (*returnAddressToFrameIndex) (GC_returnAddress ra);
objptr savedThread; /* Result of GC_copyCurrentThread.
* Thread interrupted by arrival of signal.
Copied: mlton/branches/on-20050822-x86_64-branch/runtime/gc/generational.c (from rev 4078, mlton/branches/on-20050822-x86_64-branch/runtime/gc.c)
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc.c 2005-09-07 00:47:05 UTC (rev 4078)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/generational.c 2005-09-12 02:23:11 UTC (rev 4091)
@@ -0,0 +1,224 @@
+/* Copyright (C) 1999-2005 Henry Cejtin, Matthew Fluet, Suresh
+ * Jagannathan, and Stephen Weeks.
+ * Copyright (C) 1997-2000 NEC Research Institute.
+ *
+ * MLton is released under a BSD-style license.
+ * See the file MLton-LICENSE for details.
+ */
+
+/* must agree w/ cardSizeLog2 in ssa-to-rssa.fun */
+#define CARD_SIZE_LOG2 8
+#define CARD_SIZE TWOPOWER(CARD_SIZE_LOG2)
+#define CROSS_MAP_EMPTY 255
+#define CROSS_MAP_SCALE 2
+
+static inline uintptr_t pointerToCardIndex (pointer p) {
+ return (uintptr_t)p >> CARD_SIZE_LOG2;
+}
+static inline size_t sizeToCardIndex (size_t n) {
+ return n >> CARD_SIZE_LOG2;
+}
+static inline size_t cardIndexToSize (size_t n) {
+ return n << CARD_SIZE_LOG2;
+}
+
+static inline pointer pointerToCardMapAddr (GC_state s, pointer p) {
+ pointer res;
+
+ res = &s->generational.cardMapAbsolute [pointerToCardIndex (p)];
+ if (DEBUG_CARD_MARKING)
+ fprintf (stderr, "pointerToCardMapAddr ("FMTPTR") = "FMTPTR"\n",
+ (uintptr_t)p, (uintptr_t)res);
+ return res;
+}
+
+static inline bool cardIsMarked (GC_state s, pointer p) {
+ return (*pointerToCardMapAddr (s, p) != 0);
+}
+
+static inline void markCard (GC_state s, pointer p) {
+ if (DEBUG_CARD_MARKING)
+ fprintf (stderr, "markCard ("FMTPTR")\n", (uintptr_t)p);
+ if (s->generational.mutatorMarksCards)
+ *pointerToCardMapAddr (s, p) = 0x1;
+}
+
+static inline void clearCardMap (GC_state s) {
+ if (DEBUG_GENERATIONAL and DEBUG_DETAILED)
+ fprintf (stderr, "clearCardMap ()\n");
+ memset (s->generational.cardMap, 0, s->generational.cardMapSize);
+}
+
+static inline void clearCrossMap (GC_state s) {
+ if (DEBUG_GENERATIONAL and DEBUG_DETAILED)
+ fprintf (stderr, "clearCrossMap ()\n");
+ s->generational.crossMapValidSize = 0;
+ memset (s->generational.crossMap, CROSS_MAP_EMPTY, s->generational.crossMapSize);
+}
+
+static inline void setCardMapAbsolute (GC_state s) {
+ unless (s->generational.mutatorMarksCards)
+ return;
+ /* It's OK if the subtraction below underflows because all the
+ * subsequent additions to mark the cards will overflow and put us
+ * in the right place.
+ */
+ s->generational.cardMapAbsolute =
+ s->generational.cardMap - pointerToCardIndex ( s->heap.start);
+ if (DEBUG_CARD_MARKING)
+ fprintf (stderr, "cardMapAbsolute = "FMTPTR"\n",
+ (uintptr_t)s->generational.cardMapAbsolute);
+}
+
+static inline void createCardMapAndCrossMap (GC_state s) {
+ unless (s->generational.mutatorMarksCards) {
+ s->generational.cardMapSize = 0;
+ s->generational.cardMap = NULL;
+ s->generational.cardMapAbsolute = NULL;
+ s->generational.crossMapSize = 0;
+ s->generational.crossMap = NULL;
+ return;
+ }
+ assert (isAligned (s->heap.size, s->generational.cardSize));
+ s->generational.cardMapSize =
+ align (sizeToCardIndex (s->heap.size), s->pageSize);
+ s->generational.crossMapSize = s->generational.cardMapSize;
+ if (DEBUG_MEM)
+ fprintf (stderr, "Creating card/cross map of size %zd\n",
+ /*uintToCommaString*/
+ (s->generational.cardMapSize + s->generational.crossMapSize));
+ s->generational.cardMap =
+ GC_mmapAnon (s->generational.cardMapSize + s->generational.crossMapSize);
+ s->generational.crossMap =
+ s->generational.cardMap + s->generational.cardMapSize;
+ if (DEBUG_CARD_MARKING)
+ fprintf (stderr, "cardMap = "FMTPTR" crossMap = "FMTPTR"\n",
+ (uintptr_t)s->generational.cardMap,
+ (uintptr_t)s->generational.crossMap);
+ setCardMapAbsolute (s);
+ clearCardMap (s);
+ clearCrossMap (s);
+}
+
+#if ASSERT
+
+static inline pointer crossMapCardStart (GC_state s, pointer p) {
+ /* The p - 1 is so that a pointer to the beginning of a card falls
+ * into the index for the previous crossMap entry.
+ */
+ return (p == s->heap.start)
+ ? s->heap.start
+ : (p - 1) - ((uintptr_t)(p - 1) % s->generational.cardSize);
+}
+
+/* crossMapIsOK is a slower, but easier to understand, way of
+ * computing the crossMap. updateCrossMap (below) incrementally
+ * updates the crossMap, checking only the part of the old generation
+ * that it hasn't seen before. crossMapIsOK simply walks through the
+ * entire old generation. It is useful to check that the incremental
+ * update is working correctly.
+ */
+
+static inline bool crossMapIsOK (GC_state s) {
+ static uint8_t *map;
+
+ pointer front, back;
+ size_t cardIndex;
+ pointer cardStart;
+
+ if (DEBUG)
+ fprintf (stderr, "crossMapIsOK ()\n");
+ map = GC_mmapAnon (s->generational.crossMapSize);
+ memset (map, CROSS_MAP_EMPTY, s->generational.crossMapSize);
+ back = s->heap.start + s->heap.oldGenSize;
+ cardIndex = 0;
+ front = alignFrontier (s, s->heap.start);
+loopObjects:
+ assert (front <= back);
+ cardStart = crossMapCardStart (s, front);
+ cardIndex = sizeToCardIndex (cardStart - s->heap.start);
+ map[cardIndex] = (front - cardStart) >> CROSS_MAP_SCALE;
+ if (front < back) {
+ front += objectSize (s, objectData (s, front));
+ goto loopObjects;
+ }
+ for (size_t i = 0; i < cardIndex; ++i)
+ assert (map[i] == s->generational.crossMap[i]);
+ GC_munmap (map, s->generational.crossMapSize);
+ return TRUE;
+}
+
+#endif /* ASSERT */
+
+static inline void updateCrossMap (GC_state s) {
+ size_t cardIndex;
+ pointer cardStart, cardEnd;
+
+ pointer nextObject, objectStart;
+ pointer oldGenEnd;
+
+ if (s->generational.crossMapValidSize == s->heap.oldGenSize)
+ goto done;
+ oldGenEnd = s->heap.start + s->heap.oldGenSize;
+ objectStart = s->heap.start + s->generational.crossMapValidSize;
+ if (objectStart == s->heap.start) {
+ cardIndex = 0;
+ objectStart = alignFrontier (s, objectStart);
+ } else
+ cardIndex = sizeToCardIndex (objectStart - 1 - s->heap.start);
+ cardStart = s->heap.start + cardIndexToSize (cardIndex);
+ cardEnd = cardStart + s->generational.cardSize;
+loopObjects:
+ assert (objectStart < oldGenEnd);
+ assert ((objectStart == s->heap.start or cardStart < objectStart)
+ and objectStart <= cardEnd);
+ nextObject = objectStart + objectSize (s, objectData (s, objectStart));
+ if (nextObject > cardEnd) {
+ /* We're about to move to a new card, so we are looking at the
+ * last object boundary in the current card.
+ * Store it in the crossMap.
+ */
+ size_t offset;
+
+ offset = (objectStart - cardStart) >> CROSS_MAP_SCALE;
+ assert (offset < CROSS_MAP_EMPTY);
+ if (DEBUG_GENERATIONAL)
+ fprintf (stderr, "crossMap[%zu] = %zu\n",
+ cardIndex, offset);
+ s->generational.crossMap[cardIndex] = (uint8_t)offset;
+ cardIndex = sizeToCardIndex (nextObject - 1 - s->heap.start);
+ cardStart = s->heap.start + cardIndexToSize (cardIndex);
+ cardEnd = cardStart + s->generational.cardSize;
+ }
+ objectStart = nextObject;
+ if (objectStart < oldGenEnd)
+ goto loopObjects;
+ assert (objectStart == oldGenEnd);
+ s->generational.crossMap[cardIndex] = (oldGenEnd - cardStart) >> CROSS_MAP_SCALE;
+ s->generational.crossMapValidSize = s->heap.oldGenSize;
+done:
+ assert (s->generational.crossMapValidSize == s->heap.oldGenSize);
+ assert (crossMapIsOK (s));
+}
+
+static inline void resizeCardMapAndCrossMap (GC_state s) {
+ if (s->generational.mutatorMarksCards
+ and s->generational.cardMapSize
+ != align (sizeToCardIndex (s->heap.size), s->pageSize)) {
+ uint8_t *oldCardMap;
+ size_t oldCardMapSize;
+ uint8_t *oldCrossMap;
+ size_t oldCrossMapSize;
+
+ oldCardMap = s->generational.cardMap;
+ oldCardMapSize = s->generational.cardMapSize;
+ oldCrossMap = s->generational.crossMap;
+ oldCrossMapSize = s->generational.crossMapSize;
+ createCardMapAndCrossMap (s);
+ GC_memcpy ((pointer)oldCrossMap, (pointer)s->generational.crossMap,
+ min (s->generational.crossMapSize, oldCrossMapSize));
+ if (DEBUG_MEM)
+ fprintf (stderr, "Releasing card/cross map.\n");
+ GC_munmap (oldCardMap, oldCardMapSize + oldCrossMapSize);
+ }
+}
Copied: mlton/branches/on-20050822-x86_64-branch/runtime/gc/generational.h (from rev 4078, mlton/branches/on-20050822-x86_64-branch/runtime/gc.h)
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc.h 2005-09-07 00:47:05 UTC (rev 4078)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/generational.h 2005-09-12 02:23:11 UTC (rev 4091)
@@ -0,0 +1,21 @@
+/* Copyright (C) 1999-2005 Henry Cejtin, Matthew Fluet, Suresh
+ * Jagannathan, and Stephen Weeks.
+ * Copyright (C) 1997-2000 NEC Research Institute.
+ *
+ * MLton is released under a BSD-style license.
+ * See the file MLton-LICENSE for details.
+ */
+
+struct GC_generationalInfo {
+ uint8_t *cardMap;
+ uint8_t *cardMapAbsolute;
+ size_t cardMapSize;
+ size_t cardSize;
+ uint8_t *crossMap;
+ size_t crossMapSize;
+ /* crossMapValidEnd is the size of the prefix of the old generation
+ * for which the crossMap is valid.
+ */
+ size_t crossMapValidSize;
+ /*Bool*/bool mutatorMarksCards;
+};
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/heap.c
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/heap.c 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/heap.c 2005-09-12 02:23:11 UTC (rev 4091)
@@ -22,7 +22,7 @@
static inline bool pointerIsInOldGen (GC_state s, pointer p) {
return (not (isPointer (p))
or (s->heap.start <= p
- and p < s->heap.start + s->heap.oldGenBytes));
+ and p < s->heap.start + s->heap.oldGenSize));
}
static inline bool objptrIsInOldGen (GC_state s, objptr op) {
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/heap.h
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/heap.h 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/heap.h 2005-09-12 02:23:11 UTC (rev 4091)
@@ -19,7 +19,9 @@
typedef struct GC_heap {
pointer nursery; /* start of nursery */
- size_t oldGenBytes; /* size of old generation */
+ size_t oldGenSize; /* size of old generation */
pointer start; /* start of heap (and old generation) */
- size_t totalBytes; /* size of heap */
+ size_t size; /* size of heap */
} *GC_heap;
+
+#define LIMIT_SLOP 512
Copied: mlton/branches/on-20050822-x86_64-branch/runtime/gc/invariant.c (from rev 4089, mlton/branches/on-20050822-x86_64-branch/runtime/gc.c)
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc.c 2005-09-11 16:58:45 UTC (rev 4089)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/invariant.c 2005-09-12 02:23:11 UTC (rev 4091)
@@ -0,0 +1,92 @@
+/* Copyright (C) 1999-2005 Henry Cejtin, Matthew Fluet, Suresh
+ * Jagannathan, and Stephen Weeks.
+ * Copyright (C) 1997-2000 NEC Research Institute.
+ *
+ * MLton is released under a BSD-style license.
+ * See the file MLton-LICENSE for details.
+ */
+
+#if ASSERT
+
+static bool invariant (GC_state s) {
+ if (DEBUG)
+ fprintf (stderr, "invariant\n");
+ // assert (ratiosOk (s));
+ /* Frame layouts */
+ for (unsigned int i = 0; i < s->frameLayoutsSize; ++i) {
+ GC_frameLayout *layout;
+
+ layout = &(s->frameLayouts[i]);
+ if (layout->numBytes > 0) {
+ GC_frameOffsets offsets;
+
+ assert (layout->numBytes <= s->maxFrameSize);
+ offsets = layout->offsets;
+ /* No longer correct, since handler frames have a "size"
+ * (i.e. return address) pointing into the middle of the frame.
+ */
+/* for (unsigned int j = 0; j < offsets[0]; ++j) */
+/* assert (offsets[j + 1] < layout->numBytes); */
+ }
+ }
+ /* Generational */
+ if (s->generational.mutatorMarksCards) {
+ assert (s->generational.cardMap ==
+ &s->generational.cardMapAbsolute
+ [pointerToCardIndex(s->heap.start)]);
+ assert (&s->generational.cardMapAbsolute
+ [pointerToCardIndex(s->heap.start + s->heap.size - 1)]
+ < s->generational.cardMap + s->generational.cardMapSize);
+ }
+ assert (isAligned (s->heap.size, s->pageSize));
+ assert (isAligned ((size_t)s->heap.start, s->generational.cardSize));
+ assert (isAlignedFrontier (s, s->heap.start + s->heap.oldGenSize));
+ assert (isAlignedFrontier (s, s->heap.nursery));
+ assert (isAlignedFrontier (s, s->frontier));
+ assert (s->heap.nursery <= s->frontier);
+ unless (0 == s->heap.size) {
+ assert (s->heap.nursery <= s->frontier);
+ assert (s->frontier <= s->limitPlusSlop);
+ assert (s->limit == s->limitPlusSlop - LIMIT_SLOP);
+/* assert (hasBytesFree (s, 0, 0)); */
+ }
+ assert (s->secondaryHeap.start == NULL or s->heap.size == s->secondaryHeap.size);
+/* /\* Check that all pointers are into from space. *\/ */
+/* foreachGlobal (s, assertIsInFromSpace); */
+/* back = s->heap.start + s->oldGenSize; */
+/* if (DEBUG_DETAILED) */
+/* fprintf (stderr, "Checking old generation.\n"); */
+/* foreachPointerInRange (s, alignFrontier (s, s->heap.start), &back, FALSE, */
+/* assertIsInFromSpace); */
+/* if (DEBUG_DETAILED) */
+/* fprintf (stderr, "Checking nursery.\n"); */
+/* foreachPointerInRange (s, s->nursery, &s->frontier, FALSE, */
+/* assertIsInFromSpace); */
+/* /\* Current thread. *\/ */
+/* stack = s->currentThread->stack; */
+/* assert (isAlignedReserved (s, stack->reserved)); */
+/* assert (s->stackBottom == stackBottom (s, stack)); */
+/* assert (s->stackTop == stackTop (s, stack)); */
+/* assert (s->stackLimit == stackLimit (s, stack)); */
+/* assert (stack->used == currentStackUsed (s)); */
+/* assert (stack->used <= stack->reserved); */
+/* assert (s->stackBottom <= s->stackTop); */
+ if (DEBUG)
+ fprintf (stderr, "invariant passed\n");
+ return TRUE;
+}
+
+static bool mutatorInvariant (GC_state s, bool frontier, bool stack) {
+#if FALSE
+ if (DEBUG)
+ GC_display (s, stderr);
+ if (frontier)
+ assert (mutatorFrontierInvariant(s));
+ if (stack)
+ assert (mutatorStackInvariant(s));
+#endif
+ assert (invariant (s));
+ return TRUE;
+}
+
+#endif /* #if ASSERT */
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/object.c
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/object.c 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/object.c 2005-09-12 02:23:11 UTC (rev 4091)
@@ -20,30 +20,6 @@
#define WEAK_GONE_HEADER GC_objectHeader (WEAK_GONE_TYPE_INDEX)
#define WORD8_VECTOR_HEADER GC_objectHeader (WORD8_TYPE_INDEX)
-#define SPLIT_HEADER() \
- do { \
- unsigned int objectTypeIndex; \
- GC_objectType *t; \
- \
- assert (1 == (header & GC_VALID_HEADER_MASK)); \
- objectTypeIndex = (header & TYPE_INDEX_MASK) >> TYPE_INDEX_SHIFT; \
- assert (objectTypeIndex < s->objectTypesSize); \
- t = &s->objectTypes [objectTypeIndex]; \
- tag = t->tag; \
- hasIdentity = t->hasIdentity; \
- numNonObjptrs = t->numNonObjptrs; \
- numObjptrs = t->numObjptrs; \
- if (DEBUG_DETAILED) \
- fprintf (stderr, \
- "SPLIT_HEADER ("FMTHDR")" \
- " tag = %s" \
- " hasIdentity = %u" \
- " numNonObjptrs = %"PRIu16 \
- " numObjptrs = %"PRIu16"\n", \
- header, \
- tagToString(tag), hasIdentity, numNonObjptrs, numObjptrs); \
- } while (0)
-
static char* tagToString (GC_objectTypeTag tag) {
switch (tag) {
case ARRAY_TAG:
@@ -59,10 +35,50 @@
}
}
-/* If p points at the beginning of an object, then toData p returns a
- * pointer to the start of the object data.
+static inline void splitHeader(GC_state s, GC_header header,
+ GC_objectTypeTag *tagRet, bool *hasIdentityRet,
+ uint16_t *numNonObjptrsRet, uint16_t *numObjptrsRet) {
+ unsigned int objectTypeIndex;
+ GC_objectType *objectType;
+ GC_objectTypeTag tag;
+ bool hasIdentity;
+ uint16_t numNonObjptrs, numObjptrs;
+
+ assert (1 == (header & GC_VALID_HEADER_MASK));
+ objectTypeIndex = (header & TYPE_INDEX_MASK) >> TYPE_INDEX_SHIFT;
+ assert (objectTypeIndex < s->objectTypesSize);
+ objectType = &s->objectTypes [objectTypeIndex];
+ tag = objectType->tag;
+ hasIdentity = objectType->hasIdentity;
+ numNonObjptrs = objectType->numNonObjptrs;
+ numObjptrs = objectType->numObjptrs;
+
+ if (DEBUG_DETAILED)
+ fprintf (stderr,
+ "splitHeader ("FMTHDR")"
+ " tag = %s"
+ " hasIdentity = %u"
+ " numNonObjptrs = %"PRIu16
+ " numObjptrs = %"PRIu16"\n",
+ header,
+ tagToString(tag), hasIdentity, numNonObjptrs, numObjptrs);
+
+ if (tagRet != NULL)
+ *tagRet = tag;
+ if (hasIdentityRet != NULL)
+ *hasIdentityRet = hasIdentity;
+ if (numNonObjptrsRet != NULL)
+ *numNonObjptrsRet = numNonObjptrs;
+ if (numObjptrsRet != NULL)
+ *numObjptrsRet = numObjptrs;
+}
+
+/* objectData (s, p)
+ *
+ * If p points at the beginning of an object, then objectData returns
+ * a pointer to the start of the object data.
*/
-static inline pointer toData (GC_state s, pointer p) {
+static inline pointer objectData (GC_state s, pointer p) {
GC_header header;
pointer res;
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/pointer.c
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/pointer.c 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/pointer.c 2005-09-12 02:23:11 UTC (rev 4091)
@@ -12,11 +12,9 @@
return (0 == ((uintptr_t)p & mask));
}
-static inline void copy (pointer src, pointer dst, size_t size) {
- unsigned int *to, *from, *limit;
-
+static inline void GC_memcpy (pointer src, pointer dst, size_t size) {
if (DEBUG_DETAILED)
- fprintf (stderr, "copy ("FMTPTR", "FMTPTR", %zu)\n",
+ fprintf (stderr, "GC_memcpy ("FMTPTR", "FMTPTR", %zu)\n",
(uintptr_t)src, (uintptr_t)dst, size);
assert (isAligned ((uintptr_t)src, sizeof(unsigned int)));
assert (isAligned ((uintptr_t)dst, sizeof(unsigned int)));
@@ -24,9 +22,5 @@
assert (dst <= src or src + size <= dst);
if (src == dst)
return;
- from = (unsigned int*)src;
- to = (unsigned int*)dst;
- limit = (unsigned int*)(src + size);
- until (from == limit)
- *to++ = *from++;
+ memcpy (dst, src, size);
}
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/statistics.h
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/statistics.h 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/statistics.h 2005-09-12 02:23:11 UTC (rev 4091)
@@ -13,6 +13,8 @@
uintmax_t bytesHashConsed;
uintmax_t bytesMarkCompacted;
+ uintmax_t markedCards; /* Number of marked cards seen during minor GCs. */
+
size_t maxBytesLive;
size_t maxHeapSizeSeen;
size_t maxStackSizeSeen;
Modified: mlton/branches/on-20050822-x86_64-branch/runtime/gc/util.h
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc/util.h 2005-09-11 23:26:29 UTC (rev 4090)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/util.h 2005-09-12 02:23:11 UTC (rev 4091)
@@ -24,6 +24,7 @@
#include <inttypes.h>
#include <stdlib.h>
#include <limits.h>
+#include <string.h>
#include "../assert.h"
@@ -38,6 +39,14 @@
#define unless(p) if (not (p))
#define until(p) while (not (p))
+#ifndef max
+#define max(a, b) ((a)>(b)?(a):(b))
+#endif
+
+#ifndef min
+#define min(a, b) ((a)<(b)?(a):(b))
+#endif
+
/* issue error message and exit */
extern void die (char *fmt, ...)
__attribute__ ((format(printf, 1, 2)))
Copied: mlton/branches/on-20050822-x86_64-branch/runtime/gc/virtual-memory.h (from rev 4089, mlton/branches/on-20050822-x86_64-branch/runtime/gc.h)
===================================================================
--- mlton/branches/on-20050822-x86_64-branch/runtime/gc.h 2005-09-11 16:58:45 UTC (rev 4089)
+++ mlton/branches/on-20050822-x86_64-branch/runtime/gc/virtual-memory.h 2005-09-12 02:23:11 UTC (rev 4091)
@@ -0,0 +1,11 @@
+/* Copyright (C) 1999-2005 Henry Cejtin, Matthew Fluet, Suresh
+ * Jagannathan, and Stephen Weeks.
+ * Copyright (C) 1997-2000 NEC Research Institute.
+ *
+ * MLton is released under a BSD-style license.
+ * See the file MLton-LICENSE for details.
+ */
+
+void *GC_mmap (void *start, size_t length);
+void *GC_mmapAnon (size_t length);
+void *GC_munmap (void *base, size_t length);