[pypy-svn] r77464 - in pypy/branch/minimark-free/pypy/rpython: lltypesystem memory/gc memory/gc/test
arigo at codespeak.net
arigo at codespeak.net
Wed Sep 29 13:05:48 CEST 2010
Author: arigo
Date: Wed Sep 29 13:05:46 2010
New Revision: 77464
Modified:
pypy/branch/minimark-free/pypy/rpython/lltypesystem/llarena.py
pypy/branch/minimark-free/pypy/rpython/memory/gc/minimarkpage.py
pypy/branch/minimark-free/pypy/rpython/memory/gc/test/test_minimarkpage.py
Log:
Tweak minimarkpage to free unused arenas.
Modified: pypy/branch/minimark-free/pypy/rpython/lltypesystem/llarena.py
==============================================================================
--- pypy/branch/minimark-free/pypy/rpython/lltypesystem/llarena.py (original)
+++ pypy/branch/minimark-free/pypy/rpython/lltypesystem/llarena.py Wed Sep 29 13:05:46 2010
@@ -124,6 +124,9 @@
assert self.usagemap[i] == 'x'
self.usagemap[i] = '#'
+ def mark_freed(self):
+ self.freed = True # this method is a hook for tests
+
class fakearenaaddress(llmemory.fakeaddress):
def __init__(self, arena, offset):
@@ -314,7 +317,7 @@
assert arena_addr.offset == 0
arena_addr.arena.reset(False)
assert not arena_addr.arena.objectptrs
- arena_addr.arena.freed = True
+ arena_addr.arena.mark_freed()
def arena_reset(arena_addr, size, zero):
"""Free all objects in the arena, which can then be reused.
Modified: pypy/branch/minimark-free/pypy/rpython/memory/gc/minimarkpage.py
==============================================================================
--- pypy/branch/minimark-free/pypy/rpython/memory/gc/minimarkpage.py (original)
+++ pypy/branch/minimark-free/pypy/rpython/memory/gc/minimarkpage.py Wed Sep 29 13:05:46 2010
@@ -4,15 +4,45 @@
from pypy.rlib.debug import ll_assert
WORD = LONG_BIT // 8
-WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT]
NULL = llmemory.NULL
+WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT]
+assert 1 << WORD_POWER_2 == WORD
-# Terminology: the memory is subdivided into "pages".
+# Terminology: the memory is subdivided into "arenas" containing "pages".
# A page contains a number of allocated objects, called "blocks".
-# The actual allocation occurs in whole arenas, which are subdivided
-# into pages. We don't keep track of the arenas. A page can be:
+# The actual allocation occurs in whole arenas, which are then subdivided
+# into pages. For each arena we allocate one of the following structures:
+
+ARENA_PTR = lltype.Ptr(lltype.ForwardReference())
+ARENA = lltype.Struct('ArenaReference',
+ # -- The address of the arena, as returned by malloc()
+ ('base', llmemory.Address),
+ # -- The number of free and the total number of pages in the arena
+ ('nfreepages', lltype.Signed),
+ ('totalpages', lltype.Signed),
+ # -- A chained list of free pages in the arena. Ends with NULL.
+ ('freepages', llmemory.Address),
+ # -- A linked list of arenas. See below.
+ ('nextarena', ARENA_PTR),
+ )
+ARENA_PTR.TO.become(ARENA)
+ARENA_NULL = lltype.nullptr(ARENA)
+
+# The idea is that when we need a free page, we take it from the arena
+# which currently has the *lowest* number of free pages. This allows
+# arenas with a lot of free pages to eventually become entirely free, at
+# which point they are returned to the OS. If an arena has a total of
+# 64 pages, then we have 64 global lists, arenas_lists[0] to
+# arenas_lists[63], such that arenas_lists[i] contains exactly those
+# arenas that have 'nfreepages == i'. We allocate pages out of the
+# arena in 'current_arena'; when it is exhausted we pick another arena
+# with the smallest value for nfreepages (but > 0).
+
+# ____________________________________________________________
+#
+# Each page in an arena can be:
#
# - uninitialized: never touched so far.
#
@@ -20,11 +50,12 @@
# PAGE_HEADER. The page is on the chained list of pages that still have
# room for objects of that size, unless it is completely full.
#
-# - free: used to be partially full, and is now free again. The page is
-# on the chained list of free pages.
+# - free. The page is on the chained list of free pages 'freepages' from
+# its arena.
-# Similarily, each allocated page contains blocks of a given size, which can
-# be either uninitialized, allocated or free.
+# Each allocated page contains blocks of a given size, which can be in
+# one of three states: allocated, free, or uninitialized. The uninitialized
+# blocks (initially all of them) are a tail of the page.
PAGE_PTR = lltype.Ptr(lltype.ForwardReference())
PAGE_HEADER = lltype.Struct('PageHeader',
@@ -32,13 +63,16 @@
# pages, it is a chained list of pages having the same size class,
# rooted in 'page_for_size[size_class]'. For full pages, it is a
# different chained list rooted in 'full_page_for_size[size_class]'.
+ # For free pages, it is the list 'freepages' in the arena header.
('nextpage', PAGE_PTR),
- # -- The number of free blocks, and the number of uninitialized blocks.
- # The number of allocated blocks is the rest.
- ('nuninitialized', lltype.Signed),
+ # -- The arena this page is part of.
+ ('arena', ARENA_PTR),
+ # -- The number of free blocks. The numbers of uninitialized and
+ # allocated blocks can be deduced from the context if needed.
('nfree', lltype.Signed),
- # -- The chained list of free blocks. If there are none, points to the
- # first uninitialized block.
+ # -- The chained list of free blocks. It ends as a pointer to the
+ # first uninitialized block (pointing to data that is uninitialized,
+ # or to the end of the page).
('freeblock', llmemory.Address),
# -- The structure above is 4 words, which is a good value:
# '(1024-4) % N' is zero or very small for various small N's,
@@ -72,13 +106,35 @@
self.nblocks_for_size = lltype.malloc(rffi.CArray(lltype.Signed),
length, flavor='raw')
self.hdrsize = llmemory.raw_malloc_usage(llmemory.sizeof(PAGE_HEADER))
+ assert page_size > self.hdrsize
self.nblocks_for_size[0] = 0 # unused
for i in range(1, length):
self.nblocks_for_size[i] = (page_size - self.hdrsize) // (WORD * i)
#
- self.uninitialized_pages = NULL
+ self.max_pages_per_arena = arena_size // page_size
+ self.arenas_lists = lltype.malloc(rffi.CArray(ARENA_PTR),
+ self.max_pages_per_arena,
+ flavor='raw', zero=True)
+ # this is used in mass_free() only
+ self.old_arenas_lists = lltype.malloc(rffi.CArray(ARENA_PTR),
+ self.max_pages_per_arena,
+ flavor='raw', zero=True)
+ #
+ # the arena currently consumed; it must have at least one page
+ # available, or be NULL. The arena object that we point to is
+ # not in any 'arenas_lists'. We will consume all its pages before
+ # we choose a next arena, even if there is a major collection
+ # in-between.
+ self.current_arena = ARENA_NULL
+ #
+ # guarantee that 'arenas_lists[1:min_empty_nfreepages]' are all empty
+ self.min_empty_nfreepages = self.max_pages_per_arena
+ #
+ # part of current_arena might still contain uninitialized pages
self.num_uninitialized_pages = 0
- self.free_pages = NULL
+ #
+ # the total memory used, counting every block in use, without
+ # the additional bookkeeping stuff.
self.total_memory_used = r_uint(0)
@@ -109,16 +165,12 @@
#
else:
# The 'result' is part of the uninitialized blocks.
- ll_assert(page.nuninitialized > 0,
- "fully allocated page found in the page_for_size list")
- page.nuninitialized -= 1
- if page.nuninitialized > 0:
- freeblock = result + nsize
- else:
- freeblock = NULL
+ freeblock = result + nsize
#
page.freeblock = freeblock
- if freeblock == NULL:
+ #
+ pageaddr = llarena.getfakearenaaddress(llmemory.cast_ptr_to_adr(page))
+ if freeblock - pageaddr > self.page_size - nsize:
# This was the last free block, so unlink the page from the
# chained list and put it in the 'full_page_for_size' list.
self.page_for_size[size_class] = page.nextpage
@@ -132,37 +184,83 @@
def allocate_new_page(self, size_class):
"""Allocate and return a new page for the given size_class."""
#
- if self.free_pages != NULL:
+ # Allocate a new arena if needed.
+ if self.current_arena == ARENA_NULL:
+ self.allocate_new_arena()
+ #
+ # The result is simply 'current_arena.freepages'.
+ arena = self.current_arena
+ result = arena.freepages
+ if arena.nfreepages > 0:
+ #
+ # The 'result' was part of the chained list; read the next.
+ arena.nfreepages -= 1
+ freepages = result.address[0]
+ llarena.arena_reset(result,
+ llmemory.sizeof(llmemory.Address),
+ 0)
#
- # Get the page from the chained list 'free_pages'.
- page = self.free_pages
- self.free_pages = page.address[0]
- llarena.arena_reset(page, llmemory.sizeof(llmemory.Address), 0)
else:
- # Get the next free page from the uninitialized pages.
- if self.num_uninitialized_pages == 0:
- self.allocate_new_arena() # Out of memory. Get a new arena.
- page = self.uninitialized_pages
- self.uninitialized_pages += self.page_size
+ # The 'result' is part of the uninitialized pages.
+ ll_assert(self.num_uninitialized_pages > 0,
+ "fully allocated arena found in self.current_arena")
self.num_uninitialized_pages -= 1
+ if self.num_uninitialized_pages > 0:
+ freepages = result + self.page_size
+ else:
+ freepages = NULL
#
- # Initialize the fields of the resulting page
- llarena.arena_reserve(page, llmemory.sizeof(PAGE_HEADER))
- result = llmemory.cast_adr_to_ptr(page, PAGE_PTR)
+ arena.freepages = freepages
+ if freepages == NULL:
+ # This was the last page, so put the arena away into
+ # arenas_lists[0].
+ arena.nextarena = self.arenas_lists[0]
+ self.arenas_lists[0] = arena
+ self.current_arena = ARENA_NULL
#
- result.nuninitialized = self.nblocks_for_size[size_class]
- result.nfree = 0
- result.freeblock = page + self.hdrsize
- result.nextpage = PAGE_NULL
+ # Initialize the fields of the resulting page
+ llarena.arena_reserve(result, llmemory.sizeof(PAGE_HEADER))
+ page = llmemory.cast_adr_to_ptr(result, PAGE_PTR)
+ page.arena = arena
+ page.nfree = 0
+ page.freeblock = result + self.hdrsize
+ page.nextpage = PAGE_NULL
ll_assert(self.page_for_size[size_class] == PAGE_NULL,
"allocate_new_page() called but a page is already waiting")
- self.page_for_size[size_class] = result
- return result
+ self.page_for_size[size_class] = page
+ return page
+
+
+ def _all_arenas(self):
+ """For testing. Enumerates all arenas."""
+ if self.current_arena:
+ yield self.current_arena
+ for arena in self.arenas_lists:
+ while arena:
+ yield arena
+ arena = arena.nextarena
def allocate_new_arena(self):
- ll_assert(self.num_uninitialized_pages == 0,
- "some uninitialized pages are already waiting")
+ """Return in self.current_arena the arena to allocate from next."""
+ #
+ # Pick an arena from 'arenas_lists[i]', with i as small as possible
+ # but > 0. Use caching with 'min_empty_nfreepages', which guarantees
+ # that 'arenas_lists[1:min_empty_nfreepages]' are all empty.
+ i = self.min_empty_nfreepages
+ while i < self.max_pages_per_arena:
+ #
+ if self.arenas_lists[i] != ARENA_NULL:
+ #
+ # Found it.
+ self.current_arena = self.arenas_lists[i]
+ self.arenas_lists[i] = self.current_arena.nextarena
+ return
+ #
+ i += 1
+ self.min_empty_nfreepages = i
+ #
+ # No more arena with any free page. We must allocate a new arena.
#
# 'arena_base' points to the start of malloced memory; it might not
# be a page-aligned address
@@ -177,13 +275,15 @@
# 'npages' is the number of full pages just allocated
npages = (arena_end - firstpage) // self.page_size
#
- # add these pages to the list
- self.uninitialized_pages = firstpage
+ # Allocate an ARENA object and initialize it
+ arena = lltype.malloc(ARENA, flavor='raw')
+ arena.base = arena_base
+ arena.nfreepages = 0 # they are all uninitialized pages
+ arena.totalpages = npages
+ arena.freepages = firstpage
self.num_uninitialized_pages = npages
+ self.current_arena = arena
#
- # increase a bit arena_size for the next time
- self.arena_size = (self.arena_size // 4 * 5) + (self.page_size - 1)
- self.arena_size = (self.arena_size // self.page_size) * self.page_size
allocate_new_arena._dont_inline_ = True
@@ -199,16 +299,51 @@
#
# Walk the pages in 'page_for_size[size_class]' and
# 'full_page_for_size[size_class]' and free some objects.
- # Pages completely freed are added to 'self.free_pages', and
- # become available for reuse by any size class. Pages not
- # completely freed are re-chained either in
+ # Pages completely freed are added to 'page.arena.freepages',
+ # and become available for reuse by any size class. Pages
+ # not completely freed are re-chained either in
# 'full_page_for_size[]' or 'page_for_size[]'.
- self.mass_free_in_page(size_class, ok_to_free_func)
+ self.mass_free_in_pages(size_class, ok_to_free_func)
#
size_class -= 1
+ #
+ # Rehash arenas into the correct arenas_lists[i]. If
+ # 'self.current_arena' contains an arena too, it remains there.
+ (self.old_arenas_lists, self.arenas_lists) = (
+ self.arenas_lists, self.old_arenas_lists)
+ #
+ i = 0
+ while i < self.max_pages_per_arena:
+ self.arenas_lists[i] = ARENA_NULL
+ i += 1
+ #
+ i = 0
+ while i < self.max_pages_per_arena:
+ arena = self.old_arenas_lists[i]
+ while arena != ARENA_NULL:
+ nextarena = arena.nextarena
+ #
+ if arena.nfreepages == arena.totalpages:
+ #
+ # The whole arena is empty. Free it.
+ llarena.arena_free(arena.base)
+ lltype.free(arena, flavor='raw')
+ #
+ else:
+ # Insert 'arena' in the correct arenas_lists[n]
+ n = arena.nfreepages
+ ll_assert(n < self.max_pages_per_arena,
+ "totalpages != nfreepages >= max_pages_per_arena")
+ arena.nextarena = self.arenas_lists[n]
+ self.arenas_lists[n] = arena
+ #
+ arena = nextarena
+ i += 1
+ #
+ self.min_empty_nfreepages = 1
- def mass_free_in_page(self, size_class, ok_to_free_func):
+ def mass_free_in_pages(self, size_class, ok_to_free_func):
nblocks = self.nblocks_for_size[size_class]
block_size = size_class * WORD
remaining_partial_pages = PAGE_NULL
@@ -224,8 +359,7 @@
while page != PAGE_NULL:
#
# Collect the page.
- surviving = self.walk_page(page, block_size,
- nblocks, ok_to_free_func)
+ surviving = self.walk_page(page, block_size, ok_to_free_func)
nextpage = page.nextpage
#
if surviving == nblocks:
@@ -259,19 +393,23 @@
def free_page(self, page):
"""Free a whole page."""
#
- # Done by inserting it in the 'free_pages' list.
+ # Insert the freed page in the arena's 'freepages' list.
+ # If nfreepages == totalpages, then it will be freed at the
+ # end of mass_free().
+ arena = page.arena
+ arena.nfreepages += 1
pageaddr = llmemory.cast_ptr_to_adr(page)
pageaddr = llarena.getfakearenaaddress(pageaddr)
llarena.arena_reset(pageaddr, self.page_size, 0)
llarena.arena_reserve(pageaddr, llmemory.sizeof(llmemory.Address))
- pageaddr.address[0] = self.free_pages
- self.free_pages = pageaddr
+ pageaddr.address[0] = arena.freepages
+ arena.freepages = pageaddr
- def walk_page(self, page, block_size, nblocks, ok_to_free_func):
+ def walk_page(self, page, block_size, ok_to_free_func):
"""Walk over all objects in a page, and ask ok_to_free_func()."""
#
- # 'freeblock' is the next free block, or NULL if there isn't any more.
+ # 'freeblock' is the next free block
freeblock = page.freeblock
#
# 'prevfreeblockat' is the address of where 'freeblock' was read from.
@@ -281,22 +419,28 @@
obj = llarena.getfakearenaaddress(llmemory.cast_ptr_to_adr(page))
obj += self.hdrsize
surviving = 0 # initially
+ skip_free_blocks = page.nfree
#
- nblocks -= page.nuninitialized
- index = nblocks
- while index > 0:
+ while True:
#
if obj == freeblock:
#
+ if skip_free_blocks == 0:
+ #
+ # 'obj' points to the first uninitialized block,
+ # or to the end of the page if there are none.
+ break
+ #
# 'obj' points to a free block. It means that
# 'prevfreeblockat.address[0]' does not need to be updated.
# Just read the next free block from 'obj.address[0]'.
+ skip_free_blocks -= 1
prevfreeblockat = obj
freeblock = obj.address[0]
#
else:
# 'obj' points to a valid object.
- ll_assert(not freeblock or freeblock > obj,
+ ll_assert(freeblock > obj,
"freeblocks are linked out of order")
#
if ok_to_free_func(obj):
@@ -310,15 +454,14 @@
prevfreeblockat = obj
obj.address[0] = freeblock
#
+ # Update the number of free objects in the page.
+ page.nfree += 1
+ #
else:
# The object survives.
surviving += 1
#
obj += block_size
- index -= 1
- #
- # Update the number of free objects in the page.
- page.nfree = nblocks - surviving
#
# Update the global total size of objects.
self.total_memory_used += surviving * block_size
@@ -327,6 +470,20 @@
return surviving
+ def _nuninitialized(self, page, size_class):
+ # Helper for debugging: count the number of uninitialized blocks
+ freeblock = page.freeblock
+ for i in range(page.nfree):
+ freeblock = freeblock.address[0]
+ assert freeblock != NULL
+ pageaddr = llarena.getfakearenaaddress(llmemory.cast_ptr_to_adr(page))
+ num_initialized_blocks, rem = divmod(
+ freeblock - pageaddr - self.hdrsize, size_class * WORD)
+ assert rem == 0, "page size_class misspecified?"
+ nblocks = self.nblocks_for_size[size_class]
+ return nblocks - num_initialized_blocks
+
+
# ____________________________________________________________
# Helpers to go from a pointer to the start of its page
Modified: pypy/branch/minimark-free/pypy/rpython/memory/gc/test/test_minimarkpage.py
==============================================================================
--- pypy/branch/minimark-free/pypy/rpython/memory/gc/test/test_minimarkpage.py (original)
+++ pypy/branch/minimark-free/pypy/rpython/memory/gc/test/test_minimarkpage.py Wed Sep 29 13:05:46 2010
@@ -12,17 +12,19 @@
def test_allocate_arena():
- ac = ArenaCollection(SHIFT + 16*20, 16, 1)
+ ac = ArenaCollection(SHIFT + 64*20, 64, 1)
ac.allocate_new_arena()
assert ac.num_uninitialized_pages == 20
- ac.uninitialized_pages + 16*20 # does not raise
- py.test.raises(llarena.ArenaError, "ac.uninitialized_pages + 16*20 + 1")
+ upages = ac.current_arena.freepages
+ upages + 64*20 # does not raise
+ py.test.raises(llarena.ArenaError, "upages + 64*20 + 1")
#
- ac = ArenaCollection(SHIFT + 16*20 + 7, 16, 1)
+ ac = ArenaCollection(SHIFT + 64*20 + 7, 64, 1)
ac.allocate_new_arena()
assert ac.num_uninitialized_pages == 20
- ac.uninitialized_pages + 16*20 + 7 # does not raise
- py.test.raises(llarena.ArenaError, "ac.uninitialized_pages + 16*20 + 16")
+ upages = ac.current_arena.freepages
+ upages + 64*20 + 7 # does not raise
+ py.test.raises(llarena.ArenaError, "upages + 64*20 + 64")
def test_allocate_new_page():
@@ -31,7 +33,8 @@
#
def checknewpage(page, size_class):
size = WORD * size_class
- assert page.nuninitialized == (pagesize - hdrsize) // size
+ assert (ac._nuninitialized(page, size_class) ==
+ (pagesize - hdrsize) // size)
assert page.nfree == 0
page1 = page.freeblock - hdrsize
assert llmemory.cast_ptr_to_adr(page) == page1
@@ -44,13 +47,13 @@
page = ac.allocate_new_page(5)
checknewpage(page, 5)
assert ac.num_uninitialized_pages == 2
- assert ac.uninitialized_pages - pagesize == cast_ptr_to_adr(page)
+ assert ac.current_arena.freepages - pagesize == cast_ptr_to_adr(page)
assert ac.page_for_size[5] == page
#
page = ac.allocate_new_page(3)
checknewpage(page, 3)
assert ac.num_uninitialized_pages == 1
- assert ac.uninitialized_pages - pagesize == cast_ptr_to_adr(page)
+ assert ac.current_arena.freepages - pagesize == cast_ptr_to_adr(page)
assert ac.page_for_size[3] == page
#
page = ac.allocate_new_page(4)
@@ -71,17 +74,17 @@
page = llmemory.cast_adr_to_ptr(pageaddr, PAGE_PTR)
if step == 1:
page.nfree = 0
- page.nuninitialized = nblocks - nusedblocks
+ nuninitialized = nblocks - nusedblocks
else:
page.nfree = nusedblocks
- page.nuninitialized = nblocks - 2*nusedblocks
+ nuninitialized = nblocks - 2*nusedblocks
+ page.freeblock = pageaddr + hdrsize + nusedblocks * size_block
if nusedblocks < nblocks:
- page.freeblock = pageaddr + hdrsize + nusedblocks * size_block
chainedlists = ac.page_for_size
else:
- page.freeblock = NULL
chainedlists = ac.full_page_for_size
page.nextpage = chainedlists[size_class]
+ page.arena = ac.current_arena
chainedlists[size_class] = page
if fill_with_objects:
for i in range(0, nusedblocks*step, step):
@@ -98,11 +101,15 @@
prev = 'prevhole.address[0]'
endaddr = pageaddr + hdrsize + 2*nusedblocks * size_block
exec '%s = endaddr' % prev in globals(), locals()
+ assert ac._nuninitialized(page, size_class) == nuninitialized
#
ac.allocate_new_arena()
num_initialized_pages = len(pagelayout.rstrip(" "))
- ac._startpageaddr = ac.uninitialized_pages
- ac.uninitialized_pages += pagesize * num_initialized_pages
+ ac._startpageaddr = ac.current_arena.freepages
+ if pagelayout.endswith(" "):
+ ac.current_arena.freepages += pagesize * num_initialized_pages
+ else:
+ ac.current_arena.freepages = NULL
ac.num_uninitialized_pages -= num_initialized_pages
#
for i in reversed(range(num_initialized_pages)):
@@ -115,8 +122,9 @@
link(pageaddr, size_class, size_block, nblocks, nblocks-1)
elif c == '.': # a free, but initialized, page
llarena.arena_reserve(pageaddr, llmemory.sizeof(llmemory.Address))
- pageaddr.address[0] = ac.free_pages
- ac.free_pages = pageaddr
+ pageaddr.address[0] = ac.current_arena.freepages
+ ac.current_arena.freepages = pageaddr
+ ac.current_arena.nfreepages += 1
elif c == '#': # a random full page, in the list 'full_pages'
size_class = fill_with_objects or 1
size_block = WORD * size_class
@@ -142,26 +150,29 @@
def checkpage(ac, page, expected_position):
assert llmemory.cast_ptr_to_adr(page) == pagenum(ac, expected_position)
+def freepages(ac):
+ return ac.current_arena.freepages
+
def test_simple_arena_collection():
pagesize = hdrsize + 16
ac = arena_collection_for_test(pagesize, "##....# ")
#
- assert ac.free_pages == pagenum(ac, 2)
+ assert freepages(ac) == pagenum(ac, 2)
page = ac.allocate_new_page(1); checkpage(ac, page, 2)
- assert ac.free_pages == pagenum(ac, 3)
+ assert freepages(ac) == pagenum(ac, 3)
page = ac.allocate_new_page(2); checkpage(ac, page, 3)
- assert ac.free_pages == pagenum(ac, 4)
+ assert freepages(ac) == pagenum(ac, 4)
page = ac.allocate_new_page(3); checkpage(ac, page, 4)
- assert ac.free_pages == pagenum(ac, 5)
+ assert freepages(ac) == pagenum(ac, 5)
page = ac.allocate_new_page(4); checkpage(ac, page, 5)
- assert ac.free_pages == NULL and ac.num_uninitialized_pages == 3
+ assert freepages(ac) == pagenum(ac, 7) and ac.num_uninitialized_pages == 3
page = ac.allocate_new_page(5); checkpage(ac, page, 7)
- assert ac.free_pages == NULL and ac.num_uninitialized_pages == 2
+ assert freepages(ac) == pagenum(ac, 8) and ac.num_uninitialized_pages == 2
page = ac.allocate_new_page(6); checkpage(ac, page, 8)
- assert ac.free_pages == NULL and ac.num_uninitialized_pages == 1
+ assert freepages(ac) == pagenum(ac, 9) and ac.num_uninitialized_pages == 1
page = ac.allocate_new_page(7); checkpage(ac, page, 9)
- assert ac.free_pages == NULL and ac.num_uninitialized_pages == 0
+ assert not ac.current_arena and ac.num_uninitialized_pages == 0
def chkob(ac, num_page, pos_obj, obj):
@@ -205,47 +216,47 @@
ac = arena_collection_for_test(pagesize, "/.", fill_with_objects=2)
page = getpage(ac, 0)
assert page.nfree == 3
- assert page.nuninitialized == 3
+ assert ac._nuninitialized(page, 2) == 3
chkob(ac, 0, 2*WORD, page.freeblock)
#
obj = ac.malloc(2*WORD); chkob(ac, 0, 2*WORD, obj)
obj = ac.malloc(2*WORD); chkob(ac, 0, 6*WORD, obj)
assert page.nfree == 1
- assert page.nuninitialized == 3
+ assert ac._nuninitialized(page, 2) == 3
chkob(ac, 0, 10*WORD, page.freeblock)
#
obj = ac.malloc(2*WORD); chkob(ac, 0, 10*WORD, obj)
assert page.nfree == 0
- assert page.nuninitialized == 3
+ assert ac._nuninitialized(page, 2) == 3
chkob(ac, 0, 12*WORD, page.freeblock)
#
obj = ac.malloc(2*WORD); chkob(ac, 0, 12*WORD, obj)
- assert page.nuninitialized == 2
+ assert ac._nuninitialized(page, 2) == 2
obj = ac.malloc(2*WORD); chkob(ac, 0, 14*WORD, obj)
obj = ac.malloc(2*WORD); chkob(ac, 0, 16*WORD, obj)
assert page.nfree == 0
- assert page.nuninitialized == 0
+ assert ac._nuninitialized(page, 2) == 0
obj = ac.malloc(2*WORD); chkob(ac, 1, 0*WORD, obj)
def test_malloc_new_arena():
pagesize = hdrsize + 7*WORD
ac = arena_collection_for_test(pagesize, "### ")
+ arena_size = ac.arena_size
obj = ac.malloc(2*WORD); chkob(ac, 3, 0*WORD, obj) # 3rd page -> size 2
#
del ac.allocate_new_arena # restore the one from the class
- arena_size = ac.arena_size
obj = ac.malloc(3*WORD) # need a new arena
assert ac.num_uninitialized_pages == (arena_size // ac.page_size
- - 1 # for start_of_page()
- 1 # the just-allocated page
)
class OkToFree(object):
- def __init__(self, ac, answer):
+ def __init__(self, ac, answer, multiarenas=False):
assert callable(answer) or 0.0 <= answer <= 1.0
self.ac = ac
self.answer = answer
+ self.multiarenas = multiarenas
self.lastnum = 0.0
self.seen = {}
@@ -257,7 +268,10 @@
ok_to_free = self.lastnum >= 1.0
if ok_to_free:
self.lastnum -= 1.0
- key = addr - self.ac._startpageaddr
+ if self.multiarenas:
+ key = (addr.arena, addr.offset)
+ else:
+ key = addr - self.ac._startpageaddr
assert key not in self.seen
self.seen[key] = ok_to_free
return ok_to_free
@@ -272,10 +286,10 @@
page = getpage(ac, 0)
assert page == ac.page_for_size[2]
assert page.nextpage == PAGE_NULL
- assert page.nuninitialized == 1
+ assert ac._nuninitialized(page, 2) == 1
assert page.nfree == 0
chkob(ac, 0, 4*WORD, page.freeblock)
- assert ac.free_pages == NULL
+ assert freepages(ac) == NULL
def test_mass_free_emptied_page():
pagesize = hdrsize + 7*WORD
@@ -285,7 +299,7 @@
assert ok_to_free.seen == {hdrsize + 0*WORD: True,
hdrsize + 2*WORD: True}
pageaddr = pagenum(ac, 0)
- assert pageaddr == ac.free_pages
+ assert pageaddr == freepages(ac)
assert pageaddr.address[0] == NULL
assert ac.page_for_size[2] == PAGE_NULL
@@ -300,10 +314,9 @@
page = getpage(ac, 0)
assert page == ac.full_page_for_size[2]
assert page.nextpage == PAGE_NULL
- assert page.nuninitialized == 0
+ assert ac._nuninitialized(page, 2) == 0
assert page.nfree == 0
- assert page.freeblock == NULL
- assert ac.free_pages == NULL
+ assert freepages(ac) == NULL
assert ac.page_for_size[2] == PAGE_NULL
def test_mass_free_full_is_partially_emptied():
@@ -319,19 +332,19 @@
pageaddr = pagenum(ac, 0)
assert page == ac.page_for_size[2]
assert page.nextpage == PAGE_NULL
- assert page.nuninitialized == 0
+ assert ac._nuninitialized(page, 2) == 0
assert page.nfree == 2
assert page.freeblock == pageaddr + hdrsize + 2*WORD
assert page.freeblock.address[0] == pageaddr + hdrsize + 6*WORD
- assert page.freeblock.address[0].address[0] == NULL
- assert ac.free_pages == NULL
+ assert page.freeblock.address[0].address[0] == pageaddr + hdrsize + 8*WORD
+ assert freepages(ac) == NULL
assert ac.full_page_for_size[2] == PAGE_NULL
def test_mass_free_half_page_remains():
pagesize = hdrsize + 24*WORD
ac = arena_collection_for_test(pagesize, "/", fill_with_objects=2)
page = getpage(ac, 0)
- assert page.nuninitialized == 4
+ assert ac._nuninitialized(page, 2) == 4
assert page.nfree == 4
#
ok_to_free = OkToFree(ac, False)
@@ -344,7 +357,7 @@
pageaddr = pagenum(ac, 0)
assert page == ac.page_for_size[2]
assert page.nextpage == PAGE_NULL
- assert page.nuninitialized == 4
+ assert ac._nuninitialized(page, 2) == 4
assert page.nfree == 4
assert page.freeblock == pageaddr + hdrsize + 2*WORD
assert page.freeblock.address[0] == pageaddr + hdrsize + 6*WORD
@@ -352,14 +365,14 @@
pageaddr + hdrsize + 10*WORD
assert page.freeblock.address[0].address[0].address[0] == \
pageaddr + hdrsize + 14*WORD
- assert ac.free_pages == NULL
+ assert freepages(ac) == NULL
assert ac.full_page_for_size[2] == PAGE_NULL
def test_mass_free_half_page_becomes_more_free():
pagesize = hdrsize + 24*WORD
ac = arena_collection_for_test(pagesize, "/", fill_with_objects=2)
page = getpage(ac, 0)
- assert page.nuninitialized == 4
+ assert ac._nuninitialized(page, 2) == 4
assert page.nfree == 4
#
ok_to_free = OkToFree(ac, 0.5)
@@ -372,7 +385,7 @@
pageaddr = pagenum(ac, 0)
assert page == ac.page_for_size[2]
assert page.nextpage == PAGE_NULL
- assert page.nuninitialized == 4
+ assert ac._nuninitialized(page, 2) == 4
assert page.nfree == 6
fb = page.freeblock
assert fb == pageaddr + hdrsize + 2*WORD
@@ -384,7 +397,7 @@
pageaddr + hdrsize + 12*WORD
assert fb.address[0].address[0].address[0].address[0].address[0] == \
pageaddr + hdrsize + 14*WORD
- assert ac.free_pages == NULL
+ assert freepages(ac) == NULL
assert ac.full_page_for_size[2] == PAGE_NULL
# ____________________________________________________________
@@ -392,17 +405,29 @@
def test_random():
import random
pagesize = hdrsize + 24*WORD
- num_pages = 28
+ num_pages = 3
ac = arena_collection_for_test(pagesize, " " * num_pages)
live_objects = {}
#
- # Run the test until ac.allocate_new_arena() is called.
+ # Run the test until three arenas are freed. This is a quick test
+ # that the arenas are really freed by the logic.
class DoneTesting(Exception):
- pass
- def done_testing():
- raise DoneTesting
- ac.allocate_new_arena = done_testing
- #
+ counter = 0
+ def my_allocate_new_arena():
+ # the following output looks cool on a 112-character-wide terminal.
+ lst = sorted(ac._all_arenas(), key=lambda a: a.base.arena._arena_index)
+ for a in lst:
+ print a.base.arena, a.base.arena.usagemap
+ print '-' * 80
+ ac.__class__.allocate_new_arena(ac)
+ a = ac.current_arena.base.arena
+ def my_mark_freed():
+ a.freed = True
+ DoneTesting.counter += 1
+ if DoneTesting.counter > 3:
+ raise DoneTesting
+ a.mark_freed = my_mark_freed
+ ac.allocate_new_arena = my_allocate_new_arena
try:
while True:
#
@@ -410,12 +435,13 @@
for i in range(random.randrange(50, 100)):
size_class = random.randrange(1, 7)
obj = ac.malloc(size_class * WORD)
- at = obj - ac._startpageaddr
+ at = (obj.arena, obj.offset)
assert at not in live_objects
live_objects[at] = size_class * WORD
#
# Free half the objects, randomly
- ok_to_free = OkToFree(ac, lambda obj: random.random() < 0.5)
+ ok_to_free = OkToFree(ac, lambda obj: random.random() < 0.5,
+ multiarenas=True)
ac.mass_free(ok_to_free)
#
# Check that we have seen all objects
@@ -428,5 +454,4 @@
surviving_total_size += live_objects[at]
assert ac.total_memory_used == surviving_total_size
except DoneTesting:
- # the following output looks cool on a 112-character-wide terminal.
- print ac._startpageaddr.arena.usagemap
+ pass
More information about the Pypy-commit
mailing list