[pypy-commit] pypy counter-decay: Not tested: clean up the jitcell_dicts that are used internally. This

[arigo]

Author: Armin Rigo <arigo at tunes.org>
Branch: counter-decay
Changeset: r50540:6e5fb0038df9
Date: 2011-12-15 01:01 +0100
http://bitbucket.org/pypy/pypy/changeset/6e5fb0038df9/

Log:	Not tested: clean up the jitcell_dicts that are used internally.
	This is probably important to ensure that a long-running process
	cannot "leak" arbitrary amounts of memory due to having more and
	more JitCells stored in these big dicts.

diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -38,6 +38,7 @@
         self.current_generation = r_int64(1)
         self.next_check = r_int64(-1)
         self.alive_loops = {}
+        self._cleanup_jitcell_dicts = lambda: None
 
     def set_max_age(self, max_age, check_frequency=0):
         if max_age <= 0:
@@ -53,6 +54,7 @@
         self.current_generation += 1
         if self.current_generation == self.next_check:
             self._kill_old_loops_now()
+            self._cleanup_jitcell_dicts()
             self.next_check = self.current_generation + self.check_frequency
 
     def keep_loop_alive(self, looptoken):
@@ -86,3 +88,27 @@
         """Return the current generation, possibly truncated to a uint.
         To use only as an approximation for decaying counters."""
         return r_uint(self.current_generation)
+
+    def record_jitcell_dict(self, warmstate, jitcell_dict):
+        """NOT_RPYTHON.  The given jitcell_dict is a dict that needs
+        occasional clean-ups of old cells.  A cell is old if it never
+        reached the threshold, and its counter decayed to a tiny value."""
+        # note that the various jitcell_dicts have different RPython types,
+        # so we have to make a different function for each one.  These
+        # functions are chained to each other: each calls the previous one.
+        def cleanup_dict():
+            minimum = min(warmstate.increment_threshold,
+                          warmstate.increment_function_threshold)
+            current = self.get_current_generation_uint()
+            killme = []
+            for key, cell in jitcell_dict.iteritems():
+                if cell.counter >= 0:
+                    cell.adjust_counter(current, warmstate.log_decay_factor)
+                    if cell.counter < minimum:
+                        killme.append(key)
+            for key in killme:
+                del jitcell_dict[key]
+            cleanup_previous()
+        #
+        cleanup_previous = self._cleanup_jitcell_dicts
+        self._cleanup_jitcell_dicts = cleanup_dict
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -190,7 +190,6 @@
 
 class WarmEnterState(object):
     THRESHOLD_LIMIT = sys.maxint // 2
-    default_jitcell_dict = None
 
     def __init__(self, warmrunnerdesc, jitdriver_sd):
         "NOT_RPYTHON"
@@ -491,6 +490,9 @@
             self.warmrunnerdesc.stats.jitcell_dicts.append(jitcell_dict)
         except AttributeError:
             pass
+        memmgr = self.warmrunnerdesc.memory_manager
+        if memmgr:
+            memmgr.record_jitcell_dict(self, jitcell_dict)
         #
         def get_jitcell(build, *greenargs):
             try:
@@ -509,6 +511,10 @@
         get_jitcell_at_ptr = self.jitdriver_sd._get_jitcell_at_ptr
         set_jitcell_at_ptr = self.jitdriver_sd._set_jitcell_at_ptr
         lltohlhack = {}
+        # note that there is no equivalent of record_jitcell_dict()
+        # in the case of custom getters.  We assume that the interpreter
+        # stores the JitCells on some objects that can go away by GC,
+        # like the PyCode objects in PyPy.
         #
         def get_jitcell(build, *greenargs):
             fn = support.maybe_on_top_of_llinterp(rtyper, get_jitcell_at_ptr)