[SciPy-dev] reimplementation of lfilter

[Sturla Molden]

Charles R Harris skrev:
> It might be useful to think up some way that setjmp/longjmp could be 
> used in some sort of standard error handling template.
>

One way is to use a resource pool (a la Apache), see Cython code below.

cdef int _foobar() nogil:
    cdef  mempool *mp
    cdef int rv
    mp = mempool_init()
    if (mp == NULL):
        return -1 # error
    else:
        some_C_function(mp)
        mempool_destroy(mp)
        return int(rv)

def foobar():
    if (_foobar() < 0):
        raise MemoryError, 'malloc failed'

This alleviates any need for error checking on the return value from 
mempool_malloc. It might not be obvious though. What happens is that a 
NULL return form malloc trigger a longjmp back to mempool_init, after 
which the memory error is raised. Anything left allocated by the pool is 
freed on mempool_destroy, so it cannot leak. Pools can be used to manage 
any type of resource, e.g. open file handles, not just memory. The 
advantage is that you get C code that don't have to do error checking 
everywhere.

The problem with Cython is that a longjmp can mess up refcounts, so it 
probably should only be used in pure C mode, i.e. in a function safe to 
call with nogil.

C++ exceptions are much cleaner than C resource pools though, for 
example because destructors are called automatically. And the fact that 
C++ can put objects on the stack, and references can be used instead of 
pointers, means it is very safe against resource leaks if used 
correctly. The problems people have with C++ comes from bad style, for 
example calling new outside a constructor, using new[] instead of 
std::vector, using pointers instead of references (a reference cannot be 
dangling), etc.


Sturla Molden




# memory pool
# Copyright (C) 2009 Sturla Molden

import numpy as np
cimport numpy

cdef extern from "setjmp.h":
    ctypedef struct jmp_buf:
        pass
    int setjmp(jmp_buf jb) nogil
    void longjmp(jmp_buf jb, int errcode) nogil
  
cdef extern from "stdlib.h":
    # Assume malloc, realloc and free are thread safe
    # We may need to change this, in which case a lock
    # is needed in the memory pool.
    ctypedef unsigned long size_t
    void free(void *ptr) nogil
    void *malloc(size_t size) nogil
    void *realloc(void *ptr, size_t size) nogil
  
cdef struct mempool:
    mempool *prev
    mempool *next
  
cdef public mempool *mempool_init() nogil:
    cdef jmp_buf *jb
    cdef mempool *p = <mempool *> malloc(sizeof(jmp_buf) + sizeof(mempool))
    if not p: return NULL
    jb = <jmp_buf*>(<np.npy_intp>p + sizeof(mempool))
    p.prev = NULL
    p.next = NULL
    if setjmp(jb[0]):
        # mempool_error has been called
        mempool_destroy(p)
        return NULL
    else:
        return p
  
cdef public void mempool_error(mempool *p) nogil:
    # rewind stack back to mempool_init()
    cdef jmp_buf *jb = <jmp_buf*>(<np.npy_intp>p + sizeof(mempool))
    longjmp(jb[0], 1)

cdef public void *mempool_destroy(mempool *p) nogil:
    # this releases all memory allocated to this pool
    cdef mempool *tmp
    while p:
        tmp = p
        p = p.next
        free(tmp)

cdef public void *mempool_malloc(mempool *p, size_t n) nogil:
    cdef mempool *block
    cdef void *m = malloc(sizeof(mempool) + n)
    if not m:
        mempool_error(p)
    block = <mempool*> m
    block.next = p.next
    if block.next:
        block.next.prev = block
    block.prev = p
    p.next = block
    return <void*>(<np.npy_intp>m + sizeof(mempool))

cdef public void *mempool_realloc(mempool *p, void *ptr, size_t n) nogil:
    cdef void *retval
    cdef mempool *block, *prev, *next, *new_addr
    if not n:
        # realloc(p, 0) is free(p)
        mempool_free(ptr)
        retval = NULL
    elif not ptr:
        # realloc(0, n) is malloc(n)
        retval = mempool_malloc(p,n)
    else:
        # realloc(p, n)
        block = <mempool *>(<np.npy_intp>ptr - sizeof(mempool))
        prev = block.prev
        next = block.next
        new_addr = <mempool*> realloc(block, n + sizeof(mempool))
        if not new_addr:
            mempool_error(p)
        if new_addr != block:
            prev.next = new_addr
            if next:
                next.prev = new_addr
            new_addr.prev = prev
            new_addr.next = next
        retval = <void *>(<np.npy_intp>new_addr + sizeof(mempool))
    return retval

cdef public void *mempool_free(void *ptr) nogil:
    cdef mempool *prev, *next, *cur
    cdef mempool *block = <mempool *>(<np.npy_intp>ptr - sizeof(mempool))
    prev = block.prev
    next = block.next
    if next:
        free(block)
        next.prev = prev
        prev.next = next
    else:
        free(block)
        prev.next = NULL