Pyrex and Distuils: an enhanced build_ext command

[Graham Fawcett]

I wrote:
> I suppose that one could add a distutils.command.pyrexc as a
> subcommand of "build_ext" to handle the .pyx compilation. That way the
> os.system() call in the build script could be avoided. I've never
> written a distutils command, but perhaps I'll give that a try.

Okay, I wrote the distutils command. Actually I just added a
subpackage to Pyrex that you can import into your setup script:

   from distutils.core import setup
   from distutils.extension import Extension
   import Pyrex.Distutils

   setup(name='foo', ext_modules=[Extension("foo", ["foo.pyx"])])

Note that the Extension() specifies a .pyx file, not a .c file. You
can specify .c files as well, but of course Pyrex will ignore them.

Pyrex.Distuils (my subpackage) is a build_ext replacement -- it
import-hacks itself into the distutils.command package, so that it
gets a first shot at any build_ext calls. Once it's finished, it
returns control to the "real" build_ext.

I tested on Win32 only, with mingw32 and msvc compilers, but it should
run fine on other platforms.

I've put a modified distribution of Pyrex 0.3.3 at 
   http://cfl-x.uwindsor.ca/graham/pyrex/ if anyone wants to try it
out. Here are the two key source files:

###########################################
# Pyrex.Distutils.__init__.py
# July 2002, Graham Fawcett
#
# this hack was inspired by the way Thomas Heller got py2exe
# to appear as a distutil command
#
# we replace distutils.command.build_ext with our own version
# and keep the old one under the module name _build_ext,
# so that *our* build_ext can make use of it.

import sys
import distutils.command.build_ext
sys.modules['_build_ext'] = distutils.command.build_ext
import build_ext
sys.modules['distutils.command.build_ext']=build_ext

###########################################
# Pyrex.Distutils.build_ext.py
"""
Subclasses disutils.command.build_ext,
replacing it with a Pyrex version that compiles pyx->c
before calling the original build_ext command.
"""

# July 2002, Graham Fawcett
# Pyrex is (c) Greg Ewing.

import Pyrex.Compiler.Main
from Pyrex.Compiler.Errors import PyrexError
from distutils.dep_util import newer
import os
import sys

# in Distutils.__init__, the original build_ext
# is inserted into sys.modules as _build_ext
import _build_ext

class build_ext (_build_ext.build_ext):

    description = "compile Pyrex scripts, then build C/C++ extensions
(compile/link to build directory)"

    def run (self):
        if not self.extensions:
            return

        # collect the names of the source (.pyx) files
        pyx_sources = []
        for e in self.extensions:
            pyx_sources.extend(e.sources)
        pyx_sources = [source for source in pyx_sources if
source.endswith('.pyx')]

        for pyx in pyx_sources:
            # should I raise an exception if it doesn't exist?
            if os.path.exists(pyx):
                source = pyx
                target = source.replace('.pyx', '.c')
                if newer(source, target) or self.force:
                    self.pyrex_compile(source)

                    # compiling with mingw32 gets an "initializer not
a constant" error
                    #
http://www.python.org/cgi-bin/faqw.py?req=show&file=faq03.024.htp
                    # doesn't appear to happen with MSVC
                    # so if we are compiling with mingw32, massage the
Pyrex-generated
                    # C files to compile properly

                    if self.compiler=='mingw32':
                        self.fix_c_file(target)


        # correct the extension sourcefile names in setup script
        # so that .pyx files are refered to as .c files
        # then we can call the "real" build_ext on them.
        for e in self.extensions:
            e.sources = [src.replace('.pyx', '.c') for src in
e.sources]

        # run the original build_ext.
        _build_ext.build_ext.run(self)

    def pyrex_compile(self, source):
        try:
            Pyrex.Compiler.Main.compile(source, c_only=1,
use_listing_file=0)
        except PyrexError, e:
            print e
            sys.exit(1)

    def fix_c_file(self, filename):
        print 'fixing ' + filename
        HEAD_INIT = 'PyObject_HEAD_INIT(0)'
        TYPESTART = 'statichere PyTypeObject __pyx_type_'
        GETATTR = 'PyObject_GenericGetAttr, /*tp_getattro*/'
        SETATTR = 'PyObject_GenericSetAttr, /*tp_setattro*/'
        ALLOC = 'PyType_GenericAlloc, /*tp_alloc*/'
        FREE = '_PyObject_Del, /*tp_free*/'
        TYPEINIT ='''__pyx_type_%(typename)s.ob_type = &PyType_Type;
        __pyx_type_%(typename)s.tp_getattro = PyObject_GenericGetAttr;
/*tp_getattro*/
        __pyx_type_%(typename)s.tp_setattro = PyObject_GenericSetAttr;
/*tp_setattro*/
        __pyx_type_%(typename)s.tp_alloc = PyType_GenericAlloc;
/*tp_alloc*/
        __pyx_type_%(typename)s.tp_free = _PyObject_Del; /*tp_free*/
        '''
        f = open(filename, 'r')
        lines = f.readlines()
        f.close()
        lines = [line.rstrip() for line in lines]
        types = []
        f = open(filename, 'w')
        for line in lines:
            if line.startswith(HEAD_INIT):
                print >> f,  'PyObject_HEAD_INIT(NULL)'
            elif line.startswith(TYPESTART):
                typename = line[len(TYPESTART):].split(' ')[0]
                types.append(typename)
                print >> f, line
            elif line.startswith(GETATTR):
                print >> f, 'NULL,' + GETATTR.split(',')[1]
            elif line.startswith(SETATTR):
                print >> f, 'NULL,' + SETATTR.split(',')[1]
            elif line.startswith(ALLOC):
                print >> f, 'NULL,' + ALLOC.split(',')[1]
            elif line.startswith(FREE):
                print >> f, 'NULL,' + FREE.split(',')[1]
            elif line.startswith('void init') and
line.endswith('(void) {'):
                print >> f, line
                for typename in types:
                    print >> f, TYPEINIT % locals()
            else:
                pass
                print >> f, line
        f.close()

###########################################

This was fun to do! I hope that it will be useful to others. Having
read through the Distutils source, I have a new and profound respect
for its authors.


Regards,

-- Graham